From f39e512990469ab9a84f41aa59f93ddc3e864692 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Mon, 4 May 2026 14:13:01 -0400 Subject: [PATCH 01/17] feat(web): split Queued column out of Initializing on the run board Submitted and Queued lifecycle statuses now live in a dedicated Queued column rendered to the left of Initializing; Starting stays in Initializing. The column is omitted from the board when it has no items so day-to-day boards stay compact. Co-Authored-By: Claude Opus 4.7 (1M context) --- apps/fabro-web/app/data/runs.ts | 4 ++- apps/fabro-web/app/routes/runs.test.tsx | 1 + apps/fabro-web/app/routes/runs.tsx | 9 ++++-- docs/public/api-reference/fabro-api.yaml | 1 + lib/crates/fabro-server/src/demo/mod.rs | 30 +++++++++++++++++-- .../fabro-server/src/server/handler/runs.rs | 4 ++- lib/crates/fabro-server/src/server/tests.rs | 4 +-- .../src/models/board-column.ts | 1 + 8 files changed, 46 insertions(+), 8 deletions(-) diff --git a/apps/fabro-web/app/data/runs.ts b/apps/fabro-web/app/data/runs.ts index 3922b7cd3..77d8a4711 100644 --- a/apps/fabro-web/app/data/runs.ts +++ b/apps/fabro-web/app/data/runs.ts @@ -37,9 +37,10 @@ export interface RunItem { sourceDirectory?: string; } -export type ColumnStatus = "initializing" | "running" | "blocked" | "succeeded" | "failed"; +export type ColumnStatus = "queued" | "initializing" | "running" | "blocked" | "succeeded" | "failed"; export const columnStatusDisplay: Record = { + queued: { label: "Queued", dot: "bg-fg-muted", text: "text-fg-muted" }, initializing: { label: "Initializing", dot: "bg-amber", text: "text-amber" }, running: { label: "Running", dot: "bg-teal-500", text: "text-teal-500" }, blocked: { label: "Blocked", dot: "bg-amber", text: "text-amber" }, @@ -113,6 +114,7 @@ export function columnForStatus(status: ApiRunStatus | null | undefined): Column switch (status?.kind) { case "submitted": case "queued": + return "queued"; case "starting": return "initializing"; case "running": diff --git a/apps/fabro-web/app/routes/runs.test.tsx b/apps/fabro-web/app/routes/runs.test.tsx index 948900769..3cd9dec0a 100644 --- a/apps/fabro-web/app/routes/runs.test.tsx +++ b/apps/fabro-web/app/routes/runs.test.tsx @@ -25,6 +25,7 @@ describe("runs route board mapping", () => { test("keeps blocked runs in the blocked lane and preserves question text", () => { const columns = buildBoardColumns({ columns: [ + { id: "queued", name: "Queued" }, { id: "initializing", name: "Initializing" }, { id: "running", name: "Running" }, { id: "blocked", name: "Blocked" }, diff --git a/apps/fabro-web/app/routes/runs.tsx b/apps/fabro-web/app/routes/runs.tsx index f46c19396..90ef09405 100644 --- a/apps/fabro-web/app/routes/runs.tsx +++ b/apps/fabro-web/app/routes/runs.tsx @@ -38,6 +38,7 @@ interface ColumnStyle { } const columnStyles: Record = { + queued: { iconType: "branch", actions: [] }, initializing: { iconType: "branch", actions: [] }, running: { iconType: "branch", actions: ["Watch", "Steer"] }, blocked: { iconType: "branch", actions: ["Answer Question"] }, @@ -65,6 +66,7 @@ type Column = { }; const SKELETON_STATUSES: ColumnStatus[] = [ + "queued", "initializing", "running", "blocked", @@ -754,6 +756,9 @@ export default function Runs() { (sum, col) => sum + col.items.length, 0, ); + const visibleColumns = filteredColumns.filter( + (col) => col.id !== "queued" || col.items.length > 0, + ); return ( @@ -815,7 +820,7 @@ export default function Runs() { {view === "columns" ? ( <>
- {filteredColumns.map((col) => ( + {visibleColumns.map((col) => (
@@ -838,7 +843,7 @@ export default function Runs() { ) : ( <>
- {filteredColumns.map((col) => { + {visibleColumns.map((col) => { const isCollapsed = collapsed.has(col.id); return (
diff --git a/docs/public/api-reference/fabro-api.yaml b/docs/public/api-reference/fabro-api.yaml index d1b8bc7f3..d10db4a47 100644 --- a/docs/public/api-reference/fabro-api.yaml +++ b/docs/public/api-reference/fabro-api.yaml @@ -5656,6 +5656,7 @@ components: description: Board column status for a run in the list view. type: string enum: + - queued - initializing - running - blocked diff --git a/lib/crates/fabro-server/src/demo/mod.rs b/lib/crates/fabro-server/src/demo/mod.rs index c9751cf83..787fb1ae4 100644 --- a/lib/crates/fabro-server/src/demo/mod.rs +++ b/lib/crates/fabro-server/src/demo/mod.rs @@ -795,8 +795,8 @@ mod runs { .collect() } - fn demo_run_ids() -> &'static [RunId; 6] { - static IDS: OnceLock<[RunId; 6]> = OnceLock::new(); + fn demo_run_ids() -> &'static [RunId; 7] { + static IDS: OnceLock<[RunId; 7]> = OnceLock::new(); IDS.get_or_init(|| { [ RunId::with_timestamp(ts("2026-03-06T14:30:00Z"), 1), @@ -805,6 +805,7 @@ mod runs { RunId::with_timestamp(ts("2026-03-04T10:00:00Z"), 4), RunId::with_timestamp(ts("2026-03-03T16:45:00Z"), 5), RunId::with_timestamp(ts("2026-02-28T14:00:00Z"), 6), + RunId::with_timestamp(ts("2026-03-06T14:35:00Z"), 7), ] }) } @@ -973,6 +974,10 @@ mod runs { pub(super) fn columns() -> Vec { vec![ + BoardColumnDefinition { + id: "queued".into(), + name: "Queued".into(), + }, BoardColumnDefinition { id: "initializing".into(), name: "Initializing".into(), @@ -1082,6 +1087,20 @@ mod runs { Some(720000), &[("release", "preview")], ), + summary( + 7, + "api-server", + "implement", + "Implement", + "Add audit log retention policy", + "queued", + "2026-03-06T14:35:00Z", + None, + None, + None, + None, + &[("owner", "platform")], + ), ] } @@ -1150,6 +1169,13 @@ mod runs { None, None, ), + board_item( + take_summary(&mut summaries, demo_run_id(7)), + BoardColumn::Queued, + None, + None, + None, + ), ] } diff --git a/lib/crates/fabro-server/src/server/handler/runs.rs b/lib/crates/fabro-server/src/server/handler/runs.rs index ff03b55e5..eb48c7c4f 100644 --- a/lib/crates/fabro-server/src/server/handler/runs.rs +++ b/lib/crates/fabro-server/src/server/handler/runs.rs @@ -84,7 +84,8 @@ impl ListRunsParams { fn board_column(status: RunStatus) -> Option<&'static str> { match status { - RunStatus::Submitted | RunStatus::Queued | RunStatus::Starting => Some("initializing"), + RunStatus::Submitted | RunStatus::Queued => Some("queued"), + RunStatus::Starting => Some("initializing"), RunStatus::Running | RunStatus::Paused { .. } => Some("running"), RunStatus::Blocked { .. } => Some("blocked"), RunStatus::Succeeded { .. } => Some("succeeded"), @@ -95,6 +96,7 @@ fn board_column(status: RunStatus) -> Option<&'static str> { pub(crate) fn board_columns() -> serde_json::Value { serde_json::json!([ + {"id": "queued", "name": "Queued"}, {"id": "initializing", "name": "Initializing"}, {"id": "running", "name": "Running"}, {"id": "blocked", "name": "Blocked"}, diff --git a/lib/crates/fabro-server/src/server/tests.rs b/lib/crates/fabro-server/src/server/tests.rs index 6078c362f..5480fef0f 100644 --- a/lib/crates/fabro-server/src/server/tests.rs +++ b/lib/crates/fabro-server/src/server/tests.rs @@ -6125,7 +6125,7 @@ async fn pause_run_sets_pending_control_on_board_response() { assert_eq!(body["pending_control"].as_str(), Some("pause")); // Verify the run appears on the board (store has Submitted status → - // "initializing" column) + // "queued" column) let req = Request::builder() .method("GET") .uri(api("/boards/runs")) @@ -6140,7 +6140,7 @@ async fn pause_run_sets_pending_control_on_board_response() { .find(|item| item["run_id"].as_str() == Some(run_id_str.as_str())) .expect("board item should exist"); assert!(item["status"].is_object()); - assert_eq!(item["column"].as_str(), Some("initializing")); + assert_eq!(item["column"].as_str(), Some("queued")); assert_eq!(item["pending_control"].as_str(), Some("pause")); } diff --git a/lib/packages/fabro-api-client/src/models/board-column.ts b/lib/packages/fabro-api-client/src/models/board-column.ts index 9d97dc120..66c24c107 100644 --- a/lib/packages/fabro-api-client/src/models/board-column.ts +++ b/lib/packages/fabro-api-client/src/models/board-column.ts @@ -19,6 +19,7 @@ */ export const BoardColumn = { + QUEUED: 'queued', INITIALIZING: 'initializing', RUNNING: 'running', BLOCKED: 'blocked', From ade721ae65e8b7cf0bb0b93e4bb9e6f13a78397b Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Mon, 4 May 2026 14:54:39 -0400 Subject: [PATCH 02/17] feat(web): coordinate SSE subscriptions across tabs Elect a single browser tab to own the global attach stream and broadcast run events to sibling tabs. Keep the existing per-tab EventSource path as the fallback when cross-tab coordination is unavailable. --- apps/fabro-web/app/lib/board-events.test.tsx | 84 +- apps/fabro-web/app/lib/board-events.ts | 41 +- apps/fabro-web/app/lib/cross-tab-sse.test.ts | 385 ++++++ apps/fabro-web/app/lib/cross-tab-sse.ts | 1175 ++++++++++++++++++ apps/fabro-web/app/lib/run-events.test.tsx | 128 +- apps/fabro-web/app/lib/run-events.ts | 75 +- 6 files changed, 1852 insertions(+), 36 deletions(-) create mode 100644 apps/fabro-web/app/lib/cross-tab-sse.test.ts create mode 100644 apps/fabro-web/app/lib/cross-tab-sse.ts diff --git a/apps/fabro-web/app/lib/board-events.test.tsx b/apps/fabro-web/app/lib/board-events.test.tsx index 853d300b9..9b398fdc1 100644 --- a/apps/fabro-web/app/lib/board-events.test.tsx +++ b/apps/fabro-web/app/lib/board-events.test.tsx @@ -4,6 +4,10 @@ import { shouldRefreshBoardForEvent, subscribeToBoardEvents, } from "./board-events"; +import { + createCrossTabSseCoordinator, + type BroadcastChannelLike, +} from "./cross-tab-sse"; import { queryKeys } from "./query-keys"; type MessageHandler = ((event: { data: string }) => void) | null; @@ -21,6 +25,14 @@ class FakeEventSource { } } +class FakeBroadcastChannel implements BroadcastChannelLike { + onmessage: ((event: { data: unknown }) => void) | null = null; + + postMessage() {} + + close() {} +} + describe("shouldRefreshBoardForEvent", () => { test("refreshes board for run and interview status changes only", () => { expect(shouldRefreshBoardForEvent("run.running")).toBe(true); @@ -31,10 +43,11 @@ describe("shouldRefreshBoardForEvent", () => { }); describe("subscribeToBoardEvents", () => { - test("shares one source and invalidates the board runs key", () => { + test("coordinated mode shares one global source and invalidates the board runs key", async () => { const source = new FakeEventSource(); const created: string[] = []; const keys: string[] = []; + const coordinator = createCoordinator(); const mutate = (key: string) => { keys.push(key); return Promise.resolve(); @@ -43,10 +56,13 @@ describe("subscribeToBoardEvents", () => { const firstCleanup = subscribeToBoardEvents(mutate, (url) => { created.push(url); return source; - }, { debounceMs: 0 }); + }, { debounceMs: 0, coordinator }); const secondCleanup = subscribeToBoardEvents(mutate, () => { throw new Error("source should be reused"); - }, { debounceMs: 0 }); + }, { debounceMs: 0, coordinator }); + + await waitFor(() => created.length === 1); + keys.length = 0; source.emit({ event: "run.running" }); @@ -57,5 +73,67 @@ describe("subscribeToBoardEvents", () => { expect(source.closed).toBe(false); secondCleanup(); expect(source.closed).toBe(true); + coordinator.close(); + }); + + test("fallback mode preserves the existing shared board EventSource", () => { + const source = new FakeEventSource(); + const created: string[] = []; + const keys: string[] = []; + const coordinator = createFallbackCoordinator(); + const mutate = (key: string) => { + keys.push(key); + return Promise.resolve(); + }; + + const firstCleanup = subscribeToBoardEvents(mutate, (url) => { + created.push(url); + return source; + }, { debounceMs: 0, coordinator }); + const secondCleanup = subscribeToBoardEvents(mutate, () => { + throw new Error("source should be reused"); + }, { debounceMs: 0, coordinator }); + + source.emit({ event: "run.running" }); + + expect(created).toEqual(["/api/v1/attach"]); + expect(keys).toEqual([queryKeys.boards.runs()]); + + firstCleanup(); + expect(source.closed).toBe(false); + secondCleanup(); + expect(source.closed).toBe(true); + coordinator.close(); }); }); + +function createCoordinator() { + return createCrossTabSseCoordinator({ + tabId: "board-test", + channelFactory: () => new FakeBroadcastChannel(), + addVisibilityChangeListener: () => () => {}, + addPagehideListener: () => () => {}, + timing: { + heartbeatMs: 10, + leaderStaleMs: 50, + electionJitterMs: 0, + }, + }); +} + +function createFallbackCoordinator() { + return createCrossTabSseCoordinator({ + channelFactory: () => { + throw new Error("BroadcastChannel unavailable"); + }, + }); +} + +async function waitFor(condition: () => boolean, timeoutMs = 200) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (condition()) return; + await new Promise((resolve) => setTimeout(resolve, 2)); + } + throw new Error("condition did not become true before timeout"); +} diff --git a/apps/fabro-web/app/lib/board-events.ts b/apps/fabro-web/app/lib/board-events.ts index ef30009b1..a182d9ad2 100644 --- a/apps/fabro-web/app/lib/board-events.ts +++ b/apps/fabro-web/app/lib/board-events.ts @@ -1,6 +1,10 @@ import { useEffect } from "react"; import { useSWRConfig } from "swr"; +import { + subscribeToCrossTabSse, + type CrossTabSseCoordinator, +} from "./cross-tab-sse"; import { queryKeys } from "./query-keys"; import { createBrowserEventSource, @@ -11,6 +15,11 @@ import { type SharedEventSubscription, } from "./sse"; +interface BoardEventOptions { + debounceMs?: number; + coordinator?: CrossTabSseCoordinator; +} + const BOARD_STATUS_EVENTS = new Set([ "run.submitted", "run.queued", @@ -41,23 +50,37 @@ export function shouldRefreshBoardForEvent(event: string) { export function subscribeToBoardEvents( mutate: MutateFn, eventSourceFactory: (url: string) => EventSourceLike = createBrowserEventSource, - { debounceMs = 500 }: { debounceMs?: number } = {}, + { debounceMs = 500, coordinator }: BoardEventOptions = {}, ): () => void { - return subscribeToSharedEventSource({ - subscriptions, + return subscribeToCrossTabSse({ + coordinator, subscriptionKey: BOARD_SUBSCRIPTION_KEY, - url: queryKeys.system.attach(), mutate, eventSourceFactory, debounceMs, - resolveInvalidation: (payload) => ({ - keys: payload.event && shouldRefreshBoardForEvent(payload.event) - ? [queryKeys.boards.runs()] - : [], - }), + resyncKeys: () => [queryKeys.boards.runs()], + resolveInvalidation: boardInvalidation, + fallbackSubscribe: () => + subscribeToSharedEventSource({ + subscriptions, + subscriptionKey: BOARD_SUBSCRIPTION_KEY, + url: queryKeys.system.attach(), + mutate, + eventSourceFactory, + debounceMs, + resolveInvalidation: boardInvalidation, + }), }); } +function boardInvalidation(payload: EventPayload) { + return { + keys: payload.event && shouldRefreshBoardForEvent(payload.event) + ? [queryKeys.boards.runs()] + : [], + }; +} + export function useBoardEvents() { const { mutate } = useSWRConfig(); diff --git a/apps/fabro-web/app/lib/cross-tab-sse.test.ts b/apps/fabro-web/app/lib/cross-tab-sse.test.ts new file mode 100644 index 000000000..32cff001c --- /dev/null +++ b/apps/fabro-web/app/lib/cross-tab-sse.test.ts @@ -0,0 +1,385 @@ +import { afterEach, describe, expect, test } from "bun:test"; + +import { + CROSS_TAB_SSE_CHANNEL, + createCrossTabSseCoordinator, + subscribeToCrossTabSse, + type BroadcastChannelLike, + type CrossTabSseCoordinator, + type CrossTabSseMessage, +} from "./cross-tab-sse"; +import type { EventPayload, MutateFn } from "./sse"; + +type MessageHandler = ((event: { data: string }) => void) | null; +type TabVisibility = "visible" | "hidden"; + +const TEST_TIMING = { + heartbeatMs: 10, + leaderStaleMs: 35, + electionJitterMs: 5, +}; + +class FakeEventSource { + onmessage: MessageHandler = null; + closed = false; + + constructor( + readonly url: string, + readonly owner: string, + ) {} + + emit(payload: unknown) { + this.onmessage?.({ data: JSON.stringify(payload) }); + } + + close() { + this.closed = true; + } +} + +class FakeBroadcastChannel implements BroadcastChannelLike { + static channels = new Set(); + static muted = false; + + onmessage: ((event: { data: unknown }) => void) | null = null; + closed = false; + + constructor(readonly name: string) { + FakeBroadcastChannel.channels.add(this); + } + + postMessage(message: CrossTabSseMessage) { + if (FakeBroadcastChannel.muted) return; + const recipients = [...FakeBroadcastChannel.channels].filter( + (channel) => channel !== this && !channel.closed && channel.name === this.name, + ); + queueMicrotask(() => { + for (const channel of recipients) { + if (channel.closed) continue; + channel.onmessage?.({ data: { ...message } }); + } + }); + } + + close() { + this.closed = true; + FakeBroadcastChannel.channels.delete(this); + } + + static reset() { + for (const channel of FakeBroadcastChannel.channels) { + channel.closed = true; + } + FakeBroadcastChannel.channels.clear(); + FakeBroadcastChannel.muted = false; + } +} + +class Harness { + readonly sources: FakeEventSource[] = []; + readonly coordinators = new Map(); + readonly visibility = new Map(); + readonly visibilityHandlers = new Map void>(); + now = 1000; + + createTab(tabId: string, visibility: TabVisibility = "visible") { + this.visibility.set(tabId, visibility); + const coordinator = createCrossTabSseCoordinator({ + tabId, + channelFactory: (name) => new FakeBroadcastChannel(name), + eventSourceFactory: (url) => { + const source = new FakeEventSource(url, tabId); + this.sources.push(source); + return source; + }, + getVisibility: () => this.visibility.get(tabId) ?? "visible", + addVisibilityChangeListener: (handler) => { + this.visibilityHandlers.set(tabId, handler); + return () => this.visibilityHandlers.delete(tabId); + }, + addPagehideListener: () => () => {}, + now: () => this.now, + timing: TEST_TIMING, + }); + this.coordinators.set(tabId, coordinator); + return coordinator; + } + + setVisibility(tabId: string, visibility: TabVisibility) { + this.visibility.set(tabId, visibility); + this.visibilityHandlers.get(tabId)?.(); + } + + openSources() { + return this.sources.filter((source) => !source.closed); + } + + close() { + for (const coordinator of this.coordinators.values()) { + coordinator.close(); + } + } +} + +const harnesses: Harness[] = []; + +afterEach(() => { + for (const harness of harnesses.splice(0)) { + harness.close(); + } + FakeBroadcastChannel.reset(); +}); + +describe("subscribeToCrossTabSse", () => { + test("opens one leader-owned global EventSource and keeps followers passive", async () => { + const harness = newHarness(); + const cleanups = ["a", "b", "c"].map((tabId) => { + const coordinator = harness.createTab(tabId); + return subscribeForRunEvent(coordinator, []); + }); + + await waitFor(() => harness.openSources().length === 1); + + expect(harness.openSources().map((source) => source.url)).toEqual(["/api/v1/attach"]); + expect([...FakeBroadcastChannel.channels].every((channel) => channel.name === CROSS_TAB_SSE_CHANNEL)).toBe(true); + + cleanups.forEach((cleanup) => cleanup()); + }); + + test("leader broadcasts events to all local subscribers", async () => { + const harness = newHarness(); + const keysByTab = new Map(); + + for (const tabId of ["a", "b", "c"]) { + keysByTab.set(tabId, []); + subscribeForRunEvent(harness.createTab(tabId), keysByTab.get(tabId)!); + } + + await waitFor(() => harness.openSources().length === 1); + clearRecordedKeys(keysByTab); + + harness.openSources()[0].emit(runEvent({ id: "evt-1", runId: "run-1", seq: 1 })); + await waitFor(() => [...keysByTab.values()].every((keys) => keys.length === 1)); + + expect(keysByTab.get("a")).toEqual(["event"]); + expect(keysByTab.get("b")).toEqual(["event"]); + expect(keysByTab.get("c")).toEqual(["event"]); + }); + + test("dedupes duplicate event ids until TTL or max-size eviction", async () => { + const harness = newHarness(); + const keys: string[] = []; + subscribeForRunEvent(harness.createTab("a"), keys); + + await waitFor(() => harness.openSources().length === 1); + keys.length = 0; + + const source = harness.openSources()[0]; + source.emit(runEvent({ id: "evt-dup", runId: "run-1", seq: 1 })); + source.emit(runEvent({ id: "evt-dup", runId: "run-1", seq: 1 })); + + expect(keys).toEqual(["event"]); + + harness.now += 5 * 60 * 1000 + 1; + source.emit(runEvent({ id: "evt-dup", runId: "run-1", seq: 1 })); + expect(keys).toEqual(["event", "event"]); + + keys.length = 0; + for (let i = 0; i < 1001; i += 1) { + source.emit(runEvent({ id: `evt-${i}`, runId: "run-1", seq: i + 2 })); + } + source.emit(runEvent({ id: "evt-0", runId: "run-1", seq: 2 })); + expect(keys).toHaveLength(1002); + }); + + test("visible followers take over from a fresh hidden leader and resync", async () => { + const harness = newHarness(); + const hiddenKeys: string[] = []; + const visibleKeys: string[] = []; + + subscribeForRunEvent(harness.createTab("z", "hidden"), hiddenKeys); + await waitFor(() => harness.openSources().length === 1); + const hiddenSource = harness.openSources()[0]; + + subscribeForRunEvent(harness.createTab("a", "visible"), visibleKeys); + + await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "a"); + + expect(hiddenSource.closed).toBe(true); + expect(visibleKeys).toContain("resync"); + }); + + test("visible candidates racing for the same hidden leader resolve lexically", async () => { + const harness = newHarness(); + + subscribeForRunEvent(harness.createTab("z", "hidden"), []); + await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "z"); + + subscribeForRunEvent(harness.createTab("b", "visible"), []); + subscribeForRunEvent(harness.createTab("a", "visible"), []); + + await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "a"); + }); + + test("a lower lexical follower does not preempt a fresh visible leader", async () => { + const harness = newHarness(); + + subscribeForRunEvent(harness.createTab("z", "visible"), []); + await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "z"); + + subscribeForRunEvent(harness.createTab("a", "visible"), []); + await sleep(TEST_TIMING.electionJitterMs * 4); + + expect(harness.openSources().map((source) => source.owner)).toEqual(["z"]); + }); + + test("stale leader detection opens a new leader source and resyncs followers", async () => { + const harness = newHarness(); + const followerKeys: string[] = []; + + subscribeForRunEvent(harness.createTab("a"), []); + subscribeForRunEvent(harness.createTab("b"), followerKeys); + await waitFor(() => harness.openSources().length === 1); + + const staleLeader = harness.openSources()[0]; + harness.coordinators.get(staleLeader.owner)?.close(); + harness.now += TEST_TIMING.leaderStaleMs + TEST_TIMING.heartbeatMs + 1; + + await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner !== staleLeader.owner); + + expect(followerKeys).toContain("resync"); + }); + + test("same-generation split brain converges to the higher-priority visible leader", async () => { + const harness = newHarness(); + FakeBroadcastChannel.muted = true; + + subscribeForRunEvent(harness.createTab("b"), []); + subscribeForRunEvent(harness.createTab("a"), []); + await waitFor(() => harness.openSources().length === 2); + + FakeBroadcastChannel.muted = false; + await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "a"); + }); + + test("old leader events are ignored after takeover", async () => { + const harness = newHarness(); + const keys: string[] = []; + + subscribeForRunEvent(harness.createTab("z", "hidden"), []); + await waitFor(() => harness.openSources().length === 1); + const oldSource = harness.openSources()[0]; + + subscribeForRunEvent(harness.createTab("a", "visible"), keys); + await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "a"); + keys.length = 0; + + oldSource.emit(runEvent({ id: "evt-old", runId: "run-1", seq: 1 })); + expect(keys).toEqual([]); + }); + + test("last unsubscribe closes the leader source and releases leadership", async () => { + const harness = newHarness(); + const cleanup = subscribeForRunEvent(harness.createTab("a"), []); + + await waitFor(() => harness.openSources().length === 1); + const source = harness.openSources()[0]; + + cleanup(); + + expect(source.closed).toBe(true); + expect(harness.openSources()).toEqual([]); + }); + + test("missing BroadcastChannel uses subscriber fallback", () => { + const coordinator = createCrossTabSseCoordinator({ + channelFactory: () => { + throw new Error("no channel"); + }, + }); + let fallbackStarted = 0; + let fallbackStopped = 0; + + const cleanup = subscribeToCrossTabSse({ + coordinator, + subscriptionKey: "fallback", + mutate: (() => Promise.resolve()) as MutateFn, + resolveInvalidation: () => ({ keys: [] }), + resyncKeys: () => [], + fallbackSubscribe: () => { + fallbackStarted += 1; + return () => { + fallbackStopped += 1; + }; + }, + debounceMs: 0, + }); + + cleanup(); + + expect(fallbackStarted).toBe(1); + expect(fallbackStopped).toBe(1); + }); +}); + +function newHarness() { + const harness = new Harness(); + harnesses.push(harness); + return harness; +} + +function subscribeForRunEvent(coordinator: CrossTabSseCoordinator, keys: string[]) { + return subscribeToCrossTabSse({ + coordinator, + subscriptionKey: "run-feed", + mutate: ((key: string) => { + keys.push(key); + return Promise.resolve(); + }) as MutateFn, + resolveInvalidation: (payload) => ({ + keys: payload.event === "run.running" ? ["event"] : [], + }), + resyncKeys: () => ["resync"], + fallbackSubscribe: () => { + throw new Error("fallback should not be used"); + }, + debounceMs: 0, + }); +} + +function runEvent({ + id, + runId, + seq, +}: { + id: string; + runId: string; + seq: number; +}) { + return { + id, + seq, + run_id: runId, + event: "run.running", + ts: "2026-05-04T12:00:00.000Z", + }; +} + +async function waitFor(condition: () => boolean, timeoutMs = 500) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (condition()) return; + await sleep(2); + } + throw new Error("condition did not become true before timeout"); +} + +function sleep(ms: number) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function clearRecordedKeys(keysByTab: Map) { + for (const keys of keysByTab.values()) { + keys.length = 0; + } +} diff --git a/apps/fabro-web/app/lib/cross-tab-sse.ts b/apps/fabro-web/app/lib/cross-tab-sse.ts new file mode 100644 index 000000000..255c4c7ce --- /dev/null +++ b/apps/fabro-web/app/lib/cross-tab-sse.ts @@ -0,0 +1,1175 @@ +import { queryKeys } from "./query-keys"; +import { + createBrowserEventSource, + type EventInvalidation, + type EventPayload, + type EventSourceLike, + type MutateFn, +} from "./sse"; +import { isRecord } from "./unknown"; + +export const CROSS_TAB_SSE_CHANNEL = "fabro:sse:v1"; +export const HEARTBEAT_MS = 1000; +export const LEADER_STALE_MS = 4000; +export const ELECTION_JITTER_MS = 150; + +const MESSAGE_VERSION = 1 as const; +const EVENT_DEDUPE_TTL_MS = 5 * 60 * 1000; +const EVENT_DEDUPE_MAX = 1000; + +type TabVisibility = "visible" | "hidden"; +type CandidateReason = "hidden-leader" | "stale-leader" | "release" | "no-leader"; + +interface BaseMessage { + type: string; + version: typeof MESSAGE_VERSION; + tabId: string; + sentAt: number; +} + +interface HelloMessage extends BaseMessage { + type: "hello"; +} + +interface HeartbeatMessage extends BaseMessage { + type: "heartbeat"; + leaderId: string; + generation: number; + visibility: TabVisibility; +} + +interface CandidateMessage extends BaseMessage { + type: "candidate"; + candidateId: string; + candidateGeneration: number; + visibility: TabVisibility; + observedLeaderId: string | null; + observedGeneration: number; + reason: CandidateReason; +} + +interface LeaderChangedMessage extends BaseMessage { + type: "leader-changed"; + leaderId: string; + generation: number; + visibility: TabVisibility; +} + +interface ReleaseMessage extends BaseMessage { + type: "release"; + leaderId: string; + generation: number; +} + +interface ResyncMessage extends BaseMessage { + type: "resync"; + leaderId: string | null; + generation: number; + reason: CandidateReason; +} + +interface EventMessage extends BaseMessage { + type: "event"; + leaderId: string; + generation: number; + payload: EventPayload; +} + +export type CrossTabSseMessage = + | HelloMessage + | HeartbeatMessage + | CandidateMessage + | LeaderChangedMessage + | ReleaseMessage + | ResyncMessage + | EventMessage; + +export interface BroadcastChannelLike { + onmessage: ((event: { data: unknown }) => void) | null; + postMessage(message: CrossTabSseMessage): void; + close(): void; +} + +interface TimingOptions { + heartbeatMs: number; + leaderStaleMs: number; + electionJitterMs: number; +} + +export interface CrossTabSseCoordinatorOptions { + tabId?: string; + channelFactory?: (name: string) => BroadcastChannelLike; + eventSourceFactory?: (url: string) => EventSourceLike; + getVisibility?: () => TabVisibility; + addVisibilityChangeListener?: (handler: () => void) => () => void; + addPagehideListener?: (handler: () => void) => () => void; + now?: () => number; + timing?: Partial; +} + +interface SubscribeOptions { + subscriptionKey: string; + mutate: MutateFn; + resolveInvalidation: (payload: TPayload) => EventInvalidation; + resyncKeys: () => string[]; + fallbackSubscribe: () => () => void; + eventSourceFactory?: (url: string) => EventSourceLike; + debounceMs?: number; +} + +export interface SubscribeToCrossTabSseOptions + extends SubscribeOptions { + coordinator?: CrossTabSseCoordinator; +} + +interface FallbackEntry { + count: number; + subscribe: () => () => void; + cleanup?: () => void; +} + +interface LocalSubscription { + refcount: number; + mutators: Map; + fallbacks: Map; + pendingKeys: Set; + debounceTimer: ReturnType | null; + debounceMs: number; + resolveInvalidation: (payload: EventPayload) => EventInvalidation; + resyncKeys: () => string[]; +} + +interface LeaderState { + leaderId: string; + generation: number; + visibility: TabVisibility; + lastSeen: number; +} + +class RecentEventCache { + private readonly seen = new Map(); + + constructor( + private readonly maxSize: number, + private readonly ttlMs: number, + ) {} + + remember(key: string | undefined, now: number): boolean { + if (!key) return true; + this.prune(now); + if (this.seen.has(key)) return false; + + this.seen.set(key, now); + while (this.seen.size > this.maxSize) { + const oldest = this.seen.keys().next().value; + if (oldest === undefined) break; + this.seen.delete(oldest); + } + return true; + } + + private prune(now: number) { + for (const [key, seenAt] of this.seen) { + if (now - seenAt > this.ttlMs) { + this.seen.delete(key); + } + } + } +} + +export class CrossTabSseCoordinator { + readonly tabId: string; + + private readonly channelFactory: (name: string) => BroadcastChannelLike; + private readonly getVisibility: () => TabVisibility; + private readonly addVisibilityChangeListener: (handler: () => void) => () => void; + private readonly addPagehideListener: (handler: () => void) => () => void; + private readonly now: () => number; + private readonly timing: TimingOptions; + private readonly recentEvents = new RecentEventCache(EVENT_DEDUPE_MAX, EVENT_DEDUPE_TTL_MS); + private readonly subscriptions = new Map(); + private readonly candidates = new Map(); + + private sourceFactory: (url: string) => EventSourceLike; + private channel: BroadcastChannelLike | null = null; + private source: EventSourceLike | null = null; + private initialized = false; + private coordinationUnavailable = false; + private fallbackMode = false; + private degradingToFallback = false; + private isLeader = false; + private leader: LeaderState | null = null; + private generation = 0; + private ownCandidate: CandidateMessage | null = null; + private candidateTimer: ReturnType | null = null; + private noLeaderTimer: ReturnType | null = null; + private heartbeatTimer: ReturnType | null = null; + private leaderCheckTimer: ReturnType | null = null; + private removeVisibilityListener: (() => void) | null = null; + private removePagehideListener: (() => void) | null = null; + private sourceFactoryLocked = false; + + constructor(options: CrossTabSseCoordinatorOptions = {}) { + this.tabId = options.tabId ?? createTabId(); + this.channelFactory = options.channelFactory ?? createBrowserBroadcastChannel; + this.sourceFactory = options.eventSourceFactory ?? createBrowserEventSource; + this.getVisibility = options.getVisibility ?? getBrowserVisibility; + this.addVisibilityChangeListener = + options.addVisibilityChangeListener ?? addBrowserVisibilityChangeListener; + this.addPagehideListener = options.addPagehideListener ?? addBrowserPagehideListener; + this.now = options.now ?? Date.now; + this.timing = { + heartbeatMs: options.timing?.heartbeatMs ?? HEARTBEAT_MS, + leaderStaleMs: options.timing?.leaderStaleMs ?? LEADER_STALE_MS, + electionJitterMs: options.timing?.electionJitterMs ?? ELECTION_JITTER_MS, + }; + } + + subscribe(options: SubscribeOptions): () => void { + if (options.eventSourceFactory && !this.source && !this.sourceFactoryLocked) { + this.sourceFactory = options.eventSourceFactory; + this.sourceFactoryLocked = true; + } + + if (this.coordinationUnavailable) { + return options.fallbackSubscribe(); + } + + if (!this.initialized && !this.initialize()) { + this.coordinationUnavailable = true; + return options.fallbackSubscribe(); + } + + const subscription = this.addLocalSubscription(options); + if (this.fallbackMode) { + this.startFallbacksFor(subscription); + } else { + this.ensureLeadershipProgress(); + } + + let active = true; + return () => { + if (!active) return; + active = false; + this.removeLocalSubscription(options.subscriptionKey, options.mutate); + }; + } + + close() { + this.releaseLeadership({ broadcast: false, resync: false }); + this.clearCandidate(); + this.clearNoLeaderTimer(); + this.shutdownTimersAndChannel(); + this.closeFallbacks(); + this.subscriptions.clear(); + this.leader = null; + this.initialized = false; + this.fallbackMode = false; + this.sourceFactoryLocked = false; + } + + private initialize(): boolean { + try { + this.channel = this.channelFactory(CROSS_TAB_SSE_CHANNEL); + } catch { + this.channel = null; + return false; + } + + this.channel.onmessage = (event) => this.handleMessage(event.data); + this.initialized = true; + this.removeVisibilityListener = this.addVisibilityChangeListener(() => { + this.handleVisibilityChange(); + }); + this.removePagehideListener = this.addPagehideListener(() => { + this.handlePagehide(); + }); + this.leaderCheckTimer = setInterval(() => { + this.checkLeaderFreshness(); + }, this.timing.heartbeatMs); + + return this.post({ type: "hello", version: MESSAGE_VERSION, tabId: this.tabId, sentAt: this.now() }); + } + + private addLocalSubscription( + options: SubscribeOptions, + ): LocalSubscription { + let subscription = this.subscriptions.get(options.subscriptionKey); + if (!subscription) { + subscription = { + refcount: 0, + mutators: new Map(), + fallbacks: new Map(), + pendingKeys: new Set(), + debounceTimer: null, + debounceMs: options.debounceMs ?? 300, + resolveInvalidation: options.resolveInvalidation as (payload: EventPayload) => EventInvalidation, + resyncKeys: options.resyncKeys, + }; + this.subscriptions.set(options.subscriptionKey, subscription); + } else { + subscription.resolveInvalidation = + options.resolveInvalidation as (payload: EventPayload) => EventInvalidation; + subscription.resyncKeys = options.resyncKeys; + subscription.debounceMs = options.debounceMs ?? subscription.debounceMs; + } + + subscription.refcount += 1; + subscription.mutators.set( + options.mutate, + (subscription.mutators.get(options.mutate) ?? 0) + 1, + ); + + const fallback = subscription.fallbacks.get(options.mutate); + if (fallback) { + fallback.count += 1; + fallback.subscribe = options.fallbackSubscribe; + } else { + subscription.fallbacks.set(options.mutate, { + count: 1, + subscribe: options.fallbackSubscribe, + }); + } + + return subscription; + } + + private removeLocalSubscription(subscriptionKey: string, mutate: MutateFn) { + const subscription = this.subscriptions.get(subscriptionKey); + if (!subscription) return; + + const mutateCount = subscription.mutators.get(mutate) ?? 0; + if (mutateCount <= 1) { + subscription.mutators.delete(mutate); + } else { + subscription.mutators.set(mutate, mutateCount - 1); + } + + const fallback = subscription.fallbacks.get(mutate); + if (fallback) { + fallback.count -= 1; + if (fallback.count <= 0) { + fallback.cleanup?.(); + subscription.fallbacks.delete(mutate); + } + } + + subscription.refcount -= 1; + if (subscription.refcount <= 0) { + if (subscription.debounceTimer) { + clearTimeout(subscription.debounceTimer); + } + this.subscriptions.delete(subscriptionKey); + } + + if (this.subscriptions.size === 0) { + this.releaseLeadership({ broadcast: true, resync: false }); + this.clearCandidate(); + this.clearNoLeaderTimer(); + this.shutdownTimersAndChannel(); + this.initialized = false; + this.fallbackMode = false; + this.sourceFactoryLocked = false; + } + } + + private handleMessage(data: unknown) { + const message = parseMessage(data); + if (!message || message.tabId === this.tabId) return; + + switch (message.type) { + case "hello": + if (this.isLeader) this.sendHeartbeat(); + break; + case "heartbeat": + this.handleLeaderAnnouncement(message, { resyncOnChange: true }); + break; + case "candidate": + this.handleCandidate(message); + break; + case "leader-changed": + this.handleLeaderAnnouncement(message, { resyncOnChange: true }); + break; + case "release": + this.handleRelease(message); + break; + case "resync": + this.handleResync(message); + break; + case "event": + this.handleBroadcastEvent(message); + break; + } + } + + private handleLeaderAnnouncement( + message: HeartbeatMessage | LeaderChangedMessage, + { resyncOnChange }: { resyncOnChange: boolean }, + ) { + const incoming: LeaderState = { + leaderId: message.leaderId, + generation: message.generation, + visibility: message.visibility, + lastSeen: this.now(), + }; + + if (this.isLeader && incoming.leaderId !== this.tabId) { + const own: LeaderState = { + leaderId: this.tabId, + generation: this.generation, + visibility: this.currentVisibility(), + lastSeen: this.now(), + }; + if ( + incoming.generation > own.generation || + (incoming.generation === own.generation && leaderHasHigherPriority(incoming, own)) + ) { + this.releaseLeadership({ broadcast: false, resync: true }); + } else { + return; + } + } + + const previous = this.leader; + if (this.ownCandidate && incoming.generation === this.ownCandidate.candidateGeneration) { + const sawIncomingCandidate = this.candidates.has( + `${incoming.generation}:${incoming.leaderId}`, + ); + const shouldAcceptFreshVisibleLeader = + this.ownCandidate.reason === "no-leader" && + incoming.visibility === "visible" && + !sawIncomingCandidate; + + if ( + !shouldAcceptFreshVisibleLeader && + !leaderHasHigherPriority(incoming, leaderStateForCandidate(this.ownCandidate, this.now())) + ) { + return; + } + } + + if (!this.shouldAcceptLeader(incoming)) return; + + this.leader = incoming; + this.clearNoLeaderTimer(); + this.generation = Math.max(this.generation, incoming.generation); + if (this.ownCandidate && incoming.generation >= this.ownCandidate.candidateGeneration) { + this.clearCandidate(); + } + + const changed = + !previous || + previous.leaderId !== incoming.leaderId || + previous.generation !== incoming.generation; + + if (changed && resyncOnChange) { + this.resyncAll(); + } + + if (incoming.visibility === "hidden" && this.currentVisibility() === "visible") { + this.enterCandidacy("hidden-leader", incoming); + } + } + + private handleCandidate(message: CandidateMessage) { + this.candidates.set(candidateKey(message), message); + + if ( + this.isLeader && + message.observedLeaderId === this.tabId && + message.observedGeneration >= this.generation + ) { + this.releaseLeadership({ broadcast: true, resync: true }); + } + + if ( + this.ownCandidate && + message.candidateGeneration === this.ownCandidate.candidateGeneration && + candidateHasHigherPriority(message, this.ownCandidate) + ) { + this.clearCandidate(); + } + } + + private handleRelease(message: ReleaseMessage) { + const current = this.leader; + if ( + current && + message.leaderId === current.leaderId && + message.generation >= current.generation + ) { + this.leader = null; + this.generation = Math.max(this.generation, message.generation); + this.enterCandidacy("release", { + leaderId: message.leaderId, + generation: message.generation, + visibility: "hidden", + lastSeen: this.now(), + }); + } + } + + private handleResync(message: ResyncMessage) { + if (this.leader && message.generation < this.leader.generation) return; + this.resyncAll(); + } + + private handleBroadcastEvent(message: EventMessage) { + if (!this.isCurrentLeader(message.leaderId, message.generation)) return; + if (!this.recentEvents.remember(eventDedupeKey(message.payload), this.now())) return; + this.dispatchPayload(message.payload); + } + + private ensureLeadershipProgress() { + if (this.subscriptions.size === 0 || this.isLeader || this.ownCandidate) return; + + if (!this.leader) { + this.scheduleNoLeaderCandidacy(); + return; + } + + if (this.leader.visibility === "hidden" && this.currentVisibility() === "visible") { + this.enterCandidacy("hidden-leader", this.leader); + } + } + + private checkLeaderFreshness() { + if (this.subscriptions.size === 0 || this.fallbackMode) return; + if (this.isLeader) return; + + const current = this.leader; + if (!current) { + this.scheduleNoLeaderCandidacy(); + return; + } + + if (this.now() - current.lastSeen > this.timing.leaderStaleMs) { + this.leader = null; + this.generation = Math.max(this.generation, current.generation); + this.resyncAll(); + this.enterCandidacy("stale-leader", current); + return; + } + + if (current.visibility === "hidden" && this.currentVisibility() === "visible") { + this.enterCandidacy("hidden-leader", current); + } + } + + private enterCandidacy(reason: CandidateReason, observedLeader: LeaderState | null = this.leader) { + if (this.subscriptions.size === 0 || this.fallbackMode) return; + + if ( + reason === "no-leader" && + this.leader && + this.leader.visibility === "visible" && + this.now() - this.leader.lastSeen <= this.timing.leaderStaleMs + ) { + return; + } + + const observedGeneration = observedLeader?.generation ?? this.generation; + const candidateGeneration = observedGeneration + 1; + if ( + this.ownCandidate && + this.ownCandidate.candidateGeneration >= candidateGeneration + ) { + return; + } + + this.clearCandidate(); + this.clearNoLeaderTimer(); + const candidate: CandidateMessage = { + type: "candidate", + version: MESSAGE_VERSION, + tabId: this.tabId, + sentAt: this.now(), + candidateId: this.tabId, + candidateGeneration, + visibility: this.currentVisibility(), + observedLeaderId: observedLeader?.leaderId ?? null, + observedGeneration, + reason, + }; + + this.ownCandidate = candidate; + this.candidates.set(candidateKey(candidate), candidate); + this.post(candidate); + this.candidateTimer = setTimeout(() => { + this.completeCandidacy(candidate); + }, this.timing.electionJitterMs); + } + + private completeCandidacy(candidate: CandidateMessage) { + if (this.ownCandidate !== candidate || this.fallbackMode) return; + + for (const other of this.candidates.values()) { + if ( + other.candidateGeneration === candidate.candidateGeneration && + candidateHasHigherPriority(other, candidate) + ) { + this.clearCandidate(); + return; + } + } + + if ( + this.leader && + this.now() - this.leader.lastSeen <= this.timing.leaderStaleMs + ) { + if (this.leader.generation > candidate.candidateGeneration) { + this.clearCandidate(); + return; + } + if ( + this.leader.generation === candidate.candidateGeneration && + leaderHasHigherPriority(this.leader, leaderStateForCandidate(candidate, this.now())) + ) { + this.clearCandidate(); + return; + } + } + + this.becomeLeader(candidate.candidateGeneration); + } + + private becomeLeader(generation: number) { + this.clearCandidate(); + this.closeSource(); + this.isLeader = true; + this.generation = generation; + this.leader = { + leaderId: this.tabId, + generation, + visibility: this.currentVisibility(), + lastSeen: this.now(), + }; + + const source = this.sourceFactory(queryKeys.system.attach()); + this.source = source; + source.onmessage = (message) => { + this.handleLeaderEventSourceMessage(message.data); + }; + + this.post({ + type: "leader-changed", + version: MESSAGE_VERSION, + tabId: this.tabId, + sentAt: this.now(), + leaderId: this.tabId, + generation, + visibility: this.currentVisibility(), + }); + this.startHeartbeat(); + this.resyncAll(); + } + + private handleLeaderEventSourceMessage(data: string) { + if (!this.isLeader) return; + + let payload: EventPayload; + try { + payload = JSON.parse(data) as EventPayload; + } catch { + return; + } + + if (!this.recentEvents.remember(eventDedupeKey(payload), this.now())) return; + this.dispatchPayload(payload); + this.post({ + type: "event", + version: MESSAGE_VERSION, + tabId: this.tabId, + sentAt: this.now(), + leaderId: this.tabId, + generation: this.generation, + payload, + }); + } + + private dispatchPayload(payload: EventPayload) { + for (const subscription of this.subscriptions.values()) { + const invalidation = subscription.resolveInvalidation(payload); + this.queueInvalidations(subscription, invalidation.keys, { + immediate: invalidation.immediate, + }); + } + } + + private queueInvalidations( + subscription: LocalSubscription, + keys: string[], + { immediate = false }: { immediate?: boolean } = {}, + ) { + if (keys.length === 0) return; + for (const key of keys) { + subscription.pendingKeys.add(key); + } + + if (immediate || subscription.debounceMs <= 0) { + this.flushInvalidations(subscription); + return; + } + + if (subscription.debounceTimer) { + clearTimeout(subscription.debounceTimer); + } + subscription.debounceTimer = setTimeout(() => { + subscription.debounceTimer = null; + this.flushInvalidations(subscription); + }, subscription.debounceMs); + } + + private flushInvalidations(subscription: LocalSubscription) { + if (subscription.pendingKeys.size === 0) return; + const keys = [...subscription.pendingKeys]; + subscription.pendingKeys.clear(); + + for (const mutator of subscription.mutators.keys()) { + for (const key of keys) { + void mutator(key); + } + } + } + + private resyncAll() { + for (const subscription of this.subscriptions.values()) { + this.queueInvalidations(subscription, subscription.resyncKeys(), { immediate: true }); + } + } + + private handleVisibilityChange() { + if (this.isLeader) { + this.sendHeartbeat(); + } + + if (this.currentVisibility() === "visible") { + this.resyncAll(); + if (this.leader?.visibility === "hidden") { + this.enterCandidacy("hidden-leader", this.leader); + } + } + } + + private handlePagehide() { + this.releaseLeadership({ broadcast: true, resync: false }); + this.clearCandidate(); + this.clearNoLeaderTimer(); + } + + private startHeartbeat() { + if (this.heartbeatTimer) { + clearInterval(this.heartbeatTimer); + } + this.sendHeartbeat(); + this.heartbeatTimer = setInterval(() => { + this.sendHeartbeat(); + }, this.timing.heartbeatMs); + } + + private sendHeartbeat() { + if (!this.isLeader) return; + const visibility = this.currentVisibility(); + this.leader = { + leaderId: this.tabId, + generation: this.generation, + visibility, + lastSeen: this.now(), + }; + this.post({ + type: "heartbeat", + version: MESSAGE_VERSION, + tabId: this.tabId, + sentAt: this.now(), + leaderId: this.tabId, + generation: this.generation, + visibility, + }); + } + + private releaseLeadership({ + broadcast, + resync, + }: { + broadcast: boolean; + resync: boolean; + }) { + if (!this.isLeader && !this.source) return; + + const generation = this.generation; + this.closeSource(); + this.isLeader = false; + if (this.heartbeatTimer) { + clearInterval(this.heartbeatTimer); + this.heartbeatTimer = null; + } + this.leader = null; + + if (broadcast) { + this.post({ + type: "release", + version: MESSAGE_VERSION, + tabId: this.tabId, + sentAt: this.now(), + leaderId: this.tabId, + generation, + }); + this.post({ + type: "resync", + version: MESSAGE_VERSION, + tabId: this.tabId, + sentAt: this.now(), + leaderId: this.tabId, + generation, + reason: "release", + }); + } + if (resync) this.resyncAll(); + } + + private closeSource() { + if (!this.source) return; + this.source.close(); + this.source = null; + } + + private clearCandidate() { + if (this.candidateTimer) { + clearTimeout(this.candidateTimer); + this.candidateTimer = null; + } + this.ownCandidate = null; + } + + private scheduleNoLeaderCandidacy() { + if ( + this.noLeaderTimer || + this.ownCandidate || + this.isLeader || + this.leader || + this.subscriptions.size === 0 || + this.fallbackMode + ) { + return; + } + + this.noLeaderTimer = setTimeout(() => { + this.noLeaderTimer = null; + if (!this.leader && !this.isLeader) { + this.enterCandidacy("no-leader"); + } + }, this.timing.electionJitterMs); + } + + private clearNoLeaderTimer() { + if (!this.noLeaderTimer) return; + clearTimeout(this.noLeaderTimer); + this.noLeaderTimer = null; + } + + private shutdownTimersAndChannel() { + this.clearNoLeaderTimer(); + if (this.heartbeatTimer) { + clearInterval(this.heartbeatTimer); + this.heartbeatTimer = null; + } + if (this.leaderCheckTimer) { + clearInterval(this.leaderCheckTimer); + this.leaderCheckTimer = null; + } + this.removeVisibilityListener?.(); + this.removeVisibilityListener = null; + this.removePagehideListener?.(); + this.removePagehideListener = null; + if (this.channel) { + this.channel.close(); + this.channel = null; + } + } + + private post(message: CrossTabSseMessage): boolean { + if (!this.channel) return false; + try { + this.channel.postMessage(message); + return true; + } catch { + this.degradeToFallback(); + return false; + } + } + + private degradeToFallback() { + if (this.degradingToFallback || this.fallbackMode) return; + this.degradingToFallback = true; + this.releaseLeadership({ broadcast: false, resync: false }); + this.clearCandidate(); + this.shutdownTimersAndChannel(); + this.initialized = false; + this.fallbackMode = true; + this.coordinationUnavailable = true; + for (const subscription of this.subscriptions.values()) { + this.startFallbacksFor(subscription); + } + this.degradingToFallback = false; + } + + private startFallbacksFor(subscription: LocalSubscription) { + for (const fallback of subscription.fallbacks.values()) { + if (!fallback.cleanup) { + fallback.cleanup = fallback.subscribe(); + } + } + } + + private closeFallbacks() { + for (const subscription of this.subscriptions.values()) { + for (const fallback of subscription.fallbacks.values()) { + fallback.cleanup?.(); + fallback.cleanup = undefined; + } + } + } + + private shouldAcceptLeader(incoming: LeaderState): boolean { + if (!this.leader) return true; + if (incoming.generation > this.leader.generation) return true; + if (incoming.generation < this.leader.generation) return false; + if (incoming.leaderId === this.leader.leaderId) return true; + return leaderHasHigherPriority(incoming, this.leader); + } + + private isCurrentLeader(leaderId: string, generation: number): boolean { + return Boolean( + this.leader && + this.leader.leaderId === leaderId && + this.leader.generation === generation, + ); + } + + private currentVisibility(): TabVisibility { + return this.getVisibility() === "hidden" ? "hidden" : "visible"; + } +} + +const defaultCoordinator = new CrossTabSseCoordinator(); + +export function createCrossTabSseCoordinator(options: CrossTabSseCoordinatorOptions = {}) { + return new CrossTabSseCoordinator(options); +} + +export function subscribeToCrossTabSse({ + coordinator = defaultCoordinator, + ...options +}: SubscribeToCrossTabSseOptions): () => void { + return coordinator.subscribe(options); +} + +function createBrowserBroadcastChannel(name: string): BroadcastChannelLike { + if (typeof BroadcastChannel === "undefined") { + throw new Error("BroadcastChannel is unavailable"); + } + return new BroadcastChannel(name); +} + +function createTabId(): string { + if (typeof crypto !== "undefined" && typeof crypto.randomUUID === "function") { + return crypto.randomUUID(); + } + return `tab-${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`; +} + +function getBrowserVisibility(): TabVisibility { + if (typeof document === "undefined") return "visible"; + return document.visibilityState === "visible" ? "visible" : "hidden"; +} + +function addBrowserVisibilityChangeListener(handler: () => void): () => void { + if (typeof document === "undefined") return () => {}; + document.addEventListener("visibilitychange", handler); + return () => document.removeEventListener("visibilitychange", handler); +} + +function addBrowserPagehideListener(handler: () => void): () => void { + if (typeof window === "undefined") return () => {}; + window.addEventListener("pagehide", handler); + return () => window.removeEventListener("pagehide", handler); +} + +function parseMessage(data: unknown): CrossTabSseMessage | undefined { + if (!isRecord(data)) return undefined; + if (data.version !== MESSAGE_VERSION) return undefined; + const type = data.type; + const tabId = data.tabId; + const sentAt = data.sentAt; + if (typeof type !== "string") return undefined; + if (typeof tabId !== "string") return undefined; + if (typeof sentAt !== "number") return undefined; + const base = { tabId, sentAt }; + + switch (type) { + case "hello": + return baseMessage(base, "hello"); + case "heartbeat": { + const { leaderId, generation, visibility } = data; + if (typeof leaderId === "string" && typeof generation === "number" && isVisibility(visibility)) { + return { + ...baseMessage(base, "heartbeat"), + leaderId, + generation, + visibility, + }; + } + return undefined; + } + case "candidate": { + const { + candidateId, + candidateGeneration, + visibility, + observedLeaderId, + observedGeneration, + reason, + } = data; + if ( + typeof candidateId === "string" && + typeof candidateGeneration === "number" && + isVisibility(visibility) && + (typeof observedLeaderId === "string" || observedLeaderId === null) && + typeof observedGeneration === "number" && + isCandidateReason(reason) + ) { + const normalizedObservedLeaderId = + typeof observedLeaderId === "string" ? observedLeaderId : null; + return { + ...baseMessage(base, "candidate"), + candidateId, + candidateGeneration, + visibility, + observedLeaderId: normalizedObservedLeaderId, + observedGeneration, + reason, + }; + } + return undefined; + } + case "leader-changed": { + const { leaderId, generation, visibility } = data; + if (typeof leaderId === "string" && typeof generation === "number" && isVisibility(visibility)) { + return { + ...baseMessage(base, "leader-changed"), + leaderId, + generation, + visibility, + }; + } + return undefined; + } + case "release": { + const { leaderId, generation } = data; + if (typeof leaderId === "string" && typeof generation === "number") { + return { + ...baseMessage(base, "release"), + leaderId, + generation, + }; + } + return undefined; + } + case "resync": { + const { leaderId, generation, reason } = data; + if ( + (typeof leaderId === "string" || leaderId === null) && + typeof generation === "number" && + isCandidateReason(reason) + ) { + const normalizedLeaderId = typeof leaderId === "string" ? leaderId : null; + return { + ...baseMessage(base, "resync"), + leaderId: normalizedLeaderId, + generation, + reason, + }; + } + return undefined; + } + case "event": { + const { leaderId, generation, payload } = data; + if (typeof leaderId === "string" && typeof generation === "number" && isRecord(payload)) { + return { + ...baseMessage(base, "event"), + leaderId, + generation, + payload, + }; + } + return undefined; + } + default: + return undefined; + } +} + +function baseMessage( + data: { + tabId: string; + sentAt: number; + }, + type: TType, +) { + return { + type, + version: MESSAGE_VERSION, + tabId: data.tabId, + sentAt: data.sentAt, + }; +} + +function isVisibility(value: unknown): value is TabVisibility { + return value === "visible" || value === "hidden"; +} + +function isCandidateReason(value: unknown): value is CandidateReason { + return ( + value === "hidden-leader" || + value === "stale-leader" || + value === "release" || + value === "no-leader" + ); +} + +function candidateHasHigherPriority(candidate: CandidateMessage, other: CandidateMessage): boolean { + if (candidate.visibility !== other.visibility) return candidate.visibility === "visible"; + return candidate.candidateId < other.candidateId; +} + +function leaderHasHigherPriority(candidate: LeaderState, other: LeaderState): boolean { + if (candidate.visibility !== other.visibility) return candidate.visibility === "visible"; + return candidate.leaderId < other.leaderId; +} + +function leaderStateForCandidate(candidate: CandidateMessage, lastSeen: number): LeaderState { + return { + leaderId: candidate.candidateId, + generation: candidate.candidateGeneration, + visibility: candidate.visibility, + lastSeen, + }; +} + +function candidateKey(candidate: CandidateMessage): string { + return `${candidate.candidateGeneration}:${candidate.candidateId}`; +} + +function eventDedupeKey(payload: EventPayload): string | undefined { + if (typeof payload.id === "string" && payload.id.length > 0) { + return payload.id; + } + + const runId = typeof payload.run_id === "string" ? payload.run_id : undefined; + const seq = typeof payload.seq === "number" ? payload.seq : undefined; + const event = typeof payload.event === "string" ? payload.event : undefined; + if (runId && seq != null && event) { + return `${runId}:${seq}:${event}`; + } + return undefined; +} diff --git a/apps/fabro-web/app/lib/run-events.test.tsx b/apps/fabro-web/app/lib/run-events.test.tsx index 14acf4f23..db35a757d 100644 --- a/apps/fabro-web/app/lib/run-events.test.tsx +++ b/apps/fabro-web/app/lib/run-events.test.tsx @@ -4,6 +4,10 @@ import { queryKeysForRunEvent, subscribeToRunEvents, } from "./run-events"; +import { + createCrossTabSseCoordinator, + type BroadcastChannelLike, +} from "./cross-tab-sse"; import { queryKeys } from "./query-keys"; type MessageHandler = ((event: { data: string }) => void) | null; @@ -25,6 +29,14 @@ class FakeEventSource { } } +class FakeBroadcastChannel implements BroadcastChannelLike { + onmessage: ((event: { data: unknown }) => void) | null = null; + + postMessage() {} + + close() {} +} + describe("queryKeysForRunEvent", () => { test("terminal events invalidate run-scoped resources", () => { expect(queryKeysForRunEvent("run-1", "run.completed")).toEqual([ @@ -39,10 +51,74 @@ describe("queryKeysForRunEvent", () => { }); describe("subscribeToRunEvents", () => { - test("refcounts shared sources and keeps mutators active until final unsubscribe", () => { + test("coordinated mode uses the global attach stream and filters by run_id", async () => { const source = new FakeEventSource(); const created: string[] = []; const keys: string[] = []; + const coordinator = createCoordinator(); + + const cleanup = subscribeToRunEvents( + "run-coordinated", + (key) => { + keys.push(key); + return Promise.resolve(); + }, + (url) => { + created.push(url); + return source; + }, + { debounceMs: 0, coordinator }, + ); + + await waitFor(() => created.length === 1); + keys.length = 0; + + source.emit({ event: "checkpoint.completed", run_id: "other-run" }); + source.emit({ event: "checkpoint.completed", run_id: "run-coordinated" }); + + expect(created).toEqual(["/api/v1/attach"]); + expect(keys).toEqual([queryKeys.runs.files("run-coordinated")]); + + cleanup(); + coordinator.close(); + }); + + test("coordinated terminal events invalidate without closing the global stream", async () => { + const source = new FakeEventSource(); + const keys: string[] = []; + const coordinator = createCoordinator(); + const cleanup = subscribeToRunEvents( + "run-terminal", + (key) => { + keys.push(key); + return Promise.resolve(); + }, + () => source, + { debounceMs: 0, coordinator }, + ); + + await waitFor(() => source.onmessage !== null); + keys.length = 0; + + source.emit({ event: "run.failed", run_id: "run-terminal" }); + expect(source.closed).toBe(false); + expect(keys).toContain(queryKeys.runs.files("run-terminal")); + expect(keys).toContain(queryKeys.runs.billing("run-terminal")); + + keys.length = 0; + source.emit({ event: "run.archived", run_id: "run-terminal" }); + expect(source.closed).toBe(false); + expect(keys).toEqual([queryKeys.runs.detail("run-terminal")]); + + cleanup(); + coordinator.close(); + }); + + test("fallback refcounts run-scoped sources and keeps mutators active until final unsubscribe", () => { + const source = new FakeEventSource(); + const created: string[] = []; + const keys: string[] = []; + const coordinator = createFallbackCoordinator(); const mutate = (key: string) => { keys.push(key); return Promise.resolve(); @@ -51,10 +127,10 @@ describe("subscribeToRunEvents", () => { const firstCleanup = subscribeToRunEvents("run-refcount", mutate, (url) => { created.push(url); return source; - }, { debounceMs: 0 }); + }, { debounceMs: 0, coordinator }); const secondCleanup = subscribeToRunEvents("run-refcount", mutate, () => { throw new Error("source should be reused"); - }, { debounceMs: 0 }); + }, { debounceMs: 0, coordinator }); expect(created).toEqual(["/api/v1/runs/run-refcount/attach"]); @@ -66,11 +142,13 @@ describe("subscribeToRunEvents", () => { secondCleanup(); expect(source.closed).toBe(true); + coordinator.close(); }); - test("terminal events close the source after invalidating keys", () => { + test("fallback terminal events close the source after invalidating keys", () => { const source = new FakeEventSource(); const keys: string[] = []; + const coordinator = createFallbackCoordinator(); const cleanup = subscribeToRunEvents( "run-terminal", (key) => { @@ -78,7 +156,7 @@ describe("subscribeToRunEvents", () => { return Promise.resolve(); }, () => source, - { debounceMs: 0 }, + { debounceMs: 0, coordinator }, ); source.emit({ event: "run.failed" }); @@ -88,13 +166,15 @@ describe("subscribeToRunEvents", () => { expect(keys).toContain(queryKeys.runs.billing("run-terminal")); cleanup(); + coordinator.close(); }); - test("malformed events are ignored and StrictMode-style cleanup does not underflow", () => { + test("fallback malformed events are ignored and StrictMode-style cleanup does not underflow", () => { const firstSource = new FakeEventSource(); const secondSource = new FakeEventSource(); const sources = [firstSource, secondSource]; const keys: string[] = []; + const coordinator = createFallbackCoordinator(); const firstCleanup = subscribeToRunEvents( "run-strict", @@ -103,7 +183,7 @@ describe("subscribeToRunEvents", () => { return Promise.resolve(); }, () => sources.shift()!, - { debounceMs: 0 }, + { debounceMs: 0, coordinator }, ); firstSource.emitRaw("{broken"); firstCleanup(); @@ -115,12 +195,44 @@ describe("subscribeToRunEvents", () => { return Promise.resolve(); }, () => sources.shift()!, - { debounceMs: 0 }, + { debounceMs: 0, coordinator }, ); secondCleanup(); expect(keys).toEqual([]); expect(firstSource.closed).toBe(true); expect(secondSource.closed).toBe(true); + coordinator.close(); }); }); + +function createCoordinator() { + return createCrossTabSseCoordinator({ + tabId: "run-test", + channelFactory: () => new FakeBroadcastChannel(), + addVisibilityChangeListener: () => () => {}, + addPagehideListener: () => () => {}, + timing: { + heartbeatMs: 10, + leaderStaleMs: 50, + electionJitterMs: 0, + }, + }); +} + +function createFallbackCoordinator() { + return createCrossTabSseCoordinator({ + channelFactory: () => { + throw new Error("BroadcastChannel unavailable"); + }, + }); +} + +async function waitFor(condition: () => boolean, timeoutMs = 200) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (condition()) return; + await new Promise((resolve) => setTimeout(resolve, 2)); + } + throw new Error("condition did not become true before timeout"); +} diff --git a/apps/fabro-web/app/lib/run-events.ts b/apps/fabro-web/app/lib/run-events.ts index 924264af2..80394da48 100644 --- a/apps/fabro-web/app/lib/run-events.ts +++ b/apps/fabro-web/app/lib/run-events.ts @@ -1,6 +1,10 @@ import { useEffect } from "react"; import { useSWRConfig } from "swr"; +import { + subscribeToCrossTabSse, + type CrossTabSseCoordinator, +} from "./cross-tab-sse"; import { queryKeys } from "./query-keys"; import { createBrowserEventSource, @@ -13,10 +17,16 @@ import { interface RunEventPayload extends EventPayload { event?: string; + run_id?: string; node_id?: string; properties?: Record; } +interface RunEventOptions { + debounceMs?: number; + coordinator?: CrossTabSseCoordinator; +} + const subscriptions = new Map(); const TERMINAL_EVENTS = new Set(["run.completed", "run.failed"]); @@ -104,31 +114,64 @@ export function subscribeToRunEvents( runId: string, mutate: MutateFn, eventSourceFactory: (url: string) => EventSourceLike = createBrowserEventSource, - { debounceMs = 300 }: { debounceMs?: number } = {}, + { debounceMs = 300, coordinator }: RunEventOptions = {}, ): () => void { - return subscribeToSharedEventSource({ - subscriptions, - subscriptionKey: runId, - url: queryKeys.runs.attach(runId), + return subscribeToCrossTabSse({ + coordinator, + subscriptionKey: `run:${runId}`, mutate, eventSourceFactory, debounceMs, + resyncKeys: () => resyncKeysForRun(runId), resolveInvalidation: (payload) => { - const event = payload.event; - if (!event) return { keys: [] }; - - const stageId = stageIdFromPayload(payload); - const keys = queryKeysForRunEvent(runId, event, stageId); - const terminal = TERMINAL_EVENTS.has(event); - return { - keys, - close: terminal, - immediate: terminal, - }; + if (payload.run_id !== runId) return { keys: [] }; + return runInvalidation(runId, payload, { closeOnTerminal: false }); }, + fallbackSubscribe: () => + subscribeToSharedEventSource({ + subscriptions, + subscriptionKey: runId, + url: queryKeys.runs.attach(runId), + mutate, + eventSourceFactory, + debounceMs, + resolveInvalidation: (payload) => + runInvalidation(runId, payload, { closeOnTerminal: true }), + }), }); } +function runInvalidation( + runId: string, + payload: RunEventPayload, + { closeOnTerminal }: { closeOnTerminal: boolean }, +) { + const event = payload.event; + if (!event) return { keys: [] }; + + const stageId = stageIdFromPayload(payload); + const keys = queryKeysForRunEvent(runId, event, stageId); + const terminal = TERMINAL_EVENTS.has(event); + return { + keys, + close: closeOnTerminal && terminal, + immediate: terminal, + }; +} + +function resyncKeysForRun(runId: string) { + return [ + queryKeys.runs.detail(runId), + queryKeys.runs.files(runId), + queryKeys.runs.billing(runId), + queryKeys.runs.stages(runId), + queryKeys.runs.events(runId, 1000), + queryKeys.runs.graph(runId, "LR"), + queryKeys.runs.graph(runId, "TB"), + queryKeys.runs.questions(runId, 25, 0), + ]; +} + function stageIdFromPayload(payload: RunEventPayload): string | undefined { if (typeof payload.node_id === "string") return payload.node_id; const nodeId = payload.properties?.node_id; From 38726666afecdb2b928d0b5f4a5a5fe5e0410319 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Mon, 4 May 2026 15:24:25 -0400 Subject: [PATCH 03/17] fix(web): harden cross-tab SSE fallback Stop coordinated election and leadership work when BroadcastChannel posting fails, so tabs degrade cleanly to per-subscriber fallback without stale resync or heartbeat side effects. Expand election coverage for the edge cases called out in the coordination plan. --- apps/fabro-web/app/lib/cross-tab-sse.test.ts | 183 ++++++++++++++++++- apps/fabro-web/app/lib/cross-tab-sse.ts | 21 ++- 2 files changed, 189 insertions(+), 15 deletions(-) diff --git a/apps/fabro-web/app/lib/cross-tab-sse.test.ts b/apps/fabro-web/app/lib/cross-tab-sse.test.ts index 32cff001c..573ef6589 100644 --- a/apps/fabro-web/app/lib/cross-tab-sse.test.ts +++ b/apps/fabro-web/app/lib/cross-tab-sse.test.ts @@ -40,6 +40,7 @@ class FakeEventSource { class FakeBroadcastChannel implements BroadcastChannelLike { static channels = new Set(); static muted = false; + static throwOnTypes = new Set(); onmessage: ((event: { data: unknown }) => void) | null = null; closed = false; @@ -49,6 +50,9 @@ class FakeBroadcastChannel implements BroadcastChannelLike { } postMessage(message: CrossTabSseMessage) { + if (FakeBroadcastChannel.throwOnTypes.has(message.type)) { + throw new Error(`postMessage failed for ${message.type}`); + } if (FakeBroadcastChannel.muted) return; const recipients = [...FakeBroadcastChannel.channels].filter( (channel) => channel !== this && !channel.closed && channel.name === this.name, @@ -61,6 +65,15 @@ class FakeBroadcastChannel implements BroadcastChannelLike { }); } + static broadcastExternal(message: CrossTabSseMessage) { + queueMicrotask(() => { + for (const channel of FakeBroadcastChannel.channels) { + if (channel.closed) continue; + channel.onmessage?.({ data: { ...message } }); + } + }); + } + close() { this.closed = true; FakeBroadcastChannel.channels.delete(this); @@ -72,6 +85,7 @@ class FakeBroadcastChannel implements BroadcastChannelLike { } FakeBroadcastChannel.channels.clear(); FakeBroadcastChannel.muted = false; + FakeBroadcastChannel.throwOnTypes.clear(); } } @@ -166,6 +180,40 @@ describe("subscribeToCrossTabSse", () => { expect(keysByTab.get("c")).toEqual(["event"]); }); + test("board and run subscriptions coexist on the same global stream", async () => { + const harness = newHarness(); + const coordinator = harness.createTab("a"); + const boardKeys: string[] = []; + const runKeys: string[] = []; + + subscribeForEvent(coordinator, { + subscriptionKey: "board", + keys: boardKeys, + resolveInvalidation: (payload) => ({ + keys: payload.event === "run.running" ? ["board"] : [], + }), + resyncKeys: () => ["board-resync"], + }); + subscribeForEvent(coordinator, { + subscriptionKey: "run:run-1", + keys: runKeys, + resolveInvalidation: (payload) => ({ + keys: payload.event === "run.running" && payload.run_id === "run-1" ? ["run"] : [], + }), + resyncKeys: () => ["run-resync"], + }); + + await waitFor(() => harness.openSources().length === 1); + boardKeys.length = 0; + runKeys.length = 0; + + harness.openSources()[0].emit(runEvent({ id: "evt-coexist", runId: "run-1", seq: 1 })); + + expect(boardKeys).toEqual(["board"]); + expect(runKeys).toEqual(["run"]); + expect(harness.openSources().map((source) => source.url)).toEqual(["/api/v1/attach"]); + }); + test("dedupes duplicate event ids until TTL or max-size eviction", async () => { const harness = newHarness(); const keys: string[] = []; @@ -218,7 +266,8 @@ describe("subscribeToCrossTabSse", () => { subscribeForRunEvent(harness.createTab("b", "visible"), []); subscribeForRunEvent(harness.createTab("a", "visible"), []); - await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "a"); + await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner !== "z"); + expect(harness.openSources().map((source) => source.owner)).toEqual(["a"]); }); test("a lower lexical follower does not preempt a fresh visible leader", async () => { @@ -250,6 +299,46 @@ describe("subscribeToCrossTabSse", () => { expect(followerKeys).toContain("resync"); }); + test("simultaneous stale leader elections resolve to the lexical winner", async () => { + const harness = newHarness(); + + subscribeForRunEvent(harness.createTab("z"), []); + await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "z"); + subscribeForRunEvent(harness.createTab("b"), []); + subscribeForRunEvent(harness.createTab("a"), []); + await sleep(TEST_TIMING.heartbeatMs * 2); + + harness.coordinators.get("z")?.close(); + harness.now += TEST_TIMING.leaderStaleMs + TEST_TIMING.heartbeatMs + 1; + + await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "a"); + }); + + test("hidden leader ignores candidates for old observed leadership", async () => { + const harness = newHarness(); + + subscribeForRunEvent(harness.createTab("z", "hidden"), []); + await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "z"); + const hiddenSource = harness.openSources()[0]; + + FakeBroadcastChannel.broadcastExternal({ + type: "candidate", + version: 1, + tabId: "ghost", + sentAt: harness.now, + candidateId: "ghost", + candidateGeneration: 1, + visibility: "visible", + observedLeaderId: "z", + observedGeneration: 0, + reason: "hidden-leader", + }); + await sleep(TEST_TIMING.electionJitterMs * 2); + + expect(hiddenSource.closed).toBe(false); + expect(harness.openSources().map((source) => source.owner)).toEqual(["z"]); + }); + test("same-generation split brain converges to the higher-priority visible leader", async () => { const harness = newHarness(); FakeBroadcastChannel.muted = true; @@ -278,6 +367,29 @@ describe("subscribeToCrossTabSse", () => { expect(keys).toEqual([]); }); + test("old leader heartbeats are ignored after takeover", async () => { + const harness = newHarness(); + + subscribeForRunEvent(harness.createTab("z", "hidden"), []); + await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "z"); + + subscribeForRunEvent(harness.createTab("a", "visible"), []); + await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "a"); + + FakeBroadcastChannel.broadcastExternal({ + type: "heartbeat", + version: 1, + tabId: "z", + sentAt: harness.now, + leaderId: "z", + generation: 1, + visibility: "hidden", + }); + await sleep(TEST_TIMING.heartbeatMs * 2); + + expect(harness.openSources().map((source) => source.owner)).toEqual(["a"]); + }); + test("last unsubscribe closes the leader source and releases leadership", async () => { const harness = newHarness(); const cleanup = subscribeForRunEvent(harness.createTab("a"), []); @@ -320,6 +432,41 @@ describe("subscribeToCrossTabSse", () => { expect(fallbackStarted).toBe(1); expect(fallbackStopped).toBe(1); }); + + test("postMessage failure after initialization degrades to fallback without coordinated resync", async () => { + const harness = newHarness(); + const coordinator = harness.createTab("a"); + const keys: string[] = []; + let fallbackStarted = 0; + let fallbackStopped = 0; + + FakeBroadcastChannel.throwOnTypes.add("leader-changed"); + const cleanup = subscribeToCrossTabSse({ + coordinator, + subscriptionKey: "throwing-channel", + mutate: ((key: string) => { + keys.push(key); + return Promise.resolve(); + }) as MutateFn, + resolveInvalidation: () => ({ keys: ["event"] }), + resyncKeys: () => ["resync"], + fallbackSubscribe: () => { + fallbackStarted += 1; + return () => { + fallbackStopped += 1; + }; + }, + debounceMs: 0, + }); + + await waitFor(() => fallbackStarted === 1); + + expect(harness.openSources()).toEqual([]); + expect(keys).toEqual([]); + + cleanup(); + expect(fallbackStopped).toBe(1); + }); }); function newHarness() { @@ -329,17 +476,39 @@ function newHarness() { } function subscribeForRunEvent(coordinator: CrossTabSseCoordinator, keys: string[]) { - return subscribeToCrossTabSse({ - coordinator, + return subscribeForEvent(coordinator, { subscriptionKey: "run-feed", - mutate: ((key: string) => { - keys.push(key); - return Promise.resolve(); - }) as MutateFn, + keys, resolveInvalidation: (payload) => ({ keys: payload.event === "run.running" ? ["event"] : [], }), resyncKeys: () => ["resync"], + }); +} + +function subscribeForEvent( + coordinator: CrossTabSseCoordinator, + { + subscriptionKey, + keys, + resolveInvalidation, + resyncKeys, + }: { + subscriptionKey: string; + keys: string[]; + resolveInvalidation: (payload: EventPayload) => { keys: string[] }; + resyncKeys: () => string[]; + }, +) { + return subscribeToCrossTabSse({ + coordinator, + subscriptionKey, + mutate: ((key: string) => { + keys.push(key); + return Promise.resolve(); + }) as MutateFn, + resolveInvalidation, + resyncKeys, fallbackSubscribe: () => { throw new Error("fallback should not be used"); }, diff --git a/apps/fabro-web/app/lib/cross-tab-sse.ts b/apps/fabro-web/app/lib/cross-tab-sse.ts index 255c4c7ce..0ddb6e220 100644 --- a/apps/fabro-web/app/lib/cross-tab-sse.ts +++ b/apps/fabro-web/app/lib/cross-tab-sse.ts @@ -594,7 +594,7 @@ export class CrossTabSseCoordinator { this.ownCandidate = candidate; this.candidates.set(candidateKey(candidate), candidate); - this.post(candidate); + if (!this.post(candidate)) return; this.candidateTimer = setTimeout(() => { this.completeCandidacy(candidate); }, this.timing.electionJitterMs); @@ -651,7 +651,7 @@ export class CrossTabSseCoordinator { this.handleLeaderEventSourceMessage(message.data); }; - this.post({ + const announced = this.post({ type: "leader-changed", version: MESSAGE_VERSION, tabId: this.tabId, @@ -660,6 +660,7 @@ export class CrossTabSseCoordinator { generation, visibility: this.currentVisibility(), }); + if (!announced) return; this.startHeartbeat(); this.resyncAll(); } @@ -761,14 +762,14 @@ export class CrossTabSseCoordinator { if (this.heartbeatTimer) { clearInterval(this.heartbeatTimer); } - this.sendHeartbeat(); + if (!this.sendHeartbeat()) return; this.heartbeatTimer = setInterval(() => { this.sendHeartbeat(); }, this.timing.heartbeatMs); } - private sendHeartbeat() { - if (!this.isLeader) return; + private sendHeartbeat(): boolean { + if (!this.isLeader) return false; const visibility = this.currentVisibility(); this.leader = { leaderId: this.tabId, @@ -776,7 +777,7 @@ export class CrossTabSseCoordinator { visibility, lastSeen: this.now(), }; - this.post({ + return this.post({ type: "heartbeat", version: MESSAGE_VERSION, tabId: this.tabId, @@ -806,7 +807,7 @@ export class CrossTabSseCoordinator { this.leader = null; if (broadcast) { - this.post({ + const released = this.post({ type: "release", version: MESSAGE_VERSION, tabId: this.tabId, @@ -814,6 +815,7 @@ export class CrossTabSseCoordinator { leaderId: this.tabId, generation, }); + if (!released) return; this.post({ type: "resync", version: MESSAGE_VERSION, @@ -888,7 +890,10 @@ export class CrossTabSseCoordinator { } private post(message: CrossTabSseMessage): boolean { - if (!this.channel) return false; + if (!this.channel) { + this.degradeToFallback(); + return false; + } try { this.channel.postMessage(message); return true; From 65298455545f3b43449e94e7b71e37b93535db9d Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Mon, 4 May 2026 15:29:42 -0400 Subject: [PATCH 04/17] fix(web): clean up cross-tab SSE lifecycle Prune stale election candidates as generations advance, reset coordination availability on explicit close, and keep fallback subscribers tracked so coordinator shutdown can clean them up consistently. --- apps/fabro-web/app/lib/cross-tab-sse.test.ts | 137 +++++++++++++++++++ apps/fabro-web/app/lib/cross-tab-sse.ts | 34 +++-- 2 files changed, 161 insertions(+), 10 deletions(-) diff --git a/apps/fabro-web/app/lib/cross-tab-sse.test.ts b/apps/fabro-web/app/lib/cross-tab-sse.test.ts index 573ef6589..f38c881fa 100644 --- a/apps/fabro-web/app/lib/cross-tab-sse.test.ts +++ b/apps/fabro-web/app/lib/cross-tab-sse.test.ts @@ -339,6 +339,32 @@ describe("subscribeToCrossTabSse", () => { expect(harness.openSources().map((source) => source.owner)).toEqual(["z"]); }); + test("prunes candidate records from older generations", async () => { + const harness = newHarness(); + subscribeForRunEvent(harness.createTab("z", "hidden"), []); + await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "z"); + + const coordinator = harness.createTab("a", "visible"); + subscribeForRunEvent(coordinator, []); + await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "a"); + + FakeBroadcastChannel.broadcastExternal({ + type: "candidate", + version: 1, + tabId: "old-candidate", + sentAt: harness.now, + candidateId: "old-candidate", + candidateGeneration: 1, + visibility: "visible", + observedLeaderId: "previous-leader", + observedGeneration: 0, + reason: "stale-leader", + }); + await sleep(TEST_TIMING.electionJitterMs * 2); + + expect(candidateGenerations(coordinator)).not.toContain(1); + }); + test("same-generation split brain converges to the higher-priority visible leader", async () => { const harness = newHarness(); FakeBroadcastChannel.muted = true; @@ -467,6 +493,89 @@ describe("subscribeToCrossTabSse", () => { cleanup(); expect(fallbackStopped).toBe(1); }); + + test("close resets coordination availability after an initial channel failure", async () => { + let channelUnavailable = true; + const sources: FakeEventSource[] = []; + const coordinator = createCrossTabSseCoordinator({ + tabId: "recovering", + channelFactory: (name) => { + if (channelUnavailable) throw new Error("channel unavailable"); + return new FakeBroadcastChannel(name); + }, + eventSourceFactory: (url) => { + const source = new FakeEventSource(url, "recovering"); + sources.push(source); + return source; + }, + addVisibilityChangeListener: () => () => {}, + addPagehideListener: () => () => {}, + timing: TEST_TIMING, + }); + let firstFallbackStarted = 0; + let secondFallbackStarted = 0; + + const firstCleanup = subscribeWithFallback(coordinator, { + fallbackSubscribe: () => { + firstFallbackStarted += 1; + return () => {}; + }, + }); + firstCleanup(); + coordinator.close(); + + channelUnavailable = false; + const secondCleanup = subscribeWithFallback(coordinator, { + fallbackSubscribe: () => { + secondFallbackStarted += 1; + return () => {}; + }, + }); + + await waitFor(() => sources.some((source) => !source.closed)); + + expect(firstFallbackStarted).toBe(1); + expect(secondFallbackStarted).toBe(0); + expect(sources.filter((source) => !source.closed).map((source) => source.url)).toEqual(["/api/v1/attach"]); + + secondCleanup(); + coordinator.close(); + }); + + test("close stops fallback subscriptions added after degradation", async () => { + const harness = newHarness(); + const coordinator = harness.createTab("a"); + let fallbackStarted = 0; + let fallbackStopped = 0; + + FakeBroadcastChannel.throwOnTypes.add("leader-changed"); + subscribeWithFallback(coordinator, { + subscriptionKey: "before-degrade", + fallbackSubscribe: () => { + fallbackStarted += 1; + return () => { + fallbackStopped += 1; + }; + }, + }); + await waitFor(() => fallbackStarted === 1); + + FakeBroadcastChannel.throwOnTypes.clear(); + subscribeWithFallback(coordinator, { + subscriptionKey: "after-degrade", + fallbackSubscribe: () => { + fallbackStarted += 1; + return () => { + fallbackStopped += 1; + }; + }, + }); + expect(fallbackStarted).toBe(2); + + coordinator.close(); + + expect(fallbackStopped).toBe(2); + }); }); function newHarness() { @@ -516,6 +625,34 @@ function subscribeForEvent( }); } +function subscribeWithFallback( + coordinator: CrossTabSseCoordinator, + { + subscriptionKey = "fallback-test", + fallbackSubscribe, + }: { + subscriptionKey?: string; + fallbackSubscribe: () => () => void; + }, +) { + return subscribeToCrossTabSse({ + coordinator, + subscriptionKey, + mutate: (() => Promise.resolve()) as MutateFn, + resolveInvalidation: () => ({ keys: [] }), + resyncKeys: () => [], + fallbackSubscribe, + debounceMs: 0, + }); +} + +function candidateGenerations(coordinator: CrossTabSseCoordinator): number[] { + const inspectable = coordinator as unknown as { + candidates: Map; + }; + return [...inspectable.candidates.values()].map((candidate) => candidate.candidateGeneration); +} + function runEvent({ id, runId, diff --git a/apps/fabro-web/app/lib/cross-tab-sse.ts b/apps/fabro-web/app/lib/cross-tab-sse.ts index 0ddb6e220..97ffeb9f7 100644 --- a/apps/fabro-web/app/lib/cross-tab-sse.ts +++ b/apps/fabro-web/app/lib/cross-tab-sse.ts @@ -231,17 +231,16 @@ export class CrossTabSseCoordinator { this.sourceFactoryLocked = true; } - if (this.coordinationUnavailable) { - return options.fallbackSubscribe(); - } - - if (!this.initialized && !this.initialize()) { - this.coordinationUnavailable = true; - return options.fallbackSubscribe(); - } - const subscription = this.addLocalSubscription(options); - if (this.fallbackMode) { + + if (this.coordinationUnavailable) { + this.fallbackMode = true; + this.startFallbacksFor(subscription); + } else if (!this.initialized && !this.initialize()) { + this.coordinationUnavailable = true; + this.fallbackMode = true; + this.startFallbacksFor(subscription); + } else if (this.fallbackMode) { this.startFallbacksFor(subscription); } else { this.ensureLeadershipProgress(); @@ -262,8 +261,10 @@ export class CrossTabSseCoordinator { this.shutdownTimersAndChannel(); this.closeFallbacks(); this.subscriptions.clear(); + this.candidates.clear(); this.leader = null; this.initialized = false; + this.coordinationUnavailable = false; this.fallbackMode = false; this.sourceFactoryLocked = false; } @@ -453,6 +454,7 @@ export class CrossTabSseCoordinator { this.leader = incoming; this.clearNoLeaderTimer(); this.generation = Math.max(this.generation, incoming.generation); + this.pruneStaleCandidates(); if (this.ownCandidate && incoming.generation >= this.ownCandidate.candidateGeneration) { this.clearCandidate(); } @@ -473,6 +475,7 @@ export class CrossTabSseCoordinator { private handleCandidate(message: CandidateMessage) { this.candidates.set(candidateKey(message), message); + this.pruneStaleCandidates(); if ( this.isLeader && @@ -500,6 +503,7 @@ export class CrossTabSseCoordinator { ) { this.leader = null; this.generation = Math.max(this.generation, message.generation); + this.pruneStaleCandidates(); this.enterCandidacy("release", { leaderId: message.leaderId, generation: message.generation, @@ -546,6 +550,7 @@ export class CrossTabSseCoordinator { if (this.now() - current.lastSeen > this.timing.leaderStaleMs) { this.leader = null; this.generation = Math.max(this.generation, current.generation); + this.pruneStaleCandidates(); this.resyncAll(); this.enterCandidacy("stale-leader", current); return; @@ -638,6 +643,7 @@ export class CrossTabSseCoordinator { this.closeSource(); this.isLeader = true; this.generation = generation; + this.pruneStaleCandidates(); this.leader = { leaderId: this.tabId, generation, @@ -843,6 +849,14 @@ export class CrossTabSseCoordinator { this.ownCandidate = null; } + private pruneStaleCandidates() { + for (const [key, candidate] of this.candidates) { + if (candidate.candidateGeneration < this.generation) { + this.candidates.delete(key); + } + } + } + private scheduleNoLeaderCandidacy() { if ( this.noLeaderTimer || From e4e51511e08c29a196146a92862f0b1a6457a3ce Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Mon, 4 May 2026 15:38:19 -0400 Subject: [PATCH 05/17] refactor(web): simplify cross-tab SSE message parsing and helpers Use unknown.ts helpers in parseMessage, factor out parseLeaderPair/Triple and per-variant parsers to remove repeated typeof guards. Extract leaderIsFresh() for the staleness check used in three places, and make RecentEventCache amortized O(1) by walking expired entries from the oldest instead of scanning the whole map per event. Drop the closeOnTerminal parameter in run-events; the fallback path computes close at its single call site. Co-Authored-By: Claude Opus 4.7 (1M context) --- apps/fabro-web/app/lib/cross-tab-sse.ts | 216 +++++++++++------------- apps/fabro-web/app/lib/run-events.ts | 22 +-- 2 files changed, 104 insertions(+), 134 deletions(-) diff --git a/apps/fabro-web/app/lib/cross-tab-sse.ts b/apps/fabro-web/app/lib/cross-tab-sse.ts index 97ffeb9f7..99281a8f0 100644 --- a/apps/fabro-web/app/lib/cross-tab-sse.ts +++ b/apps/fabro-web/app/lib/cross-tab-sse.ts @@ -6,7 +6,7 @@ import { type EventSourceLike, type MutateFn, } from "./sse"; -import { isRecord } from "./unknown"; +import { getNumber, getString, isRecord, type UnknownRecord } from "./unknown"; export const CROSS_TAB_SSE_CHANNEL = "fabro:sse:v1"; export const HEARTBEAT_MS = 1000; @@ -156,7 +156,7 @@ class RecentEventCache { remember(key: string | undefined, now: number): boolean { if (!key) return true; - this.prune(now); + this.evictExpired(now); if (this.seen.has(key)) return false; this.seen.set(key, now); @@ -168,11 +168,10 @@ class RecentEventCache { return true; } - private prune(now: number) { + private evictExpired(now: number) { for (const [key, seenAt] of this.seen) { - if (now - seenAt > this.ttlMs) { - this.seen.delete(key); - } + if (now - seenAt <= this.ttlMs) return; + this.seen.delete(key); } } } @@ -547,7 +546,7 @@ export class CrossTabSseCoordinator { return; } - if (this.now() - current.lastSeen > this.timing.leaderStaleMs) { + if (!this.leaderIsFresh(current)) { this.leader = null; this.generation = Math.max(this.generation, current.generation); this.pruneStaleCandidates(); @@ -568,7 +567,7 @@ export class CrossTabSseCoordinator { reason === "no-leader" && this.leader && this.leader.visibility === "visible" && - this.now() - this.leader.lastSeen <= this.timing.leaderStaleMs + this.leaderIsFresh(this.leader) ) { return; } @@ -618,10 +617,7 @@ export class CrossTabSseCoordinator { } } - if ( - this.leader && - this.now() - this.leader.lastSeen <= this.timing.leaderStaleMs - ) { + if (this.leader && this.leaderIsFresh(this.leader)) { if (this.leader.generation > candidate.candidateGeneration) { this.clearCandidate(); return; @@ -968,6 +964,10 @@ export class CrossTabSseCoordinator { private currentVisibility(): TabVisibility { return this.getVisibility() === "hidden" ? "hidden" : "visible"; } + + private leaderIsFresh(leader: LeaderState): boolean { + return this.now() - leader.lastSeen <= this.timing.leaderStaleMs; + } } const defaultCoordinator = new CrossTabSseCoordinator(); @@ -1017,132 +1017,108 @@ function addBrowserPagehideListener(handler: () => void): () => void { function parseMessage(data: unknown): CrossTabSseMessage | undefined { if (!isRecord(data)) return undefined; if (data.version !== MESSAGE_VERSION) return undefined; - const type = data.type; - const tabId = data.tabId; - const sentAt = data.sentAt; - if (typeof type !== "string") return undefined; - if (typeof tabId !== "string") return undefined; - if (typeof sentAt !== "number") return undefined; - const base = { tabId, sentAt }; + const type = getString(data, "type"); + const tabId = getString(data, "tabId"); + const sentAt = getNumber(data, "sentAt"); + if (!type || !tabId || sentAt === undefined) return undefined; + const base = { type, version: MESSAGE_VERSION, tabId, sentAt }; switch (type) { case "hello": - return baseMessage(base, "hello"); + return { ...base, type: "hello" }; case "heartbeat": { - const { leaderId, generation, visibility } = data; - if (typeof leaderId === "string" && typeof generation === "number" && isVisibility(visibility)) { - return { - ...baseMessage(base, "heartbeat"), - leaderId, - generation, - visibility, - }; - } - return undefined; - } - case "candidate": { - const { - candidateId, - candidateGeneration, - visibility, - observedLeaderId, - observedGeneration, - reason, - } = data; - if ( - typeof candidateId === "string" && - typeof candidateGeneration === "number" && - isVisibility(visibility) && - (typeof observedLeaderId === "string" || observedLeaderId === null) && - typeof observedGeneration === "number" && - isCandidateReason(reason) - ) { - const normalizedObservedLeaderId = - typeof observedLeaderId === "string" ? observedLeaderId : null; - return { - ...baseMessage(base, "candidate"), - candidateId, - candidateGeneration, - visibility, - observedLeaderId: normalizedObservedLeaderId, - observedGeneration, - reason, - }; - } - return undefined; + const triple = parseLeaderTriple(data); + return triple && { ...base, type: "heartbeat", ...triple }; } + case "candidate": + return parseCandidate(data, base); case "leader-changed": { - const { leaderId, generation, visibility } = data; - if (typeof leaderId === "string" && typeof generation === "number" && isVisibility(visibility)) { - return { - ...baseMessage(base, "leader-changed"), - leaderId, - generation, - visibility, - }; - } - return undefined; + const triple = parseLeaderTriple(data); + return triple && { ...base, type: "leader-changed", ...triple }; } case "release": { - const { leaderId, generation } = data; - if (typeof leaderId === "string" && typeof generation === "number") { - return { - ...baseMessage(base, "release"), - leaderId, - generation, - }; - } - return undefined; - } - case "resync": { - const { leaderId, generation, reason } = data; - if ( - (typeof leaderId === "string" || leaderId === null) && - typeof generation === "number" && - isCandidateReason(reason) - ) { - const normalizedLeaderId = typeof leaderId === "string" ? leaderId : null; - return { - ...baseMessage(base, "resync"), - leaderId: normalizedLeaderId, - generation, - reason, - }; - } - return undefined; - } - case "event": { - const { leaderId, generation, payload } = data; - if (typeof leaderId === "string" && typeof generation === "number" && isRecord(payload)) { - return { - ...baseMessage(base, "event"), - leaderId, - generation, - payload, - }; - } - return undefined; + const pair = parseLeaderPair(data); + return pair && { ...base, type: "release", ...pair }; } + case "resync": + return parseResync(data, base); + case "event": + return parseEvent(data, base); default: return undefined; } } -function baseMessage( - data: { - tabId: string; - sentAt: number; - }, - type: TType, -) { +interface ParsedBase { + version: typeof MESSAGE_VERSION; + tabId: string; + sentAt: number; +} + +function parseLeaderPair(data: unknown): { leaderId: string; generation: number } | undefined { + const leaderId = getString(data, "leaderId"); + const generation = getNumber(data, "generation"); + if (!leaderId || generation === undefined) return undefined; + return { leaderId, generation }; +} + +function parseLeaderTriple( + data: unknown, +): { leaderId: string; generation: number; visibility: TabVisibility } | undefined { + const pair = parseLeaderPair(data); + const visibility = getString(data, "visibility"); + if (!pair || !isVisibility(visibility)) return undefined; + return { ...pair, visibility }; +} + +function parseCandidate(data: UnknownRecord, base: ParsedBase): CandidateMessage | undefined { + const candidateId = getString(data, "candidateId"); + const candidateGeneration = getNumber(data, "candidateGeneration"); + const visibility = getString(data, "visibility"); + const observedGeneration = getNumber(data, "observedGeneration"); + const reason = getString(data, "reason"); + const observedLeaderRaw = data.observedLeaderId; + const observedLeaderId = + typeof observedLeaderRaw === "string" ? observedLeaderRaw : observedLeaderRaw === null ? null : undefined; + if ( + !candidateId || + candidateGeneration === undefined || + !isVisibility(visibility) || + observedLeaderId === undefined || + observedGeneration === undefined || + !isCandidateReason(reason) + ) { + return undefined; + } return { - type, - version: MESSAGE_VERSION, - tabId: data.tabId, - sentAt: data.sentAt, + ...base, + type: "candidate", + candidateId, + candidateGeneration, + visibility, + observedLeaderId, + observedGeneration, + reason, }; } +function parseResync(data: UnknownRecord, base: ParsedBase): ResyncMessage | undefined { + const generation = getNumber(data, "generation"); + const reason = getString(data, "reason"); + const leaderRaw = data.leaderId; + const leaderId = typeof leaderRaw === "string" ? leaderRaw : leaderRaw === null ? null : undefined; + if (generation === undefined || !isCandidateReason(reason) || leaderId === undefined) { + return undefined; + } + return { ...base, type: "resync", leaderId, generation, reason }; +} + +function parseEvent(data: UnknownRecord, base: ParsedBase): EventMessage | undefined { + const pair = parseLeaderPair(data); + if (!pair || !isRecord(data.payload)) return undefined; + return { ...base, type: "event", ...pair, payload: data.payload }; +} + function isVisibility(value: unknown): value is TabVisibility { return value === "visible" || value === "hidden"; } diff --git a/apps/fabro-web/app/lib/run-events.ts b/apps/fabro-web/app/lib/run-events.ts index 80394da48..f0a5f6a66 100644 --- a/apps/fabro-web/app/lib/run-events.ts +++ b/apps/fabro-web/app/lib/run-events.ts @@ -125,7 +125,7 @@ export function subscribeToRunEvents( resyncKeys: () => resyncKeysForRun(runId), resolveInvalidation: (payload) => { if (payload.run_id !== runId) return { keys: [] }; - return runInvalidation(runId, payload, { closeOnTerminal: false }); + return runInvalidation(runId, payload); }, fallbackSubscribe: () => subscribeToSharedEventSource({ @@ -135,28 +135,22 @@ export function subscribeToRunEvents( mutate, eventSourceFactory, debounceMs, - resolveInvalidation: (payload) => - runInvalidation(runId, payload, { closeOnTerminal: true }), + resolveInvalidation: (payload) => { + const result = runInvalidation(runId, payload); + return { ...result, close: result.immediate }; + }, }), }); } -function runInvalidation( - runId: string, - payload: RunEventPayload, - { closeOnTerminal }: { closeOnTerminal: boolean }, -) { +function runInvalidation(runId: string, payload: RunEventPayload) { const event = payload.event; - if (!event) return { keys: [] }; + if (!event) return { keys: [], immediate: false }; const stageId = stageIdFromPayload(payload); const keys = queryKeysForRunEvent(runId, event, stageId); const terminal = TERMINAL_EVENTS.has(event); - return { - keys, - close: closeOnTerminal && terminal, - immediate: terminal, - }; + return { keys, immediate: terminal }; } function resyncKeysForRun(runId: string) { From 63940fdddc1cce458f7498035e45ea3b2511ba3b Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Mon, 4 May 2026 15:52:18 -0400 Subject: [PATCH 06/17] fix(web): recover cross-tab SSE coordination after fallback Reset coordinator state when the last subscriber leaves, clear pending debounce timers on close, and keep coordinated EventSource construction owned by the coordinator while fallback subscriptions keep their local factories. --- apps/fabro-web/app/lib/board-events.test.tsx | 14 +++--- apps/fabro-web/app/lib/board-events.ts | 1 - apps/fabro-web/app/lib/cross-tab-sse.test.ts | 45 ++++++++++++++++++++ apps/fabro-web/app/lib/cross-tab-sse.ts | 41 ++++++++++-------- apps/fabro-web/app/lib/run-events.test.tsx | 16 ++++--- apps/fabro-web/app/lib/run-events.ts | 1 - 6 files changed, 87 insertions(+), 31 deletions(-) diff --git a/apps/fabro-web/app/lib/board-events.test.tsx b/apps/fabro-web/app/lib/board-events.test.tsx index 9b398fdc1..497bb6984 100644 --- a/apps/fabro-web/app/lib/board-events.test.tsx +++ b/apps/fabro-web/app/lib/board-events.test.tsx @@ -9,6 +9,7 @@ import { type BroadcastChannelLike, } from "./cross-tab-sse"; import { queryKeys } from "./query-keys"; +import type { EventSourceLike } from "./sse"; type MessageHandler = ((event: { data: string }) => void) | null; @@ -47,15 +48,17 @@ describe("subscribeToBoardEvents", () => { const source = new FakeEventSource(); const created: string[] = []; const keys: string[] = []; - const coordinator = createCoordinator(); + const coordinator = createCoordinator((url) => { + created.push(url); + return source; + }); const mutate = (key: string) => { keys.push(key); return Promise.resolve(); }; - const firstCleanup = subscribeToBoardEvents(mutate, (url) => { - created.push(url); - return source; + const firstCleanup = subscribeToBoardEvents(mutate, () => { + throw new Error("source should be created by coordinator"); }, { debounceMs: 0, coordinator }); const secondCleanup = subscribeToBoardEvents(mutate, () => { throw new Error("source should be reused"); @@ -107,10 +110,11 @@ describe("subscribeToBoardEvents", () => { }); }); -function createCoordinator() { +function createCoordinator(eventSourceFactory: (url: string) => EventSourceLike) { return createCrossTabSseCoordinator({ tabId: "board-test", channelFactory: () => new FakeBroadcastChannel(), + eventSourceFactory, addVisibilityChangeListener: () => () => {}, addPagehideListener: () => () => {}, timing: { diff --git a/apps/fabro-web/app/lib/board-events.ts b/apps/fabro-web/app/lib/board-events.ts index a182d9ad2..dc01765ee 100644 --- a/apps/fabro-web/app/lib/board-events.ts +++ b/apps/fabro-web/app/lib/board-events.ts @@ -56,7 +56,6 @@ export function subscribeToBoardEvents( coordinator, subscriptionKey: BOARD_SUBSCRIPTION_KEY, mutate, - eventSourceFactory, debounceMs, resyncKeys: () => [queryKeys.boards.runs()], resolveInvalidation: boardInvalidation, diff --git a/apps/fabro-web/app/lib/cross-tab-sse.test.ts b/apps/fabro-web/app/lib/cross-tab-sse.test.ts index f38c881fa..76fc0b16e 100644 --- a/apps/fabro-web/app/lib/cross-tab-sse.test.ts +++ b/apps/fabro-web/app/lib/cross-tab-sse.test.ts @@ -542,6 +542,51 @@ describe("subscribeToCrossTabSse", () => { coordinator.close(); }); + test("last unsubscribe retries coordination after an initial channel failure", async () => { + let channelUnavailable = true; + const sources: FakeEventSource[] = []; + const coordinator = createCrossTabSseCoordinator({ + tabId: "retry-after-unsubscribe", + channelFactory: (name) => { + if (channelUnavailable) throw new Error("channel unavailable"); + return new FakeBroadcastChannel(name); + }, + eventSourceFactory: (url) => { + const source = new FakeEventSource(url, "retry-after-unsubscribe"); + sources.push(source); + return source; + }, + addVisibilityChangeListener: () => () => {}, + addPagehideListener: () => () => {}, + timing: TEST_TIMING, + }); + let fallbackStarted = 0; + + const firstCleanup = subscribeWithFallback(coordinator, { + fallbackSubscribe: () => { + fallbackStarted += 1; + return () => {}; + }, + }); + firstCleanup(); + + channelUnavailable = false; + const secondCleanup = subscribeWithFallback(coordinator, { + fallbackSubscribe: () => { + fallbackStarted += 1; + return () => {}; + }, + }); + + await waitFor(() => sources.some((source) => !source.closed)); + + expect(fallbackStarted).toBe(1); + expect(sources.filter((source) => !source.closed).map((source) => source.url)).toEqual(["/api/v1/attach"]); + + secondCleanup(); + coordinator.close(); + }); + test("close stops fallback subscriptions added after degradation", async () => { const harness = newHarness(); const coordinator = harness.createTab("a"); diff --git a/apps/fabro-web/app/lib/cross-tab-sse.ts b/apps/fabro-web/app/lib/cross-tab-sse.ts index 99281a8f0..6975e6948 100644 --- a/apps/fabro-web/app/lib/cross-tab-sse.ts +++ b/apps/fabro-web/app/lib/cross-tab-sse.ts @@ -113,7 +113,6 @@ interface SubscribeOptions { resolveInvalidation: (payload: TPayload) => EventInvalidation; resyncKeys: () => string[]; fallbackSubscribe: () => () => void; - eventSourceFactory?: (url: string) => EventSourceLike; debounceMs?: number; } @@ -206,7 +205,6 @@ export class CrossTabSseCoordinator { private leaderCheckTimer: ReturnType | null = null; private removeVisibilityListener: (() => void) | null = null; private removePagehideListener: (() => void) | null = null; - private sourceFactoryLocked = false; constructor(options: CrossTabSseCoordinatorOptions = {}) { this.tabId = options.tabId ?? createTabId(); @@ -225,11 +223,6 @@ export class CrossTabSseCoordinator { } subscribe(options: SubscribeOptions): () => void { - if (options.eventSourceFactory && !this.source && !this.sourceFactoryLocked) { - this.sourceFactory = options.eventSourceFactory; - this.sourceFactoryLocked = true; - } - const subscription = this.addLocalSubscription(options); if (this.coordinationUnavailable) { @@ -259,13 +252,10 @@ export class CrossTabSseCoordinator { this.clearNoLeaderTimer(); this.shutdownTimersAndChannel(); this.closeFallbacks(); + this.clearSubscriptionTimers(); this.subscriptions.clear(); this.candidates.clear(); - this.leader = null; - this.initialized = false; - this.coordinationUnavailable = false; - this.fallbackMode = false; - this.sourceFactoryLocked = false; + this.resetIdleState(); } private initialize(): boolean { @@ -356,9 +346,7 @@ export class CrossTabSseCoordinator { subscription.refcount -= 1; if (subscription.refcount <= 0) { - if (subscription.debounceTimer) { - clearTimeout(subscription.debounceTimer); - } + this.clearSubscriptionTimer(subscription); this.subscriptions.delete(subscriptionKey); } @@ -367,9 +355,7 @@ export class CrossTabSseCoordinator { this.clearCandidate(); this.clearNoLeaderTimer(); this.shutdownTimersAndChannel(); - this.initialized = false; - this.fallbackMode = false; - this.sourceFactoryLocked = false; + this.resetIdleState(); } } @@ -945,6 +931,25 @@ export class CrossTabSseCoordinator { } } + private clearSubscriptionTimers() { + for (const subscription of this.subscriptions.values()) { + this.clearSubscriptionTimer(subscription); + } + } + + private clearSubscriptionTimer(subscription: LocalSubscription) { + if (!subscription.debounceTimer) return; + clearTimeout(subscription.debounceTimer); + subscription.debounceTimer = null; + } + + private resetIdleState() { + this.leader = null; + this.initialized = false; + this.coordinationUnavailable = false; + this.fallbackMode = false; + } + private shouldAcceptLeader(incoming: LeaderState): boolean { if (!this.leader) return true; if (incoming.generation > this.leader.generation) return true; diff --git a/apps/fabro-web/app/lib/run-events.test.tsx b/apps/fabro-web/app/lib/run-events.test.tsx index db35a757d..e9faa93fc 100644 --- a/apps/fabro-web/app/lib/run-events.test.tsx +++ b/apps/fabro-web/app/lib/run-events.test.tsx @@ -9,6 +9,7 @@ import { type BroadcastChannelLike, } from "./cross-tab-sse"; import { queryKeys } from "./query-keys"; +import type { EventSourceLike } from "./sse"; type MessageHandler = ((event: { data: string }) => void) | null; @@ -55,7 +56,10 @@ describe("subscribeToRunEvents", () => { const source = new FakeEventSource(); const created: string[] = []; const keys: string[] = []; - const coordinator = createCoordinator(); + const coordinator = createCoordinator((url) => { + created.push(url); + return source; + }); const cleanup = subscribeToRunEvents( "run-coordinated", @@ -63,9 +67,8 @@ describe("subscribeToRunEvents", () => { keys.push(key); return Promise.resolve(); }, - (url) => { - created.push(url); - return source; + () => { + throw new Error("source should be created by coordinator"); }, { debounceMs: 0, coordinator }, ); @@ -86,7 +89,7 @@ describe("subscribeToRunEvents", () => { test("coordinated terminal events invalidate without closing the global stream", async () => { const source = new FakeEventSource(); const keys: string[] = []; - const coordinator = createCoordinator(); + const coordinator = createCoordinator(() => source); const cleanup = subscribeToRunEvents( "run-terminal", (key) => { @@ -206,10 +209,11 @@ describe("subscribeToRunEvents", () => { }); }); -function createCoordinator() { +function createCoordinator(eventSourceFactory: (url: string) => EventSourceLike) { return createCrossTabSseCoordinator({ tabId: "run-test", channelFactory: () => new FakeBroadcastChannel(), + eventSourceFactory, addVisibilityChangeListener: () => () => {}, addPagehideListener: () => () => {}, timing: { diff --git a/apps/fabro-web/app/lib/run-events.ts b/apps/fabro-web/app/lib/run-events.ts index f0a5f6a66..1413d38d4 100644 --- a/apps/fabro-web/app/lib/run-events.ts +++ b/apps/fabro-web/app/lib/run-events.ts @@ -120,7 +120,6 @@ export function subscribeToRunEvents( coordinator, subscriptionKey: `run:${runId}`, mutate, - eventSourceFactory, debounceMs, resyncKeys: () => resyncKeysForRun(runId), resolveInvalidation: (payload) => { From b5b08e78d389a1746aa52490efe5d66f26a07eaa Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Mon, 4 May 2026 15:52:24 -0400 Subject: [PATCH 07/17] refactor(api): reuse board column contract across clients Make BoardColumnDefinition.id reference the existing BoardColumn schema and carry that typed contract through generated TypeScript, server responses, demo data, and the runs board UI. --- apps/fabro-web/app/data/runs.ts | 21 ++++++-- apps/fabro-web/app/lib/queries.ts | 9 ++-- apps/fabro-web/app/routes/runs.tsx | 17 ++---- docs/public/api-reference/fabro-api.yaml | 2 +- lib/crates/fabro-server/src/demo/mod.rs | 12 ++--- .../fabro-server/src/server/handler/runs.rs | 54 +++++++++++++------ lib/crates/fabro-server/src/server/tests.rs | 4 +- .../src/models/board-column-definition.ts | 6 ++- 8 files changed, 74 insertions(+), 51 deletions(-) diff --git a/apps/fabro-web/app/data/runs.ts b/apps/fabro-web/app/data/runs.ts index 77d8a4711..b397c5ecc 100644 --- a/apps/fabro-web/app/data/runs.ts +++ b/apps/fabro-web/app/data/runs.ts @@ -1,8 +1,10 @@ import { formatElapsedSecs, formatDurationSecs } from "../lib/format"; -import type { - RunListItem, - RunStatus as ApiRunStatus, - RunSummary, +import { + BoardColumn, + type BoardColumn as ApiBoardColumn, + type RunListItem, + type RunStatus as ApiRunStatus, + type RunSummary, } from "@qltysh/fabro-api-client"; export type CiStatus = "passing" | "failing" | "pending"; @@ -37,7 +39,16 @@ export interface RunItem { sourceDirectory?: string; } -export type ColumnStatus = "queued" | "initializing" | "running" | "blocked" | "succeeded" | "failed"; +export type ColumnStatus = ApiBoardColumn; + +export const columnStatuses = [ + BoardColumn.QUEUED, + BoardColumn.INITIALIZING, + BoardColumn.RUNNING, + BoardColumn.BLOCKED, + BoardColumn.SUCCEEDED, + BoardColumn.FAILED, +] as const satisfies readonly ColumnStatus[]; export const columnStatusDisplay: Record = { queued: { label: "Queued", dot: "bg-fg-muted", text: "text-fg-muted" }, diff --git a/apps/fabro-web/app/lib/queries.ts b/apps/fabro-web/app/lib/queries.ts index 4de23a73e..0da17785e 100644 --- a/apps/fabro-web/app/lib/queries.ts +++ b/apps/fabro-web/app/lib/queries.ts @@ -34,6 +34,9 @@ const immutableOptions: SWRConfiguration = { revalidateOnReconnect: false, }; +type BoardRunsEnvelope = PaginatedEnvelope & + Pick; + export function useAuthConfig() { return useSWR<{ methods: string[] }>(queryKeys.auth.config(), apiFetcher, immutableOptions); } @@ -61,11 +64,7 @@ export function useSystemInfo() { } export function useBoardsRuns() { - return useSWR< - PaginatedEnvelope & { - columns: { id: string; name: string }[]; - } - >(queryKeys.boards.runs(), apiPaginatedFetcher); + return useSWR(queryKeys.boards.runs(), apiPaginatedFetcher); } export function useRun(id: string | undefined) { diff --git a/apps/fabro-web/app/routes/runs.tsx b/apps/fabro-web/app/routes/runs.tsx index 90ef09405..78db03496 100644 --- a/apps/fabro-web/app/routes/runs.tsx +++ b/apps/fabro-web/app/routes/runs.tsx @@ -18,7 +18,7 @@ import { arrayMove, } from "@dnd-kit/sortable"; import { CSS } from "@dnd-kit/utilities"; -import { ciConfig, columnStatusDisplay, deriveCiStatus, mapRunListItem } from "../data/runs"; +import { ciConfig, columnStatusDisplay, columnStatuses, deriveCiStatus, mapRunListItem } from "../data/runs"; import type { CiStatus, CheckRun, CheckStatus, RunItem, RunWithStatus, ColumnStatus } from "../data/runs"; import { EmptyState } from "../components/state"; import { shouldRefreshBoardForEvent, useBoardEvents } from "../lib/board-events"; @@ -50,7 +50,7 @@ const defaultColumnStyle: ColumnStyle = { iconType: "branch", actions: [] }; const defaultColumnColors = { dot: "bg-fg-muted", text: "text-fg-muted" }; interface BoardRunsResponse { - columns: { id: string; name: string }[]; + columns: PaginatedBoardRunList["columns"]; data: PaginatedBoardRunList["data"]; meta: PaginatedBoardRunList["meta"]; } @@ -65,17 +65,8 @@ type Column = { items: RunItem[]; }; -const SKELETON_STATUSES: ColumnStatus[] = [ - "queued", - "initializing", - "running", - "blocked", - "succeeded", - "failed", -]; - function buildSkeletonColumns(): Column[] { - return SKELETON_STATUSES.map((id) => { + return columnStatuses.map((id) => { const colors = columnStatusDisplay[id]; return { id, @@ -100,7 +91,7 @@ export function buildBoardColumns(response: BoardRunsResponse): Column[] { } return response.columns.map((col) => { - const id = col.id as ColumnStatus; + const id = col.id; const colors = columnStatusDisplay[id] ?? defaultColumnColors; return { id, diff --git a/docs/public/api-reference/fabro-api.yaml b/docs/public/api-reference/fabro-api.yaml index d10db4a47..4137dfaad 100644 --- a/docs/public/api-reference/fabro-api.yaml +++ b/docs/public/api-reference/fabro-api.yaml @@ -5670,7 +5670,7 @@ components: - name properties: id: - type: string + $ref: "#/components/schemas/BoardColumn" name: type: string diff --git a/lib/crates/fabro-server/src/demo/mod.rs b/lib/crates/fabro-server/src/demo/mod.rs index 787fb1ae4..08423cd55 100644 --- a/lib/crates/fabro-server/src/demo/mod.rs +++ b/lib/crates/fabro-server/src/demo/mod.rs @@ -975,27 +975,27 @@ mod runs { pub(super) fn columns() -> Vec { vec![ BoardColumnDefinition { - id: "queued".into(), + id: BoardColumn::Queued, name: "Queued".into(), }, BoardColumnDefinition { - id: "initializing".into(), + id: BoardColumn::Initializing, name: "Initializing".into(), }, BoardColumnDefinition { - id: "running".into(), + id: BoardColumn::Running, name: "Running".into(), }, BoardColumnDefinition { - id: "blocked".into(), + id: BoardColumn::Blocked, name: "Blocked".into(), }, BoardColumnDefinition { - id: "succeeded".into(), + id: BoardColumn::Succeeded, name: "Succeeded".into(), }, BoardColumnDefinition { - id: "failed".into(), + id: BoardColumn::Failed, name: "Failed".into(), }, ] diff --git a/lib/crates/fabro-server/src/server/handler/runs.rs b/lib/crates/fabro-server/src/server/handler/runs.rs index eb48c7c4f..9f31597ad 100644 --- a/lib/crates/fabro-server/src/server/handler/runs.rs +++ b/lib/crates/fabro-server/src/server/handler/runs.rs @@ -10,7 +10,9 @@ use axum::{Json, Router}; use base64::Engine as _; use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; use bytes::Bytes; -use fabro_api::types::{RunManifest, RunStatusResponse, SubmitAnswerRequest}; +use fabro_api::types::{ + BoardColumn, BoardColumnDefinition, RunManifest, RunStatusResponse, SubmitAnswerRequest, +}; use fabro_config::Storage; use fabro_interview::AnswerSubmission; use fabro_types::{ @@ -82,27 +84,45 @@ impl ListRunsParams { } } -fn board_column(status: RunStatus) -> Option<&'static str> { +fn board_column(status: RunStatus) -> Option { match status { - RunStatus::Submitted | RunStatus::Queued => Some("queued"), - RunStatus::Starting => Some("initializing"), - RunStatus::Running | RunStatus::Paused { .. } => Some("running"), - RunStatus::Blocked { .. } => Some("blocked"), - RunStatus::Succeeded { .. } => Some("succeeded"), - RunStatus::Failed { .. } | RunStatus::Dead => Some("failed"), + RunStatus::Submitted | RunStatus::Queued => Some(BoardColumn::Queued), + RunStatus::Starting => Some(BoardColumn::Initializing), + RunStatus::Running | RunStatus::Paused { .. } => Some(BoardColumn::Running), + RunStatus::Blocked { .. } => Some(BoardColumn::Blocked), + RunStatus::Succeeded { .. } => Some(BoardColumn::Succeeded), + RunStatus::Failed { .. } | RunStatus::Dead => Some(BoardColumn::Failed), RunStatus::Removing | RunStatus::Archived { .. } => None, } } -pub(crate) fn board_columns() -> serde_json::Value { - serde_json::json!([ - {"id": "queued", "name": "Queued"}, - {"id": "initializing", "name": "Initializing"}, - {"id": "running", "name": "Running"}, - {"id": "blocked", "name": "Blocked"}, - {"id": "succeeded", "name": "Succeeded"}, - {"id": "failed", "name": "Failed"}, - ]) +pub(crate) fn board_columns() -> Vec { + vec![ + BoardColumnDefinition { + id: BoardColumn::Queued, + name: "Queued".into(), + }, + BoardColumnDefinition { + id: BoardColumn::Initializing, + name: "Initializing".into(), + }, + BoardColumnDefinition { + id: BoardColumn::Running, + name: "Running".into(), + }, + BoardColumnDefinition { + id: BoardColumn::Blocked, + name: "Blocked".into(), + }, + BoardColumnDefinition { + id: BoardColumn::Succeeded, + name: "Succeeded".into(), + }, + BoardColumnDefinition { + id: BoardColumn::Failed, + name: "Failed".into(), + }, + ] } async fn board_run_metadata( diff --git a/lib/crates/fabro-server/src/server/tests.rs b/lib/crates/fabro-server/src/server/tests.rs index 5480fef0f..74a18fc17 100644 --- a/lib/crates/fabro-server/src/server/tests.rs +++ b/lib/crates/fabro-server/src/server/tests.rs @@ -6615,8 +6615,8 @@ async fn queue_position_reported_for_queued_runs() { let first_run_id = create_and_start_run(&app, MINIMAL_DOT).await; let second_run_id = create_and_start_run(&app, MINIMAL_DOT).await; - // Queued runs are excluded from the board, so verify queue positions - // via the in-memory state directly. + // Queue position is tracked in memory even when queued runs are also + // visible on the board. let runs = state.runs.lock().expect("runs lock poisoned"); let positions = compute_queue_positions(&runs); let first_id = first_run_id.parse::().unwrap(); diff --git a/lib/packages/fabro-api-client/src/models/board-column-definition.ts b/lib/packages/fabro-api-client/src/models/board-column-definition.ts index f1730a16c..20deba1f5 100644 --- a/lib/packages/fabro-api-client/src/models/board-column-definition.ts +++ b/lib/packages/fabro-api-client/src/models/board-column-definition.ts @@ -13,9 +13,11 @@ */ +// May contain unused imports in some cases +// @ts-ignore +import type { BoardColumn } from './board-column'; export interface BoardColumnDefinition { - 'id': string; + 'id': BoardColumn; 'name': string; } - From 4b100d350d952e11cc5aff9c03423392408835d2 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Mon, 4 May 2026 16:09:12 -0400 Subject: [PATCH 08/17] chore: add plan --- .../2026-05-04-cross-tab-sse-coordination.md | 106 ++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 docs/superpowers/plans/2026-05-04-cross-tab-sse-coordination.md diff --git a/docs/superpowers/plans/2026-05-04-cross-tab-sse-coordination.md b/docs/superpowers/plans/2026-05-04-cross-tab-sse-coordination.md new file mode 100644 index 000000000..115ff9326 --- /dev/null +++ b/docs/superpowers/plans/2026-05-04-cross-tab-sse-coordination.md @@ -0,0 +1,106 @@ +# Cross-Tab SSE Coordination Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Coordinate Fabro web SSE subscriptions across tabs so one browser profile/origin opens at most one UI-owned `/api/v1/attach` EventSource in steady state when `BroadcastChannel` is available; brief overlap during election/takeover is tolerated and deduped. + +**Architecture:** Add a browser-side SSE coordinator that elects one tab as leader, has that leader own the global EventSource, and broadcasts parsed run events to follower tabs over `BroadcastChannel`. Existing board and run-detail invalidation logic becomes a consumer of that global event feed, with the current per-tab SSE behavior preserved as a compatibility fallback. + +**Tech Stack:** React, SWR, browser `BroadcastChannel`, browser `EventSource`, Bun tests, existing Fabro web API query keys. + +--- + +## Summary + +Build a browser-side SSE coordinator so Fabro web opens at most one `/api/v1/attach` EventSource per origin/browser profile in steady state when `BroadcastChannel` is available. Temporary duplicate leaders may exist during election/takeover, but event dedupe and generation checks make the overlap harmless and short-lived. The global stream becomes a shared cache-invalidation feed for both the runs board and run detail pages. No server API, OpenAPI, or Rust streaming contract changes are part of v1. + +This supersedes the earlier web SSE limitation documented in `docs/plans/2026-04-19-002-feat-web-ui-lifecycle-actions-plan.md`: the old shared hook was code reuse only; this plan adds actual socket deduplication. + +## Implementation Changes + +- Add `apps/fabro-web/app/lib/cross-tab-sse.ts`. + - Export `subscribeToCrossTabSse(...)` with the same invalidation style as `subscribeToSharedEventSource`, a `resyncKeys` callback for gap recovery, and a `fallbackSubscribe` callback used when cross-tab coordination is unavailable. + - Use `BroadcastChannel` name `fabro:sse:v1`. + - Generate `tabId` with `crypto.randomUUID()` and a safe random fallback. + - Open one leader-owned `EventSource` to `queryKeys.system.attach()` (`/api/v1/attach`) in steady state. + - Leader dispatches each parsed `EventEnvelope` locally and broadcasts it to follower tabs. + - Followers do not open EventSource while a valid visible leader heartbeat exists; lower lexical `tabId` does not preempt a healthy visible leader. + +- Implement leader election in the cross-tab module. + - Constants: `HEARTBEAT_MS = 1000`, `LEADER_STALE_MS = 4000`, `ELECTION_JITTER_MS = 150`. + - Messages: `hello`, `heartbeat`, `candidate`, `leader-changed`, `release`, `resync`, `event`. + - Define a typed message union. Every message includes `type`, `version: 1`, `tabId`, and `sentAt`. + - `heartbeat`: `{ type, version, tabId, sentAt, leaderId, generation, visibility }`. + - `candidate`: `{ type, version, tabId, sentAt, candidateId: tabId, candidateGeneration, visibility, observedLeaderId, observedGeneration, reason }`, where `reason` is `"hidden-leader" | "stale-leader" | "release" | "no-leader"`. + - `leader-changed`: `{ type, version, tabId, sentAt, leaderId, generation, visibility }`. + - `release`: `{ type, version, tabId, sentAt, leaderId, generation }`. + - `resync`: `{ type, version, tabId, sentAt, leaderId, generation, reason }`. + - `event`: `{ type, version, tabId, sentAt, leaderId, generation, payload }`. + - Use the candidacy phase for all leadership changes: hidden-leader takeover, stale-leader recovery, leader release, and no-leader startup. + - A candidate sets `candidateGeneration = observedGeneration + 1`, broadcasts `candidate`, waits jitter, and opens EventSource only if no higher-priority candidate for the same `candidateGeneration` appears. + - Candidate priority is election-scoped: visible candidates outrank hidden candidates; for equal visibility, lower lexical `candidateId` wins. This priority resolves elections and same-generation split brain only; it is not a reason to preempt a fresh visible leader. + - When a visible follower observes a fresh hidden leader heartbeat, it enters candidacy with `reason: "hidden-leader"`. + - When tabs detect a stale leader, leader release, or no known leader, they enter the same candidacy flow with the matching `reason`. + - If two visible candidates race for the same observed leader/generation, the lower lexical `candidateId` wins. + - Current leaders release when they observe a candidate whose `observedLeaderId` matches their `leaderId` and whose `observedGeneration` is current or newer. + - If same-generation split brain still occurs, lower-priority leaders release when they observe a same-generation higher-priority leader heartbeat or `leader-changed`. + - Each new leader uses `candidateGeneration`, broadcasts `leader-changed`, and followers ignore heartbeats/events from non-current leaders or stale generations. + - Hidden leader keeps the stream only when no visible candidate takes over. + - On `pagehide`/last local unsubscribe, a leader closes EventSource and broadcasts `release`. + - Brief split brain is tolerated; dedupe events by `payload.id`, falling back to `${run_id}:${seq}:${event}`. + - Keep dedupe bounded with a recent-event cache: max 1000 IDs and 5 minute TTL. Evict oldest entries when the max is exceeded and prune expired entries during event handling. Duplicate invalidations after eviction are acceptable; unbounded growth is not. + +- Migrate consumers. + - `apps/fabro-web/app/lib/board-events.ts`: subscribe through the cross-tab global stream; keep existing board event allowlist. + - `apps/fabro-web/app/lib/run-events.ts`: subscribe through the same global stream, filter by `payload.run_id === runId`, and reuse `queryKeysForRunEvent`. + - In coordinated mode, run detail pages stay subscribed while mounted, including terminal runs, so post-terminal archive/unarchive changes can reconcile live. + - Do not close the global stream on `run.completed` / `run.failed`; terminal events only invalidate run-scoped keys. + - Keep `subscribeToSharedEventSource` in `apps/fabro-web/app/lib/sse.ts` for fallback and existing local sharing behavior. + +- Gap and fallback behavior. + - If `BroadcastChannel` is unavailable or throws, call each subscriber's `fallbackSubscribe`. + - Board fallback uses the existing global `/api/v1/attach` path. + - Run detail fallback preserves the existing run-scoped `/api/v1/runs/:id/attach` path, so the old terminal-tab stale limitation remains only in fallback mode. + - Do not add replay to `/api/v1/attach`. + - On leader takeover, stale leader timeout, leader release, and new leader generation, broadcast `resync` or `leader-changed` so every tab with active local subscriptions runs its own `resyncKeys`. + - On `visibilitychange` back to visible without leadership change, run only that tab's local `resyncKeys`; do not broadcast cross-tab resync. + - Board `resyncKeys`: `queryKeys.boards.runs()`. + - Run `resyncKeys`: detail, files, billing, stages, events, LR graph, TB graph, and questions for that run. + +## Tests + +- Add `apps/fabro-web/app/lib/cross-tab-sse.test.ts` with fake `BroadcastChannel`, fake `EventSource`, and fake timers. + - One leader opens `/api/v1/attach`; followers open no EventSource. + - Leader broadcasts an event and all local subscribers receive invalidations. + - Run subscribers ignore events for other `run_id` values. + - Board and run subscriptions can coexist on the same global stream. + - Temporary duplicate leaders are allowed only during election/takeover and converge back to one leader. + - With fresh hidden-leader heartbeats, a visible tab broadcasts candidacy, hidden leader closes, visible tab opens `/api/v1/attach`, and followers resync. + - Two visible candidates racing for the same hidden leader resolve to the lower lexical `candidateId`. + - Two tabs detect the same stale leader simultaneously; only the winning candidate opens `/api/v1/attach` after jitter. + - Same-generation split brain converges to one leader by visibility, then lexical `tabId`. + - A fresh visible leader exists; a new visible follower with a lower lexical `tabId` joins and does not take leadership. + - A hidden leader does not release for a candidate that references an old `observedLeaderId` or stale `observedGeneration`. + - Stale heartbeat triggers takeover and every active tab calls its own `resyncKeys`. + - A follower tab calls its own board/run `resyncKeys` after another tab becomes leader. + - Duplicate event IDs are ignored. + - The recent-event dedupe cache evicts by TTL and max-size bound; duplicate invalidation may recur only after eviction. + - Stale heartbeat/event messages from an old leader/generation are ignored after takeover. + - Last unsubscribe closes leader EventSource and sends `release`. + - Missing/broken BroadcastChannel uses the per-tab fallback. + +- Update existing tests. + - `apps/fabro-web/app/lib/board-events.test.tsx`: assert coordinated mode uses `/api/v1/attach` once and fallback preserves current behavior. + - `apps/fabro-web/app/lib/run-events.test.tsx`: assert coordinated mode filters by `run_id`; fallback preserves current `/runs/:id/attach` behavior; terminal run events do not close the global coordinator; terminal detail tabs still receive archive/unarchive invalidations while mounted in coordinated mode. + +- Verification commands. + - `cd apps/fabro-web && bun test app/lib/cross-tab-sse.test.ts app/lib/board-events.test.tsx app/lib/run-events.test.tsx` + - `cd apps/fabro-web && bun run typecheck` + - Optional manual check: open 8 run-detail tabs for active runs; after election settles and with BroadcastChannel available, Chrome should show one active `/api/v1/attach` EventStream across the participating tabs and no UI-created `/api/v1/runs/:id/attach` streams. + +## Assumptions + +- Treat SSE as live cache invalidation, not an exact gapless event log. +- Preserve current run-specific attach endpoint for CLI, API clients, and fallback only. +- Do not touch unrelated dirty worktree files; intentionally replace or adapt the earlier interrupted SSE test edits as part of the new test suite. +- No docs or public API updates are required for v1 because this is an internal web transport change. From 7769c5cec1627d0733a08b46c65075952ccf25a1 Mon Sep 17 00:00:00 2001 From: "fabro-sh-0530[bot]" <281434857+fabro-sh-0530[bot]@users.noreply.github.com> Date: Mon, 4 May 2026 22:00:15 -0400 Subject: [PATCH 09/17] Generate Fabro PR titles and bodies with structured output (#208) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Summary Fabro now asks the LLM for a structured PR title and reviewer-sized body instead of deriving every title from the workflow goal. This ports the compound-engineering PR-writing recipe into the existing pull request pipeline while preserving Fabro's programmatically appended trailing sections. ### What changed - Replaced plain-text PR body generation with `generate_object` and a strict `{ title, body }` schema. - Added the sizing matrix, writing principles, visual-aid guidance, and duplicate-section guardrails to the PR prompt. - Added model-aware goal/plan/diff truncation caps, with unknown or smaller-context models using the conservative tier. - Kept goal-derived titles as a narrow fallback only when the LLM returns a usable body with an empty title. - Enforced a 72-character title cap across both LLM-generated and fallback titles. - Updated workflow, server, and integration tests for structured responses, fallback behavior, title truncation, and blank-body failures. ### Plan Summary - Move PR content generation to structured output. - Keep existing body assembly and appended sections intact. - Add coverage for title fallback and validation edge cases. ### Fabro Details
Ran 9 stages in 42m 43s for $44.59 | Stage | Duration | Cost | Retries | |---|---|---|---| | start | 0s | – | 0 | | toolchain | 1s | – | 0 | | preflight_compile | 2m 3s | – | 0 | | preflight_lint | 2m 13s | – | 0 | | implement | 15m 8s | $6.37 | 0 | | simplify_opus | 11m 16s | $3.53 | 0 | | simplify_gpt | 9m 5s | $34.69 | 0 | | verify | 2m 17s | – | 0 | | fmt | 2s | – | 0 | | **Total** | **42m 43s** | **$44.59** | **0** |
Ran ImplementPlan.fabro (12 nodes and 15 edges) ```dot digraph ImplementPlan { graph [ goal="Implement and simplify", model_stylesheet=" * { model: claude-opus-4-7; } " ] rankdir=LR start [shape=Mdiamond, label="Start"] exit [shape=Msquare, label="Exit"] toolchain [label="Toolchain", shape=parallelogram, script="command -v cargo >/dev/null || { curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && sudo ln -sf $HOME/.cargo/bin/* /usr/local/bin/; }; cargo --version 2>&1", max_retries=0] preflight_compile [label="Preflight Compile", shape=parallelogram, script="cargo check -q --workspace 2>&1", max_retries=0] preflight_lint [label="Preflight Lint", shape=parallelogram, script="cargo +nightly-2026-04-14 clippy -q --workspace --all-targets -- -D warnings 2>&1", max_retries=0] fix_lints [label="Fix Lints", prompt="The preflight lint step failed. Read the build output from context and fix all clippy lint warnings.", max_visits=3] implement [label="Implement", prompt="Read the plan file referenced in the goal and implement every step. Make all the code changes described in the plan. Use red/green TDD."] simplify_opus [label="Simplify (Opus)", prompt="@prompts/simplify.md"] simplify_gpt [label="Simplify (GPT-55)", prompt="@prompts/simplify.md", model="gpt-55"] verify [label="Verify", shape=parallelogram, script="cargo +nightly-2026-04-14 clippy -q --workspace --all-targets -- -D warnings 2>&1 && cargo nextest run --cargo-quiet --workspace --status-level fail 2>&1 && cargo dev docs refresh 2>&1 && cargo dev docs check 2>&1", goal_gate=true, retry_target="fixup"] fixup [label="Fixup", prompt="The verify step failed. Read the build output from context and fix all clippy lint warnings, test failures, and generated docs errors.", max_visits=3] fmt [label="Format", shape=parallelogram, script="cargo +nightly-2026-04-14 fmt --all 2>&1", max_retries=0] start -> toolchain toolchain -> preflight_compile [condition="outcome=succeeded"] toolchain -> exit preflight_compile -> preflight_lint [condition="outcome=succeeded"] preflight_compile -> exit preflight_lint -> implement [condition="outcome=succeeded"] preflight_lint -> fix_lints fix_lints -> preflight_lint implement -> simplify_opus -> simplify_gpt -> verify verify -> fmt [condition="outcome=succeeded"] verify -> fixup fixup -> verify fmt -> exit } ```
⚒️ Generated with [Fabro](https://fabro.sh) --------- Co-authored-by: Fabro --- lib/crates/fabro-server/src/server/tests.rs | 8 +- .../src/pipeline/pull_request.rs | 587 ++++++++++++++++-- .../fabro-workflow/tests/it/integration.rs | 11 +- 3 files changed, 547 insertions(+), 59 deletions(-) diff --git a/lib/crates/fabro-server/src/server/tests.rs b/lib/crates/fabro-server/src/server/tests.rs index 74a18fc17..5587e1952 100644 --- a/lib/crates/fabro-server/src/server/tests.rs +++ b/lib/crates/fabro-server/src/server/tests.rs @@ -3665,7 +3665,13 @@ async fn create_run_pull_request_creates_and_persists_record() { .header("authorization", "Bearer openai-key"); then.status(200) .header("content-type", "application/json") - .json_body(openai_responses_payload("Narrative from mock.")); + .json_body(openai_responses_payload( + &serde_json::to_string(&json!({ + "title": "Mock title", + "body": "Narrative from mock.", + })) + .unwrap(), + )); }) .await; let openai_base_url = llm.url("/v1"); diff --git a/lib/crates/fabro-workflow/src/pipeline/pull_request.rs b/lib/crates/fabro-workflow/src/pipeline/pull_request.rs index 5b9355ccf..8825f8eef 100644 --- a/lib/crates/fabro-workflow/src/pipeline/pull_request.rs +++ b/lib/crates/fabro-workflow/src/pipeline/pull_request.rs @@ -1,10 +1,11 @@ -use std::sync::Arc; +use std::sync::{Arc, LazyLock}; use fabro_auth::CredentialSource; use fabro_github::{self as github_app, ssh_url_to_https}; use fabro_graphviz::parser; use fabro_llm::client::Client; -use fabro_llm::generate::{GenerateParams, generate}; +use fabro_llm::generate::{GenerateParams, generate_object}; +use fabro_model::Catalog; use fabro_retro::retro::Retro; use fabro_store::RunProjection; use fabro_types::PullRequestRecord; @@ -18,17 +19,165 @@ use crate::outcome::{StageOutcome, format_cost as outcome_format_cost}; use crate::records::{Conclusion, RunSpec}; use crate::runtime_store::RunStoreHandle; +/// Maximum length of a PR title (Unicode scalar values). Single source of +/// truth — referenced by the structured-output schema, the system prompt, +/// and [`enforce_title_cap`]. +const PR_TITLE_MAX_CHARS: usize = 72; + +/// Structured output schema for the LLM-generated PR title and body. +/// +/// `title` is required but allows empty strings (the only signal that +/// triggers the deterministic title fallback in +/// [`maybe_open_pull_request`]). `body` requires `minLength: 1` because +/// there is no body fallback — an empty body is fatal. +static PR_CONTENT_SCHEMA: LazyLock = LazyLock::new(|| { + serde_json::json!({ + "type": "object", + "properties": { + "title": { "type": "string", "maxLength": PR_TITLE_MAX_CHARS }, + "body": { "type": "string", "minLength": 1 } + }, + "required": ["title", "body"], + "additionalProperties": false + }) +}); + +#[derive(Debug, serde::Deserialize)] +struct GeneratedPrContent { + title: String, + body: String, +} + +/// System prompt that instructs the LLM how to write a Fabro PR title and +/// body. The trailing programmatic sections (Plan `
`, Retro, +/// Fabro Details, footer) are appended after the LLM body — the prompt +/// explicitly forbids the LLM from duplicating them. +// +// The "max 72 characters" instruction must stay in sync with +// `PR_TITLE_MAX_CHARS` and the schema above; the prompt is advisory and +// `enforce_title_cap` is the actual enforcement. +const PR_BODY_SYSTEM_PROMPT: &str = "You are writing a pull request title and description for a code change produced by an AI workflow. + +OUTPUT FORMAT +Return a JSON object with exactly two fields: +- \"title\": a one-line title, max 72 characters, no trailing period. +- \"body\": the markdown body as described below. + +DO NOT INCLUDE in the body +- A `#` or `##` title heading at the top — the title goes in the `title` field. +- A \"Retro\" section, \"Fabro Details\" section, cost/duration table, or \"Generated with\" footer — those are appended programmatically after your output. +- The full plan text — the full plan is appended programmatically as a
block. +- Bare `#1`, `#2` list prefixes — GitHub auto-links those as issue references. Use plain `1.`, `2.` instead. +- A test plan unless the testing approach is non-obvious. + +SIZE THE BODY TO THE CHANGE +First classify along two axes from the diff: +- Size: how many files changed, how large the diff is. +- Complexity: trivial (rename / typo / dep bump / config) vs. design decisions / new patterns / cross-cutting concerns. + +Then write at the matching depth: + +| Profile | Body shape | +|---|---| +| Small + simple (typo, config, dep bump) | 1–2 sentences, no headers, total under ~300 characters | +| Small + non-trivial (targeted bugfix, behavioral change) | Short \"Problem / Fix\" narrative, 3–5 sentences. No headers unless two distinct concerns. | +| Medium feature or refactor | Summary paragraph, then a section explaining what changed and why. Call out design decisions. | +| Large or architecturally significant | Full narrative: problem context, approach chosen (and why), key decisions, migration/rollback notes if relevant. | +| Performance improvement | Include before/after measurements if available. A markdown table works well here. | + +Brevity matters for small changes. A 3-line bugfix with a 20-line description signals miscalibration. When in doubt, shorter is better — reviewers can read the diff. + +WRITING PRINCIPLES +- Lead with value: the first sentence tells the reviewer *why this PR exists*, not *what files changed*. +- Describe the net result, not the journey: skip intermediate failures, debugging steps, and refactors done during development. +- Trust the final diff: if the goal or plan disagree with the diff, the diff is authoritative. +- Explain the non-obvious: spend description space on what the diff doesn't show — why this approach, what was rejected, what to look at first. +- Use structure when it earns its keep: no empty sections, no template headers without content. +- If the body uses any `##` heading, the opening summary must also be under a heading (e.g. `## Summary`); otherwise a bare paragraph is fine. + +PLAN SUMMARY +The full plan is attached separately as a
block, so do not restate it. Include a brief `### Plan Summary` with bullet points only when the change is medium or larger in the sizing matrix above. Skip it for small changes. + +VISUAL AIDS +Include a visual aid only when a reviewer would struggle to reconstruct the mental model from prose alone — based on what changes structurally, not on PR size. Skip for trivial / mechanical changes, or when prose already communicates clearly. + +| PR changes... | Visual aid | +|---|---| +| 3+ interacting components or services | Mermaid component / interaction diagram | +| Multi-step workflow or pipeline with non-obvious sequencing | Mermaid flow diagram | +| 3+ behavioral modes or variants | Markdown comparison table | +| Before/after data or trade-offs | Markdown table | +| Data model changes with 3+ related entities | Mermaid ERD | + +Mermaid: prefer `TB` direction, ≤10 nodes typical. Place inline at the point of relevance, not in a separate \"Diagrams\" section."; + +/// Truncation budget for the LLM prompt's goal / plan / diff sections. +struct TruncationCaps { + goal: usize, + plan: usize, + diff: usize, +} + +/// Generous tier for models with ≥200k context windows. +const TRUNCATION_LARGE: TruncationCaps = TruncationCaps { + goal: 75_000, + plan: 75_000, + diff: 250_000, +}; + +/// Conservative tier (matches the pre-refactor values). Used for smaller +/// or unknown models. +const TRUNCATION_SMALL: TruncationCaps = TruncationCaps { + goal: 20_000, + plan: 20_000, + diff: 50_000, +}; + +/// Resolve truncation caps based on the model's context window. Unknown +/// models fall through to the conservative tier. +fn truncation_caps(model: &str) -> &'static TruncationCaps { + let large_enough = Catalog::builtin() + .get(model) + .is_some_and(|m| m.context_window() >= 200_000); + if large_enough { + &TRUNCATION_LARGE + } else { + &TRUNCATION_SMALL + } +} + +/// Truncate `s` to at most `max` Unicode scalar values without splitting a +/// UTF-8 sequence. +fn truncate_chars(s: &str, max: usize) -> &str { + s.char_indices() + .nth(max) + .map_or(s, |(boundary, _)| &s[..boundary]) +} + +/// Truncate `s` to at most `max` Unicode scalar values, replacing the +/// trailing char with `…` when truncation occurs. +fn truncate_with_ellipsis(s: &str, max: usize) -> String { + if s.chars().count() > max { + let truncated: String = s.chars().take(max - 1).collect(); + format!("{truncated}\u{2026}") + } else { + s.to_string() + } +} + +/// Cap a PR title at [`PR_TITLE_MAX_CHARS`]. +fn enforce_title_cap(title: &str) -> String { + truncate_with_ellipsis(title, PR_TITLE_MAX_CHARS) +} + /// Derive a PR title from the workflow goal. /// -/// Uses the first line, truncated to 120 characters for readability. +/// Uses the first line, truncated to 120 characters for readability. The +/// caller is expected to apply [`enforce_title_cap`] afterwards if a +/// stricter cap is required (the wider cap here is the legacy behaviour +/// for the deterministic fallback path). fn pr_title_from_goal(goal: &str) -> String { - let stripped = strip_goal_decoration(goal); - if stripped.chars().count() > 120 { - let truncated: String = stripped.chars().take(119).collect(); - format!("{truncated}…") - } else { - stripped.to_string() - } + truncate_with_ellipsis(strip_goal_decoration(goal), 120) } /// Truncate a PR body to fit GitHub's 65,536 character limit. @@ -286,8 +435,13 @@ async fn load_pull_request_diff(run_store: &RunStoreHandle) -> String { .unwrap_or_default() } -/// Build a complete PR body by combining LLM-generated narrative with -/// programmatic sections (plan, retro, fabro details). +/// Build a complete PR title and body by combining LLM-generated narrative +/// with programmatic sections (plan, retro, fabro details). +/// +/// Returns `(title, body)`. The title may be the empty string when the LLM +/// returned a usable body but no usable title — callers fall back to +/// [`pr_title_from_goal`] in that case. Every other generation failure is +/// surfaced as `Err`. pub async fn build_pr_body( diff: &str, goal: &str, @@ -295,7 +449,7 @@ pub async fn build_pr_body( run_store: &RunStoreHandle, llm_source: &dyn CredentialSource, conclusion: Option<&Conclusion>, -) -> Result { +) -> Result<(String, String), String> { let client = Client::from_source(llm_source) .await .map_err(|e| format!("Failed to create LLM client: {e}"))?; @@ -310,7 +464,7 @@ async fn build_pr_body_with_client( run_store: &RunStoreHandle, conclusion: Option<&Conclusion>, client: Arc, -) -> Result { +) -> Result<(String, String), String> { build_pr_body_with_client_and_state(diff, goal, model, run_store, conclusion, client, None) .await } @@ -323,7 +477,7 @@ async fn build_pr_body_with_source_and_state( llm_source: &dyn CredentialSource, conclusion: Option<&Conclusion>, run_state: Option<&fabro_store::RunProjection>, -) -> Result { +) -> Result<(String, String), String> { let client = Client::from_source(llm_source) .await .map_err(|e| format!("Failed to create LLM client: {e}"))?; @@ -348,7 +502,7 @@ async fn build_pr_body_with_client_and_state( conclusion: Option<&Conclusion>, client: Arc, run_state: Option<&fabro_store::RunProjection>, -) -> Result { +) -> Result<(String, String), String> { info!("Building PR body"); let loaded_run_state = if run_state.is_none() { @@ -369,45 +523,39 @@ async fn build_pr_body_with_client_and_state( let run_spec = run_state.and_then(|state| state.spec.clone()); let dot_source = run_state.and_then(|state| state.graph_source.clone()); - // Build LLM prompt - let system = if plan_text.is_some() { - "Write a PR description with: (1) 2-3 concise paragraphs explaining the change, then (2) a '### Plan Summary' section with bullet points summarizing the plan. Do not include a title. Do not include the full plan.".to_string() - } else { - "Write a concise PR description in 2-3 paragraphs explaining the change. Do not include a title.".to_string() - }; - - // Truncate diff to fit context windows (~50k chars) - let max_diff_len = 50_000; - let truncated_diff = if diff.len() > max_diff_len { - &diff[..diff.floor_char_boundary(max_diff_len)] - } else { - diff - }; + let caps = truncation_caps(model); + let truncated_goal = truncate_chars(goal, caps.goal); + let truncated_diff = truncate_chars(diff, caps.diff); let prompt = if let Some(ref plan) = plan_text { - // Truncate plan for LLM context (~20k chars) - let max_plan_len = 20_000; - let truncated_plan = if plan.len() > max_plan_len { - &plan[..plan.floor_char_boundary(max_plan_len)] - } else { - plan.as_str() - }; + let truncated_plan = truncate_chars(plan, caps.plan); format!( - "Goal: {goal}\n\nPlan:\n```\n{truncated_plan}\n```\n\nDiff:\n```\n{truncated_diff}\n```" + "Goal: {truncated_goal}\n\nPlan:\n```\n{truncated_plan}\n```\n\nDiff:\n```\n{truncated_diff}\n```" ) } else { - format!("Goal: {goal}\n\nDiff:\n```\n{truncated_diff}\n```") + format!("Goal: {truncated_goal}\n\nDiff:\n```\n{truncated_diff}\n```") }; let params = GenerateParams::new(model, client) - .system(system) + .system(PR_BODY_SYSTEM_PROMPT) .prompt(prompt); - let result = generate(params) + let result = generate_object(params, PR_CONTENT_SCHEMA.clone()) .await .map_err(|e| format!("LLM generation failed: {e}"))?; - let llm_output = result.response.text(); + let output = result + .output + .ok_or_else(|| "LLM generation returned no structured output".to_string())?; + let generated: GeneratedPrContent = serde_json::from_value(output) + .map_err(|e| format!("Failed to deserialize PR content: {e}"))?; + + if generated.body.trim().is_empty() { + return Err("LLM generated an empty PR body".to_string()); + } + + let title = enforce_title_cap(generated.title.trim()); + let llm_body = generated.body; let retro_section = retro.as_ref().map(format_retro_section).unwrap_or_default(); let arc_details_section = conclusion @@ -416,7 +564,7 @@ async fn build_pr_body_with_client_and_state( .unwrap_or_default(); let body = assemble_pr_body( - &llm_output, + &llm_body, plan_text.as_deref(), &retro_section, &arc_details_section, @@ -424,7 +572,7 @@ async fn build_pr_body_with_client_and_state( info!("PR body generated"); - Ok(body) + Ok((title, body)) } /// Auto-merge configuration for a pull request. @@ -465,7 +613,7 @@ pub async fn maybe_open_pull_request( let (owner, repo) = github_app::parse_github_owner_repo(&https_url).map_err(|err| format!("{err:#}"))?; - let body = build_pr_body_with_source_and_state( + let (llm_title, body) = build_pr_body_with_source_and_state( req.diff, req.goal, req.model, @@ -478,7 +626,12 @@ pub async fn maybe_open_pull_request( .map_err(|err| format!("{err:#}"))?; let body = truncate_pr_body(&body); - let title = pr_title_from_goal(req.goal); + let title = if llm_title.is_empty() { + pr_title_from_goal(req.goal) + } else { + llm_title + }; + let title = enforce_title_cap(&title); let created = github_app::create_pull_request( &req.github, @@ -782,6 +935,16 @@ mod tests { }) } + /// JSON string the MockProvider/openai mock returns to simulate the + /// structured-output response for `(title, body)`. + fn pr_content_json(title: &str, body: &str) -> String { + serde_json::to_string(&serde_json::json!({ + "title": title, + "body": body, + })) + .unwrap() + } + fn make_test_conclusion() -> Conclusion { Conclusion { timestamp: Utc::now(), @@ -1126,17 +1289,21 @@ mod tests { async fn build_pr_body_uses_in_memory_conclusion() { let store = test_store(); let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); - let body = build_pr_body_with_client( + let (title, body) = build_pr_body_with_client( "diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n", "Implement feature", "mock-model", &run_store.clone().into(), Some(&make_test_conclusion()), - explicit_client("mock", "Narrative from mock."), + explicit_client( + "mock", + &pr_content_json("Mock title", "Narrative from mock."), + ), ) .await .unwrap(); + assert_eq!(title, "Mock title"); assert!(body.contains("Narrative from mock.")); assert!(body.contains("### Fabro Details")); assert!(body.contains("Ran 3 stages in 2m 30s for $0.42")); @@ -1196,13 +1363,16 @@ mod tests { .await .unwrap(); - let body = build_pr_body_with_client( + let (_, body) = build_pr_body_with_client( "diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n", "Implement feature", "mock-model", &run_store.clone().into(), Some(&make_test_conclusion()), - explicit_client("mock", "Narrative from mock."), + explicit_client( + "mock", + &pr_content_json("Mock title", "Narrative from mock."), + ), ) .await .unwrap(); @@ -1283,13 +1453,16 @@ mod tests { .await .unwrap(); - let body = build_pr_body_with_client( + let (_, body) = build_pr_body_with_client( "diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n", "Implement feature", "mock-model", &run_store.clone().into(), Some(&make_test_conclusion()), - explicit_client("mock", "Narrative from mock."), + explicit_client( + "mock", + &pr_content_json("Mock title", "Narrative from mock."), + ), ) .await .unwrap(); @@ -1302,13 +1475,16 @@ mod tests { async fn build_pr_body_uses_explicit_llm_client() { let store = test_store(); let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); - let body = build_pr_body_with_client( + let (_, body) = build_pr_body_with_client( "diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n", "Implement feature", "gpt-5.4", &run_store.clone().into(), Some(&make_test_conclusion()), - explicit_client("openai", "Narrative from explicit client."), + explicit_client( + "openai", + &pr_content_json("Explicit title", "Narrative from explicit client."), + ), ) .await .unwrap(); @@ -1327,7 +1503,10 @@ mod tests { .header("authorization", "Bearer vault-openai-key"); then.status(200) .header("content-type", "application/json") - .json_body(openai_responses_payload("Narrative from vault source.")); + .json_body(openai_responses_payload(&pr_content_json( + "Vault title", + "Narrative from vault source.", + ))); }) .await; @@ -1355,7 +1534,7 @@ mod tests { let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); let run_store_handle: RunStoreHandle = run_store.into(); - let body = build_pr_body( + let (title, body) = build_pr_body( "diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n", "Implement feature", "gpt-5.4", @@ -1366,6 +1545,7 @@ mod tests { .await .unwrap(); + assert_eq!(title, "Vault title"); assert!(body.contains("Narrative from vault source.")); response_mock.assert_async().await; } @@ -1575,4 +1755,299 @@ mod tests { assert!(diff.contains("from_store")); } + + // ── Structured-output PR content tests ────────────────────────────── + + /// MockProvider returns an over-long title; builder must cap it at 72 + /// chars and end with `…`. Exercises [`enforce_title_cap`] inside + /// [`build_pr_body_with_client_and_state`]. + #[tokio::test] + async fn build_pr_body_truncates_long_title() { + let store = test_store(); + let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); + let long_title = "x".repeat(200); + let payload = pr_content_json(&long_title, "Body content."); + let (title, _) = build_pr_body_with_client( + "diff --git a/src/lib.rs b/src/lib.rs\n+fn x() {}\n", + "Implement feature", + "mock-model", + &run_store.clone().into(), + Some(&make_test_conclusion()), + explicit_client("mock", &payload), + ) + .await + .unwrap(); + + assert_eq!(title.chars().count(), 72); + assert!(title.ends_with('\u{2026}')); + } + + /// Empty bodies are fatal. Real providers may reject this via the + /// schema's `minLength`; the Rust-side trim check also catches it for + /// local/mock providers. + #[tokio::test] + async fn build_pr_body_returns_err_when_body_empty() { + let store = test_store(); + let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); + let payload = pr_content_json("Mock", ""); + let result = build_pr_body_with_client( + "diff --git a/src/lib.rs b/src/lib.rs\n+fn x() {}\n", + "Implement feature", + "mock-model", + &run_store.clone().into(), + Some(&make_test_conclusion()), + explicit_client("mock", &payload), + ) + .await; + + assert!(result.is_err(), "expected Err, got {result:?}"); + } + + /// Whitespace-only bodies pass schema validation but fail the + /// `body.trim().is_empty()` check inside the builder. + #[tokio::test] + async fn build_pr_body_returns_err_when_body_whitespace() { + let store = test_store(); + let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); + let payload = pr_content_json("Mock", " \n"); + let result = build_pr_body_with_client( + "diff --git a/src/lib.rs b/src/lib.rs\n+fn x() {}\n", + "Implement feature", + "mock-model", + &run_store.clone().into(), + Some(&make_test_conclusion()), + explicit_client("mock", &payload), + ) + .await; + + let err = result.expect_err("expected Err for whitespace-only body"); + assert!(err.contains("empty PR body"), "unexpected error: {err}"); + } + + // ── maybe_open_pull_request fallback tests ────────────────────────── + + /// Set of mock servers and credentials for the `maybe_open_pull_request` + /// fallback path. The builder's `Client::from_source` rebuilds the LLM + /// client from the credential source, so the in-process MockProvider + /// cannot intercept — we mock the OpenAI HTTP endpoint instead. + struct FallbackHarness { + _vault_dir: tempfile::TempDir, + // Held to keep the mock listener alive for the duration of the test; + // the test interacts with it via `Client::from_source` (which goes + // out via HTTP to the mock URL stored in `llm_source`). + openai_server: MockServer, + github_server: MockServer, + openai_mock_id: usize, + github_mock_id: usize, + llm_source: Arc, + creds: fabro_github::GitHubCredentials, + run_store: RunStoreHandle, + } + + impl FallbackHarness { + async fn assert_mocks_called_once(&self) { + httpmock::Mock::new(self.openai_mock_id, &self.openai_server) + .assert_async() + .await; + httpmock::Mock::new(self.github_mock_id, &self.github_server) + .assert_async() + .await; + } + } + + /// Stand up an OpenAI mock that returns the given structured-output + /// payload, a GitHub mock that accepts a PR creation, a vault-backed + /// credential source, and a run store seeded with a non-empty + /// `final_patch`. + async fn setup_fallback_test_harness(openai_payload_text: &str) -> FallbackHarness { + let openai_server = MockServer::start_async().await; + let openai_mock = openai_server + .mock_async(|when, then| { + when.method(POST) + .path("/v1/responses") + .header("authorization", "Bearer vault-openai-key"); + then.status(200) + .header("content-type", "application/json") + .json_body(openai_responses_payload(openai_payload_text)); + }) + .await; + + let github_server = MockServer::start_async().await; + let github_mock = github_server + .mock_async(|when, then| { + when.method(POST) + .path("/repos/owner/repo/pulls") + .header("authorization", "Bearer test-token"); + then.status(201) + .header("content-type", "application/json") + .json_body(serde_json::json!({ + "number": 1, + "html_url": "https://example.test/owner/repo/pull/1", + "node_id": "PR_kwTest1", + })); + }) + .await; + + let vault_dir = tempfile::tempdir().unwrap(); + let mut vault = Vault::load(vault_dir.path().join("secrets.json")).unwrap(); + vault + .set( + "openai_codex", + &serde_json::to_string(&openai_api_key_credential("vault-openai-key")).unwrap(), + SecretType::Credential, + None, + ) + .unwrap(); + let base_url = openai_server.url("/v1"); + let llm_source: Arc = + Arc::new(VaultCredentialSource::with_env_lookup( + Arc::new(AsyncRwLock::new(vault)), + move |name| match name { + "OPENAI_BASE_URL" => Some(base_url.clone()), + _ => None, + }, + )); + + let creds = fabro_github::GitHubCredentials::Token("test-token".to_string()); + + let store = test_store(); + let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); + // Seed a non-empty `final_patch` so `load_pull_request_diff` returns + // diff content and the early-return for empty diffs does not fire. + let run_spec = RunSpec { + run_id: fixtures::RUN_1, + settings: fabro_types::WorkflowSettings::default(), + graph: Graph::new("test"), + workflow_slug: None, + source_directory: None, + git: None, + labels: HashMap::new(), + provenance: None, + manifest_blob: None, + definition_blob: None, + fork_source_ref: None, + in_place: false, + }; + append_event(&run_store, &fixtures::RUN_1, &Event::RunCreated { + run_id: fixtures::RUN_1, + settings: serde_json::to_value(&run_spec.settings).unwrap(), + graph: serde_json::to_value(&run_spec.graph).unwrap(), + workflow_source: None, + workflow_config: None, + labels: run_spec.labels.clone().into_iter().collect(), + run_dir: "/tmp/x".to_string(), + source_directory: None, + workflow_slug: None, + db_prefix: None, + provenance: None, + manifest_blob: None, + git: None, + fork_source_ref: None, + in_place: false, + web_url: None, + }) + .await + .unwrap(); + append_event(&run_store, &fixtures::RUN_1, &Event::WorkflowRunCompleted { + duration_ms: 1, + artifact_count: 0, + status: "succeeded".to_string(), + reason: SuccessReason::Completed, + total_usd_micros: None, + final_git_commit_sha: None, + final_patch: Some( + "diff --git a/src/lib.rs b/src/lib.rs\n+fn from_store() {}\n".to_string(), + ), + billing: None, + }) + .await + .unwrap(); + + let openai_mock_id = openai_mock.id; + let github_mock_id = github_mock.id; + + FallbackHarness { + _vault_dir: vault_dir, + openai_server, + github_server, + openai_mock_id, + github_mock_id, + llm_source, + creds, + run_store: run_store.into(), + } + } + + /// LLM returns a usable body but an empty title; `maybe_open_pull_request` + /// must fall back to `pr_title_from_goal` (first line, decoration + /// stripped) and the PR creation must succeed with that title. + #[tokio::test] + async fn maybe_open_pull_request_falls_back_to_goal_title_when_llm_returns_empty_title() { + let payload = pr_content_json("", "Narrative."); + let harness = setup_fallback_test_harness(&payload).await; + + let github_base_url = harness.github_server.url(""); + let github = github_app::GitHubContext::new(&harness.creds, &github_base_url); + + let result = maybe_open_pull_request(OpenPullRequestRequest { + github, + origin_url: "https://github.com/owner/repo.git", + base_branch: "main", + head_branch: "fabro/run/123", + goal: "Fix telemetry leak\n\ndetails...", + diff: "diff --git a/src/lib.rs b/src/lib.rs\n+fn x() {}\n", + model: "gpt-5.4", + draft: false, + auto_merge: None, + run_store: &harness.run_store, + llm_source: harness.llm_source.as_ref(), + conclusion: None, + run_state: None, + }) + .await + .expect("PR creation should succeed"); + + let record = result.expect("PR record should be Some"); + assert_eq!(record.title, "Fix telemetry leak"); + harness.assert_mocks_called_once().await; + } + + /// LLM returns an empty title; the fallback path produces a long title + /// (close to `pr_title_from_goal`'s 120-char cap), and the unconditional + /// `enforce_title_cap` in `maybe_open_pull_request` must still bring it + /// down to 72 chars ending with `…`. + #[tokio::test] + async fn maybe_open_pull_request_caps_fallback_title_at_72_chars() { + let payload = pr_content_json("", "Narrative."); + let harness = setup_fallback_test_harness(&payload).await; + + let github_base_url = harness.github_server.url(""); + let github = github_app::GitHubContext::new(&harness.creds, &github_base_url); + + // Single ~200-char line, no `Plan:` / heading prefix, no newlines. + let goal = "x".repeat(200); + + let result = maybe_open_pull_request(OpenPullRequestRequest { + github, + origin_url: "https://github.com/owner/repo.git", + base_branch: "main", + head_branch: "fabro/run/123", + goal: &goal, + diff: "diff --git a/src/lib.rs b/src/lib.rs\n+fn x() {}\n", + model: "gpt-5.4", + draft: false, + auto_merge: None, + run_store: &harness.run_store, + llm_source: harness.llm_source.as_ref(), + conclusion: None, + run_state: None, + }) + .await + .expect("PR creation should succeed"); + + let record = result.expect("PR record should be Some"); + assert_eq!(record.title.chars().count(), 72); + assert!(record.title.ends_with('\u{2026}')); + harness.assert_mocks_called_once().await; + } } diff --git a/lib/crates/fabro-workflow/tests/it/integration.rs b/lib/crates/fabro-workflow/tests/it/integration.rs index 002d473da..742218c14 100644 --- a/lib/crates/fabro-workflow/tests/it/integration.rs +++ b/lib/crates/fabro-workflow/tests/it/integration.rs @@ -6809,7 +6809,13 @@ async fn workflow_run_with_vault_only_openai_codex_builds_pr_body() { .header("authorization", "Bearer vault-openai-key"); then.status(200) .header("content-type", "application/json") - .json_body(openai_responses_payload("Narrative from vault source.")); + .json_body(openai_responses_payload( + &serde_json::to_string(&serde_json::json!({ + "title": "Vault title", + "body": "Narrative from vault source.", + })) + .unwrap(), + )); }) .await; @@ -6890,7 +6896,7 @@ async fn workflow_run_with_vault_only_openai_codex_builds_pr_body() { let run_store = store.open_run_reader(&run_options.run_id).await.unwrap(); let run_store_handle: fabro_workflow::runtime_store::RunStoreHandle = run_store.into(); - let body = fabro_workflow::pull_request::build_pr_body( + let (title, body) = fabro_workflow::pull_request::build_pr_body( "diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n", "Implement feature", "gpt-5.4", @@ -6910,6 +6916,7 @@ async fn workflow_run_with_vault_only_openai_codex_builds_pr_body() { .await .expect("PR body should build from vault-only credentials"); + assert_eq!(title, "Vault title"); assert!(body.contains("Narrative from vault source.")); response_mock.assert_async().await; } From a2fbac1d60d76018fa934dbe0865369fc40e1127 Mon Sep 17 00:00:00 2001 From: "fabro-releases[bot]" Date: Tue, 5 May 2026 09:52:01 +0000 Subject: [PATCH 10/17] Bump version to 0.224.0-nightly.0 --- Cargo.lock | 86 +++++++++++++++++++++++++++--------------------------- Cargo.toml | 2 +- 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e4e138ffa..f88b44544 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1536,7 +1536,7 @@ dependencies = [ [[package]] name = "fabro-agent" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "async-trait", @@ -1575,7 +1575,7 @@ dependencies = [ [[package]] name = "fabro-api" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "chrono", "fabro-config", @@ -1596,7 +1596,7 @@ dependencies = [ [[package]] name = "fabro-auth" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "async-trait", @@ -1620,7 +1620,7 @@ dependencies = [ [[package]] name = "fabro-checkpoint" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "chrono", "fabro-config", @@ -1636,7 +1636,7 @@ dependencies = [ [[package]] name = "fabro-cli" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "assert_cmd", @@ -1732,7 +1732,7 @@ dependencies = [ [[package]] name = "fabro-client" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "bytes", @@ -1761,7 +1761,7 @@ dependencies = [ [[package]] name = "fabro-config" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "chrono", @@ -1788,7 +1788,7 @@ dependencies = [ [[package]] name = "fabro-core" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "async-trait", "fabro-types", @@ -1803,7 +1803,7 @@ dependencies = [ [[package]] name = "fabro-dev" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "assert_cmd", @@ -1823,7 +1823,7 @@ dependencies = [ [[package]] name = "fabro-devcontainer" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "fabro-http", "fabro-static", @@ -1840,7 +1840,7 @@ dependencies = [ [[package]] name = "fabro-dump" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "bytes", @@ -1854,7 +1854,7 @@ dependencies = [ [[package]] name = "fabro-github" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "base64", @@ -1876,7 +1876,7 @@ dependencies = [ [[package]] name = "fabro-graphviz" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "fabro-types", @@ -1890,7 +1890,7 @@ dependencies = [ [[package]] name = "fabro-hooks" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "async-trait", "fabro-agent", @@ -1914,7 +1914,7 @@ dependencies = [ [[package]] name = "fabro-http" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "fabro-static", "http", @@ -1924,7 +1924,7 @@ dependencies = [ [[package]] name = "fabro-install" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "base64", @@ -1939,7 +1939,7 @@ dependencies = [ [[package]] name = "fabro-interview" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "async-trait", "dialoguer", @@ -1954,7 +1954,7 @@ dependencies = [ [[package]] name = "fabro-llm" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "async-trait", @@ -1986,7 +1986,7 @@ dependencies = [ [[package]] name = "fabro-macros" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "clap", "fabro-options-metadata", @@ -1997,7 +1997,7 @@ dependencies = [ [[package]] name = "fabro-mcp" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "fabro-config", @@ -2013,7 +2013,7 @@ dependencies = [ [[package]] name = "fabro-model" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "fabro-static", "insta", @@ -2024,7 +2024,7 @@ dependencies = [ [[package]] name = "fabro-oauth" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "axum", @@ -2046,7 +2046,7 @@ dependencies = [ [[package]] name = "fabro-options-metadata" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "serde", "serde_json", @@ -2054,7 +2054,7 @@ dependencies = [ [[package]] name = "fabro-proc" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "cc", "libc", @@ -2063,7 +2063,7 @@ dependencies = [ [[package]] name = "fabro-redact" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "aho-corasick", "ref-cast", @@ -2079,7 +2079,7 @@ dependencies = [ [[package]] name = "fabro-retro" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "chrono", @@ -2098,7 +2098,7 @@ dependencies = [ [[package]] name = "fabro-sandbox" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "async-trait", @@ -2138,7 +2138,7 @@ dependencies = [ [[package]] name = "fabro-server" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "async-trait", @@ -2219,7 +2219,7 @@ dependencies = [ [[package]] name = "fabro-slack" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "fabro-http", "fabro-interview", @@ -2240,18 +2240,18 @@ dependencies = [ [[package]] name = "fabro-spa" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "rust-embed", ] [[package]] name = "fabro-static" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" [[package]] name = "fabro-store" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "async-trait", "bytes", @@ -2278,7 +2278,7 @@ dependencies = [ [[package]] name = "fabro-telemetry" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "base64", @@ -2304,7 +2304,7 @@ dependencies = [ [[package]] name = "fabro-template" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "fabro-util", @@ -2316,7 +2316,7 @@ dependencies = [ [[package]] name = "fabro-test" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "assert_cmd", "axum", @@ -2339,7 +2339,7 @@ dependencies = [ [[package]] name = "fabro-tracker" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "async-trait", @@ -2353,7 +2353,7 @@ dependencies = [ [[package]] name = "fabro-types" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "chrono", "clap", @@ -2374,7 +2374,7 @@ dependencies = [ [[package]] name = "fabro-util" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "console 0.15.11", @@ -2394,7 +2394,7 @@ dependencies = [ [[package]] name = "fabro-validate" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "fabro-graphviz", "fabro-model", @@ -2404,7 +2404,7 @@ dependencies = [ [[package]] name = "fabro-vault" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "chrono", "fabro-types", @@ -2416,7 +2416,7 @@ dependencies = [ [[package]] name = "fabro-workflow" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "assert_cmd", @@ -7120,7 +7120,7 @@ dependencies = [ [[package]] name = "twin-github" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "axum", "base64", @@ -7139,7 +7139,7 @@ dependencies = [ [[package]] name = "twin-openai" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "async-stream", diff --git a/Cargo.toml b/Cargo.toml index f7897daf1..52d5940be 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ resolver = "2" [workspace.package] edition = "2021" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" license = "MIT" [workspace.dependencies] From d9c8030e7448cbed2ef0b26dc5763a14279f78b7 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Tue, 5 May 2026 08:25:17 -0400 Subject: [PATCH 11/17] refactor(workflow): harden PR content generation Return named PR content from the builder and keep title/body fallback logic inside the builder. Move the PR body prompt to markdown and scale prompt truncation from model context windows. Keep PR creation resilient when generated bodies are empty by emitting a reviewer-visible skeleton body. --- lib/crates/fabro-workflow/src/pipeline/mod.rs | 3 +- .../src/pipeline/prompts/pr_body.md | 54 +++ .../src/pipeline/pull_request.rs | 447 ++++++++++-------- lib/crates/fabro-workflow/src/pull_request.rs | 2 +- .../fabro-workflow/tests/it/integration.rs | 7 +- 5 files changed, 303 insertions(+), 210 deletions(-) create mode 100644 lib/crates/fabro-workflow/src/pipeline/prompts/pr_body.md diff --git a/lib/crates/fabro-workflow/src/pipeline/mod.rs b/lib/crates/fabro-workflow/src/pipeline/mod.rs index 71aefbef9..bb35d298f 100644 --- a/lib/crates/fabro-workflow/src/pipeline/mod.rs +++ b/lib/crates/fabro-workflow/src/pipeline/mod.rs @@ -19,7 +19,8 @@ pub use initialize::initialize; pub use parse::parse; pub(crate) use persist::persist; pub use pull_request::{ - AutoMergeOptions, OpenPullRequestRequest, build_pr_body, maybe_open_pull_request, pull_request, + AutoMergeOptions, OpenPullRequestRequest, PrContent, build_pr_content, maybe_open_pull_request, + pull_request, }; pub use retro::{retro, run_retro}; pub use transform::transform; diff --git a/lib/crates/fabro-workflow/src/pipeline/prompts/pr_body.md b/lib/crates/fabro-workflow/src/pipeline/prompts/pr_body.md new file mode 100644 index 000000000..9175e62db --- /dev/null +++ b/lib/crates/fabro-workflow/src/pipeline/prompts/pr_body.md @@ -0,0 +1,54 @@ +You are writing a pull request title and description for a code change produced by an AI workflow. + +OUTPUT FORMAT +Return a JSON object with exactly two fields: +- "title": a one-line title, max 72 characters, no trailing period. +- "body": the markdown body as described below. + +DO NOT INCLUDE in the body +- A `#` or `##` title heading at the top -- the title goes in the `title` field. +- A "Retro" section, "Fabro Details" section, cost/duration table, or "Generated with" footer -- those are appended programmatically after your output. +- The full plan text -- the full plan is appended programmatically as a
block. +- Bare `#1`, `#2` list prefixes -- GitHub auto-links those as issue references. Use plain `1.`, `2.` instead. +- A test plan unless the testing approach is non-obvious. + +SIZE THE BODY TO THE CHANGE +First classify along two axes from the diff: +- Size: how many files changed, how large the diff is. +- Complexity: trivial (rename / typo / dep bump / config) vs. design decisions / new patterns / cross-cutting concerns. + +Then write at the matching depth: + +| Profile | Body shape | +|---|---| +| Small + simple (typo, config, dep bump) | 1-2 sentences, no headers, total under ~300 characters | +| Small + non-trivial (targeted bugfix, behavioral change) | Short "Problem / Fix" narrative, 3-5 sentences. No headers unless two distinct concerns. | +| Medium feature or refactor | Summary paragraph, then a section explaining what changed and why. Call out design decisions. | +| Large or architecturally significant | Full narrative: problem context, approach chosen (and why), key decisions, migration/rollback notes if relevant. | +| Performance improvement | Include before/after measurements if available. A markdown table works well here. | + +Brevity matters for small changes. A 3-line bugfix with a 20-line description signals miscalibration. When in doubt, shorter is better -- reviewers can read the diff. + +WRITING PRINCIPLES +- Lead with value: the first sentence tells the reviewer *why this PR exists*, not *what files changed*. +- Describe the net result, not the journey: skip intermediate failures, debugging steps, and refactors done during development. +- Trust the final diff: if the goal or plan disagree with the diff, the diff is authoritative. +- Explain the non-obvious: spend description space on what the diff doesn't show -- why this approach, what was rejected, what to look at first. +- Use structure when it earns its keep: no empty sections, no template headers without content. +- If the body uses any `##` heading, the opening summary must also be under a heading (e.g. `## Summary`); otherwise a bare paragraph is fine. + +PLAN SUMMARY +The full plan is attached separately as a
block, so do not restate it. Include a brief `### Plan Summary` with bullet points only when the change is medium or larger in the sizing matrix above. Skip it for small changes. + +VISUAL AIDS +Include a visual aid only when a reviewer would struggle to reconstruct the mental model from prose alone -- based on what changes structurally, not on PR size. Skip for trivial / mechanical changes, or when prose already communicates clearly. + +| PR changes... | Visual aid | +|---|---| +| 3+ interacting components or services | Mermaid component / interaction diagram | +| Multi-step workflow or pipeline with non-obvious sequencing | Mermaid flow diagram | +| 3+ behavioral modes or variants | Markdown comparison table | +| Before/after data or trade-offs | Markdown table | +| Data model changes with 3+ related entities | Mermaid ERD | + +Mermaid: prefer `TB` direction, <=10 nodes typical. Place inline at the point of relevance, not in a separate "Diagrams" section. diff --git a/lib/crates/fabro-workflow/src/pipeline/pull_request.rs b/lib/crates/fabro-workflow/src/pipeline/pull_request.rs index 8825f8eef..35f27aa92 100644 --- a/lib/crates/fabro-workflow/src/pipeline/pull_request.rs +++ b/lib/crates/fabro-workflow/src/pipeline/pull_request.rs @@ -11,7 +11,7 @@ use fabro_store::RunProjection; use fabro_types::PullRequestRecord; use fabro_types::settings::run::MergeStrategy; use fabro_util::text::strip_goal_decoration; -use tracing::{debug, info}; +use tracing::{debug, info, warn}; use super::types::{Concluded, Finalized, PullRequestOptions}; use crate::event::{Event, RunNoticeLevel}; @@ -19,130 +19,75 @@ use crate::outcome::{StageOutcome, format_cost as outcome_format_cost}; use crate::records::{Conclusion, RunSpec}; use crate::runtime_store::RunStoreHandle; -/// Maximum length of a PR title (Unicode scalar values). Single source of -/// truth — referenced by the structured-output schema, the system prompt, -/// and [`enforce_title_cap`]. +/// Maximum length of a PR title (Unicode scalar values). const PR_TITLE_MAX_CHARS: usize = 72; /// Structured output schema for the LLM-generated PR title and body. -/// -/// `title` is required but allows empty strings (the only signal that -/// triggers the deterministic title fallback in -/// [`maybe_open_pull_request`]). `body` requires `minLength: 1` because -/// there is no body fallback — an empty body is fatal. static PR_CONTENT_SCHEMA: LazyLock = LazyLock::new(|| { serde_json::json!({ "type": "object", "properties": { - "title": { "type": "string", "maxLength": PR_TITLE_MAX_CHARS }, - "body": { "type": "string", "minLength": 1 } + "title": { "type": "string" }, + "body": { "type": "string" } }, "required": ["title", "body"], "additionalProperties": false }) }); +/// Complete pull request content generated for a workflow run. #[derive(Debug, serde::Deserialize)] -struct GeneratedPrContent { - title: String, - body: String, +pub struct PrContent { + pub title: String, + pub body: String, } /// System prompt that instructs the LLM how to write a Fabro PR title and /// body. The trailing programmatic sections (Plan `
`, Retro, /// Fabro Details, footer) are appended after the LLM body — the prompt /// explicitly forbids the LLM from duplicating them. -// -// The "max 72 characters" instruction must stay in sync with -// `PR_TITLE_MAX_CHARS` and the schema above; the prompt is advisory and -// `enforce_title_cap` is the actual enforcement. -const PR_BODY_SYSTEM_PROMPT: &str = "You are writing a pull request title and description for a code change produced by an AI workflow. +const PR_BODY_SYSTEM_PROMPT: &str = include_str!("prompts/pr_body.md"); -OUTPUT FORMAT -Return a JSON object with exactly two fields: -- \"title\": a one-line title, max 72 characters, no trailing period. -- \"body\": the markdown body as described below. +const DEFAULT_PR_TITLE: &str = "Update workflow output"; +const EMPTY_BODY_NOTICE: &str = "> _The LLM did not produce a description for this change. The diff and the appended details are the source of truth for review._"; -DO NOT INCLUDE in the body -- A `#` or `##` title heading at the top — the title goes in the `title` field. -- A \"Retro\" section, \"Fabro Details\" section, cost/duration table, or \"Generated with\" footer — those are appended programmatically after your output. -- The full plan text — the full plan is appended programmatically as a
block. -- Bare `#1`, `#2` list prefixes — GitHub auto-links those as issue references. Use plain `1.`, `2.` instead. -- A test plan unless the testing approach is non-obvious. - -SIZE THE BODY TO THE CHANGE -First classify along two axes from the diff: -- Size: how many files changed, how large the diff is. -- Complexity: trivial (rename / typo / dep bump / config) vs. design decisions / new patterns / cross-cutting concerns. - -Then write at the matching depth: - -| Profile | Body shape | -|---|---| -| Small + simple (typo, config, dep bump) | 1–2 sentences, no headers, total under ~300 characters | -| Small + non-trivial (targeted bugfix, behavioral change) | Short \"Problem / Fix\" narrative, 3–5 sentences. No headers unless two distinct concerns. | -| Medium feature or refactor | Summary paragraph, then a section explaining what changed and why. Call out design decisions. | -| Large or architecturally significant | Full narrative: problem context, approach chosen (and why), key decisions, migration/rollback notes if relevant. | -| Performance improvement | Include before/after measurements if available. A markdown table works well here. | - -Brevity matters for small changes. A 3-line bugfix with a 20-line description signals miscalibration. When in doubt, shorter is better — reviewers can read the diff. - -WRITING PRINCIPLES -- Lead with value: the first sentence tells the reviewer *why this PR exists*, not *what files changed*. -- Describe the net result, not the journey: skip intermediate failures, debugging steps, and refactors done during development. -- Trust the final diff: if the goal or plan disagree with the diff, the diff is authoritative. -- Explain the non-obvious: spend description space on what the diff doesn't show — why this approach, what was rejected, what to look at first. -- Use structure when it earns its keep: no empty sections, no template headers without content. -- If the body uses any `##` heading, the opening summary must also be under a heading (e.g. `## Summary`); otherwise a bare paragraph is fine. - -PLAN SUMMARY -The full plan is attached separately as a
block, so do not restate it. Include a brief `### Plan Summary` with bullet points only when the change is medium or larger in the sizing matrix above. Skip it for small changes. - -VISUAL AIDS -Include a visual aid only when a reviewer would struggle to reconstruct the mental model from prose alone — based on what changes structurally, not on PR size. Skip for trivial / mechanical changes, or when prose already communicates clearly. - -| PR changes... | Visual aid | -|---|---| -| 3+ interacting components or services | Mermaid component / interaction diagram | -| Multi-step workflow or pipeline with non-obvious sequencing | Mermaid flow diagram | -| 3+ behavioral modes or variants | Markdown comparison table | -| Before/after data or trade-offs | Markdown table | -| Data model changes with 3+ related entities | Mermaid ERD | - -Mermaid: prefer `TB` direction, ≤10 nodes typical. Place inline at the point of relevance, not in a separate \"Diagrams\" section."; - -/// Truncation budget for the LLM prompt's goal / plan / diff sections. +/// Truncation budget for the LLM prompt's plan / diff sections. +#[derive(Debug, PartialEq, Eq)] struct TruncationCaps { - goal: usize, plan: usize, diff: usize, } -/// Generous tier for models with ≥200k context windows. -const TRUNCATION_LARGE: TruncationCaps = TruncationCaps { - goal: 75_000, - plan: 75_000, - diff: 250_000, -}; - -/// Conservative tier (matches the pre-refactor values). Used for smaller -/// or unknown models. -const TRUNCATION_SMALL: TruncationCaps = TruncationCaps { - goal: 20_000, - plan: 20_000, - diff: 50_000, -}; +const DIFF_HARD_CAP: usize = 500_000; +const PLAN_HARD_CAP: usize = 100_000; +const DIFF_FRACTION_NUM: usize = 4; +const PLAN_FRACTION_NUM: usize = 1; +const FRACTION_DEN: usize = 10; +const UNKNOWN_MODEL_CTX: usize = 200_000; /// Resolve truncation caps based on the model's context window. Unknown -/// models fall through to the conservative tier. -fn truncation_caps(model: &str) -> &'static TruncationCaps { - let large_enough = Catalog::builtin() +/// models use the baseline 200k context-window assumption. +fn truncation_caps(model: &str) -> TruncationCaps { + let ctx = Catalog::builtin() .get(model) - .is_some_and(|m| m.context_window() >= 200_000); - if large_enough { - &TRUNCATION_LARGE - } else { - &TRUNCATION_SMALL + .and_then(|m| usize::try_from(m.context_window()).ok()) + .unwrap_or(UNKNOWN_MODEL_CTX); + + truncation_caps_for_context_window(ctx) +} + +fn truncation_caps_for_context_window(ctx: usize) -> TruncationCaps { + TruncationCaps { + diff: ctx + .saturating_mul(DIFF_FRACTION_NUM) + .checked_div(FRACTION_DEN) + .unwrap_or(DIFF_HARD_CAP) + .min(DIFF_HARD_CAP), + plan: ctx + .saturating_mul(PLAN_FRACTION_NUM) + .checked_div(FRACTION_DEN) + .unwrap_or(PLAN_HARD_CAP) + .min(PLAN_HARD_CAP), } } @@ -172,12 +117,18 @@ fn enforce_title_cap(title: &str) -> String { /// Derive a PR title from the workflow goal. /// -/// Uses the first line, truncated to 120 characters for readability. The -/// caller is expected to apply [`enforce_title_cap`] afterwards if a -/// stricter cap is required (the wider cap here is the legacy behaviour -/// for the deterministic fallback path). +/// Uses the first line, truncated to the same cap as LLM-generated titles. fn pr_title_from_goal(goal: &str) -> String { - truncate_with_ellipsis(strip_goal_decoration(goal), 120) + truncate_with_ellipsis(strip_goal_decoration(goal), PR_TITLE_MAX_CHARS) +} + +fn fallback_pr_title(goal: &str) -> String { + let title = pr_title_from_goal(goal); + if title.trim().is_empty() { + DEFAULT_PR_TITLE.to_string() + } else { + title + } } /// Truncate a PR body to fit GitHub's 65,536 character limit. @@ -435,75 +386,43 @@ async fn load_pull_request_diff(run_store: &RunStoreHandle) -> String { .unwrap_or_default() } -/// Build a complete PR title and body by combining LLM-generated narrative -/// with programmatic sections (plan, retro, fabro details). -/// -/// Returns `(title, body)`. The title may be the empty string when the LLM -/// returned a usable body but no usable title — callers fall back to -/// [`pr_title_from_goal`] in that case. Every other generation failure is -/// surfaced as `Err`. -pub async fn build_pr_body( +/// Build complete PR content by combining LLM-generated narrative with +/// deterministic fallbacks and programmatic sections. +pub async fn build_pr_content( diff: &str, goal: &str, model: &str, run_store: &RunStoreHandle, llm_source: &dyn CredentialSource, conclusion: Option<&Conclusion>, -) -> Result<(String, String), String> { + run_state: Option<&RunProjection>, +) -> Result { let client = Client::from_source(llm_source) .await .map_err(|e| format!("Failed to create LLM client: {e}"))?; - build_pr_body_with_client(diff, goal, model, run_store, conclusion, Arc::new(client)).await -} - -async fn build_pr_body_with_client( - diff: &str, - goal: &str, - model: &str, - run_store: &RunStoreHandle, - conclusion: Option<&Conclusion>, - client: Arc, -) -> Result<(String, String), String> { - build_pr_body_with_client_and_state(diff, goal, model, run_store, conclusion, client, None) - .await -} - -async fn build_pr_body_with_source_and_state( - diff: &str, - goal: &str, - model: &str, - run_store: &RunStoreHandle, - llm_source: &dyn CredentialSource, - conclusion: Option<&Conclusion>, - run_state: Option<&fabro_store::RunProjection>, -) -> Result<(String, String), String> { - let client = Client::from_source(llm_source) - .await - .map_err(|e| format!("Failed to create LLM client: {e}"))?; - - build_pr_body_with_client_and_state( + build_pr_content_with_client( diff, goal, model, run_store, conclusion, - Arc::new(client), run_state, + Arc::new(client), ) .await } -async fn build_pr_body_with_client_and_state( +async fn build_pr_content_with_client( diff: &str, goal: &str, model: &str, run_store: &RunStoreHandle, conclusion: Option<&Conclusion>, + run_state: Option<&RunProjection>, client: Arc, - run_state: Option<&fabro_store::RunProjection>, -) -> Result<(String, String), String> { - info!("Building PR body"); +) -> Result { + info!("Building PR content"); let loaded_run_state = if run_state.is_none() { run_store @@ -524,16 +443,15 @@ async fn build_pr_body_with_client_and_state( let dot_source = run_state.and_then(|state| state.graph_source.clone()); let caps = truncation_caps(model); - let truncated_goal = truncate_chars(goal, caps.goal); let truncated_diff = truncate_chars(diff, caps.diff); let prompt = if let Some(ref plan) = plan_text { let truncated_plan = truncate_chars(plan, caps.plan); format!( - "Goal: {truncated_goal}\n\nPlan:\n```\n{truncated_plan}\n```\n\nDiff:\n```\n{truncated_diff}\n```" + "Goal: {goal}\n\nPlan:\n```\n{truncated_plan}\n```\n\nDiff:\n```\n{truncated_diff}\n```" ) } else { - format!("Goal: {truncated_goal}\n\nDiff:\n```\n{truncated_diff}\n```") + format!("Goal: {goal}\n\nDiff:\n```\n{truncated_diff}\n```") }; let params = GenerateParams::new(model, client) @@ -547,15 +465,22 @@ async fn build_pr_body_with_client_and_state( let output = result .output .ok_or_else(|| "LLM generation returned no structured output".to_string())?; - let generated: GeneratedPrContent = serde_json::from_value(output) + let generated: PrContent = serde_json::from_value(output) .map_err(|e| format!("Failed to deserialize PR content: {e}"))?; - if generated.body.trim().is_empty() { - return Err("LLM generated an empty PR body".to_string()); - } + let title = if generated.title.trim().is_empty() { + fallback_pr_title(goal) + } else { + generated.title.trim().to_string() + }; + let title = enforce_title_cap(&title); - let title = enforce_title_cap(generated.title.trim()); - let llm_body = generated.body; + let llm_body = if generated.body.trim().is_empty() { + warn!(model = %model, "LLM generated empty PR body; using skeleton PR body"); + EMPTY_BODY_NOTICE.to_string() + } else { + generated.body + }; let retro_section = retro.as_ref().map(format_retro_section).unwrap_or_default(); let arc_details_section = conclusion @@ -570,9 +495,9 @@ async fn build_pr_body_with_client_and_state( &arc_details_section, ); - info!("PR body generated"); + info!("PR content generated"); - Ok((title, body)) + Ok(PrContent { title, body }) } /// Auto-merge configuration for a pull request. @@ -594,7 +519,7 @@ pub struct OpenPullRequestRequest<'a> { pub run_store: &'a RunStoreHandle, pub llm_source: &'a dyn CredentialSource, pub conclusion: Option<&'a Conclusion>, - pub run_state: Option<&'a fabro_store::RunProjection>, + pub run_state: Option<&'a RunProjection>, } /// Optionally open a pull request after a successful workflow run. @@ -613,7 +538,7 @@ pub async fn maybe_open_pull_request( let (owner, repo) = github_app::parse_github_owner_repo(&https_url).map_err(|err| format!("{err:#}"))?; - let (llm_title, body) = build_pr_body_with_source_and_state( + let content = build_pr_content( req.diff, req.goal, req.model, @@ -624,14 +549,8 @@ pub async fn maybe_open_pull_request( ) .await .map_err(|err| format!("{err:#}"))?; - let body = truncate_pr_body(&body); - - let title = if llm_title.is_empty() { - pr_title_from_goal(req.goal) - } else { - llm_title - }; - let title = enforce_title_cap(&title); + let body = truncate_pr_body(&content.body); + let title = content.title; let created = github_app::create_pull_request( &req.github, @@ -1286,15 +1205,16 @@ mod tests { } #[tokio::test] - async fn build_pr_body_uses_in_memory_conclusion() { + async fn build_pr_content_uses_in_memory_conclusion() { let store = test_store(); let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); - let (title, body) = build_pr_body_with_client( + let PrContent { title, body } = build_pr_content_with_client( "diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n", "Implement feature", "mock-model", &run_store.clone().into(), Some(&make_test_conclusion()), + None, explicit_client( "mock", &pr_content_json("Mock title", "Narrative from mock."), @@ -1311,7 +1231,7 @@ mod tests { } #[tokio::test] - async fn build_pr_body_uses_store_records_without_legacy_files() { + async fn build_pr_content_uses_store_records_without_legacy_files() { let store = test_store(); let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); @@ -1363,19 +1283,21 @@ mod tests { .await .unwrap(); - let (_, body) = build_pr_body_with_client( + let body = build_pr_content_with_client( "diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n", "Implement feature", "mock-model", &run_store.clone().into(), Some(&make_test_conclusion()), + None, explicit_client( "mock", &pr_content_json("Mock title", "Narrative from mock."), ), ) .await - .unwrap(); + .unwrap() + .body; assert!(body.contains("Narrative from mock.")); assert!(body.contains("### Retro")); @@ -1384,7 +1306,7 @@ mod tests { } #[tokio::test] - async fn build_pr_body_uses_plan_text_from_store_without_response_md() { + async fn build_pr_content_uses_plan_text_from_store_without_response_md() { let store = test_store(); let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); @@ -1453,48 +1375,52 @@ mod tests { .await .unwrap(); - let (_, body) = build_pr_body_with_client( + let body = build_pr_content_with_client( "diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n", "Implement feature", "mock-model", &run_store.clone().into(), Some(&make_test_conclusion()), + None, explicit_client( "mock", &pr_content_json("Mock title", "Narrative from mock."), ), ) .await - .unwrap(); + .unwrap() + .body; assert!(body.contains("Full plan")); assert!(body.contains("Plan from store")); } #[tokio::test] - async fn build_pr_body_uses_explicit_llm_client() { + async fn build_pr_content_uses_explicit_llm_client() { let store = test_store(); let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); - let (_, body) = build_pr_body_with_client( + let body = build_pr_content_with_client( "diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n", "Implement feature", "gpt-5.4", &run_store.clone().into(), Some(&make_test_conclusion()), + None, explicit_client( "openai", &pr_content_json("Explicit title", "Narrative from explicit client."), ), ) .await - .unwrap(); + .unwrap() + .body; assert!(body.contains("Narrative from explicit client.")); assert!(!body.contains("Narrative from mock.")); } #[tokio::test] - async fn build_pr_body_uses_vault_only_openai_codex_source() { + async fn build_pr_content_uses_vault_only_openai_codex_source() { let server = MockServer::start_async().await; let response_mock = server .mock_async(|when, then| { @@ -1534,13 +1460,14 @@ mod tests { let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); let run_store_handle: RunStoreHandle = run_store.into(); - let (title, body) = build_pr_body( + let PrContent { title, body } = build_pr_content( "diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n", "Implement feature", "gpt-5.4", &run_store_handle, llm_source.as_ref(), Some(&make_test_conclusion()), + None, ) .await .unwrap(); @@ -1642,7 +1569,7 @@ mod tests { fn pr_title_truncates_long_line() { let long = "x".repeat(300); let title = pr_title_from_goal(&long); - assert_eq!(title.chars().count(), 120); + assert_eq!(title.chars().count(), 72); assert!(title.ends_with('…')); } @@ -1665,6 +1592,42 @@ mod tests { assert_eq!(pr_title_from_goal("Fix bug"), "Fix bug"); } + #[test] + fn truncation_caps_scale_with_context_window_and_clamp() { + assert_eq!( + truncation_caps_for_context_window(100_000), + TruncationCaps { + diff: 40_000, + plan: 10_000, + } + ); + assert_eq!( + truncation_caps_for_context_window(200_000), + TruncationCaps { + diff: 80_000, + plan: 20_000, + } + ); + assert_eq!( + truncation_caps_for_context_window(1_000_000), + TruncationCaps { + diff: 400_000, + plan: 100_000, + } + ); + assert_eq!( + truncation_caps_for_context_window(10_000_000), + TruncationCaps { + diff: 500_000, + plan: 100_000, + } + ); + assert_eq!(truncation_caps("unknown-model"), TruncationCaps { + diff: 80_000, + plan: 20_000, + }); + } + #[tokio::test] async fn empty_diff_returns_none() { let store = test_store(); @@ -1760,68 +1723,144 @@ mod tests { /// MockProvider returns an over-long title; builder must cap it at 72 /// chars and end with `…`. Exercises [`enforce_title_cap`] inside - /// [`build_pr_body_with_client_and_state`]. + /// [`build_pr_content_with_client`]. #[tokio::test] - async fn build_pr_body_truncates_long_title() { + async fn build_pr_content_truncates_long_title() { let store = test_store(); let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); let long_title = "x".repeat(200); let payload = pr_content_json(&long_title, "Body content."); - let (title, _) = build_pr_body_with_client( + let title = build_pr_content_with_client( "diff --git a/src/lib.rs b/src/lib.rs\n+fn x() {}\n", "Implement feature", "mock-model", &run_store.clone().into(), Some(&make_test_conclusion()), + None, explicit_client("mock", &payload), ) .await - .unwrap(); + .unwrap() + .title; assert_eq!(title.chars().count(), 72); assert!(title.ends_with('\u{2026}')); } - /// Empty bodies are fatal. Real providers may reject this via the - /// schema's `minLength`; the Rust-side trim check also catches it for - /// local/mock providers. #[tokio::test] - async fn build_pr_body_returns_err_when_body_empty() { + async fn build_pr_content_uses_default_title_when_generated_and_goal_titles_empty() { let store = test_store(); let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); - let payload = pr_content_json("Mock", ""); - let result = build_pr_body_with_client( + let payload = pr_content_json("", "Body content."); + let title = build_pr_content_with_client( "diff --git a/src/lib.rs b/src/lib.rs\n+fn x() {}\n", - "Implement feature", + "## Plan:", "mock-model", &run_store.clone().into(), Some(&make_test_conclusion()), + None, explicit_client("mock", &payload), ) - .await; + .await + .unwrap() + .title; - assert!(result.is_err(), "expected Err, got {result:?}"); + assert_eq!(title, DEFAULT_PR_TITLE); } - /// Whitespace-only bodies pass schema validation but fail the - /// `body.trim().is_empty()` check inside the builder. + /// Empty or whitespace-only bodies use the skeleton fallback instead of + /// aborting PR creation. #[tokio::test] - async fn build_pr_body_returns_err_when_body_whitespace() { + async fn build_pr_content_uses_skeleton_when_body_empty() { let store = test_store(); let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); + + let run_spec = RunSpec { + run_id: fixtures::RUN_1, + settings: fabro_types::WorkflowSettings::default(), + graph: Graph::new("test"), + workflow_slug: Some("test".to_string()), + source_directory: Some("/tmp/project".to_string()), + git: None, + labels: HashMap::new(), + provenance: None, + manifest_blob: None, + definition_blob: None, + fork_source_ref: None, + in_place: false, + }; + append_event(&run_store, &fixtures::RUN_1, &Event::RunCreated { + run_id: fixtures::RUN_1, + settings: serde_json::to_value(&run_spec.settings).unwrap(), + graph: serde_json::to_value(&run_spec.graph).unwrap(), + workflow_source: Some("digraph test { plan -> code }".to_string()), + workflow_config: None, + labels: run_spec.labels.clone().into_iter().collect(), + run_dir: "/tmp/project".to_string(), + source_directory: run_spec.source_directory.clone(), + workflow_slug: run_spec.workflow_slug.clone(), + db_prefix: None, + provenance: None, + manifest_blob: None, + git: None, + fork_source_ref: None, + in_place: false, + web_url: None, + }) + .await + .unwrap(); + append_event(&run_store, &fixtures::RUN_1, &Event::StageCompleted { + node_id: "plan".to_string(), + name: "plan".to_string(), + index: 0, + duration_ms: 1, + status: "succeeded".to_string(), + preferred_label: None, + suggested_next_ids: vec![], + billing: None, + failure: None, + notes: None, + files_touched: vec![], + context_updates: None, + jump_to_node: None, + context_values: None, + node_visits: None, + loop_failure_signatures: None, + restart_failure_signatures: None, + response: Some("Plan from store".to_string()), + attempt: 1, + max_attempts: 1, + }) + .await + .unwrap(); + append_event(&run_store, &fixtures::RUN_1, &Event::RetroCompleted { + duration_ms: 1, + response: Some(String::new()), + retro: Some(serde_json::to_value(make_test_retro()).unwrap()), + }) + .await + .unwrap(); + let payload = pr_content_json("Mock", " \n"); - let result = build_pr_body_with_client( + let body = build_pr_content_with_client( "diff --git a/src/lib.rs b/src/lib.rs\n+fn x() {}\n", "Implement feature", "mock-model", &run_store.clone().into(), Some(&make_test_conclusion()), + None, explicit_client("mock", &payload), ) - .await; + .await + .unwrap() + .body; - let err = result.expect_err("expected Err for whitespace-only body"); - assert!(err.contains("empty PR body"), "unexpected error: {err}"); + assert!(body.contains("The LLM did not produce a description")); + assert!(body.contains("Full plan")); + assert!(body.contains("Plan from store")); + assert!(body.contains("### Retro")); + assert!(body.contains("### Fabro Details")); + assert!(body.contains("Generated with [Fabro](https://fabro.sh)")); } // ── maybe_open_pull_request fallback tests ────────────────────────── @@ -1978,9 +2017,9 @@ mod tests { } } - /// LLM returns a usable body but an empty title; `maybe_open_pull_request` - /// must fall back to `pr_title_from_goal` (first line, decoration - /// stripped) and the PR creation must succeed with that title. + /// LLM returns a usable body but an empty title; the content builder + /// falls back to `pr_title_from_goal` (first line, decoration stripped) + /// and PR creation succeeds with that title. #[tokio::test] async fn maybe_open_pull_request_falls_back_to_goal_title_when_llm_returns_empty_title() { let payload = pr_content_json("", "Narrative."); @@ -2012,10 +2051,8 @@ mod tests { harness.assert_mocks_called_once().await; } - /// LLM returns an empty title; the fallback path produces a long title - /// (close to `pr_title_from_goal`'s 120-char cap), and the unconditional - /// `enforce_title_cap` in `maybe_open_pull_request` must still bring it - /// down to 72 chars ending with `…`. + /// LLM returns an empty title; the content builder fallback still caps + /// the deterministic goal title at 72 chars ending with `…`. #[tokio::test] async fn maybe_open_pull_request_caps_fallback_title_at_72_chars() { let payload = pr_content_json("", "Narrative."); diff --git a/lib/crates/fabro-workflow/src/pull_request.rs b/lib/crates/fabro-workflow/src/pull_request.rs index 3989bf63c..3b65dfcbc 100644 --- a/lib/crates/fabro-workflow/src/pull_request.rs +++ b/lib/crates/fabro-workflow/src/pull_request.rs @@ -1,4 +1,4 @@ pub use crate::pipeline::{ - AutoMergeOptions, OpenPullRequestRequest, PullRequestRecord, build_pr_body, + AutoMergeOptions, OpenPullRequestRequest, PrContent, PullRequestRecord, build_pr_content, maybe_open_pull_request, }; diff --git a/lib/crates/fabro-workflow/tests/it/integration.rs b/lib/crates/fabro-workflow/tests/it/integration.rs index 742218c14..f6ece331b 100644 --- a/lib/crates/fabro-workflow/tests/it/integration.rs +++ b/lib/crates/fabro-workflow/tests/it/integration.rs @@ -6896,7 +6896,7 @@ async fn workflow_run_with_vault_only_openai_codex_builds_pr_body() { let run_store = store.open_run_reader(&run_options.run_id).await.unwrap(); let run_store_handle: fabro_workflow::runtime_store::RunStoreHandle = run_store.into(); - let (title, body) = fabro_workflow::pull_request::build_pr_body( + let content = fabro_workflow::pull_request::build_pr_content( "diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n", "Implement feature", "gpt-5.4", @@ -6912,12 +6912,13 @@ async fn workflow_run_with_vault_only_openai_codex_builds_pr_body() { billing: None, total_retries: 0, }), + None, ) .await .expect("PR body should build from vault-only credentials"); - assert_eq!(title, "Vault title"); - assert!(body.contains("Narrative from vault source.")); + assert_eq!(content.title, "Vault title"); + assert!(content.body.contains("Narrative from vault source.")); response_mock.assert_async().await; } From 333b603f5bc6f177309df542304e904dfff203af Mon Sep 17 00:00:00 2001 From: "fabro-sh-0530[bot]" <281434857+fabro-sh-0530[bot]@users.noreply.github.com> Date: Tue, 5 May 2026 08:27:11 -0400 Subject: [PATCH 12/17] Encode stage visits in run stage URLs (#206) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Summary Stages that re-enter the same workflow node now get distinct `node@visit` identities end to end, so looped stages like `verify@1` and `verify@2` no longer collapse to the same sidebar link, event stream, graph selection, or turns view. ### What changed - `RunStage.id` now uses the full `StageId` string (`node_id@visit`), with required `node_id` and `visit` fields in the OpenAPI schema and generated clients. This intentionally replaces the old `dot_id` field. - The server builds `/runs/{id}/stages` from `RunProjection::iter_stages()` instead of checkpoint `completed_nodes`, preserving visit information and including in-flight stages from projection data. - Stage status is derived from the latest lifecycle event for each exact `stage_id`, so retrying stages do not appear failed while a retry is underway. - The frontend maps and displays visits with `(N)` suffixes, filters fallback turns by `stage_id`, invalidates suffixed stage-turn query keys from SSE, and aggregates graph nodes by `node_id` with latest-visit click targets. ### Plan Summary - Preserve per-visit stage identity across API, server projection, generated clients, and UI routing. - Keep graph nodes keyed by workflow node while routing clicks to the latest visit. - Add coverage for multi-visit stages, retrying status derivation, suffixed SSE invalidation, sidebar labels, and stage event filtering. ### Reviewer notes This is a breaking API shape change for `RunStage`: consumers should use `node_id` for graph/node identity and `id` for per-visit stage identity. The old `dot_id` field is removed rather than kept as a compatibility alias. ### Fabro Details
Ran 9 stages in 54m 55s for $41.40 | Stage | Duration | Cost | Retries | |---|---|---|---| | start | 0s | – | 0 | | toolchain | 1s | – | 0 | | preflight_compile | 2m 8s | – | 0 | | preflight_lint | 2m 14s | – | 0 | | implement | 31m 38s | $17.65 | 0 | | simplify_opus | 10m 2s | $2.40 | 0 | | simplify_gpt | 6m 9s | $21.35 | 0 | | verify | 2m 3s | – | 0 | | fmt | 2s | – | 0 | | **Total** | **54m 55s** | **$41.40** | **0** |
Ran ImplementPlan.fabro (12 nodes and 15 edges) ```dot digraph ImplementPlan { graph [ goal="Implement and simplify", model_stylesheet=" * { model: claude-opus-4-7; } " ] rankdir=LR start [shape=Mdiamond, label="Start"] exit [shape=Msquare, label="Exit"] toolchain [label="Toolchain", shape=parallelogram, script="command -v cargo >/dev/null || { curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && sudo ln -sf $HOME/.cargo/bin/* /usr/local/bin/; }; cargo --version 2>&1", max_retries=0] preflight_compile [label="Preflight Compile", shape=parallelogram, script="cargo check -q --workspace 2>&1", max_retries=0] preflight_lint [label="Preflight Lint", shape=parallelogram, script="cargo +nightly-2026-04-14 clippy -q --workspace --all-targets -- -D warnings 2>&1", max_retries=0] fix_lints [label="Fix Lints", prompt="The preflight lint step failed. Read the build output from context and fix all clippy lint warnings.", max_visits=3] implement [label="Implement", prompt="Read the plan file referenced in the goal and implement every step. Make all the code changes described in the plan. Use red/green TDD."] simplify_opus [label="Simplify (Opus)", prompt="@prompts/simplify.md"] simplify_gpt [label="Simplify (GPT-55)", prompt="@prompts/simplify.md", model="gpt-55"] verify [label="Verify", shape=parallelogram, script="cargo +nightly-2026-04-14 clippy -q --workspace --all-targets -- -D warnings 2>&1 && cargo nextest run --cargo-quiet --workspace --status-level fail 2>&1 && cargo dev docs refresh 2>&1 && cargo dev docs check 2>&1", goal_gate=true, retry_target="fixup"] fixup [label="Fixup", prompt="The verify step failed. Read the build output from context and fix all clippy lint warnings, test failures, and generated docs errors.", max_visits=3] fmt [label="Format", shape=parallelogram, script="cargo +nightly-2026-04-14 fmt --all 2>&1", max_retries=0] start -> toolchain toolchain -> preflight_compile [condition="outcome=succeeded"] toolchain -> exit preflight_compile -> preflight_lint [condition="outcome=succeeded"] preflight_compile -> exit preflight_lint -> implement [condition="outcome=succeeded"] preflight_lint -> fix_lints fix_lints -> preflight_lint implement -> simplify_opus -> simplify_gpt -> verify verify -> fmt [condition="outcome=succeeded"] verify -> fixup fixup -> verify fmt -> exit } ```
⚒️ Generated with [Fabro](https://fabro.sh) --------- Co-authored-by: Fabro Co-authored-by: Bryan Helmkamp Co-authored-by: Claude Opus 4.7 (1M context) --- .../app/components/stage-sidebar.tsx | 7 +- apps/fabro-web/app/lib/run-events.test.tsx | 65 ++ apps/fabro-web/app/lib/run-events.ts | 9 +- apps/fabro-web/app/lib/stage-sidebar.test.ts | 172 ++++ apps/fabro-web/app/lib/stage-sidebar.ts | 51 +- apps/fabro-web/app/routes/run-overview.tsx | 34 +- apps/fabro-web/app/routes/run-stages.test.ts | 96 ++- apps/fabro-web/app/routes/run-stages.tsx | 20 +- docs/public/api-reference/fabro-api.yaml | 30 +- lib/crates/fabro-dump/src/lib.rs | 7 +- lib/crates/fabro-server/src/demo/mod.rs | 61 +- lib/crates/fabro-server/src/server.rs | 127 +-- .../src/server/handler/billing.rs | 277 +++--- lib/crates/fabro-server/src/server/tests.rs | 791 ++++++++++++++++-- lib/crates/fabro-store/src/artifact_store.rs | 7 +- lib/crates/fabro-store/src/run_state.rs | 327 +++++++- .../tests/serializable_projection.rs | 30 +- lib/crates/fabro-types/src/lib.rs | 2 +- lib/crates/fabro-types/src/outcome.rs | 25 +- lib/crates/fabro-types/src/run_event/stage.rs | 2 + lib/crates/fabro-types/src/run_projection.rs | 123 ++- lib/crates/fabro-types/src/stage_id.rs | 39 +- .../fabro-workflow/src/billing_rollup.rs | 291 +++++++ lib/crates/fabro-workflow/src/error.rs | 1 + .../fabro-workflow/src/event/convert.rs | 29 +- lib/crates/fabro-workflow/src/event/events.rs | 1 + lib/crates/fabro-workflow/src/lib.rs | 189 ++++- .../fabro-workflow/src/lifecycle/event.rs | 2 + .../fabro-workflow/src/pipeline/finalize.rs | 137 ++- lib/crates/fabro-workflow/src/pipeline/mod.rs | 2 +- .../fabro-workflow/src/pipeline/retro.rs | 2 +- lib/crates/fabro-workflow/src/test_support.rs | 7 +- .../src/models/billing-by-model.ts | 2 +- .../src/models/billing-stage-ref.ts | 2 +- .../src/models/run-billing-stage.ts | 4 +- .../src/models/run-billing.ts | 2 +- .../fabro-api-client/src/models/run-stage.ts | 10 +- 37 files changed, 2508 insertions(+), 475 deletions(-) create mode 100644 apps/fabro-web/app/lib/stage-sidebar.test.ts create mode 100644 lib/crates/fabro-workflow/src/billing_rollup.rs diff --git a/apps/fabro-web/app/components/stage-sidebar.tsx b/apps/fabro-web/app/components/stage-sidebar.tsx index b72ee77bd..2d3122602 100644 --- a/apps/fabro-web/app/components/stage-sidebar.tsx +++ b/apps/fabro-web/app/components/stage-sidebar.tsx @@ -11,14 +11,15 @@ import { } from "@heroicons/react/24/solid"; import { Bars3BottomLeftIcon, DocumentTextIcon, MapIcon } from "@heroicons/react/24/outline"; import { formatDurationSecs } from "../lib/format"; -import { ACTIVE_STAGE_STATES } from "../lib/stage-sidebar"; +import { ACTIVE_STAGE_STATES, formatStageLabel } from "../lib/stage-sidebar"; export interface Stage { id: string; name: string; status: StageState; duration: string; - dotId?: string; + nodeId: string; + visit: number; } export const statusConfig: Record; color: string }> = { @@ -100,7 +101,7 @@ export function StageSidebar({ stages, runId, selectedStageId, activeLink }: Sta }`} > - {stage.name} + {formatStageLabel(stage)} {stageDuration(stage)} diff --git a/apps/fabro-web/app/lib/run-events.test.tsx b/apps/fabro-web/app/lib/run-events.test.tsx index e9faa93fc..c38ea1dd2 100644 --- a/apps/fabro-web/app/lib/run-events.test.tsx +++ b/apps/fabro-web/app/lib/run-events.test.tsx @@ -49,6 +49,14 @@ describe("queryKeysForRunEvent", () => { queryKeys.runs.graph("run-1", "TB"), ]); }); + + test("stage.retrying invalidates the same keys as other stage events", () => { + const keys = queryKeysForRunEvent("run-1", "stage.retrying", "verify@2"); + expect(keys).toContain(queryKeys.runs.stages("run-1")); + expect(keys).toContain(queryKeys.runs.events("run-1", 1000)); + expect(keys).toContain(queryKeys.runs.detail("run-1")); + expect(keys).toContain(queryKeys.runs.stageTurns("run-1", "verify@2")); + }); }); describe("subscribeToRunEvents", () => { @@ -172,6 +180,63 @@ describe("subscribeToRunEvents", () => { coordinator.close(); }); + test("envelope with suffixed stage_id invalidates stageTurns(runId, stageId)", async () => { + const source = new FakeEventSource(); + const keys: string[] = []; + const coordinator = createCoordinator(() => source); + const cleanup = subscribeToRunEvents( + "run-stage", + (key) => { + keys.push(key); + return Promise.resolve(); + }, + () => source, + { debounceMs: 0, coordinator }, + ); + + await waitFor(() => source.onmessage !== null); + source.emit({ + event: "stage.retrying", + run_id: "run-stage", + stage_id: "verify@2", + node_id: "verify", + }); + + expect(keys).toContain(queryKeys.runs.stageTurns("run-stage", "verify@2")); + expect(keys).toContain(queryKeys.runs.stages("run-stage")); + expect(keys).toContain(queryKeys.runs.events("run-stage", 1000)); + expect(keys).toContain(queryKeys.runs.graph("run-stage", "LR")); + expect(keys).toContain(queryKeys.runs.detail("run-stage")); + expect(keys).not.toContain(queryKeys.runs.stageTurns("run-stage", "verify")); + + cleanup(); + coordinator.close(); + }); + + test("falls back to node_id when an event has no stage_id", async () => { + const source = new FakeEventSource(); + const keys: string[] = []; + const coordinator = createCoordinator(() => source); + const cleanup = subscribeToRunEvents( + "run-stage-node", + (key) => { + keys.push(key); + return Promise.resolve(); + }, + () => source, + { debounceMs: 0, coordinator }, + ); + + await waitFor(() => source.onmessage !== null); + source.emit({ event: "stage.started", run_id: "run-stage-node", node_id: "verify" }); + + expect(keys).toContain(queryKeys.runs.stageTurns("run-stage-node", "verify")); + expect(keys).toContain(queryKeys.runs.stages("run-stage-node")); + + cleanup(); + coordinator.close(); + }); + test("fallback malformed events are ignored and StrictMode-style cleanup does not underflow", () => { const firstSource = new FakeEventSource(); const secondSource = new FakeEventSource(); diff --git a/apps/fabro-web/app/lib/run-events.ts b/apps/fabro-web/app/lib/run-events.ts index 1413d38d4..a81a11c2b 100644 --- a/apps/fabro-web/app/lib/run-events.ts +++ b/apps/fabro-web/app/lib/run-events.ts @@ -19,6 +19,7 @@ interface RunEventPayload extends EventPayload { event?: string; run_id?: string; node_id?: string; + stage_id?: string; properties?: Record; } @@ -42,7 +43,12 @@ const RUN_SUMMARY_EVENTS = new Set([ "run.archived", "run.unarchived", ]); -const STAGE_EVENTS = new Set(["stage.started", "stage.completed", "stage.failed"]); +const STAGE_EVENTS = new Set([ + "stage.started", + "stage.completed", + "stage.failed", + "stage.retrying", +]); const COMMAND_EVENTS = new Set(["command.started", "command.completed"]); const INTERVIEW_EVENTS = new Set([ "interview.started", @@ -166,6 +172,7 @@ function resyncKeysForRun(runId: string) { } function stageIdFromPayload(payload: RunEventPayload): string | undefined { + if (typeof payload.stage_id === "string") return payload.stage_id; if (typeof payload.node_id === "string") return payload.node_id; const nodeId = payload.properties?.node_id; return typeof nodeId === "string" ? nodeId : undefined; diff --git a/apps/fabro-web/app/lib/stage-sidebar.test.ts b/apps/fabro-web/app/lib/stage-sidebar.test.ts new file mode 100644 index 000000000..9e8891645 --- /dev/null +++ b/apps/fabro-web/app/lib/stage-sidebar.test.ts @@ -0,0 +1,172 @@ +import { describe, expect, test } from "bun:test"; +import type { PaginatedRunStageList, StageState } from "@qltysh/fabro-api-client"; + +import type { Stage } from "../components/stage-sidebar"; +import { aggregateGraphNodeStatus, formatStageLabel, mapRunStagesToSidebarStages } from "./stage-sidebar"; + +function makeStage(nodeId: string, visit: number, status: StageState): Stage { + return { + id: `${nodeId}@${visit}`, + name: nodeId, + nodeId, + visit, + status, + duration: "--", + }; +} + +describe("mapRunStagesToSidebarStages", () => { + test("maps two visits of the same node to distinct sidebar entries", () => { + const stages: PaginatedRunStageList = { + data: [ + { + id: "apply-changes@1", + name: "Apply Changes", + status: "succeeded", + duration_secs: 12.5, + node_id: "apply", + visit: 1, + }, + { + id: "apply-changes@2", + name: "Apply Changes", + status: "running", + node_id: "apply", + visit: 2, + }, + ], + meta: { has_more: false }, + }; + + const result = mapRunStagesToSidebarStages(stages); + expect(result).toHaveLength(2); + + expect(result[0].id).toBe("apply-changes@1"); + expect(result[0].nodeId).toBe("apply"); + expect(result[0].visit).toBe(1); + expect(formatStageLabel(result[0])).toBe("Apply Changes"); + + expect(result[1].id).toBe("apply-changes@2"); + expect(result[1].nodeId).toBe("apply"); + expect(result[1].visit).toBe(2); + expect(formatStageLabel(result[1])).toBe("Apply Changes (2)"); + }); + + test("filters by node_id (suffixed start@1 / exit@1 are still hidden)", () => { + const stages: PaginatedRunStageList = { + data: [ + { + id: "start@1", + name: "start", + status: "succeeded", + node_id: "start", + visit: 1, + }, + { + id: "verify@1", + name: "verify", + status: "succeeded", + node_id: "verify", + visit: 1, + }, + { + id: "exit@1", + name: "exit", + status: "succeeded", + node_id: "exit", + visit: 1, + }, + ], + meta: { has_more: false }, + }; + + const result = mapRunStagesToSidebarStages(stages); + expect(result.map((s) => s.id)).toEqual(["verify@1"]); + }); + + test("missing duration renders as '--'", () => { + const stages: PaginatedRunStageList = { + data: [ + { + id: "verify@1", + name: "verify", + status: "running", + node_id: "verify", + visit: 1, + }, + ], + meta: { has_more: false }, + }; + + expect(mapRunStagesToSidebarStages(stages)[0].duration).toBe("--"); + }); +}); + +describe("aggregateGraphNodeStatus", () => { + test("(failed, running) renders as running and clicks open the latest visit", () => { + const result = aggregateGraphNodeStatus([ + makeStage("verify", 1, "failed"), + makeStage("verify", 2, "running"), + ]); + expect(result.get("verify")).toEqual({ + displayStatus: "running", + latestStageId: "verify@2", + }); + }); + + test("(failed, succeeded) renders as succeeded — failure-then-fix shows healed", () => { + const result = aggregateGraphNodeStatus([ + makeStage("verify", 1, "failed"), + makeStage("verify", 2, "succeeded"), + ]); + expect(result.get("verify")).toEqual({ + displayStatus: "succeeded", + latestStageId: "verify@2", + }); + }); + + test("(succeeded, failed) renders as failed and clicks open the latest visit", () => { + const result = aggregateGraphNodeStatus([ + makeStage("verify", 1, "succeeded"), + makeStage("verify", 2, "failed"), + ]); + expect(result.get("verify")).toEqual({ + displayStatus: "failed", + latestStageId: "verify@2", + }); + }); + + test("(running, retrying) — latest active wins", () => { + const result = aggregateGraphNodeStatus([ + makeStage("verify", 1, "running"), + makeStage("verify", 2, "retrying"), + ]); + expect(result.get("verify")).toEqual({ + displayStatus: "retrying", + latestStageId: "verify@2", + }); + }); + + test("orders by visit even when input is shuffled", () => { + const result = aggregateGraphNodeStatus([ + makeStage("verify", 2, "running"), + makeStage("verify", 1, "failed"), + ]); + expect(result.get("verify")?.latestStageId).toBe("verify@2"); + }); + + test("single visit per node is unaffected", () => { + const result = aggregateGraphNodeStatus([ + makeStage("plan", 1, "succeeded"), + makeStage("apply", 1, "running"), + ]); + expect(result.get("plan")).toEqual({ + displayStatus: "succeeded", + latestStageId: "plan@1", + }); + expect(result.get("apply")).toEqual({ + displayStatus: "running", + latestStageId: "apply@1", + }); + }); +}); diff --git a/apps/fabro-web/app/lib/stage-sidebar.ts b/apps/fabro-web/app/lib/stage-sidebar.ts index 747c6a70e..e44e5f4a5 100644 --- a/apps/fabro-web/app/lib/stage-sidebar.ts +++ b/apps/fabro-web/app/lib/stage-sidebar.ts @@ -10,18 +10,65 @@ export const SUCCEEDED_STAGE_STATES: ReadonlySet = new Set([ "partially_succeeded", ]); +/** + * Display label for a stage. Suffixes `(N)` for visits > 1 so a looped node + * (e.g. `verify`) renders as `verify`, `verify (2)`, `verify (3)` in the + * sidebar and stage header. + */ +export function formatStageLabel(stage: { name: string; visit: number }): string { + return stage.visit > 1 ? `${stage.name} (${stage.visit})` : stage.name; +} + export function mapRunStagesToSidebarStages( stagesResult: PaginatedRunStageList | null | undefined, ): Stage[] { return (stagesResult?.data ?? []) - .filter((stage) => isVisibleStage(stage.id)) + .filter((stage) => isVisibleStage(stage.node_id)) .map((stage) => ({ id: stage.id, name: stage.name, - dotId: stage.dot_id ?? stage.id, + nodeId: stage.node_id, + visit: stage.visit, status: stage.status, duration: stage.duration_secs != null ? formatDurationSecs(stage.duration_secs) : "--", })); } + +/** + * Aggregate per-node display state for the workflow graph. + * + * Status policy: if any visit is active (running/retrying), the node renders + * that active state (latest active visit wins). Otherwise the node renders + * the latest visit's terminal state. The click target is always the latest + * visit's stageId. + */ +export function aggregateGraphNodeStatus(stages: readonly Stage[]): Map< + string, + { displayStatus: StageState; latestStageId: string } +> { + // Single pass per nodeId: track the visit with the highest `visit` overall + // (drives click target + terminal status) and the highest-visit *active* + // stage (drives display when any visit is in flight). + const latest = new Map(); + const latestActive = new Map(); + for (const stage of stages) { + const prevLatest = latest.get(stage.nodeId); + if (!prevLatest || stage.visit > prevLatest.visit) { + latest.set(stage.nodeId, stage); + } + if (ACTIVE_STAGE_STATES.has(stage.status)) { + const prevActive = latestActive.get(stage.nodeId); + if (!prevActive || stage.visit > prevActive.visit) { + latestActive.set(stage.nodeId, stage); + } + } + } + const result = new Map(); + for (const [nodeId, latestStage] of latest) { + const display = latestActive.get(nodeId) ?? latestStage; + result.set(nodeId, { displayStatus: display.status, latestStageId: latestStage.id }); + } + return result; +} diff --git a/apps/fabro-web/app/routes/run-overview.tsx b/apps/fabro-web/app/routes/run-overview.tsx index bf0c0dd93..48066eae0 100644 --- a/apps/fabro-web/app/routes/run-overview.tsx +++ b/apps/fabro-web/app/routes/run-overview.tsx @@ -3,7 +3,6 @@ import { useNavigate, useParams } from "react-router"; import { graphTheme } from "../lib/graph-theme"; import { useRun, useRunGraph, useRunStages } from "../lib/queries"; import { StageSidebar } from "../components/stage-sidebar"; -import type { Stage } from "../components/stage-sidebar"; import { GRAPH_DEFAULT_ZOOM_INDEX, GRAPH_ZOOM_STEPS, @@ -13,6 +12,7 @@ import { EmptyState } from "../components/state"; import { ACTIVE_STAGE_STATES, SUCCEEDED_STAGE_STATES, + aggregateGraphNodeStatus, mapRunStagesToSidebarStages, } from "../lib/stage-sidebar"; @@ -54,27 +54,27 @@ export default function RunOverview() { const inner = innerRef.current; if (!inner || !graphSvg) return; - let cancelled = false; - (async () => { - if (cancelled) return; inner.innerHTML = graphSvg; const svg = inner.querySelector("svg"); if (!svg) return; svgRef.current = svg; const gt = graphTheme; - const runningDotIds = new Set( - stages.filter((s: Stage) => ACTIVE_STAGE_STATES.has(s.status)).map((s: Stage) => s.dotId ?? s.id), - ); - const failedDotIds = new Set( - stages.filter((s: Stage) => s.status === "failed").map((s: Stage) => s.dotId ?? s.id), - ); - const completedDotIds = new Set( - stages.filter((s: Stage) => SUCCEEDED_STAGE_STATES.has(s.status)).map((s: Stage) => s.dotId ?? s.id), - ); - const dotIdToStageId = new Map( - stages.map((s: Stage) => [s.dotId ?? s.id, s.id]), - ); + const aggregated = aggregateGraphNodeStatus(stages); + const runningDotIds = new Set(); + const failedDotIds = new Set(); + const completedDotIds = new Set(); + const dotIdToStageId = new Map(); + for (const [nodeId, { displayStatus, latestStageId }] of aggregated) { + dotIdToStageId.set(nodeId, latestStageId); + if (ACTIVE_STAGE_STATES.has(displayStatus)) { + runningDotIds.add(nodeId); + } else if (displayStatus === "failed") { + failedDotIds.add(nodeId); + } else if (SUCCEEDED_STAGE_STATES.has(displayStatus)) { + completedDotIds.add(nodeId); + } + } const ns = "http://www.w3.org/2000/svg"; for (const group of svg.querySelectorAll(".node")) { @@ -148,8 +148,6 @@ export default function RunOverview() { } } } - })(); - return () => { cancelled = true; }; }, [stages, graphSvg, id, navigate, terminalOutcome]); const onPointerDown = useCallback((e: React.PointerEvent) => { diff --git a/apps/fabro-web/app/routes/run-stages.test.ts b/apps/fabro-web/app/routes/run-stages.test.ts index d10b5b2d2..704c5cac8 100644 --- a/apps/fabro-web/app/routes/run-stages.test.ts +++ b/apps/fabro-web/app/routes/run-stages.test.ts @@ -1,6 +1,7 @@ import { describe, expect, test } from "bun:test"; +import type { EventEnvelope } from "@qltysh/fabro-api-client"; -import { isSafeMarkdownHref } from "./run-stages"; +import { isSafeMarkdownHref, turnsFromEvents } from "./run-stages"; describe("isSafeMarkdownHref", () => { test("rejects protocol-relative URLs", () => { @@ -15,3 +16,96 @@ describe("isSafeMarkdownHref", () => { expect(isSafeMarkdownHref("mailto:test@example.com")).toBe(true); }); }); + +function makeEnvelope(overrides: Partial): EventEnvelope { + return { + seq: 1, + id: "evt", + ts: "2026-01-01T00:00:00Z", + run_id: "run-1", + event: "stage.prompt", + ...overrides, + } as EventEnvelope; +} + +describe("turnsFromEvents", () => { + test("filters events by stage_id (verify@1 vs verify@2 do not cross-contaminate)", () => { + const events: EventEnvelope[] = [ + makeEnvelope({ + seq: 1, + event: "stage.prompt", + stage_id: "verify@1", + node_id: "verify", + properties: { text: "first visit prompt" }, + }), + makeEnvelope({ + seq: 2, + event: "stage.prompt", + stage_id: "verify@2", + node_id: "verify", + properties: { text: "second visit prompt" }, + }), + makeEnvelope({ + seq: 3, + event: "agent.message", + stage_id: "verify@1", + node_id: "verify", + properties: { text: "first visit reply" }, + }), + makeEnvelope({ + seq: 4, + event: "agent.message", + stage_id: "verify@2", + node_id: "verify", + properties: { text: "second visit reply" }, + }), + ]; + + const firstVisit = turnsFromEvents(events, "verify@1"); + expect(firstVisit).toEqual([ + { kind: "system", content: "first visit prompt" }, + { kind: "assistant", content: "first visit reply" }, + ]); + + const secondVisit = turnsFromEvents(events, "verify@2"); + expect(secondVisit).toEqual([ + { kind: "system", content: "second visit prompt" }, + { kind: "assistant", content: "second visit reply" }, + ]); + }); + + test("command turn carries the requested stage_id, no @1 fallback", () => { + const events: EventEnvelope[] = [ + makeEnvelope({ + seq: 1, + event: "command.started", + stage_id: "verify@2", + node_id: "verify", + properties: { script: "echo hi", language: "shell" }, + }), + makeEnvelope({ + seq: 2, + event: "command.completed", + stage_id: "verify@2", + node_id: "verify", + properties: { + stdout: "hi", + stderr: "", + exit_code: 0, + duration_ms: 5, + termination: "exited", + }, + }), + ]; + + const turns = turnsFromEvents(events, "verify@2"); + expect(turns).toHaveLength(1); + const turn = turns[0]; + expect(turn.kind).toBe("command"); + if (turn.kind === "command") { + expect(turn.stageId).toBe("verify@2"); + expect(turn.script).toBe("echo hi"); + expect(turn.running).toBe(false); + } + }); +}); diff --git a/apps/fabro-web/app/routes/run-stages.tsx b/apps/fabro-web/app/routes/run-stages.tsx index 6d179d701..1176cb71a 100644 --- a/apps/fabro-web/app/routes/run-stages.tsx +++ b/apps/fabro-web/app/routes/run-stages.tsx @@ -41,7 +41,7 @@ import { EmptyState } from "../components/state"; import { CopyButton } from "../components/ui"; import { formatDurationSecs } from "../lib/format"; import { fetchRunCommandLog, useRunEventsList, useRunStageTurns, useRunStages } from "../lib/queries"; -import { mapRunStagesToSidebarStages } from "../lib/stage-sidebar"; +import { ACTIVE_STAGE_STATES, formatStageLabel, mapRunStagesToSidebarStages } from "../lib/stage-sidebar"; import { getNumber, getString, type UnknownRecord } from "../lib/unknown"; import { CommandOutputStream, @@ -68,8 +68,8 @@ function readTermination(props: UnknownRecord): CommandTermination { return CommandTermination.EXITED; } -function turnsFromEvents(events: EventEnvelope[], stageId: string): TurnType[] { - const stageEvents = events.filter((e) => e.node_id === stageId); +export function turnsFromEvents(events: EventEnvelope[], stageId: string): TurnType[] { + const stageEvents = events.filter((e) => e.stage_id === stageId); const turns: TurnType[] = []; // Collect tool pairs: started → completed const pendingTools = new Map(); @@ -114,7 +114,7 @@ function turnsFromEvents(events: EventEnvelope[], stageId: string): TurnType[] { } case "command.started": { pendingCommand = { - stageId: e.stage_id ?? `${stageId}@1`, + stageId, script: getString(props, "script") ?? "", language: getString(props, "language") ?? "shell", }; @@ -123,7 +123,7 @@ function turnsFromEvents(events: EventEnvelope[], stageId: string): TurnType[] { case "command.completed": { turns.push({ kind: "command", - stageId: pendingCommand?.stageId ?? e.stage_id ?? `${stageId}@1`, + stageId: pendingCommand?.stageId ?? stageId, script: pendingCommand?.script ?? "", language: pendingCommand?.language ?? "shell", stdout: getString(props, "stdout") ?? "", @@ -614,7 +614,7 @@ export default function RunStages() { () => mapTurns(turnsQuery.data, eventsQuery.data, selectedStage?.id), [eventsQuery.data, selectedStage?.id, turnsQuery.data], ); - const isRunning = selectedStage?.status === "running"; + const isActive = selectedStage ? ACTIVE_STAGE_STATES.has(selectedStage.status) : false; if (!id || !stages.length) { return ( @@ -636,11 +636,13 @@ export default function RunStages() {
- -

{selectedStage.name}

+ +

+ {formatStageLabel(selectedStage)} +

diff --git a/docs/public/api-reference/fabro-api.yaml b/docs/public/api-reference/fabro-api.yaml index 4137dfaad..a767a7262 100644 --- a/docs/public/api-reference/fabro-api.yaml +++ b/docs/public/api-reference/fabro-api.yaml @@ -6160,7 +6160,7 @@ components: example: 3501.0 BillingStageRef: - description: Reference to a billing stage. + description: Reference to a workflow node in a billing stage row. type: object required: - id @@ -6320,11 +6320,13 @@ components: - id - name - status + - node_id + - visit properties: id: type: string - description: Unique stage identifier within the run. - example: propose-changes + description: StageId in "node_id@visit" form, e.g. verify@2. + example: verify@2 name: type: string description: Human-readable stage name. @@ -6335,10 +6337,16 @@ components: type: number description: Time spent in this stage, in seconds. example: 154.0 - dot_id: + node_id: type: string - description: Node identifier in the Graphviz graph source. - example: propose + description: Node id in the workflow graph; multiple stages with different visits share the same node_id. + example: verify + visit: + type: integer + format: uint32 + minimum: 1 + description: 1-based visit count; bumped each time the workflow re-enters this node. + example: 2 ToolUse: description: A single tool invocation with its input, result, and execution metadata. @@ -6608,7 +6616,7 @@ components: # ── Billing Schemas ────────────────────────────────────────────────── RunBillingStage: - description: Token counts and billed totals for a single stage within a run. + description: Token counts and billed totals for one workflow node within a run. Rows are grouped by node; billing and runtime sum every visit of that node. type: object required: - stage @@ -6619,7 +6627,7 @@ components: stage: $ref: "#/components/schemas/BillingStageRef" model: - description: Model used for this stage; null for non-LLM stages. + description: Latest usage-bearing visit model for this node; null when no visit used an LLM model. oneOf: - $ref: "#/components/schemas/ModelReference" - type: "null" @@ -6627,7 +6635,7 @@ components: $ref: "#/components/schemas/BilledTokenCounts" runtime_secs: type: number - description: Wall-clock runtime in seconds. + description: Wall-clock runtime in seconds, summed across every visit of this node. example: 154.0 RunBillingTotals: @@ -6688,7 +6696,7 @@ components: $ref: "#/components/schemas/ModelReference" stages: type: integer - description: Number of stages that used this model. + description: Number of usage-bearing stage visits that used this model. example: 2 billing: $ref: "#/components/schemas/BilledTokenCounts" @@ -6703,7 +6711,7 @@ components: properties: stages: type: array - description: Per-stage billing breakdown. + description: Per-node billing breakdown. Each row sums billing and runtime across all visits of that node. items: $ref: "#/components/schemas/RunBillingStage" totals: diff --git a/lib/crates/fabro-dump/src/lib.rs b/lib/crates/fabro-dump/src/lib.rs index 69a670565..5b2da9403 100644 --- a/lib/crates/fabro-dump/src/lib.rs +++ b/lib/crates/fabro-dump/src/lib.rs @@ -66,18 +66,13 @@ impl RunDump { entries.push(RunDumpEntry::text("graph.fabro", graph_source.clone())); } - let mut stages: Vec<_> = state.iter_stages().collect(); + let stages: Vec<_> = state.iter_stages().collect(); if stages.len() > MAX_STAGES_IN_DUMP { bail!( "run dump supports at most {MAX_STAGES_IN_DUMP} stages with the current path prefix width (got {})", stages.len() ); } - stages.sort_by(|(left_id, left), (right_id, right)| { - left.first_event_seq - .cmp(&right.first_event_seq) - .then_with(|| left_id.cmp(right_id)) - }); let mut stage_ranks = HashMap::new(); for (index, (stage_id, _)) in stages.iter().enumerate() { diff --git a/lib/crates/fabro-server/src/demo/mod.rs b/lib/crates/fabro-server/src/demo/mod.rs index 08423cd55..bc527007d 100644 --- a/lib/crates/fabro-server/src/demo/mod.rs +++ b/lib/crates/fabro-server/src/demo/mod.rs @@ -784,9 +784,10 @@ mod runs { RunNamespace, RunPrepareSettings, RunSandboxSettings, }; use fabro_types::settings::{InterpString, ProjectNamespace, WorkflowNamespace}; - use fabro_types::{RunId, WorkflowSettings}; + use fabro_types::{RunId, StageId, WorkflowSettings}; use super::ts; + use crate::server::run_stage_from_stage_id; fn labels(entries: &[(&str, &str)]) -> HashMap { entries @@ -1181,34 +1182,36 @@ mod runs { pub(super) fn stages() -> Vec { vec![ - RunStage { - id: "detect-drift".into(), - name: "Detect Drift".into(), - status: StageState::Succeeded, - duration_secs: Some(72.0), - dot_id: Some("detect".into()), - }, - RunStage { - id: "propose-changes".into(), - name: "Propose Changes".into(), - status: StageState::Succeeded, - duration_secs: Some(154.0), - dot_id: Some("propose".into()), - }, - RunStage { - id: "review-changes".into(), - name: "Review Changes".into(), - status: StageState::Succeeded, - duration_secs: Some(45.0), - dot_id: Some("review".into()), - }, - RunStage { - id: "apply-changes".into(), - name: "Apply Changes".into(), - status: StageState::Running, - duration_secs: Some(118.0), - dot_id: Some("apply".into()), - }, + run_stage_from_stage_id( + &StageId::new("detect-drift", 1), + "Detect Drift", + StageState::Succeeded, + Some(72.0), + ), + run_stage_from_stage_id( + &StageId::new("propose-changes", 1), + "Propose Changes", + StageState::Succeeded, + Some(154.0), + ), + run_stage_from_stage_id( + &StageId::new("review-changes", 1), + "Review Changes", + StageState::Succeeded, + Some(45.0), + ), + run_stage_from_stage_id( + &StageId::new("apply-changes", 1), + "Apply Changes", + StageState::Succeeded, + Some(118.0), + ), + run_stage_from_stage_id( + &StageId::new("apply-changes", 2), + "Apply Changes", + StageState::Running, + None, + ), ] } diff --git a/lib/crates/fabro-server/src/server.rs b/lib/crates/fabro-server/src/server.rs index eb35ef8b0..26fbdd8d6 100644 --- a/lib/crates/fabro-server/src/server.rs +++ b/lib/crates/fabro-server/src/server.rs @@ -54,7 +54,7 @@ use fabro_llm::types::{ ContentPart, FinishReason, Message as LlmMessage, Request as LlmRequest, Role, ToolChoice, ToolDefinition, }; -use fabro_model::{BilledModelUsage, BilledTokenCounts, Catalog, ModelTestMode, Provider}; +use fabro_model::{BilledTokenCounts, Catalog, ModelTestMode, Provider}; use fabro_redact::redact_jsonl_line; use fabro_sandbox::daytona::{self, DaytonaSandbox}; use fabro_sandbox::reconnect::reconnect; @@ -536,17 +536,48 @@ pub(crate) struct ResolvedAppStateSettings { pub(crate) manifest_run_settings: std::result::Result, } -fn accumulate_model_billing(entry: &mut ModelBillingTotals, usage: &BilledModelUsage) { - let tokens = usage.tokens(); - entry.stages += 1; - entry.billing.input_tokens += tokens.input_tokens; - entry.billing.output_tokens += tokens.output_tokens; - entry.billing.reasoning_tokens += tokens.reasoning_tokens; - entry.billing.cache_read_tokens += tokens.cache_read_tokens; - entry.billing.cache_write_tokens += tokens.cache_write_tokens; - entry.billing.total_tokens += tokens.total_tokens(); - if let Some(value) = usage.total_usd_micros { - *entry.billing.total_usd_micros.get_or_insert(0) += value; +fn accumulate_billed_token_counts(target: &mut BilledTokenCounts, source: &BilledTokenCounts) { + target.input_tokens += source.input_tokens; + target.output_tokens += source.output_tokens; + target.reasoning_tokens += source.reasoning_tokens; + target.cache_read_tokens += source.cache_read_tokens; + target.cache_write_tokens += source.cache_write_tokens; + target.total_tokens += source.total_tokens; + if let Some(value) = source.total_usd_micros { + *target.total_usd_micros.get_or_insert(0) += value; + } +} + +fn accumulate_billing_rollup( + accumulator: &mut BillingAccumulator, + rollup: &fabro_workflow::ProjectionBillingRollup, +) { + accumulator.total_runs += 1; + accumulator.total_runtime_secs += rollup.runtime_ms as f64 / 1000.0; + for model in &rollup.by_model { + let entry = accumulator + .by_model + .entry(model.model_id.clone()) + .or_default(); + entry.stages += model.stages; + accumulate_billed_token_counts(&mut entry.billing, &model.billing); + } +} + +pub(crate) fn run_stage_from_stage_id( + stage_id: &StageId, + name: impl Into, + status: StageState, + duration_secs: Option, +) -> RunStage { + RunStage { + id: stage_id.to_string(), + name: name.into(), + status, + duration_secs, + node_id: stage_id.node_id().to_string(), + visit: std::num::NonZeroU32::new(stage_id.visit()) + .expect("StageId stores a non-zero visit"), } } @@ -2776,9 +2807,9 @@ async fn execute_run_in_process(state: Arc, run_id: RunId) { } } - // Save final checkpoint - let checkpoint = match run_store.state().await { - Ok(state) => state.checkpoint, + // Save final projection + let final_projection = match run_store.state().await { + Ok(state) => Some(state), Err(err) => { tracing::warn!(run_id = %run_id, error = %err, "Failed to load run state from store"); None @@ -2786,32 +2817,17 @@ async fn execute_run_in_process(state: Arc, run_id: RunId) { }; // Accumulate aggregate usage after execution completes. - if let Some(ref cp) = checkpoint { - let stage_durations = match run_store.list_events().await { - Ok(events) => fabro_workflow::extract_stage_durations_from_events(&events), - Err(err) => { - tracing::warn!(run_id = %run_id, error = %err, "Failed to load run events from store"); - HashMap::default() - } - }; - let mut agg = state - .aggregate_billing - .lock() - .expect("aggregate_billing lock poisoned"); - agg.total_runs += 1; - let mut run_runtime: f64 = 0.0; - for (node_id, outcome) in &cp.node_outcomes { - if let Some(usage) = &outcome.usage { - let entry = agg - .by_model - .entry(usage.model_id().to_string()) - .or_default(); - accumulate_model_billing(entry, usage); - } - let duration_ms = stage_durations.get(node_id).copied().unwrap_or(0); - run_runtime += duration_ms as f64 / 1000.0; + if let Some(ref projection) = final_projection { + if projection.checkpoint.is_some() { + let mut agg = state + .aggregate_billing + .lock() + .expect("aggregate_billing lock poisoned"); + accumulate_billing_rollup( + &mut agg, + &fabro_workflow::billing_rollup_from_projection(projection), + ); } - agg.total_runtime_secs += run_runtime; } let mut runs = state.runs.lock().expect("runs lock poisoned"); @@ -2860,7 +2876,9 @@ async fn execute_run_in_process(state: Arc, run_id: RunId) { }; } } - managed_run.checkpoint = checkpoint; + managed_run.checkpoint = final_projection + .as_ref() + .and_then(|projection| projection.checkpoint.clone()); managed_run.run_dir = Some(run_dir); clear_live_run_state(managed_run); } @@ -3103,32 +3121,15 @@ async fn execute_run_subprocess(state: Arc, run_id: RunId) { } }; - if let Some(ref checkpoint) = final_state.checkpoint { - let stage_durations = match run_store.list_events().await { - Ok(events) => fabro_workflow::extract_stage_durations_from_events(&events), - Err(err) => { - tracing::warn!(run_id = %run_id, error = %err, "Failed to load run events from store"); - HashMap::default() - } - }; + if final_state.checkpoint.is_some() { let mut agg = state .aggregate_billing .lock() .expect("aggregate_billing lock poisoned"); - agg.total_runs += 1; - let mut run_runtime: f64 = 0.0; - for (node_id, outcome) in &checkpoint.node_outcomes { - if let Some(usage) = &outcome.usage { - let entry = agg - .by_model - .entry(usage.model_id().to_string()) - .or_default(); - accumulate_model_billing(entry, usage); - } - let duration_ms = stage_durations.get(node_id).copied().unwrap_or(0); - run_runtime += duration_ms as f64 / 1000.0; - } - agg.total_runtime_secs += run_runtime; + accumulate_billing_rollup( + &mut agg, + &fabro_workflow::billing_rollup_from_projection(&final_state), + ); } let mut runs = state.runs.lock().expect("runs lock poisoned"); diff --git a/lib/crates/fabro-server/src/server/handler/billing.rs b/lib/crates/fabro-server/src/server/handler/billing.rs index dfe7d5176..193ebd78d 100644 --- a/lib/crates/fabro-server/src/server/handler/billing.rs +++ b/lib/crates/fabro-server/src/server/handler/billing.rs @@ -1,13 +1,13 @@ use std::sync::Arc; -use fabro_types::EventBody; +use fabro_store::RunProjectionReducer; +use fabro_types::{EventBody, RunProjection, StageId}; use super::super::{ - ApiError, AppState, BilledTokenCounts, BillingByModel, BillingStageRef, EventEnvelope, HashMap, - IntoResponse, Json, ListResponse, ModelBillingTotals, ModelReference, PaginationParams, Path, - Query, RequiredUser, Response, Router, RunBilling, RunBillingStage, RunBillingTotals, RunId, - RunStage, RunStatus, StageState, State, StatusCode, accumulate_model_billing, get, - parse_run_id_path, + ApiError, AppState, BillingByModel, BillingStageRef, EventEnvelope, HashMap, IntoResponse, + Json, ListResponse, ModelReference, PaginationParams, Path, Query, RequiredUser, Response, + Router, RunBilling, RunBillingStage, RunBillingTotals, RunId, StageState, State, StatusCode, + get, parse_run_id_path, run_stage_from_stage_id, }; pub(super) fn routes() -> Router> { @@ -16,25 +16,41 @@ pub(super) fn routes() -> Router> { .route("/runs/{id}/billing", get(get_run_billing)) } -fn active_stage_state_from_events(events: &[EventEnvelope], node_id: &str) -> StageState { - let latest = events.iter().rev().find(|envelope| { - envelope.event.node_id.as_deref() == Some(node_id) - && matches!( - &envelope.event.body, - EventBody::StageRetrying(_) - | EventBody::StageStarted(_) - | EventBody::StageCompleted(_) - | EventBody::StageFailed(_) - ) - }); - - if latest.is_some_and(|e| matches!(&e.event.body, EventBody::StageRetrying(_))) { - StageState::Retrying - } else { - StageState::Running +/// Map a `stage.*` lifecycle event body to the [`StageState`] it implies. +/// Returns `None` for any other variant. +fn stage_state_from_lifecycle(body: &EventBody) -> Option { + match body { + EventBody::StageStarted(_) => Some(StageState::Running), + EventBody::StageRetrying(_) => Some(StageState::Retrying), + EventBody::StageFailed(props) => Some(if props.will_retry { + StageState::Retrying + } else { + StageState::Failed + }), + EventBody::StageCompleted(props) => Some(StageState::from(props.status)), + _ => None, } } +/// Single-pass scan over `events` building the latest [`StageState`] for each +/// [`StageId`] from lifecycle events (started/retrying/completed/failed). Each +/// later lifecycle event overwrites earlier ones, leaving the latest as the +/// stored value — equivalent to "scan in reverse, take first match" but in O(E) +/// for the whole list rather than O(stages × events). +fn latest_stage_states(events: &[EventEnvelope]) -> HashMap { + let mut states = HashMap::new(); + for envelope in events { + let Some(stage_id) = envelope.event.stage_id.as_ref() else { + continue; + }; + let Some(state) = stage_state_from_lifecycle(&envelope.event.body) else { + continue; + }; + states.insert(stage_id.clone(), state); + } + states +} + async fn list_run_stages( _auth: RequiredUser, State(state): State>, @@ -46,80 +62,41 @@ async fn list_run_stages( Err(response) => return response, }; - // Try live run first. - let (checkpoint, run_is_active) = { - let runs = state.runs.lock().expect("runs lock poisoned"); - match runs.get(&id) { - Some(managed_run) => { - let active = !matches!( - managed_run.status, - RunStatus::Succeeded { .. } | RunStatus::Failed { .. } | RunStatus::Dead - ); - (managed_run.checkpoint.clone(), active) - } - None => (None, false), - } - }; - - // Fall back to stored run. - let (checkpoint, run_is_active) = if checkpoint.is_some() { - (checkpoint, run_is_active) - } else { - match state.store.open_run_reader(&id).await { - Ok(run_store) => match run_store.state().await { - Ok(run_state) => { - let active = run_state.status.is_some_and(|status| !status.is_terminal()); - (run_state.checkpoint, active) - } - Err(_) => (None, false), - }, - Err(_) => return ApiError::not_found("Run not found.").into_response(), - } - }; - - let Some(checkpoint) = checkpoint else { - return ( - StatusCode::OK, - Json(ListResponse::new(Vec::::new())), - ) - .into_response(); - }; - let events = match state.store.open_run_reader(&id).await { Ok(run_store) => run_store.list_events().await.unwrap_or_default(), - Err(_) => Vec::new(), + Err(_) => return ApiError::not_found("Run not found.").into_response(), }; - let stage_durations = fabro_workflow::extract_stage_durations_from_events(&events); + + let projection = match RunProjection::apply_events(&events) { + Ok(projection) => projection, + Err(err) => { + tracing::warn!( + run_id = %id, + error = %err, + "Failed to build run projection; returning empty stages list", + ); + RunProjection::default() + } + }; + let stage_durations = fabro_workflow::extract_stage_durations_by_stage_id(&events); + let lifecycle_states = latest_stage_states(&events); let mut stages = Vec::new(); - for node_id in &checkpoint.completed_nodes { - let duration_ms = stage_durations.get(node_id).copied().unwrap_or(0); - let status = match checkpoint.node_outcomes.get(node_id) { - Some(outcome) => StageState::from(outcome.status), - None => StageState::Succeeded, - }; - stages.push(RunStage { - id: node_id.clone(), - name: node_id.clone(), - status, - duration_secs: Some(duration_ms as f64 / 1000.0), - dot_id: Some(node_id.clone()), + for (stage_id, stage_projection) in projection.iter_stages() { + // Prefer the latest lifecycle event; fall back to the projection's + // stored completion (e.g. for runs recovered from snapshot only). + let status = lifecycle_states.get(stage_id).copied().unwrap_or_else(|| { + stage_projection + .completion + .as_ref() + .map_or(StageState::Pending, |c| StageState::from(c.outcome)) }); - } - - // Add next node as running if the run is still active. - // The checkpoint's current_node is the last *completed* stage; next_node_id - // is the stage that is currently executing. - if let Some(next_id) = &checkpoint.next_node_id { - if run_is_active && next_id != "exit" && !checkpoint.completed_nodes.contains(next_id) { - stages.push(RunStage { - id: next_id.clone(), - name: next_id.clone(), - status: active_stage_state_from_events(&events, next_id), - duration_secs: None, - dot_id: Some(next_id.clone()), - }); - } + stages.push(run_stage_from_stage_id( + stage_id, + stage_id.node_id().to_string(), + status, + stage_durations.get(stage_id).map(|ms| *ms as f64 / 1000.0), + )); } (StatusCode::OK, Json(ListResponse::new(stages))).into_response() @@ -137,91 +114,39 @@ async fn get_run_billing( } }; - let checkpoint = match run_store.state().await { - Ok(state) => state.checkpoint, + let projection = match run_store.state().await { + Ok(state) => state, Err(err) => { return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) .into_response(); } }; - - let Some(checkpoint) = checkpoint else { - let empty = RunBilling { - by_model: Vec::new(), - stages: Vec::new(), - totals: RunBillingTotals { - cache_read_tokens: 0, - cache_write_tokens: 0, - input_tokens: 0, - output_tokens: 0, - reasoning_tokens: 0, - runtime_secs: 0.0, - total_tokens: 0, - total_usd_micros: None, + let rollup = fabro_workflow::billing_rollup_from_projection(&projection); + let by_model = rollup + .by_model + .iter() + .map(|model| BillingByModel { + billing: model.billing.clone(), + model: ModelReference { + id: model.model_id.clone(), }, - }; - return (StatusCode::OK, Json(empty)).into_response(); - }; - - let stage_durations = match run_store.list_events().await { - Ok(events) => fabro_workflow::extract_stage_durations_from_events(&events), - Err(err) => { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) - .into_response(); - } - }; - - let mut by_model_totals = HashMap::::new(); - let mut billed_usages = Vec::new(); - let mut runtime_secs = 0.0_f64; - let mut stages = Vec::new(); - - for node_id in &checkpoint.completed_nodes { - let duration_ms = stage_durations.get(node_id).copied().unwrap_or(0); - runtime_secs += duration_ms as f64 / 1000.0; - - let usage = checkpoint - .node_outcomes - .get(node_id) - .and_then(|outcome| outcome.usage.as_ref()); - - let (billing, model) = if let Some(usage) = usage { - billed_usages.push(usage.clone()); - let tokens = usage.tokens(); - let billing = BilledTokenCounts { - cache_read_tokens: tokens.cache_read_tokens, - cache_write_tokens: tokens.cache_write_tokens, - input_tokens: tokens.input_tokens, - output_tokens: tokens.output_tokens, - reasoning_tokens: tokens.reasoning_tokens, - total_tokens: tokens.total_tokens(), - total_usd_micros: usage.total_usd_micros, - }; - let model_id = usage.model_id().to_string(); - accumulate_model_billing(by_model_totals.entry(model_id.clone()).or_default(), usage); - (billing, Some(ModelReference { id: model_id })) - } else { - (BilledTokenCounts::default(), None) - }; - - stages.push(RunBillingStage { - billing, - model, - runtime_secs: duration_ms as f64 / 1000.0, - stage: BillingStageRef { - id: node_id.clone(), - name: node_id.clone(), + stages: model.stages, + }) + .collect::>(); + let stages = rollup + .stages + .iter() + .map(|stage| RunBillingStage { + billing: stage.billing.clone(), + model: stage + .model_id + .as_ref() + .map(|id| ModelReference { id: id.clone() }), + runtime_secs: stage.duration_ms as f64 / 1000.0, + stage: BillingStageRef { + id: stage.node_id.clone(), + name: stage.node_id.clone(), }, - }); - } - - let totals = BilledTokenCounts::from_billed_usage(&billed_usages); - let by_model = by_model_totals - .into_iter() - .map(|(model, totals)| BillingByModel { - billing: totals.billing, - model: ModelReference { id: model }, - stages: totals.stages, }) .collect::>(); @@ -229,14 +154,14 @@ async fn get_run_billing( by_model, stages, totals: RunBillingTotals { - cache_read_tokens: totals.cache_read_tokens, - cache_write_tokens: totals.cache_write_tokens, - input_tokens: totals.input_tokens, - output_tokens: totals.output_tokens, - reasoning_tokens: totals.reasoning_tokens, - runtime_secs, - total_tokens: totals.total_tokens, - total_usd_micros: totals.total_usd_micros, + cache_read_tokens: rollup.totals.cache_read_tokens, + cache_write_tokens: rollup.totals.cache_write_tokens, + input_tokens: rollup.totals.input_tokens, + output_tokens: rollup.totals.output_tokens, + reasoning_tokens: rollup.totals.reasoning_tokens, + runtime_secs: rollup.runtime_ms as f64 / 1000.0, + total_tokens: rollup.totals.total_tokens, + total_usd_micros: rollup.totals.total_usd_micros, }, }; diff --git a/lib/crates/fabro-server/src/server/tests.rs b/lib/crates/fabro-server/src/server/tests.rs index 5587e1952..08196dbdd 100644 --- a/lib/crates/fabro-server/src/server/tests.rs +++ b/lib/crates/fabro-server/src/server/tests.rs @@ -18,8 +18,8 @@ use fabro_model::Provider; use fabro_types::settings::ServerAuthMethod; use fabro_types::{ AttrValue, AuthMethod, CommandTermination, FailureCategory, FailureDetail, Graph, - InterviewQuestionRecord, Outcome, QuestionType, RunBlobId, RunId, RunSpec, StageOutcome, - SystemActorKind, fixtures, + InterviewQuestionRecord, Outcome, QuestionType, RunBlobId, RunId, RunSpec, SystemActorKind, + fixtures, }; use fabro_util::check_report::CheckStatus; use httpmock::Method::{GET, POST}; @@ -2112,6 +2112,30 @@ async fn create_durable_run_with_events( } } +/// Append a stage lifecycle event with an explicit `StageScope`, so the +/// stored envelope carries the full `stage_id` (`node_id@visit`). The bare +/// [`workflow_event::append_event`] helper only writes `node_id` because +/// stage lifecycle variants don't carry visit in their payload — production +/// always emits via `Emitter::emit_scoped`. +async fn append_scoped_stage_event( + state: &Arc, + run_id: RunId, + node_id: &str, + visit: u32, + event: &workflow_event::Event, +) { + let scope = fabro_workflow::event::StageScope { + node_id: node_id.to_string(), + visit, + parallel_group_id: None, + parallel_branch_id: None, + }; + let stored = fabro_workflow::event::to_run_event_at(&run_id, event, Utc::now(), Some(&scope)); + let payload = fabro_workflow::event::build_redacted_event_payload(&stored, &run_id).unwrap(); + let run_store = state.store.open_run(&run_id).await.unwrap(); + run_store.append_event(&payload).await.unwrap(); +} + fn stage_status<'a>(body: &'a serde_json::Value, id: &str) -> &'a str { body["data"] .as_array() @@ -2134,7 +2158,58 @@ async fn list_run_stages_projects_retrying_until_completion() { }, workflow_event::Event::RunStarting, workflow_event::Event::RunRunning, - workflow_event::Event::StageStarted { + ]) + .await; + append_scoped_stage_event( + &state, + run_id, + "setup", + 1, + &workflow_event::Event::StageStarted { + node_id: "setup".to_string(), + name: "Setup".to_string(), + index: 0, + handler_type: "command".to_string(), + attempt: 1, + max_attempts: 1, + }, + ) + .await; + append_scoped_stage_event( + &state, + run_id, + "setup", + 1, + &workflow_event::Event::StageCompleted { + node_id: "setup".to_string(), + name: "Setup".to_string(), + index: 0, + duration_ms: 5, + status: "succeeded".to_string(), + preferred_label: None, + suggested_next_ids: Vec::new(), + billing: None, + failure: None, + notes: None, + files_touched: Vec::new(), + context_updates: None, + jump_to_node: None, + context_values: None, + node_visits: None, + loop_failure_signatures: None, + restart_failure_signatures: None, + response: None, + attempt: 1, + max_attempts: 1, + }, + ) + .await; + append_scoped_stage_event( + &state, + run_id, + "work", + 1, + &workflow_event::Event::StageStarted { node_id: "work".to_string(), name: "Work".to_string(), index: 1, @@ -2142,16 +2217,31 @@ async fn list_run_stages_projects_retrying_until_completion() { attempt: 1, max_attempts: 3, }, - workflow_event::Event::StageFailed { + ) + .await; + append_scoped_stage_event( + &state, + run_id, + "work", + 1, + &workflow_event::Event::StageFailed { node_id: "work".to_string(), name: "Work".to_string(), index: 1, failure: FailureDetail::new("try again", FailureCategory::TransientInfra), will_retry: true, duration_ms: 10, + billing: None, actor: None, }, - workflow_event::Event::StageRetrying { + ) + .await; + append_scoped_stage_event( + &state, + run_id, + "work", + 1, + &workflow_event::Event::StageRetrying { node_id: "work".to_string(), name: "Work".to_string(), index: 1, @@ -2159,41 +2249,9 @@ async fn list_run_stages_projects_retrying_until_completion() { max_attempts: 3, delay_ms: 100, }, - ]) + ) .await; - let mut node_outcomes = HashMap::new(); - node_outcomes.insert("setup".to_string(), Outcome::success()); - let mut checkpoint = Checkpoint { - timestamp: Utc::now(), - current_node: "setup".to_string(), - completed_nodes: vec!["setup".to_string()], - node_retries: HashMap::new(), - context_values: HashMap::new(), - node_outcomes, - next_node_id: Some("work".to_string()), - git_commit_sha: None, - loop_failure_signatures: HashMap::new(), - restart_failure_signatures: HashMap::new(), - node_visits: HashMap::new(), - }; - - let run_dir = std::env::temp_dir().join(format!("fabro-server-test-{run_id}")); - std::fs::create_dir_all(&run_dir).unwrap(); - let mut managed = managed_run( - MINIMAL_DOT.to_string(), - RunStatus::Running, - Utc::now(), - run_dir, - RunExecutionMode::Start, - ); - managed.checkpoint = Some(checkpoint.clone()); - state - .runs - .lock() - .expect("runs lock poisoned") - .insert(run_id, managed); - let response = app .clone() .oneshot( @@ -2206,29 +2264,14 @@ async fn list_run_stages_projects_retrying_until_completion() { .await .unwrap(); let body = response_json!(response, StatusCode::OK).await; - assert_eq!(stage_status(&body, "setup"), "succeeded"); - assert_eq!(stage_status(&body, "work"), "retrying"); + assert_eq!(stage_status(&body, "setup@1"), "succeeded"); + assert_eq!(stage_status(&body, "work@1"), "retrying"); - let mut work_outcome = Outcome::success(); - work_outcome.status = StageOutcome::PartiallySucceeded; - checkpoint.completed_nodes.push("work".to_string()); - checkpoint - .node_outcomes - .insert("work".to_string(), work_outcome); - checkpoint.current_node = "work".to_string(); - checkpoint.next_node_id = Some("exit".to_string()); - state - .runs - .lock() - .expect("runs lock poisoned") - .get_mut(&run_id) - .unwrap() - .checkpoint = Some(checkpoint); - - let run_store = state.store.open_run(&run_id).await.unwrap(); - workflow_event::append_event( - &run_store, - &run_id, + append_scoped_stage_event( + &state, + run_id, + "work", + 1, &workflow_event::Event::StageCompleted { node_id: "work".to_string(), name: "Work".to_string(), @@ -2252,8 +2295,7 @@ async fn list_run_stages_projects_retrying_until_completion() { max_attempts: 3, }, ) - .await - .unwrap(); + .await; let response = app .oneshot( @@ -2266,7 +2308,574 @@ async fn list_run_stages_projects_retrying_until_completion() { .await .unwrap(); let body = response_json!(response, StatusCode::OK).await; - assert_eq!(stage_status(&body, "work"), "partially_succeeded"); + assert_eq!(stage_status(&body, "work@1"), "partially_succeeded"); +} + +fn stage_entry<'a>(body: &'a serde_json::Value, id: &str) -> &'a serde_json::Value { + body["data"] + .as_array() + .unwrap() + .iter() + .find(|stage| stage["id"] == id) + .unwrap_or_else(|| panic!("stage {id} not found in {body:#?}")) +} + +fn test_billed_usage( + model_id: &str, + input_tokens: i64, + output_tokens: i64, +) -> fabro_model::BilledModelUsage { + serde_json::from_value(json!({ + "input": { + "usage": { + "model": { + "provider": "openai", + "model_id": model_id + }, + "tokens": { + "input_tokens": input_tokens, + "output_tokens": output_tokens + } + }, + "facts": { + "provider": "open_ai" + } + }, + "total_usd_micros": input_tokens + output_tokens + })) + .unwrap() +} + +#[tokio::test] +async fn list_run_stages_distinguishes_visits() { + let state = test_app_state_with_isolated_storage(); + let app = crate::test_support::build_test_router(Arc::clone(&state)); + let run_id = RunId::new(); + + create_durable_run_with_events(&state, run_id, &[ + workflow_event::Event::RunSubmitted { + definition_blob: None, + }, + workflow_event::Event::RunStarting, + workflow_event::Event::RunRunning, + ]) + .await; + + // First visit of `verify` — failed. + append_scoped_stage_event( + &state, + run_id, + "verify", + 1, + &workflow_event::Event::StageStarted { + node_id: "verify".to_string(), + name: "Verify".to_string(), + index: 1, + handler_type: "command".to_string(), + attempt: 1, + max_attempts: 1, + }, + ) + .await; + append_scoped_stage_event( + &state, + run_id, + "verify", + 1, + &workflow_event::Event::StageCompleted { + node_id: "verify".to_string(), + name: "Verify".to_string(), + index: 1, + duration_ms: 1500, + status: "failed".to_string(), + preferred_label: None, + suggested_next_ids: Vec::new(), + billing: None, + failure: None, + notes: None, + files_touched: Vec::new(), + context_updates: None, + jump_to_node: None, + context_values: None, + node_visits: None, + loop_failure_signatures: None, + restart_failure_signatures: None, + response: None, + attempt: 1, + max_attempts: 1, + }, + ) + .await; + + // Second visit of `verify` — running. + append_scoped_stage_event( + &state, + run_id, + "verify", + 2, + &workflow_event::Event::StageStarted { + node_id: "verify".to_string(), + name: "Verify".to_string(), + index: 1, + handler_type: "command".to_string(), + attempt: 1, + max_attempts: 1, + }, + ) + .await; + + let response = app + .clone() + .oneshot( + Request::builder() + .method("GET") + .uri(api(&format!("/runs/{run_id}/stages"))) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let body = response_json!(response, StatusCode::OK).await; + + let data = body["data"].as_array().unwrap(); + let verify_entries: Vec<_> = data.iter().filter(|s| s["node_id"] == "verify").collect(); + assert_eq!(verify_entries.len(), 2, "expected two verify visits"); + + let first = stage_entry(&body, "verify@1"); + assert_eq!(first["node_id"], "verify"); + assert_eq!(first["visit"], 1); + assert_eq!(first["status"], "failed"); + assert_eq!(first["duration_secs"], 1.5); + + let second = stage_entry(&body, "verify@2"); + assert_eq!(second["node_id"], "verify"); + assert_eq!(second["visit"], 2); + assert_eq!(second["status"], "running"); + + // Old `dot_id` field must be gone. + assert!(first.get("dot_id").is_none(), "dot_id should be removed"); +} + +/// `checkpoint.completed_nodes` records every visit, so a looped node appears +/// once per re-entry. Billing must dedup so a retried node renders as one row +/// and `runtime_secs` is summed across all visits exactly once. +#[tokio::test] +async fn run_billing_dedups_retried_nodes_and_sums_their_durations() { + let state = test_app_state_with_isolated_storage(); + let app = crate::test_support::build_test_router(Arc::clone(&state)); + let run_id = RunId::new(); + + create_durable_run_with_events(&state, run_id, &[ + workflow_event::Event::RunSubmitted { + definition_blob: None, + }, + workflow_event::Event::RunStarting, + workflow_event::Event::RunRunning, + ]) + .await; + + // Visit 1 of `verify` — completed in 1.5s. + append_scoped_stage_event( + &state, + run_id, + "verify", + 1, + &workflow_event::Event::StageCompleted { + node_id: "verify".to_string(), + name: "Verify".to_string(), + index: 1, + duration_ms: 1500, + status: "failed".to_string(), + preferred_label: None, + suggested_next_ids: Vec::new(), + billing: None, + failure: None, + notes: None, + files_touched: Vec::new(), + context_updates: None, + jump_to_node: None, + context_values: None, + node_visits: None, + loop_failure_signatures: None, + restart_failure_signatures: None, + response: None, + attempt: 1, + max_attempts: 1, + }, + ) + .await; + + // Visit 2 of `verify` — completed in 0.8s. + append_scoped_stage_event( + &state, + run_id, + "verify", + 2, + &workflow_event::Event::StageCompleted { + node_id: "verify".to_string(), + name: "Verify".to_string(), + index: 1, + duration_ms: 800, + status: "succeeded".to_string(), + preferred_label: None, + suggested_next_ids: Vec::new(), + billing: None, + failure: None, + notes: None, + files_touched: Vec::new(), + context_updates: None, + jump_to_node: None, + context_values: None, + node_visits: None, + loop_failure_signatures: None, + restart_failure_signatures: None, + response: None, + attempt: 1, + max_attempts: 1, + }, + ) + .await; + + // Checkpoint records `verify` twice (once per visit) — this is what makes + // the dedup necessary. + let run_store = state.store.open_run(&run_id).await.unwrap(); + workflow_event::append_event( + &run_store, + &run_id, + &workflow_event::Event::CheckpointCompleted { + node_id: "verify".to_string(), + status: "running".to_string(), + current_node: "verify".to_string(), + completed_nodes: vec!["verify".to_string(), "verify".to_string()], + node_retries: std::collections::BTreeMap::new(), + context_values: std::collections::BTreeMap::new(), + node_outcomes: std::collections::BTreeMap::from([( + "verify".to_string(), + Outcome::default(), + )]), + next_node_id: Some("done".to_string()), + git_commit_sha: None, + loop_failure_signatures: std::collections::BTreeMap::new(), + restart_failure_signatures: std::collections::BTreeMap::new(), + node_visits: std::collections::BTreeMap::from([("verify".to_string(), 2usize)]), + diff: None, + }, + ) + .await + .unwrap(); + + let response = app + .clone() + .oneshot( + Request::builder() + .method("GET") + .uri(api(&format!("/runs/{run_id}/billing"))) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let body = response_json!(response, StatusCode::OK).await; + + let stages = body["stages"].as_array().unwrap(); + assert_eq!( + stages.len(), + 1, + "expected one row for the retried verify node" + ); + assert_eq!(stages[0]["stage"]["id"], "verify"); + // Duration on the row is the sum across visits (1.5s + 0.8s = 2.3s). + assert!( + (stages[0]["runtime_secs"].as_f64().unwrap() - 2.3).abs() < f64::EPSILON, + "row runtime_secs should sum visits, got {}", + stages[0]["runtime_secs"] + ); + + // Totals must not double-count: a single 2.3s, not 4.6s. + assert!( + (body["totals"]["runtime_secs"].as_f64().unwrap() - 2.3).abs() < f64::EPSILON, + "totals.runtime_secs should sum visits exactly once, got {}", + body["totals"]["runtime_secs"] + ); +} + +#[tokio::test] +async fn run_billing_sums_usage_across_retry_visits_and_uses_latest_model() { + let state = test_app_state_with_isolated_storage(); + let app = crate::test_support::build_test_router(Arc::clone(&state)); + let run_id = RunId::new(); + let failed_usage = test_billed_usage("gpt-old", 100, 10); + let success_usage = test_billed_usage("gpt-new", 200, 20); + + create_durable_run_with_events(&state, run_id, &[ + workflow_event::Event::RunSubmitted { + definition_blob: None, + }, + workflow_event::Event::RunStarting, + workflow_event::Event::RunRunning, + ]) + .await; + + append_scoped_stage_event( + &state, + run_id, + "verify", + 1, + &workflow_event::Event::StageFailed { + node_id: "verify".to_string(), + name: "Verify".to_string(), + index: 1, + failure: FailureDetail::new("try again", FailureCategory::TransientInfra), + will_retry: true, + duration_ms: 1200, + billing: Some(failed_usage), + actor: None, + }, + ) + .await; + append_scoped_stage_event( + &state, + run_id, + "verify", + 2, + &workflow_event::Event::StageCompleted { + node_id: "verify".to_string(), + name: "Verify".to_string(), + index: 1, + duration_ms: 800, + status: "succeeded".to_string(), + preferred_label: None, + suggested_next_ids: Vec::new(), + billing: Some(success_usage.clone()), + failure: None, + notes: None, + files_touched: Vec::new(), + context_updates: None, + jump_to_node: None, + context_values: None, + node_visits: None, + loop_failure_signatures: None, + restart_failure_signatures: None, + response: None, + attempt: 2, + max_attempts: 2, + }, + ) + .await; + + let mut latest_outcome: Outcome> = Outcome::success(); + latest_outcome.usage = Some(success_usage); + latest_outcome.duration_ms = Some(800); + let run_store = state.store.open_run(&run_id).await.unwrap(); + workflow_event::append_event( + &run_store, + &run_id, + &workflow_event::Event::CheckpointCompleted { + node_id: "verify".to_string(), + status: "running".to_string(), + current_node: "verify".to_string(), + completed_nodes: vec!["verify".to_string(), "verify".to_string()], + node_retries: std::collections::BTreeMap::from([("verify".to_string(), 2)]), + context_values: std::collections::BTreeMap::new(), + node_outcomes: std::collections::BTreeMap::from([( + "verify".to_string(), + latest_outcome, + )]), + next_node_id: None, + git_commit_sha: None, + loop_failure_signatures: std::collections::BTreeMap::new(), + restart_failure_signatures: std::collections::BTreeMap::new(), + node_visits: std::collections::BTreeMap::from([("verify".to_string(), 2usize)]), + diff: None, + }, + ) + .await + .unwrap(); + + let response = app + .clone() + .oneshot( + Request::builder() + .method("GET") + .uri(api(&format!("/runs/{run_id}/billing"))) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let body = response_json!(response, StatusCode::OK).await; + + let stages = body["stages"].as_array().unwrap(); + assert_eq!(stages.len(), 1); + assert_eq!(stages[0]["stage"]["id"], "verify"); + assert_eq!(stages[0]["model"]["id"], "gpt-new"); + assert_eq!(stages[0]["billing"]["input_tokens"], 300); + assert_eq!(stages[0]["billing"]["output_tokens"], 30); + assert_eq!(stages[0]["billing"]["total_usd_micros"], 330); + assert!((stages[0]["runtime_secs"].as_f64().unwrap() - 2.0).abs() < f64::EPSILON); + + assert_eq!(body["totals"]["input_tokens"], 300); + assert_eq!(body["totals"]["output_tokens"], 30); + assert_eq!(body["totals"]["total_usd_micros"], 330); + assert!((body["totals"]["runtime_secs"].as_f64().unwrap() - 2.0).abs() < f64::EPSILON); + + let by_model = body["by_model"].as_array().unwrap(); + assert_eq!(by_model.len(), 2); + let old_model = by_model + .iter() + .find(|entry| entry["model"]["id"] == "gpt-old") + .unwrap(); + let new_model = by_model + .iter() + .find(|entry| entry["model"]["id"] == "gpt-new") + .unwrap(); + assert_eq!(old_model["stages"], 1); + assert_eq!(old_model["billing"]["input_tokens"], 100); + assert_eq!(new_model["stages"], 1); + assert_eq!(new_model["billing"]["input_tokens"], 200); +} + +#[tokio::test] +async fn list_run_stages_shows_retrying_after_failed_event() { + let state = test_app_state_with_isolated_storage(); + let app = crate::test_support::build_test_router(Arc::clone(&state)); + let run_id = RunId::new(); + + create_durable_run_with_events(&state, run_id, &[ + workflow_event::Event::RunSubmitted { + definition_blob: None, + }, + workflow_event::Event::RunStarting, + workflow_event::Event::RunRunning, + ]) + .await; + + append_scoped_stage_event( + &state, + run_id, + "work", + 1, + &workflow_event::Event::StageStarted { + node_id: "work".to_string(), + name: "Work".to_string(), + index: 0, + handler_type: "command".to_string(), + attempt: 1, + max_attempts: 3, + }, + ) + .await; + append_scoped_stage_event( + &state, + run_id, + "work", + 1, + &workflow_event::Event::StageFailed { + node_id: "work".to_string(), + name: "Work".to_string(), + index: 0, + failure: FailureDetail::new("flake", FailureCategory::TransientInfra), + will_retry: true, + duration_ms: 5, + billing: None, + actor: None, + }, + ) + .await; + append_scoped_stage_event( + &state, + run_id, + "work", + 1, + &workflow_event::Event::StageRetrying { + node_id: "work".to_string(), + name: "Work".to_string(), + index: 0, + attempt: 2, + max_attempts: 3, + delay_ms: 50, + }, + ) + .await; + + let response = app + .clone() + .oneshot( + Request::builder() + .method("GET") + .uri(api(&format!("/runs/{run_id}/stages"))) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let body = response_json!(response, StatusCode::OK).await; + assert_eq!(stage_status(&body, "work@1"), "retrying"); +} + +#[tokio::test] +async fn list_run_stages_shows_retrying_when_failed_will_retry() { + let state = test_app_state_with_isolated_storage(); + let app = crate::test_support::build_test_router(Arc::clone(&state)); + let run_id = RunId::new(); + + create_durable_run_with_events(&state, run_id, &[ + workflow_event::Event::RunSubmitted { + definition_blob: None, + }, + workflow_event::Event::RunStarting, + workflow_event::Event::RunRunning, + ]) + .await; + + append_scoped_stage_event( + &state, + run_id, + "work", + 1, + &workflow_event::Event::StageStarted { + node_id: "work".to_string(), + name: "Work".to_string(), + index: 0, + handler_type: "command".to_string(), + attempt: 1, + max_attempts: 3, + }, + ) + .await; + // Only StageFailed, no StageRetrying yet — should still render retrying + // because props.will_retry is true. + append_scoped_stage_event( + &state, + run_id, + "work", + 1, + &workflow_event::Event::StageFailed { + node_id: "work".to_string(), + name: "Work".to_string(), + index: 0, + failure: FailureDetail::new("flake", FailureCategory::TransientInfra), + will_retry: true, + duration_ms: 5, + billing: None, + actor: None, + }, + ) + .await; + + let response = app + .clone() + .oneshot( + Request::builder() + .method("GET") + .uri(api(&format!("/runs/{run_id}/stages"))) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let body = response_json!(response, StatusCode::OK).await; + assert_eq!(stage_status(&body, "work@1"), "retrying"); } async fn append_raw_run_event( @@ -5835,6 +6444,62 @@ async fn get_aggregate_billing_returns_zeros_initially() { assert!(body["by_model"].as_array().unwrap().is_empty()); } +#[test] +fn aggregate_billing_counts_projection_rollup_usage_visits() { + let mut accumulator = BillingAccumulator::default(); + let rollup = fabro_workflow::ProjectionBillingRollup { + stages: Vec::new(), + totals: BilledTokenCounts { + input_tokens: 300, + output_tokens: 30, + total_tokens: 330, + reasoning_tokens: 0, + cache_read_tokens: 0, + cache_write_tokens: 0, + total_usd_micros: Some(330), + }, + by_model: vec![ + fabro_workflow::ProjectionBillingByModel { + model_id: "gpt-old".to_string(), + stages: 1, + billing: BilledTokenCounts { + input_tokens: 100, + output_tokens: 10, + total_tokens: 110, + reasoning_tokens: 0, + cache_read_tokens: 0, + cache_write_tokens: 0, + total_usd_micros: Some(110), + }, + }, + fabro_workflow::ProjectionBillingByModel { + model_id: "gpt-new".to_string(), + stages: 1, + billing: BilledTokenCounts { + input_tokens: 200, + output_tokens: 20, + total_tokens: 220, + reasoning_tokens: 0, + cache_read_tokens: 0, + cache_write_tokens: 0, + total_usd_micros: Some(220), + }, + }, + ], + runtime_ms: 2000, + billed_visit_count: 2, + }; + + accumulate_billing_rollup(&mut accumulator, &rollup); + + assert_eq!(accumulator.total_runs, 1); + assert_eq!(accumulator.total_runtime_secs, 2.0); + assert_eq!(accumulator.by_model["gpt-old"].stages, 1); + assert_eq!(accumulator.by_model["gpt-old"].billing.input_tokens, 100); + assert_eq!(accumulator.by_model["gpt-new"].stages, 1); + assert_eq!(accumulator.by_model["gpt-new"].billing.input_tokens, 200); +} + #[tokio::test] async fn post_runs_returns_submitted_status() { let state = test_app_state(); diff --git a/lib/crates/fabro-store/src/artifact_store.rs b/lib/crates/fabro-store/src/artifact_store.rs index 4cd4f85e1..5be00d792 100644 --- a/lib/crates/fabro-store/src/artifact_store.rs +++ b/lib/crates/fabro-store/src/artifact_store.rs @@ -297,8 +297,13 @@ fn decode_artifact_location( )) })?; let (retry, filename) = decode_retry_and_filename(location, &mut parts)?; + let stage_id = StageId::try_new(node_id, visit).map_err(|err| { + Error::Other(format!( + "artifact location {location} has an invalid stage id: {err}" + )) + })?; Ok(NodeArtifact { - node: StageId::new(node_id, visit), + node: stage_id, retry, filename, size, diff --git a/lib/crates/fabro-store/src/run_state.rs b/lib/crates/fabro-store/src/run_state.rs index 77b36796e..af467f557 100644 --- a/lib/crates/fabro-store/src/run_state.rs +++ b/lib/crates/fabro-store/src/run_state.rs @@ -9,8 +9,8 @@ use fabro_types::run_event::{ use fabro_types::{ BilledModelUsage, Checkpoint, Conclusion, EventBody, FailureSignature, InterviewQuestionRecord, Outcome, PendingInterviewRecord, PullRequestRecord, RunControlAction, RunEvent, RunId, - RunProjection, RunSpec, RunStatus, RunSummary, SandboxRecord, StageCompletion, StageOutcome, - StageProjection, StartRecord, TerminalStatus, first_event_seq, + RunProjection, RunSpec, RunStatus, RunSummary, SandboxRecord, StageCompletion, StageId, + StageOutcome, StageProjection, StartRecord, TerminalStatus, first_event_seq, }; use fabro_util::error::render_with_causes; use serde_json::Value; @@ -297,52 +297,59 @@ impl RunProjectionReducer for RunProjection { ); } EventBody::StagePrompt(props) => { - let Some(stage) = stage_at_visit(self, stored, props.visit, event.seq) else { + let Some(stage) = stage_at_stored_or_visit(self, stored, props.visit, event.seq) + else { return Ok(()); }; stage.prompt = Some(props.text.clone()); stage.provider_used = provider_used_from_prompt(props); } EventBody::PromptCompleted(props) => { - let Some(stage) = stage_at_current_visit(self, stored, event.seq) else { + let Some(stage) = stage_at_stored_or_current_visit(self, stored, event.seq) else { return Ok(()); }; stage.response = Some(props.response.clone()); } EventBody::StageCompleted(props) => { - let Some(node_id) = stored.node_id.as_deref() else { - return Ok(()); - }; - let visit = stage_visit(node_id, props.node_visits.as_ref(), self).unwrap_or(1); let response = props.response.clone(); let outcome = stage_outcome_from_props(props); let completion = stage_completion_from_outcome(&outcome, ts); - let stage = self.stage_entry(node_id, visit, first_event_seq(event.seq)); + let Some(stage) = + stage_at_completed_visit(self, stored, props.node_visits.as_ref(), event.seq) + else { + return Ok(()); + }; stage.response = response; stage.completion = Some(completion); + stage.duration_ms = Some(props.duration_ms); + stage.usage.clone_from(&props.billing); } EventBody::StageFailed(props) => { let failure_reason = props.failure.as_ref().map(|detail| detail.message.clone()); - let Some(stage) = stage_at_current_visit(self, stored, event.seq) else { + let Some(stage) = stage_at_stored_or_current_visit(self, stored, event.seq) else { return Ok(()); }; stage.completion = Some(StageCompletion { outcome: StageOutcome::Failed { - retry_requested: false, + retry_requested: props.will_retry, }, notes: None, failure_reason, timestamp: ts, }); + stage.duration_ms = Some(props.duration_ms); + stage.usage.clone_from(&props.billing); } EventBody::AgentSessionStarted(props) => { - let Some(stage) = stage_at_visit(self, stored, props.visit, event.seq) else { + let Some(stage) = stage_at_stored_or_visit(self, stored, props.visit, event.seq) + else { return Ok(()); }; stage.provider_used = Some(provider_used_from_agent_session_started(props)); } EventBody::AgentCliStarted(props) => { - let Some(stage) = stage_at_visit(self, stored, props.visit, event.seq) else { + let Some(stage) = stage_at_stored_or_visit(self, stored, props.visit, event.seq) + else { return Ok(()); }; stage.provider_used = Some(provider_used_from_agent_cli_started(props)); @@ -351,7 +358,7 @@ impl RunProjectionReducer for RunProjection { let script_invocation = serde_json::to_value(props).map_err(|err| { Error::InvalidEvent(format!("invalid command.started payload: {err}")) })?; - let Some(stage) = stage_at_current_visit(self, stored, event.seq) else { + let Some(stage) = stage_at_stored_or_current_visit(self, stored, event.seq) else { return Ok(()); }; stage.script_invocation = Some(script_invocation); @@ -360,7 +367,7 @@ impl RunProjectionReducer for RunProjection { let script_timing = serde_json::to_value(props).map_err(|err| { Error::InvalidEvent(format!("invalid command.completed payload: {err}")) })?; - let Some(stage) = stage_at_current_visit(self, stored, event.seq) else { + let Some(stage) = stage_at_stored_or_current_visit(self, stored, event.seq) else { return Ok(()); }; stage.stdout = Some(props.stdout.clone()); @@ -376,7 +383,7 @@ impl RunProjectionReducer for RunProjection { let parallel_results = serde_json::to_value(&props.results).map_err(|err| { Error::InvalidEvent(format!("invalid parallel.completed payload: {err}")) })?; - let Some(stage) = stage_at_current_visit(self, stored, event.seq) else { + let Some(stage) = stage_at_stored_or_current_visit(self, stored, event.seq) else { return Ok(()); }; stage.parallel_results = Some(parallel_results); @@ -394,6 +401,9 @@ fn stage_at_visit<'a>( visit: u32, seq: u32, ) -> Option<&'a mut StageProjection> { + if visit == 0 { + return None; + } let node_id = stored.node_id.as_deref()?; Some(state.stage_entry(node_id, visit, first_event_seq(seq))) } @@ -408,6 +418,51 @@ fn stage_at_current_visit<'a>( Some(state.stage_entry(node_id, visit, first_event_seq(seq))) } +fn stage_at_stored_stage_id<'a>( + state: &'a mut RunProjection, + stage_id: &StageId, + seq: u32, +) -> &'a mut StageProjection { + state.stage_entry(stage_id.node_id(), stage_id.visit(), first_event_seq(seq)) +} + +fn stage_at_stored_or_visit<'a>( + state: &'a mut RunProjection, + stored: &RunEvent, + visit: u32, + seq: u32, +) -> Option<&'a mut StageProjection> { + if let Some(stage_id) = stored.stage_id.as_ref() { + return Some(stage_at_stored_stage_id(state, stage_id, seq)); + } + stage_at_visit(state, stored, visit, seq) +} + +fn stage_at_stored_or_current_visit<'a>( + state: &'a mut RunProjection, + stored: &RunEvent, + seq: u32, +) -> Option<&'a mut StageProjection> { + if let Some(stage_id) = stored.stage_id.as_ref() { + return Some(stage_at_stored_stage_id(state, stage_id, seq)); + } + stage_at_current_visit(state, stored, seq) +} + +fn stage_at_completed_visit<'a>( + state: &'a mut RunProjection, + stored: &RunEvent, + node_visits: Option<&BTreeMap>, + seq: u32, +) -> Option<&'a mut StageProjection> { + if let Some(stage_id) = stored.stage_id.as_ref() { + return Some(stage_at_stored_stage_id(state, stage_id, seq)); + } + let node_id = stored.node_id.as_deref()?; + let visit = stage_visit(node_id, node_visits, state).unwrap_or(1); + Some(state.stage_entry(node_id, visit, first_event_seq(seq))) +} + pub(crate) fn build_summary(state: &RunProjection, run_id: &RunId) -> RunSummary { let workflow_name = state.spec.as_ref().map(|spec| { if spec.graph.name.is_empty() { @@ -529,6 +584,7 @@ fn stage_visit( node_visits .and_then(|visits| visits.get(node_id).copied()) .and_then(|visit| u32::try_from(visit).ok()) + .filter(|visit| *visit > 0) .or_else(|| state.current_visit_for(node_id)) } @@ -613,12 +669,13 @@ mod tests { use fabro_types::run_event::run::RunFailedProps; use fabro_types::run_event::{ CheckpointCompletedProps, InterviewCompletedProps, InterviewOption, InterviewStartedProps, - RunControlEffectProps, StagePromptProps, StageStartedProps, + RunControlEffectProps, StageCompletedProps, StageFailedProps, StagePromptProps, + StageStartedProps, }; use fabro_types::{ - BlockedReason, Checkpoint, EventBody, FailureReason, Outcome, QuestionType, RunBlobId, - RunControlAction, RunEvent, RunStatus, StageOutcome, SuccessReason, TerminalStatus, - WorkflowSettings, first_event_seq, fixtures, + BilledModelUsage, BlockedReason, Checkpoint, EventBody, FailureReason, Outcome, + QuestionType, RunBlobId, RunControlAction, RunEvent, RunStatus, StageOutcome, + SuccessReason, TerminalStatus, WorkflowSettings, first_event_seq, fixtures, }; use serde_json::json; @@ -651,6 +708,32 @@ mod tests { event } + fn test_usage(model_id: &str, input_tokens: i64, output_tokens: i64) -> BilledModelUsage { + serde_json::from_value(json!({ + "input": { + "usage": { + "model": { + "provider": "openai", + "model_id": model_id + }, + "tokens": { + "input_tokens": input_tokens, + "output_tokens": output_tokens + } + }, + "facts": { + "provider": "open_ai" + } + }, + "total_usd_micros": input_tokens + output_tokens + })) + .unwrap() + } + + fn usage_json(usage: &BilledModelUsage) -> serde_json::Value { + serde_json::to_value(usage).unwrap() + } + fn test_raw_event( seq: u32, event: &str, @@ -861,6 +944,210 @@ mod tests { assert_eq!(stage.prompt.as_deref(), Some("prompt")); } + #[test] + fn stage_completed_event_captures_duration_and_usage_per_visit() { + let mut state = RunProjection::default(); + let usage = test_usage("gpt-5.2", 123, 45); + + state + .apply_event(&test_event( + 3, + EventBody::StageCompleted(StageCompletedProps { + index: 0, + duration_ms: 789, + status: StageOutcome::Succeeded, + preferred_label: None, + suggested_next_ids: Vec::new(), + billing: Some(usage.clone()), + failure: None, + notes: None, + files_touched: Vec::new(), + context_updates: None, + jump_to_node: None, + context_values: None, + node_visits: None, + loop_failure_signatures: None, + restart_failure_signatures: None, + response: Some("done".to_string()), + attempt: 1, + max_attempts: 1, + }), + Some("build"), + )) + .unwrap(); + + let stage = state.stage(&StageId::new("build", 1)).unwrap(); + assert_eq!(stage.duration_ms, Some(789)); + assert_eq!(stage.usage.as_ref(), Some(&usage)); + } + + #[test] + fn stage_failed_event_captures_duration_and_usage_per_visit() { + let mut state = RunProjection::default(); + let stage_id = StageId::new("build", 1); + let usage = test_usage("gpt-5.2", 321, 54); + + state + .apply_event(&test_stage_event( + 2, + EventBody::StageStarted(StageStartedProps { + index: 0, + handler_type: "agent".to_string(), + attempt: 1, + max_attempts: 1, + }), + stage_id.clone(), + )) + .unwrap(); + state + .apply_event(&test_raw_event( + 3, + "stage.failed", + &json!({ + "index": 0, + "failure": { + "message": "provider failed", + "failure_class": "transient_infra" + }, + "will_retry": false, + "duration_ms": 654, + "billing": usage_json(&usage) + }), + Some("build"), + )) + .unwrap(); + + let stage = state.stage(&stage_id).unwrap(); + assert_eq!(stage.duration_ms, Some(654)); + assert_eq!(stage.usage.as_ref(), Some(&usage)); + } + + #[test] + fn two_visits_of_one_node_retain_distinct_usage() { + let mut state = RunProjection::default(); + let first_usage = test_usage("gpt-5.2", 100, 10); + let second_usage = test_usage("gpt-5.2", 200, 20); + + for (seq, visit, duration_ms, usage) in [ + (3, 1usize, 111, first_usage.clone()), + (4, 2usize, 222, second_usage.clone()), + ] { + state + .apply_event(&test_event( + seq, + EventBody::StageCompleted(StageCompletedProps { + index: 0, + duration_ms, + status: StageOutcome::Succeeded, + preferred_label: None, + suggested_next_ids: Vec::new(), + billing: Some(usage), + failure: None, + notes: None, + files_touched: Vec::new(), + context_updates: None, + jump_to_node: None, + context_values: None, + node_visits: Some(BTreeMap::from([("build".to_string(), visit)])), + loop_failure_signatures: None, + restart_failure_signatures: None, + response: None, + attempt: 1, + max_attempts: 1, + }), + Some("build"), + )) + .unwrap(); + } + + let first_stage = state.stage(&StageId::new("build", 1)).unwrap(); + let second_stage = state.stage(&StageId::new("build", 2)).unwrap(); + assert_eq!(first_stage.duration_ms, Some(111)); + assert_eq!(first_stage.usage.as_ref(), Some(&first_usage)); + assert_eq!(second_stage.duration_ms, Some(222)); + assert_eq!(second_stage.usage.as_ref(), Some(&second_usage)); + } + + #[test] + fn stage_completed_prefers_stored_stage_id_over_legacy_node_visits() { + let mut state = RunProjection::default(); + let usage = test_usage("gpt-5.2", 300, 30); + let scoped_stage_id = StageId::new("build", 2); + + state + .apply_event(&test_stage_event( + 3, + EventBody::StageCompleted(StageCompletedProps { + index: 0, + duration_ms: 333, + status: StageOutcome::Succeeded, + preferred_label: None, + suggested_next_ids: Vec::new(), + billing: Some(usage.clone()), + failure: None, + notes: None, + files_touched: Vec::new(), + context_updates: None, + jump_to_node: None, + context_values: None, + node_visits: Some(BTreeMap::from([("build".to_string(), 1usize)])), + loop_failure_signatures: None, + restart_failure_signatures: None, + response: Some("done".to_string()), + attempt: 1, + max_attempts: 1, + }), + scoped_stage_id.clone(), + )) + .unwrap(); + + assert!( + state.stage(&StageId::new("build", 1)).is_none(), + "legacy node_visits must not override stored stage_id" + ); + let stage = state.stage(&scoped_stage_id).unwrap(); + assert_eq!(stage.duration_ms, Some(333)); + assert_eq!(stage.usage.as_ref(), Some(&usage)); + assert_eq!(stage.response.as_deref(), Some("done")); + } + + #[test] + fn stage_failed_prefers_stored_stage_id_and_preserves_retry_request() { + let mut state = RunProjection::default(); + let usage = test_usage("gpt-5.2", 400, 40); + let scoped_stage_id = StageId::new("build", 2); + + state + .apply_event(&test_stage_event( + 3, + EventBody::StageFailed(StageFailedProps { + index: 0, + failure: Some(fabro_types::FailureDetail::new( + "try again", + fabro_types::FailureCategory::TransientInfra, + )), + will_retry: true, + duration_ms: 444, + billing: Some(usage.clone()), + }), + scoped_stage_id.clone(), + )) + .unwrap(); + + assert!( + state.stage(&StageId::new("build", 1)).is_none(), + "current-visit fallback must not override stored stage_id" + ); + let stage = state.stage(&scoped_stage_id).unwrap(); + assert_eq!(stage.duration_ms, Some(444)); + assert_eq!(stage.usage.as_ref(), Some(&usage)); + let completion = stage.completion.as_ref().unwrap(); + assert_eq!(completion.outcome, StageOutcome::Failed { + retry_requested: true, + }); + assert_eq!(completion.failure_reason.as_deref(), Some("try again")); + } + #[test] fn checkpoint_completed_creates_projection_entry_for_skipped_stage() { let mut state = RunProjection::default(); diff --git a/lib/crates/fabro-store/tests/serializable_projection.rs b/lib/crates/fabro-store/tests/serializable_projection.rs index 6d51065ba..2b24cd54e 100644 --- a/lib/crates/fabro-store/tests/serializable_projection.rs +++ b/lib/crates/fabro-store/tests/serializable_projection.rs @@ -5,8 +5,8 @@ use fabro_store::{RunProjection, SerializableProjection, StageId}; use fabro_types::graph::Graph; use fabro_types::run::RunSpec; use fabro_types::{ - Checkpoint, RunStatus, SandboxRecord, StageCompletion, StageOutcome, StartRecord, - TerminalStatus, WorkflowSettings, first_event_seq, fixtures, + BilledModelUsage, Checkpoint, RunStatus, SandboxRecord, StageCompletion, StageOutcome, + StartRecord, TerminalStatus, WorkflowSettings, first_event_seq, fixtures, }; use serde_json::json; @@ -52,6 +52,28 @@ fn sample_checkpoint() -> Checkpoint { } } +fn sample_usage() -> BilledModelUsage { + serde_json::from_value(json!({ + "input": { + "usage": { + "model": { + "provider": "openai", + "model_id": "gpt-5.2" + }, + "tokens": { + "input_tokens": 123, + "output_tokens": 45 + } + }, + "facts": { + "provider": "open_ai" + } + }, + "total_usd_micros": 168 + })) + .expect("sample usage should deserialize") +} + #[test] fn serializable_projection_round_trips_and_trims_bulky_node_fields() { let stage_id = StageId::new("build", 2); @@ -94,6 +116,8 @@ fn serializable_projection_round_trips_and_trims_bulky_node_fields() { stage.script_invocation = Some(json!({ "command": "cargo test" })); stage.script_timing = Some(json!({ "duration_ms": 10 })); stage.parallel_results = Some(json!([{ "stage": "fanout@1" }])); + stage.duration_ms = Some(1234); + stage.usage = Some(sample_usage()); stage.stdout = Some("stdout".to_string()); stage.stderr = Some("stderr".to_string()); @@ -138,6 +162,8 @@ fn serializable_projection_round_trips_and_trims_bulky_node_fields() { node.parallel_results, Some(json!([{ "stage": "fanout@1" }])) ); + assert_eq!(node.duration_ms, Some(1234)); + assert_eq!(node.usage, Some(sample_usage())); } #[test] diff --git a/lib/crates/fabro-types/src/lib.rs b/lib/crates/fabro-types/src/lib.rs index c04702c1e..22675855c 100644 --- a/lib/crates/fabro-types/src/lib.rs +++ b/lib/crates/fabro-types/src/lib.rs @@ -80,7 +80,7 @@ pub use run_summary::RunSummary; pub use sandbox_record::SandboxRecord; pub use secret::{SecretMetadata, SecretType}; pub use stage_completion::StageCompletion; -pub use stage_id::{ParallelBranchId, StageId}; +pub use stage_id::{InvalidStageVisit, ParallelBranchId, StageId}; pub use start::StartRecord; pub use status::{ BlockedReason, FailureReason, InvalidTransition, ParseFailureReasonError, diff --git a/lib/crates/fabro-types/src/outcome.rs b/lib/crates/fabro-types/src/outcome.rs index af34cd10b..8ef0c67dc 100644 --- a/lib/crates/fabro-types/src/outcome.rs +++ b/lib/crates/fabro-types/src/outcome.rs @@ -138,12 +138,32 @@ impl From for StageState { match outcome { StageOutcome::Succeeded => Self::Succeeded, StageOutcome::PartiallySucceeded => Self::PartiallySucceeded, - StageOutcome::Failed { .. } => Self::Failed, + StageOutcome::Failed { + retry_requested: true, + } => Self::Retrying, + StageOutcome::Failed { + retry_requested: false, + } => Self::Failed, StageOutcome::Skipped => Self::Skipped, } } } +#[cfg(test)] +mod stage_state_tests { + use super::{StageOutcome, StageState}; + + #[test] + fn retry_requested_failure_projects_as_retrying() { + assert_eq!( + StageState::from(StageOutcome::Failed { + retry_requested: true, + }), + StageState::Retrying + ); + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum FailureCategory { @@ -340,9 +360,10 @@ mod tests { StageState::from(StageOutcome::Failed { retry_requested: true, }), - StageState::Failed + StageState::Retrying ); assert!(StageState::Cancelled.is_terminal()); + assert!(!StageState::Retrying.is_terminal()); assert!(!StageState::Running.is_terminal()); } } diff --git a/lib/crates/fabro-types/src/run_event/stage.rs b/lib/crates/fabro-types/src/run_event/stage.rs index 9f1781d82..1b6609884 100644 --- a/lib/crates/fabro-types/src/run_event/stage.rs +++ b/lib/crates/fabro-types/src/run_event/stage.rs @@ -57,6 +57,8 @@ pub struct StageFailedProps { pub will_retry: bool, #[serde(default)] pub duration_ms: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub billing: Option, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] diff --git a/lib/crates/fabro-types/src/run_projection.rs b/lib/crates/fabro-types/src/run_projection.rs index de2e4a1a5..87321d348 100644 --- a/lib/crates/fabro-types/src/run_projection.rs +++ b/lib/crates/fabro-types/src/run_projection.rs @@ -4,9 +4,9 @@ use std::num::NonZeroU32; use chrono::{DateTime, Utc}; use crate::{ - Checkpoint, Conclusion, InterviewQuestionRecord, InvalidTransition, PullRequestRecord, Retro, - RunControlAction, RunId, RunSpec, RunStatus, SandboxRecord, StageCompletion, StageId, - StartRecord, + BilledModelUsage, Checkpoint, Conclusion, InterviewQuestionRecord, InvalidTransition, + PullRequestRecord, Retro, RunControlAction, RunId, RunSpec, RunStatus, SandboxRecord, + StageCompletion, StageId, StartRecord, }; #[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] @@ -44,6 +44,10 @@ pub struct StageProjection { pub prompt: Option, pub response: Option, pub completion: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub duration_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub usage: Option, pub provider_used: Option, pub diff: Option, pub script_invocation: Option, @@ -78,6 +82,8 @@ impl StageProjection { prompt: None, response: None, completion: None, + duration_ms: None, + usage: None, provider_used: None, diff: None, script_invocation: None, @@ -99,12 +105,32 @@ impl RunProjection { self.stages.get(stage) } + /// Iterate stages in `first_event_seq` order (the chronological order in + /// which each stage's first lifecycle event was recorded). Internal + /// storage is a `HashMap`, so iteration would otherwise be + /// non-deterministic; every caller wants chronological order, so we sort + /// here once instead of asking each caller to remember. pub fn iter_stages(&self) -> impl Iterator { - self.stages.iter() + let mut entries: Vec<(&StageId, &StageProjection)> = self.stages.iter().collect(); + entries.sort_by(|(left_id, left_stage), (right_id, right_stage)| { + left_stage + .first_event_seq + .cmp(&right_stage.first_event_seq) + .then_with(|| left_id.cmp(right_id)) + }); + entries.into_iter() } + /// Mutable counterpart of [`iter_stages`]. Same chronological ordering. pub fn iter_stages_mut(&mut self) -> impl Iterator { - self.stages.iter_mut() + let mut entries: Vec<(&StageId, &mut StageProjection)> = self.stages.iter_mut().collect(); + entries.sort_by(|(left_id, left_stage), (right_id, right_stage)| { + left_stage + .first_event_seq + .cmp(&right_stage.first_event_seq) + .then_with(|| left_id.cmp(right_id)) + }); + entries.into_iter() } pub fn is_empty(&self) -> bool { @@ -186,3 +212,90 @@ impl RunProjection { } } } + +#[cfg(test)] +mod iter_stages_tests { + use std::num::NonZeroU32; + + use super::RunProjection; + + fn seq(n: u32) -> NonZeroU32 { + NonZeroU32::new(n).unwrap() + } + + #[test] + fn iter_stages_yields_chronological_order_across_nodes() { + let mut p = RunProjection::default(); + // Insert in non-monotonic seq order to exercise the sort. + p.stage_entry("c", 1, seq(30)); + p.stage_entry("a", 1, seq(10)); + p.stage_entry("b", 1, seq(20)); + + let order: Vec<&str> = p + .iter_stages() + .map(|(stage_id, _)| stage_id.node_id()) + .collect(); + assert_eq!(order, vec!["a", "b", "c"]); + } + + #[test] + fn iter_stages_orders_visits_within_a_node() { + let mut p = RunProjection::default(); + // Visit 2 inserted first; visit 1's earlier first_event_seq must still + // win the chronological ordering. + p.stage_entry("verify", 2, seq(50)); + p.stage_entry("verify", 1, seq(20)); + + let visits: Vec = p + .iter_stages() + .map(|(stage_id, _)| stage_id.visit()) + .collect(); + assert_eq!(visits, vec![1, 2]); + } + + #[test] + fn iter_stages_mut_yields_chronological_order() { + let mut p = RunProjection::default(); + p.stage_entry("c", 1, seq(30)); + p.stage_entry("a", 1, seq(10)); + p.stage_entry("b", 1, seq(20)); + + let order: Vec = p + .iter_stages_mut() + .map(|(stage_id, _)| stage_id.node_id().to_string()) + .collect(); + assert_eq!(order, vec!["a", "b", "c"]); + } + + #[test] + fn iter_stages_tie_breaks_same_first_event_seq_by_stage_id() { + for _ in 0..128 { + let mut p = RunProjection::default(); + p.stage_entry("verify", 2, seq(10)); + p.stage_entry("build", 1, seq(10)); + p.stage_entry("verify", 1, seq(10)); + + let order: Vec = p + .iter_stages() + .map(|(stage_id, _)| stage_id.to_string()) + .collect(); + assert_eq!(order, vec!["build@1", "verify@1", "verify@2"]); + } + } + + #[test] + fn iter_stages_mut_tie_breaks_same_first_event_seq_by_stage_id() { + for _ in 0..128 { + let mut p = RunProjection::default(); + p.stage_entry("verify", 2, seq(10)); + p.stage_entry("build", 1, seq(10)); + p.stage_entry("verify", 1, seq(10)); + + let order: Vec = p + .iter_stages_mut() + .map(|(stage_id, _)| stage_id.to_string()) + .collect(); + assert_eq!(order, vec!["build@1", "verify@1", "verify@2"]); + } + } +} diff --git a/lib/crates/fabro-types/src/stage_id.rs b/lib/crates/fabro-types/src/stage_id.rs index baae58864..91711cb79 100644 --- a/lib/crates/fabro-types/src/stage_id.rs +++ b/lib/crates/fabro-types/src/stage_id.rs @@ -1,4 +1,5 @@ use std::fmt; +use std::num::NonZeroU32; use std::str::FromStr; use serde::de::Error as _; @@ -7,16 +8,21 @@ use serde::{Deserialize, Deserializer, Serialize, Serializer}; #[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct StageId { node_id: String, - visit: u32, + visit: NonZeroU32, } impl StageId { #[must_use] pub fn new(node_id: impl Into, visit: u32) -> Self { - Self { + Self::try_new(node_id, visit).expect("stage id visit must be greater than zero") + } + + pub fn try_new(node_id: impl Into, visit: u32) -> Result { + let visit = NonZeroU32::new(visit).ok_or(InvalidStageVisit)?; + Ok(Self { node_id: node_id.into(), visit, - } + }) } #[must_use] @@ -26,7 +32,7 @@ impl StageId { #[must_use] pub fn visit(&self) -> u32 { - self.visit + self.visit.get() } } @@ -47,6 +53,17 @@ impl fmt::Display for ParseStageIdError { impl std::error::Error for ParseStageIdError {} +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct InvalidStageVisit; + +impl fmt::Display for InvalidStageVisit { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("stage id visit must be greater than zero") + } +} + +impl std::error::Error for InvalidStageVisit {} + impl FromStr for StageId { type Err = ParseStageIdError; @@ -67,7 +84,7 @@ impl FromStr for StageId { let visit = visit .parse() .map_err(|err| ParseStageIdError(format!("invalid stage id visit: {err}")))?; - Ok(Self::new(node_id, visit)) + Self::try_new(node_id, visit).map_err(|err| ParseStageIdError(err.to_string())) } } @@ -224,6 +241,18 @@ mod tests { assert!(err.to_string().starts_with("invalid stage id visit:")); } + #[test] + fn parse_rejects_zero_visit() { + let err = "code@0".parse::().unwrap_err(); + assert_eq!(err.to_string(), "stage id visit must be greater than zero"); + } + + #[test] + fn try_new_rejects_zero_visit() { + let err = StageId::try_new("code", 0).unwrap_err(); + assert_eq!(err.to_string(), "stage id visit must be greater than zero"); + } + #[test] fn parse_rejects_empty_node_id() { let err = "@3".parse::().unwrap_err(); diff --git a/lib/crates/fabro-workflow/src/billing_rollup.rs b/lib/crates/fabro-workflow/src/billing_rollup.rs new file mode 100644 index 000000000..03d5207d3 --- /dev/null +++ b/lib/crates/fabro-workflow/src/billing_rollup.rs @@ -0,0 +1,291 @@ +use std::collections::{BTreeMap, HashMap}; + +use fabro_types::{BilledModelUsage, BilledTokenCounts, RunProjection}; + +#[derive(Debug, Clone, PartialEq)] +pub struct ProjectionBillingStage { + pub node_id: String, + pub billing: BilledTokenCounts, + pub duration_ms: u64, + pub model_id: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ProjectionBillingByModel { + pub model_id: String, + pub stages: i64, + pub billing: BilledTokenCounts, +} + +#[derive(Debug, Clone, Default, PartialEq)] +pub struct ProjectionBillingRollup { + pub stages: Vec, + pub totals: BilledTokenCounts, + pub by_model: Vec, + pub runtime_ms: u64, + pub billed_visit_count: usize, +} + +impl ProjectionBillingRollup { + #[must_use] + pub fn billing_if_present(&self) -> Option { + (self.billed_visit_count > 0).then(|| self.totals.clone()) + } +} + +#[must_use] +pub fn billing_rollup_from_projection(projection: &RunProjection) -> ProjectionBillingRollup { + let mut stage_indices = HashMap::::new(); + let mut stages = Vec::::new(); + let mut by_model = BTreeMap::::new(); + let mut totals = BilledTokenCounts::default(); + let mut runtime_ms = 0_u64; + let mut billed_visit_count = 0_usize; + + for (stage_id, stage) in projection.iter_stages() { + if is_exit_stage(projection, stage_id.node_id()) { + continue; + } + if stage.completion.is_none() && stage.duration_ms.is_none() && stage.usage.is_none() { + continue; + } + + let node_id = stage_id.node_id(); + let index = *stage_indices.entry(node_id.to_string()).or_insert_with(|| { + let index = stages.len(); + stages.push(ProjectionBillingStage { + node_id: node_id.to_string(), + billing: BilledTokenCounts::default(), + duration_ms: 0, + model_id: None, + }); + index + }); + let row = &mut stages[index]; + + if let Some(duration_ms) = stage.duration_ms { + row.duration_ms = row.duration_ms.saturating_add(duration_ms); + runtime_ms = runtime_ms.saturating_add(duration_ms); + } + + if let Some(usage) = stage.usage.as_ref() { + billed_visit_count += 1; + row.model_id = Some(usage.model_id().to_string()); + accumulate_usage(&mut row.billing, usage); + accumulate_usage(&mut totals, usage); + + let model_id = usage.model_id().to_string(); + let model_entry = + by_model + .entry(model_id.clone()) + .or_insert_with(|| ProjectionBillingByModel { + model_id, + stages: 0, + billing: BilledTokenCounts::default(), + }); + model_entry.stages += 1; + accumulate_usage(&mut model_entry.billing, usage); + } + } + + ProjectionBillingRollup { + stages, + totals, + by_model: by_model.into_values().collect(), + runtime_ms, + billed_visit_count, + } +} + +fn is_exit_stage(projection: &RunProjection, node_id: &str) -> bool { + projection + .spec() + .and_then(|spec| spec.graph().nodes.get(node_id)) + .is_some_and(|node| node.handler_type() == Some("exit")) +} + +fn accumulate_usage(counts: &mut BilledTokenCounts, usage: &BilledModelUsage) { + let tokens = usage.tokens(); + counts.input_tokens += tokens.input_tokens; + counts.output_tokens += tokens.output_tokens; + counts.reasoning_tokens += tokens.reasoning_tokens; + counts.cache_read_tokens += tokens.cache_read_tokens; + counts.cache_write_tokens += tokens.cache_write_tokens; + counts.total_tokens += tokens.total_tokens(); + if let Some(value) = usage.total_usd_micros { + *counts.total_usd_micros.get_or_insert(0) += value; + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use fabro_types::{ + AttrValue, BilledModelUsage, Graph, Node, RunProjection, RunSpec, StageCompletion, + StageOutcome, WorkflowSettings, first_event_seq, fixtures, + }; + use serde_json::json; + + use super::billing_rollup_from_projection; + + fn test_usage(model_id: &str, input_tokens: i64, output_tokens: i64) -> BilledModelUsage { + serde_json::from_value(json!({ + "input": { + "usage": { + "model": { + "provider": "openai", + "model_id": model_id + }, + "tokens": { + "input_tokens": input_tokens, + "output_tokens": output_tokens + } + }, + "facts": { + "provider": "open_ai" + } + }, + "total_usd_micros": input_tokens + output_tokens + })) + .unwrap() + } + + #[test] + fn rollup_groups_stage_rows_by_node_and_sums_retry_visit_usage() { + let mut projection = RunProjection::default(); + let failed_usage = test_usage("gpt-old", 100, 10); + let success_usage = test_usage("gpt-new", 200, 20); + let first = projection.stage_entry("verify", 1, first_event_seq(1)); + first.duration_ms = Some(1200); + first.usage = Some(failed_usage); + first.completion = Some(StageCompletion { + outcome: StageOutcome::Failed { + retry_requested: true, + }, + notes: None, + failure_reason: Some("try again".to_string()), + timestamp: chrono::Utc::now(), + }); + let second = projection.stage_entry("verify", 2, first_event_seq(2)); + second.duration_ms = Some(800); + second.usage = Some(success_usage); + second.completion = Some(StageCompletion { + outcome: StageOutcome::Succeeded, + notes: None, + failure_reason: None, + timestamp: chrono::Utc::now(), + }); + + let rollup = billing_rollup_from_projection(&projection); + + assert_eq!(rollup.stages.len(), 1); + assert_eq!(rollup.stages[0].node_id, "verify"); + assert_eq!(rollup.stages[0].model_id.as_deref(), Some("gpt-new")); + assert_eq!(rollup.stages[0].duration_ms, 2000); + assert_eq!(rollup.stages[0].billing.input_tokens, 300); + assert_eq!(rollup.stages[0].billing.output_tokens, 30); + assert_eq!(rollup.stages[0].billing.total_usd_micros, Some(330)); + + assert_eq!(rollup.runtime_ms, 2000); + assert_eq!(rollup.totals.input_tokens, 300); + assert_eq!(rollup.totals.output_tokens, 30); + assert_eq!(rollup.totals.total_usd_micros, Some(330)); + assert_eq!(rollup.billed_visit_count, 2); + + assert_eq!(rollup.by_model.len(), 2); + assert_eq!(rollup.by_model[0].model_id, "gpt-new"); + assert_eq!(rollup.by_model[0].stages, 1); + assert_eq!(rollup.by_model[0].billing.input_tokens, 200); + assert_eq!(rollup.by_model[1].model_id, "gpt-old"); + assert_eq!(rollup.by_model[1].stages, 1); + assert_eq!(rollup.by_model[1].billing.input_tokens, 100); + } + + #[test] + fn rollup_includes_completed_non_llm_stage_rows_with_zero_billing() { + let mut projection = RunProjection::default(); + let stage = projection.stage_entry("start", 1, first_event_seq(1)); + stage.duration_ms = Some(25); + stage.completion = Some(StageCompletion { + outcome: StageOutcome::Succeeded, + notes: None, + failure_reason: None, + timestamp: chrono::Utc::now(), + }); + + let rollup = billing_rollup_from_projection(&projection); + + assert_eq!(rollup.stages.len(), 1); + assert_eq!(rollup.stages[0].node_id, "start"); + assert_eq!(rollup.stages[0].duration_ms, 25); + assert!(rollup.stages[0].model_id.is_none()); + assert_eq!(rollup.stages[0].billing.input_tokens, 0); + assert_eq!(rollup.runtime_ms, 25); + assert!(rollup.by_model.is_empty()); + assert!(rollup.billing_if_present().is_none()); + } + + #[test] + fn rollup_excludes_terminal_exit_stage_rows() { + let mut projection = RunProjection::default(); + projection.spec = Some(run_spec_with_exit_node()); + let start = projection.stage_entry("start", 1, first_event_seq(1)); + start.duration_ms = Some(25); + start.completion = Some(StageCompletion { + outcome: StageOutcome::Succeeded, + notes: None, + failure_reason: None, + timestamp: chrono::Utc::now(), + }); + let exit = projection.stage_entry("exit", 1, first_event_seq(2)); + exit.duration_ms = Some(7); + exit.completion = Some(StageCompletion { + outcome: StageOutcome::Succeeded, + notes: None, + failure_reason: None, + timestamp: chrono::Utc::now(), + }); + + let rollup = billing_rollup_from_projection(&projection); + + assert_eq!(rollup.stages.len(), 1); + assert_eq!(rollup.stages[0].node_id, "start"); + assert_eq!(rollup.runtime_ms, 25); + } + + fn run_spec_with_exit_node() -> RunSpec { + let mut graph = Graph::new("test"); + graph.nodes.insert("start".to_string(), { + let mut node = Node::new("start"); + node.attrs.insert( + "shape".to_string(), + AttrValue::String("Mdiamond".to_string()), + ); + node + }); + graph.nodes.insert("exit".to_string(), { + let mut node = Node::new("exit"); + node.attrs.insert( + "shape".to_string(), + AttrValue::String("Msquare".to_string()), + ); + node + }); + + RunSpec { + run_id: fixtures::RUN_1, + settings: WorkflowSettings::default(), + graph, + workflow_slug: None, + source_directory: None, + labels: HashMap::new(), + provenance: None, + manifest_blob: None, + definition_blob: None, + git: None, + fork_source_ref: None, + in_place: false, + } + } +} diff --git a/lib/crates/fabro-workflow/src/error.rs b/lib/crates/fabro-workflow/src/error.rs index 1d08b2777..6a3ff0d06 100644 --- a/lib/crates/fabro-workflow/src/error.rs +++ b/lib/crates/fabro-workflow/src/error.rs @@ -1838,6 +1838,7 @@ mod tests { failure: failure.clone(), will_retry: false, duration_ms: 0, + billing: None, actor: None, }; diff --git a/lib/crates/fabro-workflow/src/event/convert.rs b/lib/crates/fabro-workflow/src/event/convert.rs index 815aaec11..4aa9df5b1 100644 --- a/lib/crates/fabro-workflow/src/event/convert.rs +++ b/lib/crates/fabro-workflow/src/event/convert.rs @@ -290,12 +290,14 @@ fn event_body_from_event(event: &Event) -> EventBody { failure, will_retry, duration_ms, + billing, .. } => EventBody::StageFailed(fabro_types::StageFailedProps { index: *index, failure: Some(failure.clone()), will_retry: *will_retry, duration_ms: *duration_ms, + billing: billing.clone(), }), Event::StageRetrying { index, @@ -1178,7 +1180,7 @@ mod tests { use crate::error::Error; use crate::event::test_support::user_principal; use crate::event::{Event, StageScope}; - use crate::outcome::FailureDetail; + use crate::outcome::{BilledModelUsage, FailureDetail}; #[derive(Debug)] struct EventTestCause; @@ -1200,6 +1202,28 @@ mod tests { } } + fn test_usage(model_id: &str, input_tokens: i64, output_tokens: i64) -> BilledModelUsage { + serde_json::from_value(serde_json::json!({ + "input": { + "usage": { + "model": { + "provider": "openai", + "model_id": model_id + }, + "tokens": { + "input_tokens": input_tokens, + "output_tokens": output_tokens + } + }, + "facts": { + "provider": "open_ai" + } + }, + "total_usd_micros": input_tokens + output_tokens + })) + .unwrap() + } + #[test] fn run_event_stage_completed_places_node_fields_in_header() { let stored = to_run_event_at( @@ -1279,6 +1303,7 @@ mod tests { #[test] fn run_event_stage_failure_keeps_failure_detail() { + let usage = test_usage("gpt-5.2", 321, 54); let stored = to_run_event(&fixtures::RUN_3, &Event::StageFailed { node_id: "code".to_string(), name: "Code".to_string(), @@ -1289,6 +1314,7 @@ mod tests { ), will_retry: true, duration_ms: 5000, + billing: Some(usage.clone()), actor: None, }); @@ -1297,6 +1323,7 @@ mod tests { assert_eq!(properties["failure"]["message"], "lint failed"); assert_eq!(properties["failure"]["failure_class"], "deterministic"); assert_eq!(properties["will_retry"], true); + assert_eq!(properties["billing"], serde_json::to_value(&usage).unwrap()); } #[test] diff --git a/lib/crates/fabro-workflow/src/event/events.rs b/lib/crates/fabro-workflow/src/event/events.rs index a3b2e2b5b..c753e20c6 100644 --- a/lib/crates/fabro-workflow/src/event/events.rs +++ b/lib/crates/fabro-workflow/src/event/events.rs @@ -206,6 +206,7 @@ pub enum Event { failure: FailureDetail, will_retry: bool, duration_ms: u64, + billing: Option, #[serde(default, skip_serializing_if = "Option::is_none")] actor: Option, }, diff --git a/lib/crates/fabro-workflow/src/lib.rs b/lib/crates/fabro-workflow/src/lib.rs index 9af074d97..5478e611d 100644 --- a/lib/crates/fabro-workflow/src/lib.rs +++ b/lib/crates/fabro-workflow/src/lib.rs @@ -20,6 +20,7 @@ use std::sync::Arc; use fabro_retro::retro::CompletedStage; use fabro_store::EventEnvelope; +use fabro_types::{EventBody, StageId}; /// Callback invoked when a workflow node starts executing. pub type OnNodeCallback = Option>; @@ -86,34 +87,190 @@ pub fn build_completed_stages(cp: &records::Checkpoint, run_failed: bool) -> Vec stages } -pub fn extract_stage_durations_from_events(events: &[EventEnvelope]) -> HashMap { +/// Extract the `duration_ms` from a `stage.completed` / `stage.failed` +/// event body, or `None` for any other variant. +fn stage_completion_duration_ms(body: &EventBody) -> Option { + match body { + EventBody::StageCompleted(props) => Some(props.duration_ms), + EventBody::StageFailed(props) => Some(props.duration_ms), + _ => None, + } +} + +/// Extract per-stage (node_id, visit) durations from `stage.completed` / +/// `stage.failed` events. Keys on the full [`StageId`] so multi-visit stages +/// (e.g. a looped `verify` node) keep distinct durations. +/// +/// This is the canonical primitive; [`total_stage_duration_by_node`] and +/// [`latest_stage_duration_by_node`] are explicit rollups built on top of it. +pub fn extract_stage_durations_by_stage_id(events: &[EventEnvelope]) -> HashMap { let mut durations = HashMap::new(); for envelope in events { - let event = &envelope.event; - let event_name = event.event_name(); - if event_name != "stage.completed" && event_name != "stage.failed" { - continue; - } - let Some(node_id) = event.node_id.as_deref() else { + let Some(duration_ms) = stage_completion_duration_ms(&envelope.event.body) else { continue; }; - let Some(duration_ms) = event - .properties() - .ok() - .and_then(|properties| properties.get("duration_ms").cloned()) - .and_then(|duration| duration.as_u64()) - else { + let Some(stage_id) = envelope.event.stage_id.as_ref() else { continue; }; - durations.insert(node_id.to_string(), duration_ms); + durations.insert(stage_id.clone(), duration_ms); } durations } +/// Total duration spent in each node, summed across every visit. Use for +/// billing/usage where a retried node should count its full time. +pub fn total_stage_duration_by_node(events: &[EventEnvelope]) -> HashMap { + let mut totals: HashMap = HashMap::new(); + for (stage_id, duration_ms) in extract_stage_durations_by_stage_id(events) { + *totals.entry(stage_id.node_id().to_string()).or_default() += duration_ms; + } + totals +} + +/// Duration of each node's most recent visit (the highest visit number). Use +/// for run summaries and retros where the table shows one row per node and +/// "the last attempt" is the right representative. +pub fn latest_stage_duration_by_node(events: &[EventEnvelope]) -> HashMap { + let mut entries: Vec<(StageId, u64)> = extract_stage_durations_by_stage_id(events) + .into_iter() + .collect(); + entries.sort_by_key(|(stage_id, _)| stage_id.visit()); + let mut latest = HashMap::new(); + for (stage_id, duration_ms) in entries { + latest.insert(stage_id.node_id().to_string(), duration_ms); + } + latest +} + +#[cfg(test)] +mod duration_tests { + use chrono::{TimeZone, Utc}; + use fabro_store::EventEnvelope; + use fabro_types::run_event::{StageCompletedProps, StageFailedProps}; + use fabro_types::{EventBody, RunEvent, StageId, StageOutcome, fixtures}; + + use super::{ + extract_stage_durations_by_stage_id, latest_stage_duration_by_node, + total_stage_duration_by_node, + }; + + fn completed_event(seq: u32, node: &str, visit: u32, duration_ms: u64) -> EventEnvelope { + let event = RunEvent { + id: format!("evt_{seq}"), + ts: Utc.with_ymd_and_hms(2026, 1, 1, 0, 0, 0).unwrap(), + run_id: fixtures::RUN_1, + node_id: Some(node.to_string()), + node_label: None, + stage_id: Some(StageId::new(node, visit)), + parallel_group_id: None, + parallel_branch_id: None, + session_id: None, + parent_session_id: None, + tool_call_id: None, + actor: None, + body: EventBody::StageCompleted(StageCompletedProps { + index: 0, + duration_ms, + status: StageOutcome::Succeeded, + preferred_label: None, + suggested_next_ids: vec![], + billing: None, + failure: None, + notes: None, + files_touched: vec![], + context_updates: None, + jump_to_node: None, + context_values: None, + node_visits: None, + loop_failure_signatures: None, + restart_failure_signatures: None, + response: None, + attempt: 1, + max_attempts: 1, + }), + }; + EventEnvelope { seq, event } + } + + fn failed_event(seq: u32, node: &str, visit: u32, duration_ms: u64) -> EventEnvelope { + let event = RunEvent { + id: format!("evt_{seq}"), + ts: Utc.with_ymd_and_hms(2026, 1, 1, 0, 0, 0).unwrap(), + run_id: fixtures::RUN_1, + node_id: Some(node.to_string()), + node_label: None, + stage_id: Some(StageId::new(node, visit)), + parallel_group_id: None, + parallel_branch_id: None, + session_id: None, + parent_session_id: None, + tool_call_id: None, + actor: None, + body: EventBody::StageFailed(StageFailedProps { + index: 0, + failure: None, + will_retry: true, + duration_ms, + billing: None, + }), + }; + EventEnvelope { seq, event } + } + + #[test] + fn extract_keys_durations_by_full_stage_id() { + let events = vec![ + completed_event(1, "verify", 1, 100), + completed_event(2, "verify", 2, 200), + ]; + let durations = extract_stage_durations_by_stage_id(&events); + assert_eq!( + durations.get(&StageId::new("verify", 1)).copied(), + Some(100) + ); + assert_eq!( + durations.get(&StageId::new("verify", 2)).copied(), + Some(200) + ); + } + + #[test] + fn total_sums_across_visits_per_node() { + let events = vec![ + completed_event(1, "verify", 1, 100), + completed_event(2, "verify", 2, 200), + completed_event(3, "build", 1, 50), + ]; + let totals = total_stage_duration_by_node(&events); + assert_eq!(totals.get("verify").copied(), Some(300)); + assert_eq!(totals.get("build").copied(), Some(50)); + } + + #[test] + fn latest_picks_highest_visit_regardless_of_input_order() { + // Visit 2 appears in the events vector before visit 1; the result + // must still reflect visit 2's duration (the latest visit). + let events = vec![ + completed_event(1, "verify", 2, 999), + completed_event(2, "verify", 1, 100), + ]; + let latest = latest_stage_duration_by_node(&events); + assert_eq!(latest.get("verify").copied(), Some(999)); + } + + #[test] + fn stage_failed_durations_are_included() { + let events = vec![failed_event(1, "verify", 1, 75)]; + let durations = extract_stage_durations_by_stage_id(&events); + assert_eq!(durations.get(&StageId::new("verify", 1)).copied(), Some(75)); + } +} + #[doc(hidden)] pub mod artifact; pub mod artifact_snapshot; pub mod artifact_upload; +pub mod billing_rollup; pub mod command_log; pub(crate) mod condition; pub mod context; @@ -142,6 +299,10 @@ pub mod run_control; pub(crate) mod run_dir; pub mod run_lookup; +pub use billing_rollup::{ + ProjectionBillingByModel, ProjectionBillingRollup, ProjectionBillingStage, + billing_rollup_from_projection, +}; pub use error::{Error, FailureCategory, FailureSignature, FailureSignatureExt, Result}; pub use manifest_path::ManifestPath; pub mod run_materialization; diff --git a/lib/crates/fabro-workflow/src/lifecycle/event.rs b/lib/crates/fabro-workflow/src/lifecycle/event.rs index 482f0311a..82dbd0501 100644 --- a/lib/crates/fabro-workflow/src/lifecycle/event.rs +++ b/lib/crates/fabro-workflow/src/lifecycle/event.rs @@ -224,6 +224,7 @@ impl RunLifecycle for EventLifecycle { failure, will_retry: true, duration_ms, + billing: outcome.usage.clone(), actor, }, &scope, @@ -275,6 +276,7 @@ impl RunLifecycle for EventLifecycle { failure, will_retry: false, duration_ms, + billing: outcome.usage.clone(), actor, }, &scope, diff --git a/lib/crates/fabro-workflow/src/pipeline/finalize.rs b/lib/crates/fabro-workflow/src/pipeline/finalize.rs index 921697369..927d2d833 100644 --- a/lib/crates/fabro-workflow/src/pipeline/finalize.rs +++ b/lib/crates/fabro-workflow/src/pipeline/finalize.rs @@ -19,6 +19,7 @@ use crate::run_status::{FailureReason, RunStatus, SuccessReason}; use crate::runtime_store::RunStoreHandle; use crate::sandbox_git::git_diff_with_timeout; use crate::services::RunServices; +use crate::{ProjectionBillingRollup, billing_rollup_from_projection}; pub fn classify_engine_result( engine_result: &Result, @@ -68,22 +69,22 @@ pub(crate) async fn build_conclusion_from_store( run_duration_ms: u64, final_git_commit_sha: Option, ) -> Conclusion { - let (state_result, events_result) = tokio::join!(run_store.state(), run_store.list_events()); - let projection = state_result.ok(); + let projection = run_store.state().await.ok(); let projection_order = projection .as_ref() .map(stage_projection_order) .unwrap_or_default(); + let projection_billing = projection + .as_ref() + .map(billing_rollup_from_projection) + .unwrap_or_default(); let checkpoint = projection .as_ref() .and_then(|state| state.checkpoint.as_ref()); - let stage_durations = events_result - .map(|events| crate::extract_stage_durations_from_events(&events)) - .unwrap_or_default(); build_conclusion_from_parts( checkpoint, - &stage_durations, + &projection_billing, &projection_order, status, failure_reason, @@ -94,7 +95,7 @@ pub(crate) async fn build_conclusion_from_store( fn build_conclusion_from_parts( checkpoint: Option<&Checkpoint>, - stage_durations: &HashMap, + projection_billing: &ProjectionBillingRollup, projection_order: &HashMap, status: StageOutcome, failure_reason: Option, @@ -105,6 +106,11 @@ fn build_conclusion_from_parts( // while the other checkpoint maps are keyed by node_id. Dedupe to one row // per node so the stages table matches the deduped billing total. let (stages, total_retries) = if let Some(cp) = checkpoint { + let billing_by_node = projection_billing + .stages + .iter() + .map(|stage| (stage.node_id.as_str(), stage)) + .collect::>(); let mut stage_rows = Vec::new(); let mut seen = std::collections::HashSet::new(); let mut retries_sum: u32 = 0; @@ -130,7 +136,6 @@ fn build_conclusion_from_parts( } for (original_checkpoint_order, node_id) in stage_order { - let outcome = cp.node_outcomes.get(node_id); let retries = cp .node_retries .get(node_id) @@ -138,14 +143,13 @@ fn build_conclusion_from_parts( .unwrap_or(1) .saturating_sub(1); retries_sum += retries; + let billing = billing_by_node.get(node_id); let summary = StageSummary { stage_id: node_id.to_string(), stage_label: node_id.to_string(), - duration_ms: stage_durations.get(node_id).copied().unwrap_or(0), - billing_usd_micros: outcome - .and_then(|o| o.usage.as_ref()) - .and_then(|usage| usage.total_usd_micros), + duration_ms: billing.map_or(0, |stage| stage.duration_ms), + billing_usd_micros: billing.and_then(|stage| stage.billing.total_usd_micros), retries, }; stage_rows.push(( @@ -176,7 +180,7 @@ fn build_conclusion_from_parts( failure_reason, final_git_commit_sha, stages, - billing: checkpoint.and_then(billing_from_checkpoint), + billing: projection_billing.billing_if_present(), total_retries, } } @@ -391,15 +395,8 @@ async fn compute_final_patch( } } -/// Iterates `node_outcomes.values()` rather than `completed_nodes` to avoid -/// over-counting the last visit's usage on looping workflows. -pub(crate) fn billing_from_checkpoint(cp: &Checkpoint) -> Option { - let usage: Vec<_> = cp - .node_outcomes - .values() - .filter_map(|o| o.usage.clone()) - .collect(); - (!usage.is_empty()).then(|| BilledTokenCounts::from_billed_usage(&usage)) +pub(crate) fn billing_from_projection(projection: &RunProjection) -> Option { + billing_rollup_from_projection(projection).billing_if_present() } pub(crate) fn build_terminal_event( @@ -503,7 +500,6 @@ pub async fn finalize(retroed: Retroed, options: &FinalizeOptions) -> Result Result BilledModelUsage { + serde_json::from_value(serde_json::json!({ + "input": { + "usage": { + "model": { + "provider": "openai", + "model_id": model_id + }, + "tokens": { + "input_tokens": input_tokens, + "output_tokens": output_tokens + } + }, + "facts": { + "provider": "open_ai" + } + }, + "total_usd_micros": input_tokens + output_tokens + })) + .unwrap() + } + #[test] fn conclusion_stage_order_follows_projection_first_event_order() { let mut projection = RunProjection::default(); @@ -753,7 +776,7 @@ mod tests { let conclusion = build_conclusion_from_parts( Some(&checkpoint), - &HashMap::new(), + &ProjectionBillingRollup::default(), &projection_order, StageOutcome::Succeeded, None, @@ -788,7 +811,7 @@ mod tests { let conclusion = build_conclusion_from_parts( Some(&checkpoint), - &HashMap::new(), + &ProjectionBillingRollup::default(), &projection_order, StageOutcome::Succeeded, None, @@ -804,6 +827,66 @@ mod tests { assert_eq!(stage_ids, vec!["skipped", "finished"]); } + #[test] + fn conclusion_billing_sums_retry_visit_usage_from_projection() { + let mut projection = RunProjection::default(); + let failed_usage = test_usage("gpt-old", 100, 10); + let success_usage = test_usage("gpt-new", 200, 20); + let failed = projection.stage_entry("verify", 1, first_event_seq(1)); + failed.duration_ms = Some(1200); + failed.usage = Some(failed_usage); + failed.completion = Some(StageCompletion { + outcome: StageOutcome::Failed { + retry_requested: true, + }, + notes: None, + failure_reason: Some("try again".to_string()), + timestamp: chrono::Utc::now(), + }); + let succeeded = projection.stage_entry("verify", 2, first_event_seq(2)); + succeeded.duration_ms = Some(800); + succeeded.usage = Some(success_usage.clone()); + succeeded.completion = Some(StageCompletion { + outcome: StageOutcome::Succeeded, + notes: None, + failure_reason: None, + timestamp: chrono::Utc::now(), + }); + + let projection_order = stage_projection_order(&projection); + let projection_billing = billing_rollup_from_projection(&projection); + let mut latest_outcome = Outcome::success(); + latest_outcome.usage = Some(success_usage); + latest_outcome.duration_ms = Some(800); + let mut checkpoint = checkpoint_with( + vec!["verify", "verify"], + HashMap::from([("verify".to_string(), latest_outcome)]), + ); + checkpoint.node_retries.insert("verify".to_string(), 2); + + let conclusion = build_conclusion_from_parts( + Some(&checkpoint), + &projection_billing, + &projection_order, + StageOutcome::Succeeded, + None, + 10, + None, + ); + + assert_eq!(conclusion.billing.as_ref().unwrap().input_tokens, 300); + assert_eq!(conclusion.billing.as_ref().unwrap().output_tokens, 30); + assert_eq!( + conclusion.billing.as_ref().unwrap().total_usd_micros, + Some(330) + ); + assert_eq!(conclusion.stages.len(), 1); + assert_eq!(conclusion.stages[0].stage_id, "verify"); + assert_eq!(conclusion.stages[0].duration_ms, 2000); + assert_eq!(conclusion.stages[0].billing_usd_micros, Some(330)); + assert_eq!(conclusion.stages[0].retries, 1); + } + fn test_services( run_store: RunStoreHandle, emitter: Arc, diff --git a/lib/crates/fabro-workflow/src/pipeline/mod.rs b/lib/crates/fabro-workflow/src/pipeline/mod.rs index 71aefbef9..284c32a1f 100644 --- a/lib/crates/fabro-workflow/src/pipeline/mod.rs +++ b/lib/crates/fabro-workflow/src/pipeline/mod.rs @@ -12,7 +12,7 @@ mod validate; pub use execute::execute; pub use fabro_types::PullRequestRecord; pub(crate) use finalize::{ - billing_from_checkpoint, build_conclusion_from_store, build_terminal_event, + billing_from_projection, build_conclusion_from_store, build_terminal_event, }; pub use finalize::{classify_engine_result, finalize, write_finalize_commit}; pub use initialize::initialize; diff --git a/lib/crates/fabro-workflow/src/pipeline/retro.rs b/lib/crates/fabro-workflow/src/pipeline/retro.rs index 649435c2e..d4c492373 100644 --- a/lib/crates/fabro-workflow/src/pipeline/retro.rs +++ b/lib/crates/fabro-workflow/src/pipeline/retro.rs @@ -53,7 +53,7 @@ pub async fn run_retro(options: &RetroOptions, dry_run: bool) -> Option { return None; } }; - let stage_durations = crate::extract_stage_durations_from_events(&events); + let stage_durations = crate::latest_stage_duration_by_node(&events); let mut retro = derive_retro( options.run_id, &options.workflow_name, diff --git a/lib/crates/fabro-workflow/src/test_support.rs b/lib/crates/fabro-workflow/src/test_support.rs index 12aa40f10..0bf6377a6 100644 --- a/lib/crates/fabro-workflow/src/test_support.rs +++ b/lib/crates/fabro-workflow/src/test_support.rs @@ -17,7 +17,7 @@ use crate::handler::HandlerRegistry; use crate::outcome::Outcome; use crate::pipeline; use crate::pipeline::types::{Executed, Initialized}; -use crate::pipeline::{billing_from_checkpoint, build_terminal_event}; +use crate::pipeline::{billing_from_projection, build_terminal_event}; use crate::records::Checkpoint; use crate::run_metadata::RunMetadataRuntime; use crate::run_options::RunOptions; @@ -36,10 +36,7 @@ async fn execute_and_emit_terminal(initialized: InitializedState) -> Executed { let executed = Box::pin(pipeline::execute(initialized.initialized)).await; initialized.store_logger.flush().await; let state = executed.engine.run.run_store.state().await.ok(); - let billing = state - .as_ref() - .and_then(|s| s.checkpoint.as_ref()) - .and_then(billing_from_checkpoint); + let billing = state.as_ref().and_then(billing_from_projection); let event = build_terminal_event( &executed.outcome, executed.duration_ms, diff --git a/lib/packages/fabro-api-client/src/models/billing-by-model.ts b/lib/packages/fabro-api-client/src/models/billing-by-model.ts index d8d802575..74f074401 100644 --- a/lib/packages/fabro-api-client/src/models/billing-by-model.ts +++ b/lib/packages/fabro-api-client/src/models/billing-by-model.ts @@ -26,7 +26,7 @@ import type { ModelReference } from './model-reference'; export interface BillingByModel { 'model': ModelReference; /** - * Number of stages that used this model. + * Number of usage-bearing stage visits that used this model. */ 'stages': number; 'billing': BilledTokenCounts; diff --git a/lib/packages/fabro-api-client/src/models/billing-stage-ref.ts b/lib/packages/fabro-api-client/src/models/billing-stage-ref.ts index bcc94a656..7b87ef631 100644 --- a/lib/packages/fabro-api-client/src/models/billing-stage-ref.ts +++ b/lib/packages/fabro-api-client/src/models/billing-stage-ref.ts @@ -15,7 +15,7 @@ /** - * Reference to a billing stage. + * Reference to a workflow node in a billing stage row. */ export interface BillingStageRef { /** diff --git a/lib/packages/fabro-api-client/src/models/run-billing-stage.ts b/lib/packages/fabro-api-client/src/models/run-billing-stage.ts index b6d807d8f..5d376b7cc 100644 --- a/lib/packages/fabro-api-client/src/models/run-billing-stage.ts +++ b/lib/packages/fabro-api-client/src/models/run-billing-stage.ts @@ -24,14 +24,14 @@ import type { BillingStageRef } from './billing-stage-ref'; import type { ModelReference } from './model-reference'; /** - * Token counts and billed totals for a single stage within a run. + * Token counts and billed totals for one workflow node within a run. Rows are grouped by node; billing and runtime sum every visit of that node. */ export interface RunBillingStage { 'stage': BillingStageRef; 'model': ModelReference | null; 'billing': BilledTokenCounts; /** - * Wall-clock runtime in seconds. + * Wall-clock runtime in seconds, summed across every visit of this node. */ 'runtime_secs': number; } diff --git a/lib/packages/fabro-api-client/src/models/run-billing.ts b/lib/packages/fabro-api-client/src/models/run-billing.ts index 88544b1cc..09cfed9e3 100644 --- a/lib/packages/fabro-api-client/src/models/run-billing.ts +++ b/lib/packages/fabro-api-client/src/models/run-billing.ts @@ -28,7 +28,7 @@ import type { RunBillingTotals } from './run-billing-totals'; */ export interface RunBilling { /** - * Per-stage billing breakdown. + * Per-node billing breakdown. Each row sums billing and runtime across all visits of that node. */ 'stages': Array; 'totals': RunBillingTotals; diff --git a/lib/packages/fabro-api-client/src/models/run-stage.ts b/lib/packages/fabro-api-client/src/models/run-stage.ts index c98ec9bee..ad01b9791 100644 --- a/lib/packages/fabro-api-client/src/models/run-stage.ts +++ b/lib/packages/fabro-api-client/src/models/run-stage.ts @@ -22,7 +22,7 @@ import type { StageState } from './stage-state'; */ export interface RunStage { /** - * Unique stage identifier within the run. + * StageId in \"node_id@visit\" form, e.g. verify@2. */ 'id': string; /** @@ -35,9 +35,13 @@ export interface RunStage { */ 'duration_secs'?: number; /** - * Node identifier in the Graphviz graph source. + * Node id in the workflow graph; multiple stages with different visits share the same node_id. */ - 'dot_id'?: string; + 'node_id': string; + /** + * 1-based visit count; bumped each time the workflow re-enters this node. + */ + 'visit': number; } From 6e36d8350ed859949f8d0c39d97ef31f04f246c3 Mon Sep 17 00:00:00 2001 From: "fabro-sh-0530[bot]" <281434857+fabro-sh-0530[bot]@users.noreply.github.com> Date: Tue, 5 May 2026 09:16:52 -0400 Subject: [PATCH 13/17] Render stage activity from scoped events endpoint (#212) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Stage detail now loads activity from a canonical stage-scoped events endpoint instead of falling back to the first 1000 run-wide events. This fixes empty panes for late stages in long runs and removes the presentation-shaped `StageTurn` API from the wire. ### Plan Summary - Add `GET /runs/{id}/stages/{stageId}/events` with cursor pagination and server-side `node_id` filtering. - Replace frontend stage-turn/fallback loading with paginated stage-events loading and local event-to-activity projection. - Broaden SSE/SWR invalidation so every activity event consumed by the reducer refreshes the per-stage cache. - Remove `StageTurn` schemas/client models and update demo fixtures plus pagination/handler/reducer tests. ## What changed and why The store now scans the run event prefix and filters by `node_id` before applying the `limit + 1` cutoff. That preserves sparse late-stage matches that would otherwise be dropped if we reused the run-wide limited scan and filtered afterward. The real-mode handler returns an empty page for an unknown stage id in an existing run, while preserving 404 for missing runs. On the frontend, `run-stages` fetches all pages for the selected stage and feeds them through `eventsToActivity`, keeping `TurnType` as a local presentation model. Invalidation now targets `runs.stageEvents(runId, stageId)` for lifecycle and reducer-consumed activity events (`stage.prompt`, agent messages/tools, and command events), so active panes refresh from the existing run event subscription. The OpenAPI document and generated TS client now expose `listStageEvents` and drop stale `StageTurn` models. Demo mode serves a `detect-drift` stage-events fixture using the same cursor semantics as the real endpoint. ## API notes `/runs/{id}/stages/{stageId}/turns` is removed; clients should use `/runs/{id}/stages/{stageId}/events?since_seq=&limit=` and project events locally. The `stageId` path segment for this endpoint is the workflow node id, not the visit-qualified `node_id@visit` form used by command logs/artifacts. ⚒️ Generated with [Fabro](https://fabro.sh) --------- Co-authored-by: Fabro Co-authored-by: Bryan Helmkamp --- apps/fabro-web/app/lib/api-client.ts | 65 +++- apps/fabro-web/app/lib/queries.ts | 25 +- apps/fabro-web/app/lib/query-keys.test.ts | 26 +- apps/fabro-web/app/lib/query-keys.ts | 9 +- apps/fabro-web/app/lib/run-events.test.tsx | 10 +- apps/fabro-web/app/lib/run-events.ts | 33 +- apps/fabro-web/app/routes/run-stages.test.ts | 139 ++++++-- apps/fabro-web/app/routes/run-stages.tsx | 88 +++-- docs/public/api-reference/fabro-api.yaml | 129 +------- lib/crates/fabro-server/src/demo/mod.rs | 159 ++++++++- .../fabro-server/src/principal_middleware.rs | 18 + lib/crates/fabro-server/src/server.rs | 3 +- .../fabro-server/src/server/handler/events.rs | 311 +++++++++++++++++- .../fabro-server/src/server/handler/mod.rs | 5 +- .../fabro-server/tests/it/event_pagination.rs | 75 +++++ lib/crates/fabro-server/tests/it/main.rs | 1 + .../fabro-server/tests/it/pagination.rs | 4 - lib/crates/fabro-store/src/slate/run_store.rs | 285 +++++++++++++++- .../src/.openapi-generator/FILES | 6 - .../src/api/run-internals-api.ts | 64 ++-- .../src/models/assistant-stage-turn.ts | 34 -- .../src/models/board-column-definition.ts | 3 + .../fabro-api-client/src/models/index.ts | 6 - .../src/models/manifest-git.ts | 41 --- .../models/manifest-pre-run-push-outcome.ts | 36 -- .../src/models/paginated-stage-turn-list.ts | 30 -- .../src/models/pre-run-git-context.ts | 30 -- .../src/models/set-secret-request.ts | 25 -- .../fabro-api-client/src/models/stage-turn.ts | 35 -- .../src/models/system-stage-turn.ts | 34 -- .../src/models/tool-stage-turn.ts | 41 --- .../fabro-api-client/src/models/tool-use.ts | 46 --- 32 files changed, 1153 insertions(+), 663 deletions(-) create mode 100644 lib/crates/fabro-server/tests/it/event_pagination.rs delete mode 100644 lib/packages/fabro-api-client/src/models/assistant-stage-turn.ts delete mode 100644 lib/packages/fabro-api-client/src/models/manifest-git.ts delete mode 100644 lib/packages/fabro-api-client/src/models/manifest-pre-run-push-outcome.ts delete mode 100644 lib/packages/fabro-api-client/src/models/paginated-stage-turn-list.ts delete mode 100644 lib/packages/fabro-api-client/src/models/pre-run-git-context.ts delete mode 100644 lib/packages/fabro-api-client/src/models/set-secret-request.ts delete mode 100644 lib/packages/fabro-api-client/src/models/stage-turn.ts delete mode 100644 lib/packages/fabro-api-client/src/models/system-stage-turn.ts delete mode 100644 lib/packages/fabro-api-client/src/models/tool-stage-turn.ts delete mode 100644 lib/packages/fabro-api-client/src/models/tool-use.ts diff --git a/apps/fabro-web/app/lib/api-client.ts b/apps/fabro-web/app/lib/api-client.ts index bd990361b..87792ed69 100644 --- a/apps/fabro-web/app/lib/api-client.ts +++ b/apps/fabro-web/app/lib/api-client.ts @@ -217,6 +217,69 @@ export async function apiPaginatedFetcher( } } +function stageEventsPagePath(key: string, sinceSeq: number, limit: number): string { + const url = new URL(apiPath(key), "http://fabro.local"); + url.searchParams.set("since_seq", String(sinceSeq)); + url.searchParams.set("limit", String(limit)); + return `${url.pathname}${url.search}`; +} + +/** + * Cursor-paginated fetcher for `/runs/{id}/stages/{stageId}/events`. + * + * Loops from `since_seq=1` with a 1000-event page size, advancing the cursor + * to `highestSeq + 1` until the server reports `meta.has_more === false`. + * The empty-page guard mirrors `apiPaginatedFetcher`: if the server claims + * `has_more` but returns no rows we exit and `console.warn` to surface the + * server invariant violation without spinning the UI. + */ +export async function fetchAllStageEvents( + key: string, +): Promise { + const PAGE_LIMIT = 1000; + const MAX_PAGES = 50; + const data: TItem[] = []; + let sinceSeq = 1; + let pagesLoaded = 0; + + while (true) { + const response = await apiRequest(stageEventsPagePath(key, sinceSeq, PAGE_LIMIT)); + if (!response.ok) { + throw await apiErrorFromResponse(response); + } + const page = (await response.json()) as PaginatedEnvelope; + pagesLoaded += 1; + + if (page.data.length === 0) { + if (page.meta.has_more) { + console.warn( + `Stage events fetch for ${key} returned an empty page with has_more=true; stopping at ${data.length} items to avoid spinning.`, + ); + } + return data; + } + + data.push(...page.data); + if (!page.meta.has_more) return data; + + if (pagesLoaded >= MAX_PAGES) { + console.warn( + `Stopped stage events fetch for ${key} after ${pagesLoaded} pages and ${data.length} items because the safety cap was reached.`, + ); + return data; + } + + const highestSeq = page.data.reduce((max, event) => Math.max(max, event.seq), sinceSeq - 1); + if (highestSeq < sinceSeq) { + console.warn( + `Stage events fetch for ${key} returned a non-advancing page at since_seq=${sinceSeq}; stopping at ${data.length} items to avoid spinning.`, + ); + return data; + } + sinceSeq = highestSeq + 1; + } +} + export async function apiJsonMutation( key: string, { arg }: { arg: TArg }, @@ -233,4 +296,4 @@ export async function apiJsonMutation( } if (response.status === 204) return undefined as TResponse; return response.json() as Promise; -} +} \ No newline at end of file diff --git a/apps/fabro-web/app/lib/queries.ts b/apps/fabro-web/app/lib/queries.ts index 0da17785e..880cf97b6 100644 --- a/apps/fabro-web/app/lib/queries.ts +++ b/apps/fabro-web/app/lib/queries.ts @@ -1,12 +1,11 @@ import useSWR, { type SWRConfiguration } from "swr"; import type { ApiQuestion, + EventEnvelope, PaginatedBoardRunList, - PaginatedEventList, PaginatedRunFileList, PaginatedRunList, PaginatedRunStageList, - PaginatedStageTurnList, CommandLogResponse, CommandOutputStream, RunBilling, @@ -24,6 +23,7 @@ import { apiNullableTextFetcher, apiPaginatedFetcher, apiTextFetcher, + fetchAllStageEvents, type PaginatedEnvelope, } from "./api-client"; import { queryKeys } from "./query-keys"; @@ -140,21 +140,10 @@ export function useRunQuestions(id: string | undefined, enabled: boolean) { ); } -export function useRunStageTurns( - id: string | undefined, - stageId: string | undefined, - enabled = true, -) { - return useSWR( - id && stageId && enabled ? queryKeys.runs.stageTurns(id, stageId) : null, - apiNullableFetcher, - ); -} - -export function useRunEventsList(id: string | undefined, enabled = true) { - return useSWR( - id && enabled ? queryKeys.runs.events(id, 1000) : null, - apiNullableFetcher, +export function useRunStageEvents(id: string | undefined, stageId: string | undefined) { + return useSWR( + id && stageId ? queryKeys.runs.stageEvents(id, stageId) : null, + fetchAllStageEvents, ); } @@ -205,4 +194,4 @@ export function useServerSettings() { return useSWR(queryKeys.settings.server(), apiFetcher, immutableOptions); } -export { apiTextFetcher }; +export { apiTextFetcher }; \ No newline at end of file diff --git a/apps/fabro-web/app/lib/query-keys.test.ts b/apps/fabro-web/app/lib/query-keys.test.ts index 8f036ac3e..5d7b3a53a 100644 --- a/apps/fabro-web/app/lib/query-keys.test.ts +++ b/apps/fabro-web/app/lib/query-keys.test.ts @@ -11,6 +11,9 @@ describe("queryKeys", () => { expect(queryKeys.runs.stageLog("run 1", "build step@2", "stderr", 12, 34)).toBe( "/api/v1/runs/run%201/stages/build%20step%402/logs/stderr?offset=12&limit=34", ); + expect(queryKeys.runs.stageEvents("run 1", "build step", 7, 25)).toBe( + "/api/v1/runs/run%201/stages/build%20step/events?since_seq=7&limit=25", + ); }); test("event-mapped keys match query hook resources", () => { @@ -23,7 +26,26 @@ describe("queryKeys", () => { queryKeys.runs.graph("run-1", "LR"), queryKeys.runs.graph("run-1", "TB"), queryKeys.runs.detail("run-1"), - queryKeys.runs.stageTurns("run-1", "stage-1"), + queryKeys.runs.stageEvents("run-1", "stage-1"), ]); }); -}); + + test("agent activity events invalidate the per-stage events key", () => { + for (const event of [ + "stage.prompt", + "agent.message", + "agent.tool.started", + "agent.tool.completed", + "command.started", + "command.completed", + ]) { + expect(queryKeysForRunEvent("run-1", event, "stage-1")).toEqual([ + queryKeys.runs.stageEvents("run-1", "stage-1"), + ]); + } + }); + + test("agent activity events without a node_id invalidate nothing", () => { + expect(queryKeysForRunEvent("run-1", "agent.message")).toEqual([]); + }); +}); \ No newline at end of file diff --git a/apps/fabro-web/app/lib/query-keys.ts b/apps/fabro-web/app/lib/query-keys.ts index 7d3e847e7..402739897 100644 --- a/apps/fabro-web/app/lib/query-keys.ts +++ b/apps/fabro-web/app/lib/query-keys.ts @@ -44,8 +44,11 @@ export const queryKeys = { }), events: (id: string, limit = 1000) => withQuery(`/api/v1/runs/${pathSegment(id)}/events`, { limit }), - stageTurns: (id: string, stageId: string) => - `/api/v1/runs/${pathSegment(id)}/stages/${pathSegment(stageId)}/turns`, + stageEvents: (id: string, stageId: string, sinceSeq?: number, limit?: number) => + withQuery(`/api/v1/runs/${pathSegment(id)}/stages/${pathSegment(stageId)}/events`, { + since_seq: sinceSeq, + limit, + }), stageLog: ( id: string, stageId: string, @@ -75,4 +78,4 @@ export const queryKeys = { settings: { server: () => "/api/v1/settings", }, -}; +}; \ No newline at end of file diff --git a/apps/fabro-web/app/lib/run-events.test.tsx b/apps/fabro-web/app/lib/run-events.test.tsx index c38ea1dd2..14db0e3f2 100644 --- a/apps/fabro-web/app/lib/run-events.test.tsx +++ b/apps/fabro-web/app/lib/run-events.test.tsx @@ -55,7 +55,7 @@ describe("queryKeysForRunEvent", () => { expect(keys).toContain(queryKeys.runs.stages("run-1")); expect(keys).toContain(queryKeys.runs.events("run-1", 1000)); expect(keys).toContain(queryKeys.runs.detail("run-1")); - expect(keys).toContain(queryKeys.runs.stageTurns("run-1", "verify@2")); + expect(keys).toContain(queryKeys.runs.stageEvents("run-1", "verify@2")); }); }); @@ -180,7 +180,7 @@ describe("subscribeToRunEvents", () => { coordinator.close(); }); - test("envelope with suffixed stage_id invalidates stageTurns(runId, stageId)", async () => { + test("envelope with suffixed stage_id invalidates stageEvents(runId, stageId)", async () => { const source = new FakeEventSource(); const keys: string[] = []; const coordinator = createCoordinator(() => source); @@ -202,12 +202,12 @@ describe("subscribeToRunEvents", () => { node_id: "verify", }); - expect(keys).toContain(queryKeys.runs.stageTurns("run-stage", "verify@2")); + expect(keys).toContain(queryKeys.runs.stageEvents("run-stage", "verify@2")); expect(keys).toContain(queryKeys.runs.stages("run-stage")); expect(keys).toContain(queryKeys.runs.events("run-stage", 1000)); expect(keys).toContain(queryKeys.runs.graph("run-stage", "LR")); expect(keys).toContain(queryKeys.runs.detail("run-stage")); - expect(keys).not.toContain(queryKeys.runs.stageTurns("run-stage", "verify")); + expect(keys).not.toContain(queryKeys.runs.stageEvents("run-stage", "verify")); cleanup(); coordinator.close(); @@ -230,7 +230,7 @@ describe("subscribeToRunEvents", () => { await waitFor(() => source.onmessage !== null); source.emit({ event: "stage.started", run_id: "run-stage-node", node_id: "verify" }); - expect(keys).toContain(queryKeys.runs.stageTurns("run-stage-node", "verify")); + expect(keys).toContain(queryKeys.runs.stageEvents("run-stage-node", "verify")); expect(keys).toContain(queryKeys.runs.stages("run-stage-node")); cleanup(); diff --git a/apps/fabro-web/app/lib/run-events.ts b/apps/fabro-web/app/lib/run-events.ts index a81a11c2b..1546d1578 100644 --- a/apps/fabro-web/app/lib/run-events.ts +++ b/apps/fabro-web/app/lib/run-events.ts @@ -49,7 +49,25 @@ const STAGE_EVENTS = new Set([ "stage.failed", "stage.retrying", ]); -const COMMAND_EVENTS = new Set(["command.started", "command.completed"]); +// Single source of truth: every event type the `eventsToActivity` reducer in +// `routes/run-stages.tsx` consumes. When any of these arrive for a stage we +// currently view, the stage-events SWR key for that stage must be invalidated +// so the panel refetches. The reducer imports this list so the switch stays +// in sync with the invalidation set; if the reducer grows a new case, this +// list is the single edit point. +// +// The lifecycle `STAGE_EVENTS` set is kept separate because it also fans out +// to run-scoped invalidations (stages list, graph, detail). +export const STAGE_ACTIVITY_EVENT_TYPES = [ + "stage.prompt", + "agent.message", + "agent.tool.started", + "agent.tool.completed", + "command.started", + "command.completed", +] as const; +export type StageActivityEventType = (typeof STAGE_ACTIVITY_EVENT_TYPES)[number]; +const STAGE_ACTIVITY_EVENTS = new Set(STAGE_ACTIVITY_EVENT_TYPES); const INTERVIEW_EVENTS = new Set([ "interview.started", "interview.completed", @@ -97,20 +115,13 @@ export function queryKeysForRunEvent( queryKeys.runs.detail(runId), ]; if (stageId) { - keys.push(queryKeys.runs.stageTurns(runId, stageId)); + keys.push(queryKeys.runs.stageEvents(runId, stageId)); } return keys; } - if (COMMAND_EVENTS.has(event)) { - const keys = [ - queryKeys.runs.stages(runId), - queryKeys.runs.events(runId, 1000), - ]; - if (stageId) { - keys.push(queryKeys.runs.stageTurns(runId, stageId)); - } - return keys; + if (STAGE_ACTIVITY_EVENTS.has(event)) { + return stageId ? [queryKeys.runs.stageEvents(runId, stageId)] : []; } return []; diff --git a/apps/fabro-web/app/routes/run-stages.test.ts b/apps/fabro-web/app/routes/run-stages.test.ts index 704c5cac8..52f10ed05 100644 --- a/apps/fabro-web/app/routes/run-stages.test.ts +++ b/apps/fabro-web/app/routes/run-stages.test.ts @@ -1,7 +1,7 @@ import { describe, expect, test } from "bun:test"; import type { EventEnvelope } from "@qltysh/fabro-api-client"; -import { isSafeMarkdownHref, turnsFromEvents } from "./run-stages"; +import { eventsToActivity, isSafeMarkdownHref } from "./run-stages"; describe("isSafeMarkdownHref", () => { test("rejects protocol-relative URLs", () => { @@ -17,43 +17,39 @@ describe("isSafeMarkdownHref", () => { }); }); -function makeEnvelope(overrides: Partial): EventEnvelope { +function envelope(seq: number, partial: Partial): EventEnvelope { return { - seq: 1, - id: "evt", - ts: "2026-01-01T00:00:00Z", + seq, + id: `evt-${seq}`, + ts: "2026-04-09T12:00:00Z", run_id: "run-1", event: "stage.prompt", - ...overrides, + ...partial, } as EventEnvelope; } -describe("turnsFromEvents", () => { +describe("eventsToActivity", () => { test("filters events by stage_id (verify@1 vs verify@2 do not cross-contaminate)", () => { const events: EventEnvelope[] = [ - makeEnvelope({ - seq: 1, + envelope(1, { event: "stage.prompt", stage_id: "verify@1", node_id: "verify", properties: { text: "first visit prompt" }, }), - makeEnvelope({ - seq: 2, + envelope(2, { event: "stage.prompt", stage_id: "verify@2", node_id: "verify", properties: { text: "second visit prompt" }, }), - makeEnvelope({ - seq: 3, + envelope(3, { event: "agent.message", stage_id: "verify@1", node_id: "verify", properties: { text: "first visit reply" }, }), - makeEnvelope({ - seq: 4, + envelope(4, { event: "agent.message", stage_id: "verify@2", node_id: "verify", @@ -61,30 +57,61 @@ describe("turnsFromEvents", () => { }), ]; - const firstVisit = turnsFromEvents(events, "verify@1"); + const firstVisit = eventsToActivity(events, "verify@1"); expect(firstVisit).toEqual([ { kind: "system", content: "first visit prompt" }, { kind: "assistant", content: "first visit reply" }, ]); - const secondVisit = turnsFromEvents(events, "verify@2"); + const secondVisit = eventsToActivity(events, "verify@2"); expect(secondVisit).toEqual([ { kind: "system", content: "second visit prompt" }, { kind: "assistant", content: "second visit reply" }, ]); }); + test("pairs command.started + command.completed into a single command turn", () => { + const events: EventEnvelope[] = [ + envelope(1, { + event: "command.started", + node_id: "fmt", + properties: { script: "cargo fmt", language: "shell" }, + }), + envelope(2, { + event: "command.completed", + node_id: "fmt", + properties: { + stdout: "ok", + stderr: "", + exit_code: 0, + duration_ms: 12, + termination: "exited", + }, + }), + ]; + + const turns = eventsToActivity(events, "fmt"); + expect(turns).toHaveLength(1); + expect(turns[0]).toMatchObject({ + kind: "command", + stageId: "fmt", + script: "cargo fmt", + language: "shell", + stdout: "ok", + exitCode: 0, + running: false, + }); + }); + test("command turn carries the requested stage_id, no @1 fallback", () => { const events: EventEnvelope[] = [ - makeEnvelope({ - seq: 1, + envelope(1, { event: "command.started", stage_id: "verify@2", node_id: "verify", properties: { script: "echo hi", language: "shell" }, }), - makeEnvelope({ - seq: 2, + envelope(2, { event: "command.completed", stage_id: "verify@2", node_id: "verify", @@ -98,7 +125,7 @@ describe("turnsFromEvents", () => { }), ]; - const turns = turnsFromEvents(events, "verify@2"); + const turns = eventsToActivity(events, "verify@2"); expect(turns).toHaveLength(1); const turn = turns[0]; expect(turn.kind).toBe("command"); @@ -108,4 +135,72 @@ describe("turnsFromEvents", () => { expect(turn.running).toBe(false); } }); + + test("pairs agent.tool.started + agent.tool.completed into a single tool turn", () => { + const events: EventEnvelope[] = [ + envelope(1, { + event: "agent.tool.started", + node_id: "detect-drift", + properties: { + tool_call_id: "call-1", + tool_name: "read_file", + arguments: { path: "config.toml" }, + }, + }), + envelope(2, { + event: "agent.tool.completed", + node_id: "detect-drift", + properties: { + tool_call_id: "call-1", + tool_name: "read_file", + output: "[redis]", + is_error: false, + }, + }), + ]; + + const turns = eventsToActivity(events, "detect-drift"); + expect(turns).toHaveLength(1); + expect(turns[0].kind).toBe("tool"); + if (turns[0].kind === "tool") { + expect(turns[0].tools).toHaveLength(1); + expect(turns[0].tools[0]).toMatchObject({ + id: "call-1", + toolName: "read_file", + result: "[redis]", + isError: false, + }); + } + }); + + test("ignores unknown event types and events for other stages", () => { + const events: EventEnvelope[] = [ + envelope(1, { + event: "stage.started", + node_id: "detect-drift", + properties: {}, + }), + envelope(2, { + event: "agent.message", + node_id: "detect-drift", + properties: { text: "signal" }, + }), + envelope(3, { + event: "run.running", + node_id: "detect-drift", + properties: {}, + }), + envelope(4, { + event: "agent.message", + node_id: "other-stage", + properties: { text: "wrong stage" }, + }), + ]; + + const turns = eventsToActivity(events, "detect-drift"); + expect(turns).toHaveLength(1); + if (turns[0].kind === "assistant") { + expect(turns[0].content).toBe("signal"); + } + }); }); diff --git a/apps/fabro-web/app/routes/run-stages.tsx b/apps/fabro-web/app/routes/run-stages.tsx index 1176cb71a..a8fc69168 100644 --- a/apps/fabro-web/app/routes/run-stages.tsx +++ b/apps/fabro-web/app/routes/run-stages.tsx @@ -40,16 +40,14 @@ import type { Stage } from "../components/stage-sidebar"; import { EmptyState } from "../components/state"; import { CopyButton } from "../components/ui"; import { formatDurationSecs } from "../lib/format"; -import { fetchRunCommandLog, useRunEventsList, useRunStageTurns, useRunStages } from "../lib/queries"; +import { fetchRunCommandLog, useRunStageEvents, useRunStages } from "../lib/queries"; +import { STAGE_ACTIVITY_EVENT_TYPES, type StageActivityEventType } from "../lib/run-events"; import { ACTIVE_STAGE_STATES, formatStageLabel, mapRunStagesToSidebarStages } from "../lib/stage-sidebar"; import { getNumber, getString, type UnknownRecord } from "../lib/unknown"; import { CommandOutputStream, CommandTermination, type EventEnvelope, - type StageTurn as ApiStageTurn, - type PaginatedStageTurnList, - type PaginatedEventList, } from "@qltysh/fabro-api-client"; export const handle = { wide: true }; @@ -68,17 +66,40 @@ function readTermination(props: UnknownRecord): CommandTermination { return CommandTermination.EXITED; } -export function turnsFromEvents(events: EventEnvelope[], stageId: string): TurnType[] { - const stageEvents = events.filter((e) => e.stage_id === stageId); +const STAGE_ACTIVITY_EVENT_SET = new Set(STAGE_ACTIVITY_EVENT_TYPES); + +function assertNever(value: never): never { + throw new Error(`Unhandled stage activity event type: ${value}`); +} + +function activityEventStageId(event: EventEnvelope): string | undefined { + if (typeof event.stage_id === "string") return event.stage_id; + if (typeof event.node_id === "string") return event.node_id; + return getString(event.properties ?? {}, "node_id"); +} + +export function eventsToActivity(events: EventEnvelope[], stageId: string): TurnType[] { const turns: TurnType[] = []; // Collect tool pairs: started → completed const pendingTools = new Map(); // Track pending command for pairing started → completed let pendingCommand: { stageId: string; script: string; language: string } | undefined; - for (const e of stageEvents) { + for (const e of events) { + const eventName = e.event; + if ( + activityEventStageId(e) !== stageId || + !eventName || + !STAGE_ACTIVITY_EVENT_SET.has(eventName) + ) { + continue; + } + // Exhaustive switch over StageActivityEventType: adding a new variant to + // STAGE_ACTIVITY_EVENT_TYPES forces a TS error here until the case is + // handled, keeping the SWR invalidation set and the reducer in sync. + const eventType = eventName as StageActivityEventType; const props = e.properties ?? {}; - switch (e.event) { + switch (eventType) { case "stage.prompt": turns.push({ kind: "system", content: getString(props, "text") ?? e.text ?? "" }); break; @@ -136,6 +157,8 @@ export function turnsFromEvents(events: EventEnvelope[], stageId: string): TurnT pendingCommand = undefined; break; } + default: + assertNever(eventType); } } @@ -153,41 +176,6 @@ export function turnsFromEvents(events: EventEnvelope[], stageId: string): TurnT return turns; } -function mapApiStageTurn(t: ApiStageTurn): TurnType { - switch (t.kind) { - case "tool": - return { - kind: "tool", - tools: (t.tools ?? []).map((tu) => ({ - id: tu.id, - toolName: tu.tool_name, - input: tu.input, - result: tu.result, - isError: tu.is_error, - durationMs: tu.duration_ms, - })), - }; - case "system": - case "assistant": - return { kind: t.kind, content: t.content ?? "" }; - } -} - -function mapTurns( - turnsResult: PaginatedStageTurnList | null | undefined, - eventsResult: PaginatedEventList | null | undefined, - selectedStageId: string | undefined, -): TurnType[] { - if (!selectedStageId) return []; - if (turnsResult?.data?.length) { - return turnsResult.data.map(mapApiStageTurn); - } - if (eventsResult?.data) { - return turnsFromEvents(eventsResult.data, selectedStageId); - } - return []; -} - function Markdown({ content }: { content: string }) { const html = useMemo(() => markedSafe.parse(content, { async: false }) as string, [content]); return ( @@ -605,14 +593,14 @@ export default function RunStages() { ); const selectedStage = stages.find((s: Stage) => s.id === stageId) ?? stages[0]; - const turnsQuery = useRunStageTurns(id, selectedStage?.id); - const hasStageTurns = (turnsQuery.data?.data.length ?? 0) > 0; - const shouldLoadEventFallback = - !!selectedStage?.id && !turnsQuery.isLoading && !turnsQuery.error && !hasStageTurns; - const eventsQuery = useRunEventsList(id, shouldLoadEventFallback); + const selectedStageId = selectedStage?.id; + const stageEventsQuery = useRunStageEvents(id, selectedStageId); const turns = useMemo( - () => mapTurns(turnsQuery.data, eventsQuery.data, selectedStage?.id), - [eventsQuery.data, selectedStage?.id, turnsQuery.data], + () => + selectedStageId + ? eventsToActivity(stageEventsQuery.data ?? [], selectedStageId) + : [], + [stageEventsQuery.data, selectedStageId], ); const isActive = selectedStage ? ACTIVE_STAGE_STATES.has(selectedStage.status) : false; diff --git a/docs/public/api-reference/fabro-api.yaml b/docs/public/api-reference/fabro-api.yaml index a767a7262..4319f6774 100644 --- a/docs/public/api-reference/fabro-api.yaml +++ b/docs/public/api-reference/fabro-api.yaml @@ -1906,26 +1906,26 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" - /api/v1/runs/{id}/stages/{stageId}/turns: + /api/v1/runs/{id}/stages/{stageId}/events: get: - operationId: listStageTurns + operationId: listStageEvents tags: [Run Internals] - summary: List Stage Turns - description: Returns a paginated list of conversation turns within a specific stage, including system prompts, assistant responses, and tool invocations. + summary: List Stage Events + description: Returns a paginated JSON list of stored run events scoped to a single stage visit. parameters: - $ref: "#/components/parameters/RunId" - $ref: "#/components/parameters/StageId" - - $ref: "#/components/parameters/PageLimit" - - $ref: "#/components/parameters/PageOffset" + - $ref: "#/components/parameters/SinceSeq" + - $ref: "#/components/parameters/EventLimit" responses: "200": - description: Paginated list of conversation turns + description: Paginated list of stage events content: application/json: schema: - $ref: "#/components/schemas/PaginatedStageTurnList" + $ref: "#/components/schemas/PaginatedEventList" "404": - description: Run or stage not found + description: Run not found. headers: x-request-id: $ref: "#/components/headers/XRequestId" @@ -3856,20 +3856,6 @@ components: meta: $ref: "#/components/schemas/PaginationMeta" - PaginatedStageTurnList: - description: Paginated list of stage turns. - type: object - required: - - data - - meta - properties: - data: - type: array - items: - $ref: "#/components/schemas/StageTurn" - meta: - $ref: "#/components/schemas/PaginationMeta" - PaginatedApiQuestionList: description: Paginated list of pending questions. type: object @@ -6348,103 +6334,6 @@ components: description: 1-based visit count; bumped each time the workflow re-enters this node. example: 2 - ToolUse: - description: A single tool invocation with its input, result, and execution metadata. - type: object - required: - - id - - tool_name - - input - - result - - is_error - properties: - id: - type: string - description: Unique identifier for this tool invocation. Enables correlation in parallel tool use. - example: toolu_01A09q90qw90lq917835lq9 - tool_name: - type: string - description: Name of the tool that was invoked. - example: read_file - input: - type: string - description: JSON-encoded input passed to the tool. - example: '{ "path": "src/routes/auth.ts" }' - result: - type: string - description: Output returned by the tool. Contains the error message when is_error is true. - example: 'import { Router } from "express";' - is_error: - type: boolean - description: Whether the tool invocation failed. When true, the result field contains the error message. - example: false - duration_ms: - type: integer - description: Wall-clock execution time of the tool invocation in milliseconds. - example: 142 - - StageTurn: - description: A single turn in a stage conversation — a system prompt, assistant response, or tool invocation block. - discriminator: - propertyName: kind - mapping: - system: "#/components/schemas/SystemStageTurn" - assistant: "#/components/schemas/AssistantStageTurn" - tool: "#/components/schemas/ToolStageTurn" - oneOf: - - $ref: "#/components/schemas/SystemStageTurn" - - $ref: "#/components/schemas/AssistantStageTurn" - - $ref: "#/components/schemas/ToolStageTurn" - - SystemStageTurn: - description: A system prompt turn that sets the stage's instructions. - type: object - required: - - kind - - content - properties: - kind: - type: string - enum: [system] - content: - type: string - description: System prompt text. - example: You are a drift detection agent. Compare the production and staging environments. - - AssistantStageTurn: - description: An assistant response turn within a stage. - type: object - required: - - kind - - content - properties: - kind: - type: string - enum: [assistant] - content: - type: string - description: Assistant response text. - example: I'll start by loading the environment configurations for both production and staging. - - ToolStageTurn: - description: A tool invocation turn containing one or more tool calls. - type: object - required: - - kind - - tools - properties: - kind: - type: string - enum: [tool] - content: - type: string - description: Text accompanying the tool invocations, or null when the turn contains only tool calls. - tools: - type: array - description: Tool invocations executed in this turn. - items: - $ref: "#/components/schemas/ToolUse" - # ── File Diff Schemas ────────────────────────────────────────────── FileCheckpoint: diff --git a/lib/crates/fabro-server/src/demo/mod.rs b/lib/crates/fabro-server/src/demo/mod.rs index bc527007d..c900c8b91 100644 --- a/lib/crates/fabro-server/src/demo/mod.rs +++ b/lib/crates/fabro-server/src/demo/mod.rs @@ -15,15 +15,16 @@ use axum::http::StatusCode; use axum::response::sse::{Event, Sse}; use axum::response::{IntoResponse, Response}; use fabro_api::types::{ - CreateSecretRequest, DeleteSecretRequest, DiffFile, DiffStats, FileDiff, FileDiffChangeKind, - PaginatedRunFileList, RunArtifactListResponse, RunFilesMeta, + CreateSecretRequest, DeleteSecretRequest, DiffFile, DiffStats, EventEnvelope, FileDiff, + FileDiffChangeKind, PaginatedEventList, PaginatedRunFileList, PaginationMeta, + RunArtifactListResponse, RunFilesMeta, }; use serde_json::json; use crate::error::ApiError; use crate::principal_middleware::RequiredUser; use crate::run_selector::{ResolveRunError, resolve_run_by_selector}; -use crate::server::{AppState, PaginationParams}; +use crate::server::{AppState, EventListParams, PaginationParams, parse_stage_id_path}; fn paginated_response( items: Vec, @@ -133,13 +134,39 @@ pub(crate) async fn get_run_stages( paginated_response(runs::stages(), &pagination) } -pub(crate) async fn get_stage_turns( +pub(crate) async fn get_stage_events( _auth: RequiredUser, State(_state): State>, - Path((_id, _stage_id)): Path<(String, String)>, - Query(pagination): Query, + Path((_id, stage_id)): Path<(String, String)>, + Query(params): Query, ) -> Response { - paginated_response(runs::turns(), &pagination) + let stage_id = match parse_stage_id_path(&stage_id) { + Ok(stage_id) => stage_id, + Err(response) => return response, + }; + let since_seq = params.since_seq(); + let limit = params.limit(); + let mut matches: Vec = runs::stage_events() + .into_iter() + .filter(|envelope| { + envelope.seq >= since_seq + && (envelope.event.stage_id.as_ref() == Some(&stage_id) + || (envelope.event.stage_id.is_none() + && stage_id.visit() == 1 + && envelope.event.node_id.as_deref() == Some(stage_id.node_id()))) + }) + .take(limit + 1) + .collect(); + let has_more = matches.len() > limit; + matches.truncate(limit); + ( + StatusCode::OK, + Json(PaginatedEventList { + data: matches, + meta: PaginationMeta { has_more }, + }), + ) + .into_response() } pub(crate) async fn list_run_artifacts_stub( @@ -1215,18 +1242,114 @@ mod runs { ] } - pub(super) fn turns() -> Vec { + pub(super) fn stage_events() -> Vec { + use fabro_model::BilledTokenCounts; + use fabro_types::run_event::agent::{ + AgentMessageProps, AgentToolCompletedProps, AgentToolStartedProps, + }; + use fabro_types::run_event::stage::StagePromptProps; + use fabro_types::{EventBody, EventEnvelope, RunEvent}; + + let run_id = demo_run_id(1); + let node_id = "detect-drift"; + let stage_id = fabro_types::StageId::new(node_id, 1); + let ts = ts("2026-03-06T14:30:00Z"); + + let make_envelope = |seq: u32, id: &str, body: EventBody| EventEnvelope { + seq, + event: RunEvent { + id: id.into(), + ts, + run_id, + node_id: Some(node_id.into()), + node_label: Some("Detect Drift".into()), + stage_id: Some(stage_id.clone()), + parallel_group_id: None, + parallel_branch_id: None, + session_id: None, + parent_session_id: None, + tool_call_id: None, + actor: None, + body, + }, + }; + vec![ - StageTurn::SystemStageTurn(SystemStageTurn { kind: SystemStageTurnKind::System, content: "You are a drift detection agent. Compare the production and staging environments and identify any configuration or code drift.".into() }), - StageTurn::AssistantStageTurn(AssistantStageTurn { kind: AssistantStageTurnKind::Assistant, content: "I'll start by loading the environment configurations for both production and staging to compare them.".into() }), - StageTurn::ToolStageTurn(ToolStageTurn { - kind: ToolStageTurnKind::Tool, content: None, - tools: vec![ - ToolUse { id: "toolu_01".into(), tool_name: "read_file".into(), input: r#"{ "path": "environments/production/config.toml" }"#.into(), result: "[redis]\nhost = \"redis-prod.internal\"\nport = 6379".into(), is_error: false, duration_ms: Some(45) }, - ToolUse { id: "toolu_02".into(), tool_name: "read_file".into(), input: r#"{ "path": "environments/staging/config.toml" }"#.into(), result: "[redis]\nhost = \"redis-staging.internal\"\nport = 6379".into(), is_error: false, duration_ms: Some(38) }, - ], - }), - StageTurn::AssistantStageTurn(AssistantStageTurn { kind: AssistantStageTurnKind::Assistant, content: "I've detected drift in 3 resources between production and staging:\n\n1. **redis.max_connections** — production has 200, staging has 100\n2. **redis.tls** — enabled in production, disabled in staging\n3. **iam.session_duration** — production uses 3600s, staging uses 1800s".into() }), + make_envelope( + 1, + "evt-detect-drift-1", + EventBody::StagePrompt(StagePromptProps { + visit: 1, + text: "You are a drift detection agent. Compare the production and staging environments and identify any configuration or code drift.".into(), + mode: None, + provider: None, + model: None, + }), + ), + make_envelope( + 2, + "evt-detect-drift-2", + EventBody::AgentMessage(AgentMessageProps { + text: "I'll start by loading the environment configurations for both production and staging to compare them.".into(), + model: "Opus 4.6".into(), + billing: BilledTokenCounts::default(), + tool_call_count: 0, + visit: 1, + }), + ), + make_envelope( + 3, + "evt-detect-drift-3", + EventBody::AgentToolStarted(AgentToolStartedProps { + tool_name: "read_file".into(), + tool_call_id: "toolu_01".into(), + arguments: serde_json::json!({ "path": "environments/production/config.toml" }), + visit: 1, + }), + ), + make_envelope( + 4, + "evt-detect-drift-4", + EventBody::AgentToolCompleted(AgentToolCompletedProps { + tool_name: "read_file".into(), + tool_call_id: "toolu_01".into(), + output: serde_json::json!("[redis]\nhost = \"redis-prod.internal\"\nport = 6379"), + is_error: false, + visit: 1, + }), + ), + make_envelope( + 5, + "evt-detect-drift-5", + EventBody::AgentToolStarted(AgentToolStartedProps { + tool_name: "read_file".into(), + tool_call_id: "toolu_02".into(), + arguments: serde_json::json!({ "path": "environments/staging/config.toml" }), + visit: 1, + }), + ), + make_envelope( + 6, + "evt-detect-drift-6", + EventBody::AgentToolCompleted(AgentToolCompletedProps { + tool_name: "read_file".into(), + tool_call_id: "toolu_02".into(), + output: serde_json::json!("[redis]\nhost = \"redis-staging.internal\"\nport = 6379"), + is_error: false, + visit: 1, + }), + ), + make_envelope( + 7, + "evt-detect-drift-7", + EventBody::AgentMessage(AgentMessageProps { + text: "I've detected drift in 3 resources between production and staging:\n\n1. **redis.max_connections** — production has 200, staging has 100\n2. **redis.tls** — enabled in production, disabled in staging\n3. **iam.session_duration** — production uses 3600s, staging uses 1800s".into(), + model: "Opus 4.6".into(), + billing: BilledTokenCounts::default(), + tool_call_count: 0, + visit: 1, + }), + ), ] } diff --git a/lib/crates/fabro-server/src/principal_middleware.rs b/lib/crates/fabro-server/src/principal_middleware.rs index e8971831c..822d87dc6 100644 --- a/lib/crates/fabro-server/src/principal_middleware.rs +++ b/lib/crates/fabro-server/src/principal_middleware.rs @@ -52,6 +52,7 @@ pub(crate) struct RequestAuth(pub(crate) AuthContextSlot); pub(crate) struct RequiredUser(pub(crate) UserPrincipal); pub(crate) struct RequireRunScoped(pub(crate) RunId); pub(crate) struct RequireRunBlob(pub(crate) RunId, pub(crate) RunBlobId); +pub(crate) struct RequireRunStageScoped(pub(crate) RunId, pub(crate) String); pub(crate) struct RequireStageArtifact(pub(crate) RunId, pub(crate) StageId); pub(crate) struct RequireCommandLog( pub(crate) RunId, @@ -203,6 +204,23 @@ impl FromRequestParts> for RequireRunBlob { } } +impl FromRequestParts> for RequireRunStageScoped { + type Rejection = Response; + + async fn from_request_parts( + parts: &mut Parts, + state: &Arc, + ) -> Result { + let Path((id, stage_id)): Path<(String, String)> = Path::from_request_parts(parts, state) + .await + .map_err(IntoResponse::into_response)?; + let run_id = parse_run_id_path(&id)?; + require_worker_or_user_for_run(&auth_slot_from_parts(parts), &run_id) + .map_err(IntoResponse::into_response)?; + Ok(Self(run_id, stage_id)) + } +} + impl FromRequestParts> for RequireStageArtifact { type Rejection = Response; diff --git a/lib/crates/fabro-server/src/server.rs b/lib/crates/fabro-server/src/server.rs index 26fbdd8d6..965989c8c 100644 --- a/lib/crates/fabro-server/src/server.rs +++ b/lib/crates/fabro-server/src/server.rs @@ -125,7 +125,7 @@ use crate::ip_allowlist::{IpAllowlistConfig, ip_allowlist_middleware}; use crate::jwt_auth::{self, AuthMode}; use crate::principal_middleware::{ AuthContextSlot, RequestAuth, RequestAuthContext, RequireRunBlob, RequireRunScoped, - RequireStageArtifact, RequiredUser, principal_middleware, + RequireRunStageScoped, RequireStageArtifact, RequiredUser, principal_middleware, }; use crate::request_id::{self, RequestId}; use crate::run_files::{FilesInFlight, new_files_in_flight}; @@ -138,6 +138,7 @@ use crate::{ mod handler; +pub(crate) use handler::events::EventListParams; #[cfg(test)] pub(in crate::server) use handler::events::filtered_global_events; pub(crate) use handler::graph::render_graph_bytes; diff --git a/lib/crates/fabro-server/src/server/handler/events.rs b/lib/crates/fabro-server/src/server/handler/events.rs index ff3889cd5..fce4a4d11 100644 --- a/lib/crates/fabro-server/src/server/handler/events.rs +++ b/lib/crates/fabro-server/src/server/handler/events.rs @@ -3,9 +3,10 @@ use std::sync::Arc; use super::super::{ ApiError, AppState, AppendEventResponse, BroadcastStream, Event, EventBody, EventEnvelope, EventPayload, HashSet, IntoResponse, Json, KeepAlive, PaginatedEventList, PaginationMeta, Path, - Query, RequireRunScoped, RequiredUser, Response, Router, RunEvent, RunId, RunStatus, Sse, - State, StatusCode, StreamExt, UnboundedReceiverStream, broadcast, get, mpsc, parse_run_id_path, - redact_jsonl_line, reject_if_archived, update_live_run_from_event, + Query, RequireRunScoped, RequireRunStageScoped, RequiredUser, Response, Router, RunEvent, + RunId, RunStatus, Sse, State, StatusCode, StreamExt, UnboundedReceiverStream, broadcast, get, + mpsc, parse_run_id_path, parse_stage_id_path, redact_jsonl_line, reject_if_archived, + update_live_run_from_event, }; pub(super) fn routes() -> Router> { @@ -15,11 +16,15 @@ pub(super) fn routes() -> Router> { "/runs/{id}/events", get(list_run_events).post(append_run_event), ) + .route( + "/runs/{id}/stages/{stageId}/events", + get(list_run_stage_events), + ) .route("/runs/{id}/attach", get(attach_run_events)) } #[derive(serde::Deserialize)] -struct EventListParams { +pub(crate) struct EventListParams { #[serde(default)] since_seq: Option, #[serde(default)] @@ -27,11 +32,11 @@ struct EventListParams { } impl EventListParams { - fn since_seq(&self) -> u32 { + pub(crate) fn since_seq(&self) -> u32 { self.since_seq.unwrap_or(1).max(1) } - fn limit(&self) -> usize { + pub(crate) fn limit(&self) -> usize { self.limit.unwrap_or(100).clamp(1, 1000) } } @@ -200,6 +205,39 @@ async fn list_run_events( } } +async fn list_run_stage_events( + RequireRunStageScoped(id, stage_id): RequireRunStageScoped, + State(state): State>, + Query(params): Query, +) -> Response { + let stage_id = match parse_stage_id_path(&stage_id) { + Ok(stage_id) => stage_id, + Err(response) => return response, + }; + let since_seq = params.since_seq(); + let limit = params.limit(); + match state.store.open_run_reader(&id).await { + Ok(run_store) => match run_store + .list_events_for_stage_from_with_limit(&stage_id, since_seq, limit) + .await + { + Ok(mut events) => { + let has_more = events.len() > limit; + events.truncate(limit); + Json(PaginatedEventList { + data: events, + meta: PaginationMeta { has_more }, + }) + .into_response() + } + Err(err) => { + ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() + } + }, + Err(_) => ApiError::not_found("Run not found.").into_response(), + } +} + async fn attach_run_events( _auth: RequiredUser, State(state): State>, @@ -342,3 +380,264 @@ fn denied_lifecycle_event_name(body: &EventBody) -> Option<&'static str> { _ => None, } } + +#[cfg(test)] +mod stage_events_tests { + use axum::body::{Body, to_bytes}; + use axum::http::{Request, StatusCode, header}; + use fabro_store::EventPayload; + use fabro_types::RunId; + use serde_json::json; + use tower::ServiceExt; + + use crate::test_support::{build_test_router, test_app_state}; + + fn req_get(uri: &str) -> Request { + Request::builder() + .method("GET") + .uri(uri) + .body(Body::empty()) + .expect("stage events GET request should build") + } + + fn make_event(run_id: &RunId, idx: u32, node_id: Option<&str>) -> EventPayload { + make_event_with_stage_id(run_id, idx, node_id, None) + } + + fn make_event_with_stage_id( + run_id: &RunId, + idx: u32, + node_id: Option<&str>, + stage_id: Option<&str>, + ) -> EventPayload { + let mut value = json!({ + "id": format!("evt-{idx}"), + "ts": "2026-04-09T12:00:00Z", + "run_id": run_id.to_string(), + "event": "stage.prompt", + "properties": { + "visit": 1, + "text": format!("prompt {idx}"), + }, + }); + if let Some(node) = node_id { + value + .as_object_mut() + .unwrap() + .insert("node_id".into(), json!(node)); + } + if let Some(stage_id) = stage_id { + value + .as_object_mut() + .unwrap() + .insert("stage_id".into(), json!(stage_id)); + } + EventPayload::new(value, run_id).expect("event payload should validate") + } + + async fn body_json(response: axum::response::Response) -> serde_json::Value { + let bytes = to_bytes(response.into_body(), usize::MAX) + .await + .expect("response body should fit in memory"); + serde_json::from_slice(&bytes).expect("response body should be valid JSON") + } + + async fn seed_run_with_mixed_events() -> (RunId, axum::Router) { + let state = test_app_state(); + let app = build_test_router(state.clone()); + let run_id = RunId::new(); + let run_store = state + .store_ref() + .create_run(&run_id) + .await + .expect("test run should be creatable"); + + // Seed 200 unrelated 'beta' events first so any node-blind + // truncation would lose the sparse 'alpha' tail. Then 3 'alpha' + // events past seq 100, plus a couple with no node_id at all. + for idx in 1..=200_u32 { + run_store + .append_event(&make_event(&run_id, idx, Some("beta"))) + .await + .expect("append should succeed"); + } + run_store + .append_event(&make_event(&run_id, 201, None)) + .await + .expect("append should succeed"); + for idx in 202..=204_u32 { + run_store + .append_event(&make_event(&run_id, idx, Some("alpha"))) + .await + .expect("append should succeed"); + } + + (run_id, app) + } + + #[tokio::test] + async fn returns_only_matching_node_events_in_seq_order() { + let (run_id, app) = seed_run_with_mixed_events().await; + let response = app + .oneshot(req_get(&format!( + "/api/v1/runs/{run_id}/stages/alpha@1/events" + ))) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let body = body_json(response).await; + let data = body["data"].as_array().expect("data is array"); + let seqs: Vec = data.iter().map(|e| e["seq"].as_u64().unwrap()).collect(); + assert_eq!(seqs, vec![202, 203, 204]); + assert_eq!(body["meta"]["has_more"], false); + } + + #[tokio::test] + async fn since_seq_filters_to_events_with_seq_at_least_k() { + let (run_id, app) = seed_run_with_mixed_events().await; + let response = app + .oneshot(req_get(&format!( + "/api/v1/runs/{run_id}/stages/alpha@1/events?since_seq=203" + ))) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let body = body_json(response).await; + let seqs: Vec = body["data"] + .as_array() + .unwrap() + .iter() + .map(|e| e["seq"].as_u64().unwrap()) + .collect(); + assert_eq!(seqs, vec![203, 204]); + } + + #[tokio::test] + async fn limit_one_returns_first_envelope_with_has_more_true() { + let (run_id, app) = seed_run_with_mixed_events().await; + let response = app + .oneshot(req_get(&format!( + "/api/v1/runs/{run_id}/stages/alpha@1/events?limit=1" + ))) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let body = body_json(response).await; + let data = body["data"].as_array().unwrap(); + assert_eq!(data.len(), 1); + assert_eq!(data[0]["seq"].as_u64().unwrap(), 202); + assert_eq!(body["meta"]["has_more"], true); + } + + #[tokio::test] + async fn unknown_stage_in_existing_run_returns_empty_list_with_no_more() { + let (run_id, app) = seed_run_with_mixed_events().await; + let response = app + .oneshot(req_get(&format!( + "/api/v1/runs/{run_id}/stages/unknown-stage@1/events" + ))) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let body = body_json(response).await; + assert_eq!(body["data"].as_array().unwrap().len(), 0); + assert_eq!(body["meta"]["has_more"], false); + } + + #[tokio::test] + async fn missing_run_returns_404_with_run_not_found() { + let app = build_test_router(test_app_state()); + // A syntactically valid RunId that the store has never seen, so + // `parse_run_id_path` succeeds but `open_run_reader` fails — that + // exercises the handler's not-found branch rather than the path + // parser's 400 branch. + let absent = RunId::new(); + let response = app + .oneshot(req_get(&format!( + "/api/v1/runs/{absent}/stages/alpha@1/events" + ))) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::NOT_FOUND); + + let body = body_json(response).await; + let detail = body["errors"][0]["detail"] + .as_str() + .expect("error detail string"); + assert!( + detail.contains("Run not found."), + "unexpected error body: {body}" + ); + } + + #[tokio::test] + async fn unauthenticated_request_is_rejected() { + let state = test_app_state(); + // Bypass `build_test_router`'s auto-injected bearer token by + // building the raw router directly. The principal middleware sees + // a missing Authorization header and the extractor enforces auth. + let app = crate::server::build_router(state, crate::test_support::test_auth_mode()); + let run_id = RunId::new(); + + let request = Request::builder() + .method("GET") + .uri(format!("/api/v1/runs/{run_id}/stages/alpha@1/events")) + .header(header::ACCEPT, "application/json") + .body(Body::empty()) + .unwrap(); + let response = app.oneshot(request).await.unwrap(); + + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + } + + #[tokio::test] + async fn returns_only_requested_visit_when_stage_id_is_present() { + let state = test_app_state(); + let app = build_test_router(state.clone()); + let run_id = RunId::new(); + let run_store = state + .store_ref() + .create_run(&run_id) + .await + .expect("test run should be creatable"); + run_store + .append_event(&make_event_with_stage_id( + &run_id, + 1, + Some("verify"), + Some("verify@1"), + )) + .await + .expect("append should succeed"); + run_store + .append_event(&make_event_with_stage_id( + &run_id, + 2, + Some("verify"), + Some("verify@2"), + )) + .await + .expect("append should succeed"); + + let response = app + .oneshot(req_get(&format!( + "/api/v1/runs/{run_id}/stages/verify@2/events" + ))) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let body = body_json(response).await; + let seqs: Vec = body["data"] + .as_array() + .unwrap() + .iter() + .map(|e| e["seq"].as_u64().unwrap()) + .collect(); + assert_eq!(seqs, vec![2]); + } +} diff --git a/lib/crates/fabro-server/src/server/handler/mod.rs b/lib/crates/fabro-server/src/server/handler/mod.rs index d2625395b..79a3d8246 100644 --- a/lib/crates/fabro-server/src/server/handler/mod.rs +++ b/lib/crates/fabro-server/src/server/handler/mod.rs @@ -57,8 +57,8 @@ pub(super) fn demo_routes() -> Router> { .route("/runs/{id}/artifacts", get(demo::list_run_artifacts_stub)) .route("/runs/{id}/files", get(demo::list_run_files_stub)) .route( - "/runs/{id}/stages/{stageId}/turns", - get(demo::get_stage_turns), + "/runs/{id}/stages/{stageId}/events", + get(demo::get_stage_events), ) .route( "/runs/{id}/stages/{stageId}/artifacts", @@ -113,7 +113,6 @@ pub(super) fn demo_routes() -> Router> { pub(super) fn real_routes() -> Router> { Router::new() - .route("/runs/{id}/stages/{stageId}/turns", get(not_implemented)) .route("/runs/{id}/steer", post(not_implemented)) .route("/workflows", get(not_implemented)) .route("/workflows/{name}", get(not_implemented)) diff --git a/lib/crates/fabro-server/tests/it/event_pagination.rs b/lib/crates/fabro-server/tests/it/event_pagination.rs new file mode 100644 index 000000000..cd6ceec94 --- /dev/null +++ b/lib/crates/fabro-server/tests/it/event_pagination.rs @@ -0,0 +1,75 @@ +//! Cursor-pagination tests for the per-stage events endpoint (demo mode). +//! +//! The stage-events route uses `since_seq=` + `limit=` (cursor-based) instead +//! of the offset-based `page[limit]/page[offset]` pagination used by other +//! list endpoints, so it gets its own test rather than living in the generic +//! offset-shape matrix. + +#![allow( + clippy::absolute_paths, + reason = "This test module prefers explicit type paths over extra imports." +)] + +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use tower::ServiceExt; + +use super::helpers::{response_json, test_app_state}; + +async fn get_json(app: &axum::Router, uri: &str) -> serde_json::Value { + let req = Request::builder() + .method("GET") + .uri(uri) + .header("x-fabro-demo", "1") + .body(Body::empty()) + .expect("event pagination request should build"); + let response = app.clone().oneshot(req).await.unwrap(); + response_json(response, StatusCode::OK, format!("GET {uri}")).await +} + +#[tokio::test] +async fn demo_stage_events_default_returns_all_fixture_events_with_no_more() { + let app = fabro_server::test_support::build_test_router(test_app_state()); + + let body = get_json(&app, "/api/v1/runs/run-1/stages/detect-drift@1/events").await; + let data = body["data"].as_array().expect("data is an array"); + + assert_eq!(data.len(), 7, "all seven fixture events should be returned"); + assert_eq!(body["meta"]["has_more"], false); +} + +#[tokio::test] +async fn demo_stage_events_limit_one_signals_has_more() { + let app = fabro_server::test_support::build_test_router(test_app_state()); + + let body = get_json( + &app, + "/api/v1/runs/run-1/stages/detect-drift@1/events?limit=1", + ) + .await; + let data = body["data"].as_array().expect("data is an array"); + + assert_eq!(data.len(), 1); + assert_eq!(body["meta"]["has_more"], true); +} + +#[tokio::test] +async fn demo_stage_events_since_seq_filters_out_earlier_events() { + let app = fabro_server::test_support::build_test_router(test_app_state()); + + // The fixture seqs are 1..=7. since_seq=4 should skip the first three. + let body = get_json( + &app, + "/api/v1/runs/run-1/stages/detect-drift@1/events?since_seq=4", + ) + .await; + let data = body["data"].as_array().expect("data is an array"); + + assert_eq!(data.len(), 4); + let seqs: Vec = data + .iter() + .map(|envelope| envelope["seq"].as_u64().expect("seq is a number")) + .collect(); + assert_eq!(seqs, vec![4, 5, 6, 7]); + assert_eq!(body["meta"]["has_more"], false); +} diff --git a/lib/crates/fabro-server/tests/it/main.rs b/lib/crates/fabro-server/tests/it/main.rs index 9e5ddab9f..865bb8132 100644 --- a/lib/crates/fabro-server/tests/it/main.rs +++ b/lib/crates/fabro-server/tests/it/main.rs @@ -4,6 +4,7 @@ )] mod api; +mod event_pagination; mod helpers; mod openapi_conformance; mod pagination; diff --git a/lib/crates/fabro-server/tests/it/pagination.rs b/lib/crates/fabro-server/tests/it/pagination.rs index 64174fd78..f773ac207 100644 --- a/lib/crates/fabro-server/tests/it/pagination.rs +++ b/lib/crates/fabro-server/tests/it/pagination.rs @@ -56,10 +56,6 @@ const ENDPOINTS: &[PaginatedEndpoint] = &[ path: "/api/v1/models", name: "listModels", }, - PaginatedEndpoint { - path: "/api/v1/runs/run-1/stages/detect-drift/turns", - name: "listStageTurns", - }, PaginatedEndpoint { path: "/api/v1/runs/run-1/questions", name: "listRunQuestions", diff --git a/lib/crates/fabro-store/src/slate/run_store.rs b/lib/crates/fabro-store/src/slate/run_store.rs index a334281b9..f28e52341 100644 --- a/lib/crates/fabro-store/src/slate/run_store.rs +++ b/lib/crates/fabro-store/src/slate/run_store.rs @@ -12,7 +12,7 @@ use tokio_stream::wrappers::UnboundedReceiverStream; use super::blob_store::BlobStore; use crate::run_state::{EventProjectionCache, RunProjectionReducer, build_summary}; -use crate::{Error, EventEnvelope, EventPayload, Result, RunProjection, keys}; +use crate::{Error, EventEnvelope, EventPayload, Result, RunProjection, StageId, keys}; const DEFAULT_EVENT_TAIL_LIMIT: usize = 1024; #[derive(Clone)] @@ -213,6 +213,30 @@ impl RunDatabase { list_events_from_with_limit(&self.inner.db, &self.inner.run_id, start_seq, limit).await } + /// Returns up to `limit + 1` events for the given stage visit, + /// starting at `start_seq`. The `+1` lets callers compute `has_more`. + /// + /// Implementation note: scans the unbounded run-event prefix and + /// filters by stage identity *before* applying `limit`, so a stage with + /// matches sparsely scattered late in the event log still returns its + /// full slice (no premature truncation from a generic `limit`-bounded + /// scan). + pub async fn list_events_for_stage_from_with_limit( + &self, + stage_id: &StageId, + start_seq: u32, + limit: usize, + ) -> Result> { + list_events_for_stage_from_with_limit( + &self.inner.db, + &self.inner.run_id, + stage_id, + start_seq, + limit, + ) + .await + } + pub fn watch_events_from( &self, seq: u32, @@ -348,6 +372,74 @@ where Ok(events) } +async fn list_events_for_stage_from_with_limit( + db: &R, + run_id: &RunId, + stage_id: &StageId, + start_seq: u32, + limit: usize, +) -> Result> +where + R: DbRead + Sync, +{ + // Unbounded scan first: filtering by stage identity with a generic + // limit-bounded scan would silently drop matches whenever the stage's + // events are sparse late in the event log. + // + // We probe just the stage identity fields with a small partial deserialize and + // only run the full `RunEvent` parse on matches. Most events in a run + // belong to other nodes, so this avoids deserializing large payloads + // (`agent.tool.completed.output`, `agent.message.text`, …) we'd discard. + #[derive(serde::Deserialize)] + struct StageIdProbe<'a> { + #[serde(default, borrow)] + stage_id: Option<&'a str>, + #[serde(default, borrow)] + node_id: Option<&'a str>, + } + + let stage_id_string = stage_id.to_string(); + let max_events = limit.saturating_add(1); + let mut iter = db.scan_prefix(keys::run_events_prefix(run_id)).await?; + let mut events: Vec = Vec::new(); + while let Some(entry) = iter.next().await? { + let key = key_to_string(&entry.key)?; + let Some(seq) = keys::parse_event_seq(&key) else { + continue; + }; + if seq < start_seq { + continue; + } + let probe: StageIdProbe = serde_json::from_slice(&entry.value)?; + let matches_stage_id = probe.stage_id == Some(stage_id_string.as_str()); + let matches_legacy_node_id = probe.stage_id.is_none() + && stage_id.visit() == 1 + && probe.node_id == Some(stage_id.node_id()); + if !matches_stage_id && !matches_legacy_node_id { + continue; + } + let event: RunEvent = serde_json::from_slice(&entry.value)?; + let envelope = EventEnvelope { seq, event }; + if events.len() < max_events { + events.push(envelope); + continue; + } + + if let Some((max_index, max_seq)) = events + .iter() + .enumerate() + .max_by_key(|(_, existing)| existing.seq) + .map(|(index, existing)| (index, existing.seq)) + { + if seq < max_seq { + events[max_index] = envelope; + } + } + } + events.sort_by_key(|event| event.seq); + Ok(events) +} + async fn list_blobs(db: &R) -> Result> where R: DbRead + Sync, @@ -375,9 +467,12 @@ mod tests { use std::sync::Arc; use std::time::Duration; + use fabro_types::{RunId, StageId}; use object_store::memory::InMemory; + use serde_json::json; + + use crate::{Database, EventPayload}; - use crate::Database; #[tokio::test] async fn list_blobs_reads_global_cas_namespace() { let object_store = Arc::new(InMemory::new()); @@ -394,4 +489,190 @@ mod tests { assert_eq!(blob_ids, vec![first_id, second_id]); } + + fn stage_prompt_payload(run_id: &RunId, idx: u32, node_id: Option<&str>) -> EventPayload { + stage_prompt_payload_for_stage(run_id, idx, node_id, None) + } + + fn stage_prompt_payload_for_stage( + run_id: &RunId, + idx: u32, + node_id: Option<&str>, + stage_id: Option<&StageId>, + ) -> EventPayload { + let mut value = json!({ + "id": format!("evt-{idx}"), + "ts": "2026-04-09T12:00:00Z", + "run_id": run_id.to_string(), + "event": "stage.prompt", + "properties": { + "visit": 1, + "text": format!("prompt {idx}"), + }, + }); + if let Some(node_id) = node_id { + value + .as_object_mut() + .unwrap() + .insert("node_id".into(), json!(node_id)); + } + if let Some(stage_id) = stage_id { + value + .as_object_mut() + .unwrap() + .insert("stage_id".into(), json!(stage_id.to_string())); + } + EventPayload::new(value, run_id).unwrap() + } + + async fn fresh_run() -> super::RunDatabase { + let object_store = Arc::new(InMemory::new()); + let store = Database::new(object_store, "", Duration::from_millis(1), None); + let run_id: RunId = "01JT56VE4Z5NZ814GZN2JZD65A".parse().unwrap(); + store.create_run(&run_id).await.unwrap() + } + + #[tokio::test] + async fn list_events_for_stage_returns_only_matching_events_in_seq_order() { + let run = fresh_run().await; + let run_id = run.run_id(); + run.append_event(&stage_prompt_payload(&run_id, 1, Some("alpha"))) + .await + .unwrap(); + run.append_event(&stage_prompt_payload(&run_id, 2, Some("beta"))) + .await + .unwrap(); + run.append_event(&stage_prompt_payload(&run_id, 3, Some("alpha"))) + .await + .unwrap(); + + let events = run + .list_events_for_stage_from_with_limit(&StageId::new("alpha", 1), 1, 100) + .await + .unwrap(); + + let seqs: Vec = events.iter().map(|e| e.seq).collect(); + assert_eq!(seqs, vec![1, 3]); + } + + #[tokio::test] + async fn list_events_for_stage_skips_events_with_no_stage_identity() { + let run = fresh_run().await; + let run_id = run.run_id(); + run.append_event(&stage_prompt_payload(&run_id, 1, None)) + .await + .unwrap(); + run.append_event(&stage_prompt_payload(&run_id, 2, Some("alpha"))) + .await + .unwrap(); + + let events = run + .list_events_for_stage_from_with_limit(&StageId::new("alpha", 1), 1, 100) + .await + .unwrap(); + + let seqs: Vec = events.iter().map(|e| e.seq).collect(); + assert_eq!(seqs, vec![2]); + } + + #[tokio::test] + async fn list_events_for_stage_paginates_via_start_seq_on_filtered_slice() { + let run = fresh_run().await; + let run_id = run.run_id(); + for idx in 1..=5 { + let node = if idx % 2 == 0 { "beta" } else { "alpha" }; + run.append_event(&stage_prompt_payload(&run_id, idx, Some(node))) + .await + .unwrap(); + } + + // alpha events live at seqs 1, 3, 5. Start at seq=2 should skip seq=1. + let events = run + .list_events_for_stage_from_with_limit(&StageId::new("alpha", 1), 2, 100) + .await + .unwrap(); + + let seqs: Vec = events.iter().map(|e| e.seq).collect(); + assert_eq!(seqs, vec![3, 5]); + } + + #[tokio::test] + async fn list_events_for_stage_walks_past_unrelated_events_for_sparse_matches() { + let run = fresh_run().await; + let run_id = run.run_id(); + // 200 unrelated events first. + for idx in 1..=200 { + run.append_event(&stage_prompt_payload(&run_id, idx, Some("noise"))) + .await + .unwrap(); + } + // Then 3 sparse "alpha" events at the tail. + for idx in 201..=203 { + run.append_event(&stage_prompt_payload(&run_id, idx, Some("alpha"))) + .await + .unwrap(); + } + + // limit smaller than the number of unrelated events would have + // truncated the upstream scan if we had post-filtered. + let events = run + .list_events_for_stage_from_with_limit(&StageId::new("alpha", 1), 1, 5) + .await + .unwrap(); + + let seqs: Vec = events.iter().map(|e| e.seq).collect(); + assert_eq!(seqs, vec![201, 202, 203]); + } + + #[tokio::test] + async fn list_events_for_stage_returns_limit_plus_one_for_has_more_signal() { + let run = fresh_run().await; + let run_id = run.run_id(); + for idx in 1..=5 { + run.append_event(&stage_prompt_payload(&run_id, idx, Some("alpha"))) + .await + .unwrap(); + } + + let events = run + .list_events_for_stage_from_with_limit(&StageId::new("alpha", 1), 1, 2) + .await + .unwrap(); + + // With limit=2, we expect up to limit+1 = 3 envelopes so the + // caller can compute has_more. + assert_eq!(events.len(), 3); + } + + #[tokio::test] + async fn list_events_for_stage_prefers_stage_id_over_node_id() { + let run = fresh_run().await; + let run_id = run.run_id(); + let first_visit = StageId::new("verify", 1); + let second_visit = StageId::new("verify", 2); + run.append_event(&stage_prompt_payload_for_stage( + &run_id, + 1, + Some("verify"), + Some(&first_visit), + )) + .await + .unwrap(); + run.append_event(&stage_prompt_payload_for_stage( + &run_id, + 2, + Some("verify"), + Some(&second_visit), + )) + .await + .unwrap(); + + let events = run + .list_events_for_stage_from_with_limit(&second_visit, 1, 100) + .await + .unwrap(); + + let seqs: Vec = events.iter().map(|e| e.seq).collect(); + assert_eq!(seqs, vec![2]); + } } diff --git a/lib/packages/fabro-api-client/src/.openapi-generator/FILES b/lib/packages/fabro-api-client/src/.openapi-generator/FILES index 3113a0298..2dffdba54 100644 --- a/lib/packages/fabro-api-client/src/.openapi-generator/FILES +++ b/lib/packages/fabro-api-client/src/.openapi-generator/FILES @@ -30,7 +30,6 @@ models/artifact-batch-upload-manifest.ts models/artifact-entry.ts models/artifact-list-response.ts models/artifacts-settings.ts -models/assistant-stage-turn.ts models/auth-method.ts models/billed-token-counts.ts models/billing-by-model.ts @@ -168,7 +167,6 @@ models/paginated-run-file-list.ts models/paginated-run-list.ts models/paginated-run-stage-list.ts models/paginated-saved-query-list.ts -models/paginated-stage-turn-list.ts models/pagination-meta.ts models/pending-interview-record.ts models/pre-run-push-outcome-failed.ts @@ -300,7 +298,6 @@ models/stage-completion.ts models/stage-outcome.ts models/stage-projection.ts models/stage-state.ts -models/stage-turn.ts models/start-run-request.ts models/submit-answer-request.ts models/success-reason.ts @@ -308,13 +305,10 @@ models/system-actor-kind.ts models/system-features.ts models/system-info-response.ts models/system-run-counts.ts -models/system-stage-turn.ts models/teams-integration-settings.ts models/terminal-status.ts models/timeline-entry-response.ts models/tls-mode.ts -models/tool-stage-turn.ts -models/tool-use.ts models/user-response.ts models/validate-response.ts models/webhook-strategy.ts diff --git a/lib/packages/fabro-api-client/src/api/run-internals-api.ts b/lib/packages/fabro-api-client/src/api/run-internals-api.ts index 0852b16f6..6520b0fc6 100644 --- a/lib/packages/fabro-api-client/src/api/run-internals-api.ts +++ b/lib/packages/fabro-api-client/src/api/run-internals-api.ts @@ -36,8 +36,6 @@ import type { PaginatedEventList } from '../models'; // @ts-ignore import type { PaginatedRunStageList } from '../models'; // @ts-ignore -import type { PaginatedStageTurnList } from '../models'; -// @ts-ignore import type { RunArtifactListResponse } from '../models'; // @ts-ignore import type { RunCheckpoint } from '../models'; @@ -527,21 +525,21 @@ export const RunInternalsApiAxiosParamCreator = function (configuration?: Config }; }, /** - * Returns a paginated list of conversation turns within a specific stage, including system prompts, assistant responses, and tool invocations. - * @summary List Stage Turns + * Returns a paginated JSON list of stored run events scoped to a single stage visit. + * @summary List Stage Events * @param {string} id Unique run identifier (ULID). * @param {string} stageId Identifier of a stage within a run\'s workflow graph, serialized as `node_id@visit`. - * @param {number} [pageLimit] Maximum number of items to return per page. - * @param {number} [pageOffset] Number of items to skip before returning results. + * @param {number} [sinceSeq] First event sequence number to include. + * @param {number} [limit] Maximum number of events to return. * @param {*} [options] Override http request option. * @throws {RequiredError} */ - listStageTurns: async (id: string, stageId: string, pageLimit?: number, pageOffset?: number, options: RawAxiosRequestConfig = {}): Promise => { + listStageEvents: async (id: string, stageId: string, sinceSeq?: number, limit?: number, options: RawAxiosRequestConfig = {}): Promise => { // verify required parameter 'id' is not null or undefined - assertParamExists('listStageTurns', 'id', id) + assertParamExists('listStageEvents', 'id', id) // verify required parameter 'stageId' is not null or undefined - assertParamExists('listStageTurns', 'stageId', stageId) - const localVarPath = `/api/v1/runs/{id}/stages/{stageId}/turns` + assertParamExists('listStageEvents', 'stageId', stageId) + const localVarPath = `/api/v1/runs/{id}/stages/{stageId}/events` .replace(`{${"id"}}`, encodeURIComponent(String(id))) .replace(`{${"stageId"}}`, encodeURIComponent(String(stageId))); // use dummy base URL string because the URL constructor only accepts absolute URLs. @@ -561,12 +559,12 @@ export const RunInternalsApiAxiosParamCreator = function (configuration?: Config // http bearer authentication required await setBearerAuthToObject(localVarHeaderParameter, configuration) - if (pageLimit !== undefined) { - localVarQueryParameter['page[limit]'] = pageLimit; + if (sinceSeq !== undefined) { + localVarQueryParameter['since_seq'] = sinceSeq; } - if (pageOffset !== undefined) { - localVarQueryParameter['page[offset]'] = pageOffset; + if (limit !== undefined) { + localVarQueryParameter['limit'] = limit; } localVarHeaderParameter['Accept'] = 'application/json'; @@ -964,19 +962,19 @@ export const RunInternalsApiFp = function(configuration?: Configuration) { return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, localVarOperationServerBasePath || basePath); }, /** - * Returns a paginated list of conversation turns within a specific stage, including system prompts, assistant responses, and tool invocations. - * @summary List Stage Turns + * Returns a paginated JSON list of stored run events scoped to a single stage visit. + * @summary List Stage Events * @param {string} id Unique run identifier (ULID). * @param {string} stageId Identifier of a stage within a run\'s workflow graph, serialized as `node_id@visit`. - * @param {number} [pageLimit] Maximum number of items to return per page. - * @param {number} [pageOffset] Number of items to skip before returning results. + * @param {number} [sinceSeq] First event sequence number to include. + * @param {number} [limit] Maximum number of events to return. * @param {*} [options] Override http request option. * @throws {RequiredError} */ - async listStageTurns(id: string, stageId: string, pageLimit?: number, pageOffset?: number, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise> { - const localVarAxiosArgs = await localVarAxiosParamCreator.listStageTurns(id, stageId, pageLimit, pageOffset, options); + async listStageEvents(id: string, stageId: string, sinceSeq?: number, limit?: number, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise> { + const localVarAxiosArgs = await localVarAxiosParamCreator.listStageEvents(id, stageId, sinceSeq, limit, options); const localVarOperationServerIndex = configuration?.serverIndex ?? 0; - const localVarOperationServerBasePath = operationServerMap['RunInternalsApi.listStageTurns']?.[localVarOperationServerIndex]?.url; + const localVarOperationServerBasePath = operationServerMap['RunInternalsApi.listStageEvents']?.[localVarOperationServerIndex]?.url; return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, localVarOperationServerBasePath || basePath); }, /** @@ -1174,17 +1172,17 @@ export const RunInternalsApiFactory = function (configuration?: Configuration, b return localVarFp.listStageArtifacts(id, stageId, options).then((request) => request(axios, basePath)); }, /** - * Returns a paginated list of conversation turns within a specific stage, including system prompts, assistant responses, and tool invocations. - * @summary List Stage Turns + * Returns a paginated JSON list of stored run events scoped to a single stage visit. + * @summary List Stage Events * @param {string} id Unique run identifier (ULID). * @param {string} stageId Identifier of a stage within a run\'s workflow graph, serialized as `node_id@visit`. - * @param {number} [pageLimit] Maximum number of items to return per page. - * @param {number} [pageOffset] Number of items to skip before returning results. + * @param {number} [sinceSeq] First event sequence number to include. + * @param {number} [limit] Maximum number of events to return. * @param {*} [options] Override http request option. * @throws {RequiredError} */ - listStageTurns(id: string, stageId: string, pageLimit?: number, pageOffset?: number, options?: RawAxiosRequestConfig): AxiosPromise { - return localVarFp.listStageTurns(id, stageId, pageLimit, pageOffset, options).then((request) => request(axios, basePath)); + listStageEvents(id: string, stageId: string, sinceSeq?: number, limit?: number, options?: RawAxiosRequestConfig): AxiosPromise { + return localVarFp.listStageEvents(id, stageId, sinceSeq, limit, options).then((request) => request(axios, basePath)); }, /** * Uploads one or more artifacts for a stage. Intended for trusted internal callers. The server accepts both: - `application/octet-stream` for single-file uploads with the `filename` query parameter - strict manifest-first `multipart/form-data` uploads documented by `ArtifactBatchUploadManifest` The generated Rust client currently exposes the octet-stream variant because the OpenAPI code generator in this repo does not support multiple request media types on one operation. @@ -1374,17 +1372,17 @@ export class RunInternalsApi extends BaseAPI { } /** - * Returns a paginated list of conversation turns within a specific stage, including system prompts, assistant responses, and tool invocations. - * @summary List Stage Turns + * Returns a paginated JSON list of stored run events scoped to a single stage visit. + * @summary List Stage Events * @param {string} id Unique run identifier (ULID). * @param {string} stageId Identifier of a stage within a run\'s workflow graph, serialized as `node_id@visit`. - * @param {number} [pageLimit] Maximum number of items to return per page. - * @param {number} [pageOffset] Number of items to skip before returning results. + * @param {number} [sinceSeq] First event sequence number to include. + * @param {number} [limit] Maximum number of events to return. * @param {*} [options] Override http request option. * @throws {RequiredError} */ - public listStageTurns(id: string, stageId: string, pageLimit?: number, pageOffset?: number, options?: RawAxiosRequestConfig) { - return RunInternalsApiFp(this.configuration).listStageTurns(id, stageId, pageLimit, pageOffset, options).then((request) => request(this.axios, this.basePath)); + public listStageEvents(id: string, stageId: string, sinceSeq?: number, limit?: number, options?: RawAxiosRequestConfig) { + return RunInternalsApiFp(this.configuration).listStageEvents(id, stageId, sinceSeq, limit, options).then((request) => request(this.axios, this.basePath)); } /** diff --git a/lib/packages/fabro-api-client/src/models/assistant-stage-turn.ts b/lib/packages/fabro-api-client/src/models/assistant-stage-turn.ts deleted file mode 100644 index 8e0a8edcf..000000000 --- a/lib/packages/fabro-api-client/src/models/assistant-stage-turn.ts +++ /dev/null @@ -1,34 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ -/** - * Fabro Run API - * HTTP API for managing Fabro workflow run executions. - * - * The version of the OpenAPI document: 0.1.0 - * - * - * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). - * https://openapi-generator.tech - * Do not edit the class manually. - */ - - - -/** - * An assistant response turn within a stage. - */ -export interface AssistantStageTurn { - 'kind': AssistantStageTurnKindEnum; - /** - * Assistant response text. - */ - 'content': string; -} - -export const AssistantStageTurnKindEnum = { - ASSISTANT: 'assistant' -} as const; - -export type AssistantStageTurnKindEnum = typeof AssistantStageTurnKindEnum[keyof typeof AssistantStageTurnKindEnum]; - - diff --git a/lib/packages/fabro-api-client/src/models/board-column-definition.ts b/lib/packages/fabro-api-client/src/models/board-column-definition.ts index 20deba1f5..314cbcde1 100644 --- a/lib/packages/fabro-api-client/src/models/board-column-definition.ts +++ b/lib/packages/fabro-api-client/src/models/board-column-definition.ts @@ -21,3 +21,6 @@ export interface BoardColumnDefinition { 'id': BoardColumn; 'name': string; } + + + diff --git a/lib/packages/fabro-api-client/src/models/index.ts b/lib/packages/fabro-api-client/src/models/index.ts index 0b72c59bd..fcb9e05ef 100644 --- a/lib/packages/fabro-api-client/src/models/index.ts +++ b/lib/packages/fabro-api-client/src/models/index.ts @@ -10,7 +10,6 @@ export * from './artifact-batch-upload-manifest'; export * from './artifact-entry'; export * from './artifact-list-response'; export * from './artifacts-settings'; -export * from './assistant-stage-turn'; export * from './auth-method'; export * from './billed-token-counts'; export * from './billing-by-model'; @@ -147,7 +146,6 @@ export * from './paginated-run-file-list'; export * from './paginated-run-list'; export * from './paginated-run-stage-list'; export * from './paginated-saved-query-list'; -export * from './paginated-stage-turn-list'; export * from './pagination-meta'; export * from './pending-interview-record'; export * from './pre-run-push-outcome'; @@ -279,7 +277,6 @@ export * from './stage-completion'; export * from './stage-outcome'; export * from './stage-projection'; export * from './stage-state'; -export * from './stage-turn'; export * from './start-run-request'; export * from './submit-answer-request'; export * from './success-reason'; @@ -287,13 +284,10 @@ export * from './system-actor-kind'; export * from './system-features'; export * from './system-info-response'; export * from './system-run-counts'; -export * from './system-stage-turn'; export * from './teams-integration-settings'; export * from './terminal-status'; export * from './timeline-entry-response'; export * from './tls-mode'; -export * from './tool-stage-turn'; -export * from './tool-use'; export * from './user-response'; export * from './validate-response'; export * from './webhook-strategy'; diff --git a/lib/packages/fabro-api-client/src/models/manifest-git.ts b/lib/packages/fabro-api-client/src/models/manifest-git.ts deleted file mode 100644 index bef882689..000000000 --- a/lib/packages/fabro-api-client/src/models/manifest-git.ts +++ /dev/null @@ -1,41 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ -/** - * Fabro Run API - * HTTP API for managing Fabro workflow run executions. - * - * The version of the OpenAPI document: 0.1.0 - * - * - * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). - * https://openapi-generator.tech - * Do not edit the class manually. - */ - - -// May contain unused imports in some cases -// @ts-ignore -import type { ManifestPreRunPushOutcome } from './manifest-pre-run-push-outcome'; - -/** - * Observable git state from the CLI working directory. - */ -export interface ManifestGit { - /** - * Remote origin URL with any embedded credentials removed. - */ - 'origin_url': string; - /** - * Current branch name. - */ - 'branch': string; - /** - * Current commit SHA. - */ - 'sha': string; - /** - * Whether the working tree has uncommitted changes. - */ - 'clean': boolean; - 'push_outcome': ManifestPreRunPushOutcome; -} diff --git a/lib/packages/fabro-api-client/src/models/manifest-pre-run-push-outcome.ts b/lib/packages/fabro-api-client/src/models/manifest-pre-run-push-outcome.ts deleted file mode 100644 index 93f6be234..000000000 --- a/lib/packages/fabro-api-client/src/models/manifest-pre-run-push-outcome.ts +++ /dev/null @@ -1,36 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ -/** - * Fabro Run API - * HTTP API for managing Fabro workflow run executions. - * - * The version of the OpenAPI document: 0.1.0 - * - * - * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). - * https://openapi-generator.tech - * Do not edit the class manually. - */ - - - -/** - * Outcome of the CLI\'s best-effort pre-run push. - */ -export interface ManifestPreRunPushOutcome { - 'type': ManifestPreRunPushOutcomeTypeEnum; - 'remote'?: string | null; - 'branch'?: string | null; - 'message'?: string | null; - 'repo_origin_url'?: string | null; -} - -export const ManifestPreRunPushOutcomeTypeEnum = { - NOT_ATTEMPTED: 'not_attempted', - SUCCEEDED: 'succeeded', - FAILED: 'failed', - SKIPPED_NO_REMOTE: 'skipped_no_remote', - SKIPPED_REMOTE_MISMATCH: 'skipped_remote_mismatch' -} as const; - -export type ManifestPreRunPushOutcomeTypeEnum = typeof ManifestPreRunPushOutcomeTypeEnum[keyof typeof ManifestPreRunPushOutcomeTypeEnum]; diff --git a/lib/packages/fabro-api-client/src/models/paginated-stage-turn-list.ts b/lib/packages/fabro-api-client/src/models/paginated-stage-turn-list.ts deleted file mode 100644 index 26f8d9cf2..000000000 --- a/lib/packages/fabro-api-client/src/models/paginated-stage-turn-list.ts +++ /dev/null @@ -1,30 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ -/** - * Fabro Run API - * HTTP API for managing Fabro workflow run executions. - * - * The version of the OpenAPI document: 0.1.0 - * - * - * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). - * https://openapi-generator.tech - * Do not edit the class manually. - */ - - -// May contain unused imports in some cases -// @ts-ignore -import type { PaginationMeta } from './pagination-meta'; -// May contain unused imports in some cases -// @ts-ignore -import type { StageTurn } from './stage-turn'; - -/** - * Paginated list of stage turns. - */ -export interface PaginatedStageTurnList { - 'data': Array; - 'meta': PaginationMeta; -} - diff --git a/lib/packages/fabro-api-client/src/models/pre-run-git-context.ts b/lib/packages/fabro-api-client/src/models/pre-run-git-context.ts deleted file mode 100644 index aaddc93e7..000000000 --- a/lib/packages/fabro-api-client/src/models/pre-run-git-context.ts +++ /dev/null @@ -1,30 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ -/** - * Fabro Run API - * HTTP API for managing Fabro workflow run executions. - * - * The version of the OpenAPI document: 0.1.0 - * - * - * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). - * https://openapi-generator.tech - * Do not edit the class manually. - */ - - -// May contain unused imports in some cases -// @ts-ignore -import type { DirtyStatus } from './dirty-status'; -// May contain unused imports in some cases -// @ts-ignore -import type { PreRunPushOutcome } from './pre-run-push-outcome'; - -/** - * Submitter-side git context captured before run creation. - */ -export interface PreRunGitContext { - 'display_base_sha'?: string | null; - 'local_dirty': DirtyStatus; - 'push_outcome': PreRunPushOutcome; -} diff --git a/lib/packages/fabro-api-client/src/models/set-secret-request.ts b/lib/packages/fabro-api-client/src/models/set-secret-request.ts deleted file mode 100644 index b343832c1..000000000 --- a/lib/packages/fabro-api-client/src/models/set-secret-request.ts +++ /dev/null @@ -1,25 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ -/** - * Fabro Run API - * HTTP API for managing Fabro workflow run executions. - * - * The version of the OpenAPI document: 0.1.0 - * - * - * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). - * https://openapi-generator.tech - * Do not edit the class manually. - */ - - - -/** - * Request to store a secret value. - */ -export interface SetSecretRequest { - /** - * The secret value to store. - */ - 'value': string; -} diff --git a/lib/packages/fabro-api-client/src/models/stage-turn.ts b/lib/packages/fabro-api-client/src/models/stage-turn.ts deleted file mode 100644 index 3c5ffdf3d..000000000 --- a/lib/packages/fabro-api-client/src/models/stage-turn.ts +++ /dev/null @@ -1,35 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ -/** - * Fabro Run API - * HTTP API for managing Fabro workflow run executions. - * - * The version of the OpenAPI document: 0.1.0 - * - * - * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). - * https://openapi-generator.tech - * Do not edit the class manually. - */ - - -// May contain unused imports in some cases -// @ts-ignore -import type { AssistantStageTurn } from './assistant-stage-turn'; -// May contain unused imports in some cases -// @ts-ignore -import type { SystemStageTurn } from './system-stage-turn'; -// May contain unused imports in some cases -// @ts-ignore -import type { ToolStageTurn } from './tool-stage-turn'; -// May contain unused imports in some cases -// @ts-ignore -import type { ToolUse } from './tool-use'; - -/** - * @type StageTurn - * A single turn in a stage conversation — a system prompt, assistant response, or tool invocation block. - */ -export type StageTurn = { kind: 'assistant' } & AssistantStageTurn | { kind: 'system' } & SystemStageTurn | { kind: 'tool' } & ToolStageTurn; - - diff --git a/lib/packages/fabro-api-client/src/models/system-stage-turn.ts b/lib/packages/fabro-api-client/src/models/system-stage-turn.ts deleted file mode 100644 index 68ad6386f..000000000 --- a/lib/packages/fabro-api-client/src/models/system-stage-turn.ts +++ /dev/null @@ -1,34 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ -/** - * Fabro Run API - * HTTP API for managing Fabro workflow run executions. - * - * The version of the OpenAPI document: 0.1.0 - * - * - * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). - * https://openapi-generator.tech - * Do not edit the class manually. - */ - - - -/** - * A system prompt turn that sets the stage\'s instructions. - */ -export interface SystemStageTurn { - 'kind': SystemStageTurnKindEnum; - /** - * System prompt text. - */ - 'content': string; -} - -export const SystemStageTurnKindEnum = { - SYSTEM: 'system' -} as const; - -export type SystemStageTurnKindEnum = typeof SystemStageTurnKindEnum[keyof typeof SystemStageTurnKindEnum]; - - diff --git a/lib/packages/fabro-api-client/src/models/tool-stage-turn.ts b/lib/packages/fabro-api-client/src/models/tool-stage-turn.ts deleted file mode 100644 index e637a0520..000000000 --- a/lib/packages/fabro-api-client/src/models/tool-stage-turn.ts +++ /dev/null @@ -1,41 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ -/** - * Fabro Run API - * HTTP API for managing Fabro workflow run executions. - * - * The version of the OpenAPI document: 0.1.0 - * - * - * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). - * https://openapi-generator.tech - * Do not edit the class manually. - */ - - -// May contain unused imports in some cases -// @ts-ignore -import type { ToolUse } from './tool-use'; - -/** - * A tool invocation turn containing one or more tool calls. - */ -export interface ToolStageTurn { - 'kind': ToolStageTurnKindEnum; - /** - * Text accompanying the tool invocations, or null when the turn contains only tool calls. - */ - 'content'?: string; - /** - * Tool invocations executed in this turn. - */ - 'tools': Array; -} - -export const ToolStageTurnKindEnum = { - TOOL: 'tool' -} as const; - -export type ToolStageTurnKindEnum = typeof ToolStageTurnKindEnum[keyof typeof ToolStageTurnKindEnum]; - - diff --git a/lib/packages/fabro-api-client/src/models/tool-use.ts b/lib/packages/fabro-api-client/src/models/tool-use.ts deleted file mode 100644 index c6f5e721e..000000000 --- a/lib/packages/fabro-api-client/src/models/tool-use.ts +++ /dev/null @@ -1,46 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ -/** - * Fabro Run API - * HTTP API for managing Fabro workflow run executions. - * - * The version of the OpenAPI document: 0.1.0 - * - * - * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). - * https://openapi-generator.tech - * Do not edit the class manually. - */ - - - -/** - * A single tool invocation with its input, result, and execution metadata. - */ -export interface ToolUse { - /** - * Unique identifier for this tool invocation. Enables correlation in parallel tool use. - */ - 'id': string; - /** - * Name of the tool that was invoked. - */ - 'tool_name': string; - /** - * JSON-encoded input passed to the tool. - */ - 'input': string; - /** - * Output returned by the tool. Contains the error message when is_error is true. - */ - 'result': string; - /** - * Whether the tool invocation failed. When true, the result field contains the error message. - */ - 'is_error': boolean; - /** - * Wall-clock execution time of the tool invocation in milliseconds. - */ - 'duration_ms'?: number; -} - From e901cd3a813dc82f16738ab6bba9154b55ae1fea Mon Sep 17 00:00:00 2001 From: "fabro-sh-0530[bot]" <281434857+fabro-sh-0530[bot]@users.noreply.github.com> Date: Tue, 5 May 2026 09:18:05 -0400 Subject: [PATCH 14/17] Surface silent fallback warnings in runs and logs (#205) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Summary Silent fallback paths now emit stable warnings instead of degrading without a user-visible signal. The fallback behavior is unchanged; runs still continue, but worktree, Git, checkpoint, and LLM failover issues now show up in the run feed and logs. ### Plan Summary - Emit run notices for workflow Git/worktree fallback paths. - Reuse the existing failover event for one-shot LLM provider fallback. - Add tracing for sandbox pipe drain failures. ### What changed - Added `worktree_skipped_no_git` and gated `sandbox_git_unavailable` notices during initialization. - Added `git_push_failed` and `parallel_base_checkpoint_failed` notices, including redacted output tails where available. - Logged GitHub token mint failures with a structured `error` field before the existing notice. - Plumbed `Emitter` and `StageScope` through `CodergenBackend::one_shot` so the API backend emits the existing `agent.failover` event instead of a duplicate tracing-only warning. - Extracted sandbox pipe draining into a helper that warns on stdout/stderr read failures, with unit coverage for the error path. - Updated CLI snapshots for the new worktree warning in stderr and JSON event output. ⚒️ Generated with [Fabro](https://fabro.sh) --------- Co-authored-by: Fabro Co-authored-by: Bryan Helmkamp --- lib/crates/fabro-cli/src/commands/run/logs.rs | 51 +++++++++++++--- .../src/commands/run/run_progress/event.rs | 7 ++- .../src/commands/run/run_progress/mod.rs | 14 +++-- lib/crates/fabro-cli/tests/it/cmd/attach.rs | 17 ++++++ lib/crates/fabro-cli/tests/it/cmd/dump.rs | 6 +- lib/crates/fabro-cli/tests/it/cmd/run.rs | 1 + .../tests/it/workflow/dry_run_examples.rs | 5 ++ lib/crates/fabro-sandbox/src/local.rs | 59 ++++++++++++++----- lib/crates/fabro-types/src/lib.rs | 2 +- lib/crates/fabro-types/src/run_event/infra.rs | 53 +++++++++++++---- lib/crates/fabro-types/src/run_event/mod.rs | 4 +- lib/crates/fabro-workflow/src/event.rs | 2 +- .../fabro-workflow/src/event/convert.rs | 6 +- .../fabro-workflow/src/event/emitter.rs | 15 ++--- .../fabro-workflow/src/handler/agent.rs | 2 + .../fabro-workflow/src/handler/llm/api.rs | 20 ++++--- .../fabro-workflow/src/handler/llm/cli.rs | 6 +- .../fabro-workflow/src/handler/parallel.rs | 8 ++- .../fabro-workflow/src/handler/prompt.rs | 23 ++++++-- .../fabro-workflow/src/lifecycle/artifact.rs | 10 ++-- .../fabro-workflow/src/lifecycle/git.rs | 43 +++++++++----- .../fabro-workflow/src/pipeline/finalize.rs | 42 +++++++++---- .../fabro-workflow/src/pipeline/initialize.rs | 59 +++++++++++++++---- .../src/pipeline/pull_request.rs | 4 +- .../fabro-workflow/tests/it/integration.rs | 2 + 25 files changed, 337 insertions(+), 124 deletions(-) diff --git a/lib/crates/fabro-cli/src/commands/run/logs.rs b/lib/crates/fabro-cli/src/commands/run/logs.rs index 8f01378cc..7d7e9063b 100644 --- a/lib/crates/fabro-cli/src/commands/run/logs.rs +++ b/lib/crates/fabro-cli/src/commands/run/logs.rs @@ -14,7 +14,7 @@ use std::time::Duration; use anyhow::{Context, Result, bail}; use chrono::{DateTime, Utc}; use fabro_redact::redact_jsonl_line; -use fabro_types::run_event::is_metadata_snapshot_compat_notice_code; +use fabro_types::RunNoticeCode; use fabro_util::json::normalize_json_value; use fabro_util::terminal::Styles; use tokio::time; @@ -801,7 +801,9 @@ fn format_event_pretty_value(envelope: &serde_json::Value, styles: &Styles) -> O } fn is_metadata_snapshot_compat_notice(envelope: &serde_json::Value) -> bool { - prop_str_field(envelope, "code").is_some_and(is_metadata_snapshot_compat_notice_code) + prop_str_field(envelope, "code") + .and_then(|code| code.parse::().ok()) + .is_some_and(RunNoticeCode::is_metadata_snapshot_compat) } fn str_field<'a>(value: &'a serde_json::Value, key: &str) -> Option<&'a str> { @@ -1150,14 +1152,27 @@ mod tests { #[test] fn pretty_run_notice_warn() { let styles = no_color_styles(); - let line = r#"{"ts":"2026-01-01T14:25:00Z","event":"run.notice","properties":{"level":"warn","code":"sandbox_cleanup_failed","message":"sandbox cleanup failed: boom"}}"#; - let result = format_event_pretty(line, &styles).unwrap(); + let code = RunNoticeCode::SandboxCleanupFailed.to_string(); + let line = serde_json::json!({ + "ts": "2026-01-01T14:25:00Z", + "event": "run.notice", + "properties": { + "level": "warn", + "code": code, + "message": "sandbox cleanup failed: boom", + }, + }) + .to_string(); + let result = format_event_pretty(&line, &styles).unwrap(); assert!(result.contains("Warning:"), "got: {result}"); assert!( result.contains("sandbox cleanup failed: boom"), "got: {result}" ); - assert!(result.contains("[sandbox_cleanup_failed]"), "got: {result}"); + assert!( + result.contains(&format!("[{}]", RunNoticeCode::SandboxCleanupFailed)), + "got: {result}" + ); } #[test] @@ -1235,13 +1250,31 @@ mod tests { fn pretty_stream_suppresses_metadata_compat_notice_only() { let styles = no_color_styles(); let failed = r#"{"ts":"2026-01-01T14:25:00Z","event":"metadata.snapshot.failed","properties":{"phase":"checkpoint","branch":"fabro/meta","duration_ms":900,"failure_kind":"write","error":"write failed"}}"#; - let compat_notice = r#"{"ts":"2026-01-01T14:25:01Z","event":"run.notice","properties":{"level":"warn","code":"checkpoint_metadata_write_failed","message":"legacy metadata warning"}}"#; - let degraded_notice = r#"{"ts":"2026-01-01T14:25:02Z","event":"run.notice","properties":{"level":"warn","code":"checkpoint_metadata_degraded","message":"metadata snapshots disabled"}}"#; + let compat_notice = serde_json::json!({ + "ts": "2026-01-01T14:25:01Z", + "event": "run.notice", + "properties": { + "level": "warn", + "code": RunNoticeCode::CheckpointMetadataWriteFailed, + "message": "legacy metadata warning", + }, + }) + .to_string(); + let degraded_notice = serde_json::json!({ + "ts": "2026-01-01T14:25:02Z", + "event": "run.notice", + "properties": { + "level": "warn", + "code": RunNoticeCode::CheckpointMetadataDegraded, + "message": "metadata snapshots disabled", + }, + }) + .to_string(); let mut state = PrettyEventState::default(); assert!(format_event_pretty_streamed(failed, &styles, &mut state).is_some()); - assert!(format_event_pretty_streamed(compat_notice, &styles, &mut state).is_none()); - let degraded = format_event_pretty_streamed(degraded_notice, &styles, &mut state).unwrap(); + assert!(format_event_pretty_streamed(&compat_notice, &styles, &mut state).is_none()); + let degraded = format_event_pretty_streamed(°raded_notice, &styles, &mut state).unwrap(); assert!( degraded.contains("metadata snapshots disabled"), "got: {degraded}" diff --git a/lib/crates/fabro-cli/src/commands/run/run_progress/event.rs b/lib/crates/fabro-cli/src/commands/run/run_progress/event.rs index 9e40f0d8b..ec56dce46 100644 --- a/lib/crates/fabro-cli/src/commands/run/run_progress/event.rs +++ b/lib/crates/fabro-cli/src/commands/run/run_progress/event.rs @@ -527,7 +527,7 @@ fn display_value(value: &Value) -> Option { mod tests { use fabro_agent::AgentEvent; use fabro_types::{MetadataSnapshotFailureKind, MetadataSnapshotPhase, fixtures}; - use fabro_workflow::event::{Event, to_run_event}; + use fabro_workflow::event::{Event, RunNoticeCode, to_run_event}; use super::*; @@ -804,10 +804,11 @@ mod tests { fn round_trip_run_notice() { let event = Event::RunNotice { level: RunNoticeLevel::Warn, - code: "sandbox_cleanup_failed".into(), + code: RunNoticeCode::SandboxCleanupFailed.to_string(), message: "sandbox cleanup failed".into(), exec_output_tail: None, }; + let expected_code = RunNoticeCode::SandboxCleanupFailed.to_string(); let stored = to_run_event(&fixtures::RUN_1, &event); let parsed = from_run_event(&stored).unwrap(); @@ -817,7 +818,7 @@ mod tests { level: RunNoticeLevel::Warn, code, message, - } if code == "sandbox_cleanup_failed" && message == "sandbox cleanup failed" + } if code == expected_code && message == "sandbox cleanup failed" )); } diff --git a/lib/crates/fabro-cli/src/commands/run/run_progress/mod.rs b/lib/crates/fabro-cli/src/commands/run/run_progress/mod.rs index 2004ac0ba..7e7df7f2d 100644 --- a/lib/crates/fabro-cli/src/commands/run/run_progress/mod.rs +++ b/lib/crates/fabro-cli/src/commands/run/run_progress/mod.rs @@ -3,8 +3,7 @@ reason = "sync CLI run-progress renderer: writes to std::io::stderr directly" )] -use fabro_types::RunEvent; -use fabro_types::run_event::is_metadata_snapshot_compat_notice_code; +use fabro_types::{RunEvent, RunNoticeCode}; mod event; mod info_display; @@ -444,7 +443,10 @@ impl ProgressUI { message, } => { if self.saw_metadata_snapshot_failure - && is_metadata_snapshot_compat_notice_code(&code) + && code + .parse::() + .ok() + .is_some_and(RunNoticeCode::is_metadata_snapshot_compat) { return; } @@ -1208,7 +1210,7 @@ mod tests { emit(&mut ui, Event::RunNotice { level: RunNoticeLevel::Warn, - code: "sandbox_cleanup_failed".into(), + code: RunNoticeCode::SandboxCleanupFailed.to_string(), message: "sandbox cleanup failed".into(), exec_output_tail: None, }); @@ -1279,13 +1281,13 @@ mod tests { }); emit(&mut ui, Event::RunNotice { level: RunNoticeLevel::Warn, - code: "checkpoint_metadata_write_failed".into(), + code: RunNoticeCode::CheckpointMetadataWriteFailed.to_string(), message: "legacy metadata warning".into(), exec_output_tail: None, }); emit(&mut ui, Event::RunNotice { level: RunNoticeLevel::Warn, - code: "checkpoint_metadata_degraded".into(), + code: RunNoticeCode::CheckpointMetadataDegraded.to_string(), message: "metadata snapshots are disabled for this run".into(), exec_output_tail: None, }); diff --git a/lib/crates/fabro-cli/tests/it/cmd/attach.rs b/lib/crates/fabro-cli/tests/it/cmd/attach.rs index e703e3207..7c6728f39 100644 --- a/lib/crates/fabro-cli/tests/it/cmd/attach.rs +++ b/lib/crates/fabro-cli/tests/it/cmd/attach.rs @@ -160,6 +160,7 @@ fn attach_replays_completed_detached_run() { ----- stdout ----- ----- stderr ----- Web UI: http://localhost:3000/runs/[ULID] + Warning: Worktree mode `always` requested but no Git repository was found; running without a worktree. [worktree_skipped_no_git] Sandbox: local (ready in [TIME]) ✓ Start [TIME] ✓ Run Tests [TIME] @@ -267,6 +268,7 @@ fn attach_before_completion_streams_to_finished_state() { ----- stdout ----- ----- stderr ----- Web UI: http://localhost:3000/runs/[ULID] + Warning: Worktree mode `always` requested but no Git repository was found; running without a worktree. [worktree_skipped_no_git] Sandbox: local (ready in [TIME]) ✓ start [DURATION] ✓ wait [DURATION] @@ -699,6 +701,21 @@ fn attach_json_errors_without_prompting_for_human_input() { "run_id": "[ULID]", "ts": "[TIMESTAMP]" }, + { + "actor": { + "kind": "worker", + "run_id": "[ULID]" + }, + "event": "run.notice", + "id": "[EVENT_ID]", + "properties": { + "code": "worktree_skipped_no_git", + "level": "warn", + "message": "Worktree mode `always` requested but no Git repository was found; running without a worktree." + }, + "run_id": "[ULID]", + "ts": "[TIMESTAMP]" + }, { "actor": { "kind": "worker", diff --git a/lib/crates/fabro-cli/tests/it/cmd/dump.rs b/lib/crates/fabro-cli/tests/it/cmd/dump.rs index 12b13ee9a..c541b59eb 100644 --- a/lib/crates/fabro-cli/tests/it/cmd/dump.rs +++ b/lib/crates/fabro-cli/tests/it/cmd/dump.rs @@ -278,9 +278,9 @@ fn dump_exports_completed_run_snapshot() { "); assert_snapshot!(dump_file_summary(&output_dir), @" - checkpoints/0013.json - checkpoints/0017.json - checkpoints/0021.json + checkpoints/0014.json + checkpoints/0018.json + checkpoints/0022.json events.jsonl graph.fabro run.json diff --git a/lib/crates/fabro-cli/tests/it/cmd/run.rs b/lib/crates/fabro-cli/tests/it/cmd/run.rs index 19d876cfc..2aa614979 100644 --- a/lib/crates/fabro-cli/tests/it/cmd/run.rs +++ b/lib/crates/fabro-cli/tests/it/cmd/run.rs @@ -684,6 +684,7 @@ fn dry_run_simple() { Run: [ULID] Web UI: http://localhost:3000/runs/[ULID] + Warning: Worktree mode `always` requested but no Git repository was found; running without a worktree. [worktree_skipped_no_git] Sandbox: local (ready in [TIME]) ✓ Start [TIME] ✓ Run Tests [TIME] diff --git a/lib/crates/fabro-cli/tests/it/workflow/dry_run_examples.rs b/lib/crates/fabro-cli/tests/it/workflow/dry_run_examples.rs index bb8a50c85..c3f8d836c 100644 --- a/lib/crates/fabro-cli/tests/it/workflow/dry_run_examples.rs +++ b/lib/crates/fabro-cli/tests/it/workflow/dry_run_examples.rs @@ -21,6 +21,7 @@ fn dry_run_branching() { warning [node: implement]: Node 'implement' has goal_gate=true but no retry_target or fallback_retry_target (goal_gate_has_retry) Run: [ULID] Web UI: http://localhost:3000/runs/[ULID] + Warning: Worktree mode `always` requested but no Git repository was found; running without a worktree. [worktree_skipped_no_git] Sandbox: local (ready in [TIME]) ✓ Start [TIME] ✓ Plan [TIME] @@ -57,6 +58,7 @@ fn dry_run_conditions() { Run: [ULID] Web UI: http://localhost:3000/runs/[ULID] + Warning: Worktree mode `always` requested but no Git repository was found; running without a worktree. [worktree_skipped_no_git] Sandbox: local (ready in [TIME]) ✓ start [TIME] ✓ Decide [TIME] @@ -91,6 +93,7 @@ fn dry_run_parallel() { Run: [ULID] Web UI: http://localhost:3000/runs/[ULID] + Warning: Worktree mode `always` requested but no Git repository was found; running without a worktree. [worktree_skipped_no_git] Sandbox: local (ready in [TIME]) ✓ start [TIME] ✓ Fork Work [TIME] @@ -126,6 +129,7 @@ fn dry_run_styled() { Run: [ULID] Web UI: http://localhost:3000/runs/[ULID] + Warning: Worktree mode `always` requested but no Git repository was found; running without a worktree. [worktree_skipped_no_git] Sandbox: local (ready in [TIME]) ✓ start [TIME] ✓ Plan [TIME] @@ -161,6 +165,7 @@ fn dry_run_legacy_tool() { Run: [ULID] Web UI: http://localhost:3000/runs/[ULID] + Warning: Worktree mode `always` requested but no Git repository was found; running without a worktree. [worktree_skipped_no_git] Sandbox: local (ready in [TIME]) ✓ Start [TIME] ✓ Echo [TIME] diff --git a/lib/crates/fabro-sandbox/src/local.rs b/lib/crates/fabro-sandbox/src/local.rs index 8b69b7377..66d084b5f 100644 --- a/lib/crates/fabro-sandbox/src/local.rs +++ b/lib/crates/fabro-sandbox/src/local.rs @@ -126,6 +126,19 @@ fn process_env_vars() -> Vec<(String, String)> { std::env::vars().collect() } +async fn drain_pipe(mut pipe: Option, stream: CommandOutputStream) -> String +where + R: AsyncRead + Unpin, +{ + let mut buf = String::new(); + if let Some(ref mut reader) = pipe { + if let Err(err) = reader.read_to_string(&mut buf).await { + tracing::warn!(error = %err, ?stream, "Failed to drain child output"); + } + } + buf +} + #[async_trait] impl Sandbox for LocalSandbox { async fn read_file( @@ -277,22 +290,12 @@ impl Sandbox for LocalSandbox { // it writes more than the OS pipe buffer (~64 KB) the write() syscall // blocks until the parent drains the pipe, but the parent is blocked // on child.wait(). - let mut stdout_pipe = child.stdout.take(); - let mut stderr_pipe = child.stderr.take(); - let stdout_task = tokio::spawn(async move { - let mut buf = String::new(); - if let Some(ref mut r) = stdout_pipe { - let _ = r.read_to_string(&mut buf).await; - } - buf - }); - let stderr_task = tokio::spawn(async move { - let mut buf = String::new(); - if let Some(ref mut r) = stderr_pipe { - let _ = r.read_to_string(&mut buf).await; - } - buf - }); + let stdout_pipe = child.stdout.take(); + let stderr_pipe = child.stderr.take(); + let stdout_task = + tokio::spawn(async move { drain_pipe(stdout_pipe, CommandOutputStream::Stdout).await }); + let stderr_task = + tokio::spawn(async move { drain_pipe(stderr_pipe, CommandOutputStream::Stderr).await }); let (termination, exit_code) = tokio::select! { status_result = child.wait() => { @@ -712,7 +715,12 @@ where )] mod tests { use std::collections::HashMap; + use std::io; use std::path::PathBuf; + use std::pin::Pin; + use std::task::{Context as TaskContext, Poll}; + + use tokio::io::ReadBuf; use super::*; @@ -722,6 +730,25 @@ mod tests { dir } + #[tokio::test] + async fn drain_pipe_returns_empty_buffer_after_read_failure() { + struct FailingReader; + + impl AsyncRead for FailingReader { + fn poll_read( + self: Pin<&mut Self>, + _cx: &mut TaskContext<'_>, + _buf: &mut ReadBuf<'_>, + ) -> Poll> { + Poll::Ready(Err(io::Error::other("simulated read failure"))) + } + } + + let output = drain_pipe(Some(FailingReader), CommandOutputStream::Stdout).await; + + assert!(output.is_empty()); + } + #[tokio::test] async fn read_file_with_line_numbers() { let dir = temp_dir(); diff --git a/lib/crates/fabro-types/src/lib.rs b/lib/crates/fabro-types/src/lib.rs index 22675855c..e5d3b65a5 100644 --- a/lib/crates/fabro-types/src/lib.rs +++ b/lib/crates/fabro-types/src/lib.rs @@ -72,7 +72,7 @@ pub use run::{ pub use run_blob_id::RunBlobId; pub use run_event::{ EventBody, ExecOutputTail, InterviewOption, MetadataSnapshotFailureKind, MetadataSnapshotPhase, - RunEvent, RunNoticeLevel, + RunEvent, RunNoticeCode, RunNoticeLevel, }; pub use run_id::{RunId, fixtures}; pub use run_projection::{PendingInterviewRecord, RunProjection, StageProjection, first_event_seq}; diff --git a/lib/crates/fabro-types/src/run_event/infra.rs b/lib/crates/fabro-types/src/run_event/infra.rs index 8ee1bc32c..77689087b 100644 --- a/lib/crates/fabro-types/src/run_event/infra.rs +++ b/lib/crates/fabro-types/src/run_event/infra.rs @@ -1,17 +1,48 @@ use serde::{Deserialize, Serialize}; -/// Legacy `run.notice` codes paired with the new `metadata.snapshot.failed` -/// event for backward compatibility. Display layers suppress these so the -/// typed event renders without a duplicate raw warning. -pub const NOTICE_CODE_CHECKPOINT_METADATA_WRITE_FAILED: &str = "checkpoint_metadata_write_failed"; -pub const NOTICE_CODE_CHECKPOINT_METADATA_PUSH_FAILED: &str = "checkpoint_metadata_push_failed"; +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + Hash, + Serialize, + Deserialize, + strum::Display, + strum::EnumString, + strum::IntoStaticStr, +)] +#[serde(rename_all = "snake_case")] +#[strum(serialize_all = "snake_case")] +pub enum RunNoticeCode { + ArtifactCollectionFailed, + ArtifactOffloadFailed, + ArtifactSyncFailed, + ArtifactUploadFailed, + CheckpointMetadataDegraded, + CheckpointMetadataPushFailed, + CheckpointMetadataWriteFailed, + DirtyWorktree, + GitDiffFailed, + GitPushFailed, + GithubTokenFailed, + ParallelBaseCheckpointFailed, + PullRequestFailed, + SandboxCleanupFailed, + SandboxGitUnavailable, + SandboxPreserved, + WorktreeSkippedNoGit, +} -#[must_use] -pub fn is_metadata_snapshot_compat_notice_code(code: &str) -> bool { - matches!( - code, - NOTICE_CODE_CHECKPOINT_METADATA_WRITE_FAILED | NOTICE_CODE_CHECKPOINT_METADATA_PUSH_FAILED - ) +impl RunNoticeCode { + #[must_use] + pub fn is_metadata_snapshot_compat(self) -> bool { + matches!( + self, + Self::CheckpointMetadataWriteFailed | Self::CheckpointMetadataPushFailed + ) + } } #[derive( diff --git a/lib/crates/fabro-types/src/run_event/mod.rs b/lib/crates/fabro-types/src/run_event/mod.rs index 9d5fbf11d..647efc381 100644 --- a/lib/crates/fabro-types/src/run_event/mod.rs +++ b/lib/crates/fabro-types/src/run_event/mod.rs @@ -1366,7 +1366,7 @@ mod tests { for body in [ EventBody::RunNotice(RunNoticeProps { level: RunNoticeLevel::Warn, - code: "git_diff_failed".to_string(), + code: RunNoticeCode::GitDiffFailed.to_string(), message: "git diff failed".to_string(), exec_output_tail: Some(tail.clone()), }), @@ -1402,7 +1402,7 @@ mod tests { for body in [ EventBody::RunNotice(RunNoticeProps { level: RunNoticeLevel::Warn, - code: "git_diff_failed".to_string(), + code: RunNoticeCode::GitDiffFailed.to_string(), message: "git diff failed".to_string(), exec_output_tail: None, }), diff --git a/lib/crates/fabro-workflow/src/event.rs b/lib/crates/fabro-workflow/src/event.rs index f5120e62e..e64b2ae30 100644 --- a/lib/crates/fabro-workflow/src/event.rs +++ b/lib/crates/fabro-workflow/src/event.rs @@ -8,7 +8,7 @@ mod stored_fields; #[cfg(test)] mod test_support; -pub use fabro_types::{EventBody, RunNoticeLevel}; +pub use fabro_types::{EventBody, RunNoticeCode, RunNoticeLevel}; pub use self::convert::{to_run_event, to_run_event_at}; pub use self::emitter::Emitter; diff --git a/lib/crates/fabro-workflow/src/event/convert.rs b/lib/crates/fabro-workflow/src/event/convert.rs index 4aa9df5b1..255d235ea 100644 --- a/lib/crates/fabro-workflow/src/event/convert.rs +++ b/lib/crates/fabro-workflow/src/event/convert.rs @@ -1169,8 +1169,8 @@ mod tests { use std::collections::BTreeMap; use ::fabro_types::{ - EventBody, FailureReason, ParallelBranchId, Principal, RunNoticeLevel, RunProvenance, - StageId, SystemActorKind, fixtures, run_event as fabro_types, + EventBody, FailureReason, ParallelBranchId, Principal, RunNoticeCode, RunNoticeLevel, + RunProvenance, StageId, SystemActorKind, fixtures, run_event as fabro_types, }; use chrono::Utc; use fabro_agent::{AgentEvent, SandboxEvent}; @@ -1643,7 +1643,7 @@ mod tests { fn run_notice_maps_exec_output_tail_to_props() { let stored = to_run_event(&fixtures::RUN_1, &Event::RunNotice { level: RunNoticeLevel::Warn, - code: "git_diff_failed".to_string(), + code: RunNoticeCode::GitDiffFailed.to_string(), message: "git diff failed".to_string(), exec_output_tail: Some(exec_tail()), }); diff --git a/lib/crates/fabro-workflow/src/event/emitter.rs b/lib/crates/fabro-workflow/src/event/emitter.rs index a1bba727a..dc50a56dd 100644 --- a/lib/crates/fabro-workflow/src/event/emitter.rs +++ b/lib/crates/fabro-workflow/src/event/emitter.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use std::sync::atomic::{AtomicI64, Ordering}; -use ::fabro_types::{ExecOutputTail, RunEvent, RunId, RunNoticeLevel}; +use ::fabro_types::{ExecOutputTail, RunEvent, RunId, RunNoticeCode, RunNoticeLevel}; use chrono::Utc; use fabro_agent::{WorktreeEvent, WorktreeEventCallback}; @@ -76,15 +76,10 @@ impl Emitter { self.emit_with_scope(event, Some(scope)); } - pub fn notice( - &self, - level: RunNoticeLevel, - code: impl Into, - message: impl Into, - ) { + pub fn notice(&self, level: RunNoticeLevel, code: RunNoticeCode, message: impl Into) { self.emit(&Event::RunNotice { level, - code: code.into(), + code: code.to_string(), message: message.into(), exec_output_tail: None, }); @@ -93,13 +88,13 @@ impl Emitter { pub fn notice_with_tail( &self, level: RunNoticeLevel, - code: impl Into, + code: RunNoticeCode, message: impl Into, exec_output_tail: Option, ) { self.emit(&Event::RunNotice { level, - code: code.into(), + code: code.to_string(), message: message.into(), exec_output_tail, }); diff --git a/lib/crates/fabro-workflow/src/handler/agent.rs b/lib/crates/fabro-workflow/src/handler/agent.rs index 676e2f142..7e3a4be0c 100644 --- a/lib/crates/fabro-workflow/src/handler/agent.rs +++ b/lib/crates/fabro-workflow/src/handler/agent.rs @@ -52,6 +52,8 @@ pub trait CodergenBackend: Send + Sync { _node: &Node, _prompt: &str, _system_prompt: Option<&str>, + _emitter: &Arc, + _stage_scope: &StageScope, ) -> Result { Err(Error::Validation( "one_shot mode not supported by this backend".into(), diff --git a/lib/crates/fabro-workflow/src/handler/llm/api.rs b/lib/crates/fabro-workflow/src/handler/llm/api.rs index 5d21c7d32..9247c19ec 100644 --- a/lib/crates/fabro-workflow/src/handler/llm/api.rs +++ b/lib/crates/fabro-workflow/src/handler/llm/api.rs @@ -285,6 +285,8 @@ impl CodergenBackend for AgentApiBackend { node: &Node, prompt: &str, system_prompt: Option<&str>, + emitter: &Arc, + stage_scope: &StageScope, ) -> Result { let client = Client::from_source(self.source.as_ref()) .await @@ -358,14 +360,16 @@ impl CodergenBackend for AgentApiBackend { let mut found = None; for target in fallback_chain { - tracing::warn!( - stage = node.id.as_str(), - from_provider = from_provider.as_str(), - from_model = from_model.as_str(), - to_provider = target.provider.as_str(), - to_model = target.model.as_str(), - error = error_msg.as_str(), - "LLM provider failover (prompt)" + emitter.emit_scoped( + &Event::Failover { + stage: node.id.clone(), + from_provider: from_provider.clone(), + from_model: from_model.clone(), + to_provider: target.provider.clone(), + to_model: target.model.clone(), + error: error_msg.clone(), + }, + stage_scope, ); let max_tokens = node.max_tokens().or_else(|| { diff --git a/lib/crates/fabro-workflow/src/handler/llm/cli.rs b/lib/crates/fabro-workflow/src/handler/llm/cli.rs index 5036eec97..7fcf60ed8 100644 --- a/lib/crates/fabro-workflow/src/handler/llm/cli.rs +++ b/lib/crates/fabro-workflow/src/handler/llm/cli.rs @@ -810,9 +810,13 @@ impl CodergenBackend for BackendRouter { node: &Node, prompt: &str, system_prompt: Option<&str>, + emitter: &Arc, + stage_scope: &StageScope, ) -> Result { // CLI backend doesn't support one_shot, always route to API - self.api_backend.one_shot(node, prompt, system_prompt).await + self.api_backend + .one_shot(node, prompt, system_prompt, emitter, stage_scope) + .await } } diff --git a/lib/crates/fabro-workflow/src/handler/parallel.rs b/lib/crates/fabro-workflow/src/handler/parallel.rs index 39862586c..c58104f83 100644 --- a/lib/crates/fabro-workflow/src/handler/parallel.rs +++ b/lib/crates/fabro-workflow/src/handler/parallel.rs @@ -12,7 +12,7 @@ use tokio::sync::Semaphore; use super::{EngineServices, Handler}; use crate::context::{Context, WorkflowContext, keys}; use crate::error::Error; -use crate::event::{Event, StageScope}; +use crate::event::{Event, RunNoticeCode, RunNoticeLevel, StageScope}; use crate::git::sanitize_ref_component; use crate::hook_context::set_hook_node; use crate::millis_u64; @@ -207,6 +207,12 @@ impl Handler for ParallelHandler { error = %fabro_sandbox::display_for_log(&e), "parallel base checkpoint failed" ); + services.run.emitter.notice_with_tail( + RunNoticeLevel::Warn, + RunNoticeCode::ParallelBaseCheckpointFailed, + format!("Could not checkpoint base state before parallel branches: {e}"), + fabro_sandbox::default_redacted_output_tail(&e), + ); None } } diff --git a/lib/crates/fabro-workflow/src/handler/prompt.rs b/lib/crates/fabro-workflow/src/handler/prompt.rs index a76099b80..caad1e058 100644 --- a/lib/crates/fabro-workflow/src/handler/prompt.rs +++ b/lib/crates/fabro-workflow/src/handler/prompt.rs @@ -105,7 +105,13 @@ impl Handler for PromptHandler { let (response_text, stage_usage, backend_files_touched) = if let Some(backend) = &self.backend { let result = backend - .one_shot(node, &prompt, system_prompt.as_deref()) + .one_shot( + node, + &prompt, + system_prompt.as_deref(), + &services.run.emitter, + &stage_scope, + ) .await; match result { Ok(CodergenResult::Full(outcome)) => return Ok(outcome), @@ -187,6 +193,7 @@ mod tests { use tempfile::TempDir; use super::*; + use crate::event::Emitter; fn make_services() -> EngineServices { EngineServices::test_default() @@ -211,7 +218,7 @@ mod tests { let mut services = EngineServices::test_default(); services.run = services .run - .with_emitter(Arc::new(crate::event::Emitter::new(fixtures::RUN_1))) + .with_emitter(Arc::new(Emitter::new(fixtures::RUN_1))) .with_run_store(run_store.clone().into()); let logger = crate::event::StoreProgressLogger::new(run_store.clone()); logger.register(services.run.emitter.as_ref()); @@ -267,7 +274,7 @@ mod tests { _prompt: &str, _context: &Context, _thread_id: Option<&str>, - _emitter: &Arc, + _emitter: &Arc, _sandbox: &Arc, _tool_hooks: Option>, ) -> Result { @@ -279,6 +286,8 @@ mod tests { _node: &Node, _prompt: &str, _system_prompt: Option<&str>, + _emitter: &Arc, + _stage_scope: &StageScope, ) -> Result { Ok(CodergenResult::Text { text: "one-shot response".to_string(), @@ -327,7 +336,7 @@ mod tests { _prompt: &str, _context: &Context, _thread_id: Option<&str>, - _emitter: &Arc, + _emitter: &Arc, _sandbox: &Arc, _tool_hooks: Option>, ) -> Result { @@ -339,6 +348,8 @@ mod tests { _node: &Node, _prompt: &str, _system_prompt: Option<&str>, + _emitter: &Arc, + _stage_scope: &StageScope, ) -> Result { Ok(CodergenResult::Text { text: "one-shot response".to_string(), @@ -384,7 +395,7 @@ mod tests { _prompt: &str, _context: &Context, _thread_id: Option<&str>, - _emitter: &Arc, + _emitter: &Arc, _sandbox: &Arc, _tool_hooks: Option>, ) -> Result { @@ -396,6 +407,8 @@ mod tests { _node: &Node, prompt: &str, system_prompt: Option<&str>, + _emitter: &Arc, + _stage_scope: &StageScope, ) -> Result { *self.captured_prompt.lock().unwrap() = Some(prompt.to_string()); *self.captured_system_prompt.lock().unwrap() = Some(system_prompt.map(String::from)); diff --git a/lib/crates/fabro-workflow/src/lifecycle/artifact.rs b/lib/crates/fabro-workflow/src/lifecycle/artifact.rs index f65fcee86..50d6ccded 100644 --- a/lib/crates/fabro-workflow/src/lifecycle/artifact.rs +++ b/lib/crates/fabro-workflow/src/lifecycle/artifact.rs @@ -16,7 +16,7 @@ use tokio::time::sleep; use crate::artifact::{normalize_durable_updates, offload_large_values, sync_artifacts_to_env}; use crate::artifact_snapshot::collect_artifacts; use crate::artifact_upload::ArtifactSink; -use crate::event::{Emitter, Event, RunNoticeLevel}; +use crate::event::{Emitter, Event, RunNoticeCode, RunNoticeLevel}; use crate::graph::{WorkflowGraph, WorkflowNode}; use crate::lifecycle::event::{stage_scope_for, stage_visit}; use crate::outcome::BilledModelUsage; @@ -125,7 +125,7 @@ impl RunLifecycle for ArtifactLifecycle { { self.emitter.notice( RunNoticeLevel::Warn, - "artifact_upload_failed", + RunNoticeCode::ArtifactUploadFailed, format!("[node: {node_id}] artifact upload failed: {err}"), ); return Ok(()); @@ -151,7 +151,7 @@ impl RunLifecycle for ArtifactLifecycle { Err(e) => { self.emitter.notice( RunNoticeLevel::Warn, - "artifact_collection_failed", + RunNoticeCode::ArtifactCollectionFailed, format!("[node: {node_id}] artifact collection failed: {e}"), ); } @@ -174,7 +174,7 @@ impl RunLifecycle for ArtifactLifecycle { { self.emitter.notice( RunNoticeLevel::Warn, - "artifact_offload_failed", + RunNoticeCode::ArtifactOffloadFailed, format!("[node: {node_id}] artifact offload failed: {e}"), ); } @@ -187,7 +187,7 @@ impl RunLifecycle for ArtifactLifecycle { { self.emitter.notice( RunNoticeLevel::Warn, - "artifact_sync_failed", + RunNoticeCode::ArtifactSyncFailed, format!("[node: {node_id}] artifact sync failed: {e}"), ); } diff --git a/lib/crates/fabro-workflow/src/lifecycle/git.rs b/lib/crates/fabro-workflow/src/lifecycle/git.rs index 0e0e9f195..2f9d5bfdb 100644 --- a/lib/crates/fabro-workflow/src/lifecycle/git.rs +++ b/lib/crates/fabro-workflow/src/lifecycle/git.rs @@ -14,7 +14,7 @@ use fabro_util::error::collect_causes; use fabro_util::time::elapsed_ms; use crate::artifact; -use crate::event::{Emitter, Event, RunNoticeLevel, StageScope}; +use crate::event::{Emitter, Event, RunNoticeCode, RunNoticeLevel, StageScope}; use crate::graph::{WorkflowGraph, WorkflowNode}; use crate::lifecycle::event::stage_scope_for; use crate::outcome::BilledModelUsage; @@ -128,7 +128,10 @@ impl RunLifecycle for GitLifecycle { None, None, ); - self.emit_metadata_warning("checkpoint_metadata_write_failed", message); + self.emit_metadata_warning( + RunNoticeCode::CheckpointMetadataWriteFailed, + message, + ); } }, Err(err) => { @@ -145,7 +148,10 @@ impl RunLifecycle for GitLifecycle { None, None, ); - self.emit_metadata_warning("checkpoint_metadata_write_failed", message); + self.emit_metadata_warning( + RunNoticeCode::CheckpointMetadataWriteFailed, + message, + ); } } } @@ -217,7 +223,7 @@ impl RunLifecycle for GitLifecycle { Some(&scope), ); self.emit_metadata_warning( - "checkpoint_metadata_write_failed", + RunNoticeCode::CheckpointMetadataWriteFailed, message, ); None @@ -239,7 +245,10 @@ impl RunLifecycle for GitLifecycle { None, Some(&scope), ); - self.emit_metadata_warning("checkpoint_metadata_write_failed", message); + self.emit_metadata_warning( + RunNoticeCode::CheckpointMetadataWriteFailed, + message, + ); None } } @@ -292,6 +301,12 @@ impl RunLifecycle for GitLifecycle { error = %fabro_sandbox::display_for_log(&err), "git push from run lifecycle failed" ); + self.emitter.notice_with_tail( + RunNoticeLevel::Warn, + RunNoticeCode::GitPushFailed, + format!("Failed to push run branch {branch}: {err}"), + exec_output_tail.clone(), + ); (false, exec_output_tail) } }; @@ -321,7 +336,7 @@ impl RunLifecycle for GitLifecycle { fabro_sandbox::default_redacted_output_tail(&err); self.emitter.notice_with_tail( RunNoticeLevel::Warn, - "git_diff_failed", + RunNoticeCode::GitDiffFailed, format!("[node: {node_id}] git diff failed: {err}"), exec_output_tail, ); @@ -395,7 +410,10 @@ impl GitLifecycle { Some(snapshot.bytes), scope, ); - self.emit_metadata_warning("checkpoint_metadata_push_failed", message); + self.emit_metadata_warning( + RunNoticeCode::CheckpointMetadataPushFailed, + message, + ); } else { self.emit_metadata_snapshot_completed( phase, @@ -421,7 +439,7 @@ impl GitLifecycle { None, scope, ); - self.emit_metadata_warning("checkpoint_metadata_write_failed", message); + self.emit_metadata_warning(RunNoticeCode::CheckpointMetadataWriteFailed, message); None } } @@ -506,14 +524,9 @@ impl GitLifecycle { } } - fn emit_metadata_warning(&self, code: &str, message: String) { + fn emit_metadata_warning(&self, code: RunNoticeCode, message: String) { if self.metadata_runtime.mark_metadata_degraded() { - self.emitter.emit(&Event::RunNotice { - level: RunNoticeLevel::Warn, - code: code.to_string(), - message, - exec_output_tail: None, - }); + self.emitter.notice(RunNoticeLevel::Warn, code, message); } } } diff --git a/lib/crates/fabro-workflow/src/pipeline/finalize.rs b/lib/crates/fabro-workflow/src/pipeline/finalize.rs index 927d2d833..cb58bd345 100644 --- a/lib/crates/fabro-workflow/src/pipeline/finalize.rs +++ b/lib/crates/fabro-workflow/src/pipeline/finalize.rs @@ -10,7 +10,7 @@ use fabro_util::time::elapsed_ms; use super::types::{Concluded, FinalizeOptions, Retroed}; use crate::error::Error; -use crate::event::{Event, RunNoticeLevel}; +use crate::event::{Event, RunNoticeCode, RunNoticeLevel}; use crate::outcome::{Outcome, OutcomeExt, StageOutcome}; use crate::records::{Checkpoint, Conclusion, StageSummary}; use crate::run_metadata::MetadataSnapshot; @@ -239,7 +239,11 @@ pub async fn write_finalize_commit( None, None, ); - emit_metadata_warning(services, "checkpoint_metadata_write_failed", message); + emit_metadata_warning( + services, + RunNoticeCode::CheckpointMetadataWriteFailed, + message, + ); return; } }; @@ -260,7 +264,11 @@ pub async fn write_finalize_commit( None, None, ); - emit_metadata_warning(services, "checkpoint_metadata_write_failed", message); + emit_metadata_warning( + services, + RunNoticeCode::CheckpointMetadataWriteFailed, + message, + ); return; } }; @@ -281,7 +289,11 @@ pub async fn write_finalize_commit( Some(snapshot.entry_count), Some(snapshot.bytes), ); - emit_metadata_warning(services, "checkpoint_metadata_push_failed", message); + emit_metadata_warning( + services, + RunNoticeCode::CheckpointMetadataPushFailed, + message, + ); } else { emit_metadata_snapshot_completed(services, phase, meta_branch, started, &snapshot); } @@ -300,7 +312,11 @@ pub async fn write_finalize_commit( None, None, ); - emit_metadata_warning(services, "checkpoint_metadata_write_failed", message); + emit_metadata_warning( + services, + RunNoticeCode::CheckpointMetadataWriteFailed, + message, + ); } } } @@ -363,7 +379,7 @@ fn emit_metadata_snapshot_failed( }); } -fn emit_metadata_warning(services: &RunServices, code: &str, message: String) { +fn emit_metadata_warning(services: &RunServices, code: RunNoticeCode, message: String) { if services.metadata_runtime.mark_metadata_degraded() { services.emitter.notice(RunNoticeLevel::Warn, code, message); } @@ -387,7 +403,7 @@ async fn compute_final_patch( Err(err) => { services.emitter.notice( RunNoticeLevel::Warn, - "git_diff_failed", + RunNoticeCode::GitDiffFailed, format!("final diff failed: {err}"), ); None @@ -534,7 +550,7 @@ pub async fn finalize(retroed: Retroed, options: &FinalizeOptions) -> Result Result Result Option { if let Some(env_name) = env_name { options.emitter.notice( RunNoticeLevel::Warn, - "dirty_worktree", + RunNoticeCode::DirtyWorktree, format!("Uncommitted changes will not be included in the {env_name}."), ); } @@ -200,6 +200,14 @@ fn resolve_worktree_plan(options: &mut InitOptions) -> Option { }) } +fn worktree_skipped_notice(mode: Option) -> Option<(RunNoticeCode, &'static str)> { + matches!(mode, Some(WorktreeMode::Always)).then_some(( + RunNoticeCode::WorktreeSkippedNoGit, + "Worktree mode `always` requested but no Git repository was found; running without a \ + worktree.", + )) +} + fn git_setup_intent(run_options: &RunOptions) -> GitSetupIntent { if let Some(source) = run_options.fork_source_ref.as_ref() { GitSetupIntent::ForkFromCheckpoint { @@ -239,11 +247,14 @@ async fn build_sandbox_env( Ok(token) => { env.insert("GITHUB_TOKEN".to_string(), token); } - Err(e) => emitter.notice( - RunNoticeLevel::Warn, - "github_token_failed", - format!("Failed to mint GitHub token: {e}"), - ), + Err(e) => { + tracing::warn!(error = %e, "Failed to mint GitHub token"); + emitter.notice( + RunNoticeLevel::Warn, + RunNoticeCode::GithubTokenFailed, + format!("Failed to mint GitHub token: {e}"), + ); + } } } } @@ -516,6 +527,13 @@ pub async fn initialize( )) }; if worktree_plan.is_some() && !worktree_created { + if let Some((code, message)) = worktree_skipped_notice(options.worktree_mode) { + tracing::warn!( + worktree_mode = ?options.worktree_mode, + "worktree skipped: cwd is not a git repository" + ); + options.emitter.notice(RunNoticeLevel::Warn, code, message); + } options.run_options.git = None; } let cleanup_guard = scopeguard::guard(Arc::clone(&sandbox), |sandbox| { @@ -593,7 +611,8 @@ pub async fn initialize( .is_some(); if !has_run_branch { let intent = git_setup_intent(&options.run_options); - if sandbox.origin_url().is_some() { + let sandbox_has_origin = sandbox.origin_url().is_some(); + if sandbox_has_origin { sandbox_git .ensure_git_available(&*sandbox) .await @@ -619,7 +638,16 @@ pub async fn initialize( options.run_options.base_branch = info.base_branch; } } - Ok(None) => {} + Ok(None) => { + if sandbox_has_origin { + options.emitter.notice( + RunNoticeLevel::Warn, + RunNoticeCode::SandboxGitUnavailable, + "Sandbox could not set up Git despite a configured origin; running \ + without checkpointing or PR support.", + ); + } + } Err(e) => { return Err(Error::engine_with_source("Sandbox git setup failed", &e)); } @@ -702,7 +730,7 @@ pub async fn initialize( if metadata_runtime.mark_metadata_degraded() { options.emitter.notice( RunNoticeLevel::Warn, - "checkpoint_metadata_write_failed", + RunNoticeCode::CheckpointMetadataWriteFailed, message, ); } @@ -1011,6 +1039,17 @@ mod tests { assert!(options.run_options.git.is_none()); } + #[test] + fn worktree_skipped_notice_only_warns_for_always() { + assert!(worktree_skipped_notice(None).is_none()); + assert!(worktree_skipped_notice(Some(WorktreeMode::Clean)).is_none()); + assert!(worktree_skipped_notice(Some(WorktreeMode::Dirty)).is_none()); + assert!(worktree_skipped_notice(Some(WorktreeMode::Never)).is_none()); + + let (code, _) = worktree_skipped_notice(Some(WorktreeMode::Always)).unwrap(); + assert_eq!(code, RunNoticeCode::WorktreeSkippedNoGit); + } + #[tokio::test] async fn initialize_prepares_sandbox_and_uses_persisted_run_dir() { let temp = tempfile::tempdir().unwrap(); diff --git a/lib/crates/fabro-workflow/src/pipeline/pull_request.rs b/lib/crates/fabro-workflow/src/pipeline/pull_request.rs index 35f27aa92..37aec323e 100644 --- a/lib/crates/fabro-workflow/src/pipeline/pull_request.rs +++ b/lib/crates/fabro-workflow/src/pipeline/pull_request.rs @@ -14,7 +14,7 @@ use fabro_util::text::strip_goal_decoration; use tracing::{debug, info, warn}; use super::types::{Concluded, Finalized, PullRequestOptions}; -use crate::event::{Event, RunNoticeLevel}; +use crate::event::{Event, RunNoticeCode, RunNoticeLevel}; use crate::outcome::{StageOutcome, format_cost as outcome_format_cost}; use crate::records::{Conclusion, RunSpec}; use crate::runtime_store::RunStoreHandle; @@ -675,7 +675,7 @@ pub async fn pull_request(concluded: Concluded, options: &PullRequestOptions) -> .emit(&Event::PullRequestFailed { error: e.clone() }); services.emitter.notice( RunNoticeLevel::Warn, - "pull_request_failed", + RunNoticeCode::PullRequestFailed, format!("PR creation failed: {e}"), ); } diff --git a/lib/crates/fabro-workflow/tests/it/integration.rs b/lib/crates/fabro-workflow/tests/it/integration.rs index f6ece331b..b89fc7f5a 100644 --- a/lib/crates/fabro-workflow/tests/it/integration.rs +++ b/lib/crates/fabro-workflow/tests/it/integration.rs @@ -6221,6 +6221,8 @@ mod real_llm { _node: &Node, prompt: &str, _system_prompt: Option<&str>, + _emitter: &Arc, + _stage_scope: &fabro_workflow::event::StageScope, ) -> Result { self.complete(prompt).await } From 786a6f67e15dc16eb5da10cf29f7b2438879485d Mon Sep 17 00:00:00 2001 From: "fabro-sh-0530[bot]" <281434857+fabro-sh-0530[bot]@users.noreply.github.com> Date: Tue, 5 May 2026 09:32:33 -0400 Subject: [PATCH 15/17] Read billing and stages from RunProjection with live runtimes (#213) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Summary Billing and stage lists now use the event-sourced `RunProjection` as their source of truth, so running and retrying stages appear immediately and runtimes keep advancing in the UI. This removes the checkpoint completed-node bypass that hid in-flight work and froze totals until the next server response. ### Plan Summary - Store stage `started_at`, terminal `duration_ms`, server-internal `usage`, and lifecycle `state` on `StageProjection`. - Populate those fields from stage lifecycle events, including retry transitions and per-attempt reset on new starts. - Render `/runs/{id}/stages` and `/runs/{id}/billing` from `RunProjection.iter_stages()`. - Expose the new API/client fields and tick in-flight billing runtimes on the web UI. ```mermaid flowchart TB Events["Stage lifecycle events"] --> Projection["RunProjection StageProjection"] Projection --> StagesAPI["GET /runs/{id}/stages"] Projection --> BillingAPI["GET /runs/{id}/billing"] StagesAPI --> StageUI["Stage sidebar/stages view"] BillingAPI --> BillingUI["Billing tab live totals"] ``` ### Key decisions Retry and revisit handling stays one row per node id: latest visit data wins, while first-seen event sequence keeps ordering stable with finalize output. `state` is stored rather than derived so `Retrying` is representable, and old serialized projections still work through the `effective_state()` fallback. Billing `usage` remains server-internal and is skipped on the wire; public schemas only expose the fields needed by `/stages`, `/billing`, and the frontend live timer. Added focused reducer, server retry/revisit, API round-trip, billing UI, and event invalidation coverage. ⚒️ Generated with [Fabro](https://fabro.sh) --------- Co-authored-by: Fabro Co-authored-by: Bryan Helmkamp --- .../app/components/stage-sidebar.tsx | 13 +- apps/fabro-web/app/lib/query-keys.test.ts | 3 +- apps/fabro-web/app/lib/run-events.test.tsx | 16 +- apps/fabro-web/app/lib/run-events.ts | 1 + apps/fabro-web/app/lib/stage-sidebar.ts | 17 +- apps/fabro-web/app/lib/time.ts | 19 +- .../fabro-web/app/routes/run-billing.test.tsx | 62 ++++- apps/fabro-web/app/routes/run-billing.tsx | 161 ++++++----- apps/fabro-web/app/routes/run-stages.tsx | 10 +- docs/public/api-reference/fabro-api.yaml | 29 ++ .../tests/run_billing_stage_round_trip.rs | 57 ++++ .../tests/stage_projection_round_trip.rs | 5 +- lib/crates/fabro-server/src/demo/mod.rs | 13 + lib/crates/fabro-server/src/server.rs | 2 + .../src/server/handler/billing.rs | 214 ++++++++------- lib/crates/fabro-server/src/server/tests.rs | 192 ++++++++++++++ lib/crates/fabro-store/src/run_state.rs | 249 +++++++++++++++++- .../tests/serializable_projection.rs | 6 +- lib/crates/fabro-types/src/run_projection.rs | 64 ++++- .../src/models/board-column-definition.ts | 3 - .../src/models/run-billing-stage.ts | 9 +- .../fabro-api-client/src/models/run-stage.ts | 7 +- .../src/models/stage-projection.ts | 15 +- 23 files changed, 954 insertions(+), 213 deletions(-) diff --git a/apps/fabro-web/app/components/stage-sidebar.tsx b/apps/fabro-web/app/components/stage-sidebar.tsx index 2d3122602..e2cfe88f8 100644 --- a/apps/fabro-web/app/components/stage-sidebar.tsx +++ b/apps/fabro-web/app/components/stage-sidebar.tsx @@ -1,4 +1,4 @@ -import { useState, useEffect, useRef, type ComponentType } from "react"; +import { useEffect, useRef, type ComponentType } from "react"; import { Link } from "react-router"; import type { StageState } from "@qltysh/fabro-api-client"; import { @@ -12,6 +12,7 @@ import { import { Bars3BottomLeftIcon, DocumentTextIcon, MapIcon } from "@heroicons/react/24/outline"; import { formatDurationSecs } from "../lib/format"; import { ACTIVE_STAGE_STATES, formatStageLabel } from "../lib/stage-sidebar"; +import { useTickingNow } from "../lib/time"; export interface Stage { id: string; @@ -43,7 +44,6 @@ interface StageSidebarProps { export function StageSidebar({ stages, runId, selectedStageId, activeLink }: StageSidebarProps) { // Track when we first observed each running stage (for ticking timer) const runningStartRef = useRef>(new Map()); - const [, setTick] = useState(0); // Track start times for running stages useEffect(() => { @@ -63,16 +63,13 @@ export function StageSidebar({ stages, runId, selectedStageId, activeLink }: Sta }, [stages]); // Tick every second while any stage is running - useEffect(() => { - if (!stages.some((s) => ACTIVE_STAGE_STATES.has(s.status))) return; - const interval = setInterval(() => setTick((t) => t + 1), 1000); - return () => clearInterval(interval); - }, [stages]); + const hasActive = stages.some((s) => ACTIVE_STAGE_STATES.has(s.status)); + const now = useTickingNow(hasActive); function stageDuration(stage: Stage): string { if (ACTIVE_STAGE_STATES.has(stage.status)) { const start = runningStartRef.current.get(stage.id); - if (start) return formatDurationSecs(Math.floor((Date.now() - start) / 1000)); + if (start) return formatDurationSecs(Math.floor((now - start) / 1000)); return "0s"; } return stage.duration; diff --git a/apps/fabro-web/app/lib/query-keys.test.ts b/apps/fabro-web/app/lib/query-keys.test.ts index 5d7b3a53a..a06acaedb 100644 --- a/apps/fabro-web/app/lib/query-keys.test.ts +++ b/apps/fabro-web/app/lib/query-keys.test.ts @@ -22,6 +22,7 @@ describe("queryKeys", () => { ]); expect(queryKeysForRunEvent("run-1", "stage.completed", "stage-1")).toEqual([ queryKeys.runs.stages("run-1"), + queryKeys.runs.billing("run-1"), queryKeys.runs.events("run-1", 1000), queryKeys.runs.graph("run-1", "LR"), queryKeys.runs.graph("run-1", "TB"), @@ -48,4 +49,4 @@ describe("queryKeys", () => { test("agent activity events without a node_id invalidate nothing", () => { expect(queryKeysForRunEvent("run-1", "agent.message")).toEqual([]); }); -}); \ No newline at end of file +}); diff --git a/apps/fabro-web/app/lib/run-events.test.tsx b/apps/fabro-web/app/lib/run-events.test.tsx index 14db0e3f2..432791352 100644 --- a/apps/fabro-web/app/lib/run-events.test.tsx +++ b/apps/fabro-web/app/lib/run-events.test.tsx @@ -50,12 +50,16 @@ describe("queryKeysForRunEvent", () => { ]); }); - test("stage.retrying invalidates the same keys as other stage events", () => { - const keys = queryKeysForRunEvent("run-1", "stage.retrying", "verify@2"); - expect(keys).toContain(queryKeys.runs.stages("run-1")); - expect(keys).toContain(queryKeys.runs.events("run-1", 1000)); - expect(keys).toContain(queryKeys.runs.detail("run-1")); - expect(keys).toContain(queryKeys.runs.stageEvents("run-1", "verify@2")); + test("stage.retrying invalidates stages, billing, events, graph, detail, and stage events", () => { + expect(queryKeysForRunEvent("run-1", "stage.retrying", "verify@2")).toEqual([ + queryKeys.runs.stages("run-1"), + queryKeys.runs.billing("run-1"), + queryKeys.runs.events("run-1", 1000), + queryKeys.runs.graph("run-1", "LR"), + queryKeys.runs.graph("run-1", "TB"), + queryKeys.runs.detail("run-1"), + queryKeys.runs.stageEvents("run-1", "verify@2"), + ]); }); }); diff --git a/apps/fabro-web/app/lib/run-events.ts b/apps/fabro-web/app/lib/run-events.ts index 1546d1578..63855296e 100644 --- a/apps/fabro-web/app/lib/run-events.ts +++ b/apps/fabro-web/app/lib/run-events.ts @@ -109,6 +109,7 @@ export function queryKeysForRunEvent( if (STAGE_EVENTS.has(event)) { const keys = [ queryKeys.runs.stages(runId), + queryKeys.runs.billing(runId), queryKeys.runs.events(runId, 1000), queryKeys.runs.graph(runId, "LR"), queryKeys.runs.graph(runId, "TB"), diff --git a/apps/fabro-web/app/lib/stage-sidebar.ts b/apps/fabro-web/app/lib/stage-sidebar.ts index e44e5f4a5..84a5de2de 100644 --- a/apps/fabro-web/app/lib/stage-sidebar.ts +++ b/apps/fabro-web/app/lib/stage-sidebar.ts @@ -1,13 +1,22 @@ -import type { PaginatedRunStageList, StageState } from "@qltysh/fabro-api-client"; +import { StageState } from "@qltysh/fabro-api-client"; +import type { PaginatedRunStageList } from "@qltysh/fabro-api-client"; import type { Stage } from "../components/stage-sidebar"; import { isVisibleStage } from "../data/runs"; import { formatDurationSecs } from "./format"; -export const ACTIVE_STAGE_STATES: ReadonlySet = new Set(["running", "retrying"]); +export const ACTIVE_STAGE_STATES: ReadonlySet = new Set([ + StageState.RUNNING, + StageState.RETRYING, +]); +export const IN_FLIGHT_STAGE_STATES: ReadonlySet = new Set([ + StageState.PENDING, + StageState.RUNNING, + StageState.RETRYING, +]); export const SUCCEEDED_STAGE_STATES: ReadonlySet = new Set([ - "succeeded", - "partially_succeeded", + StageState.SUCCEEDED, + StageState.PARTIALLY_SUCCEEDED, ]); /** diff --git a/apps/fabro-web/app/lib/time.ts b/apps/fabro-web/app/lib/time.ts index 1d8a818d4..fb6f9b87f 100644 --- a/apps/fabro-web/app/lib/time.ts +++ b/apps/fabro-web/app/lib/time.ts @@ -1,3 +1,21 @@ +import { useEffect, useState } from "react"; + +/** + * Re-renders the calling component every `intervalMs` milliseconds while + * `active` is true, returning the current `Date.now()` value at each tick. + * Returns the captured value when paused, so renders are stable. + */ +export function useTickingNow(active: boolean, intervalMs = 1000): number { + const [now, setNow] = useState(() => Date.now()); + useEffect(() => { + if (!active) return; + setNow(Date.now()); + const interval = setInterval(() => setNow(Date.now()), intervalMs); + return () => clearInterval(interval); + }, [active, intervalMs]); + return now; +} + function relativeTime(seconds: number, past: boolean): string { if (seconds < 60) return past ? "just now" : "in <1m"; const minutes = Math.floor(seconds / 60); @@ -21,4 +39,3 @@ export function timeAgo(iso: string): string { export function timeUntil(iso: string): string { return relativeTime(Math.floor((new Date(iso).getTime() - Date.now()) / 1000), false); } - diff --git a/apps/fabro-web/app/routes/run-billing.test.tsx b/apps/fabro-web/app/routes/run-billing.test.tsx index ed9e3040c..c35ab52cb 100644 --- a/apps/fabro-web/app/routes/run-billing.test.tsx +++ b/apps/fabro-web/app/routes/run-billing.test.tsx @@ -75,12 +75,14 @@ describe("RunBilling", () => { model: null, billing: zeroBilling(), runtime_secs: 0, + state: "succeeded", }, { stage: { id: "command", name: "command" }, model: null, billing: zeroBilling(), runtime_secs: 61, + state: "succeeded", }, ], totals: { @@ -96,7 +98,7 @@ describe("RunBilling", () => { expect(text).toMatch(/—\s*\/\s*—/); expect(text).toContain("1m 1s"); expect(text).not.toContain("By model"); - expect(text).not.toContain("No completed stages yet"); + expect(text).not.toContain("No stages yet"); }); test("renders mixed LLM and non-LLM rows while counting only LLM rows by model", () => { @@ -108,6 +110,7 @@ describe("RunBilling", () => { model: null, billing: zeroBilling(), runtime_secs: 0, + state: "succeeded", }, { stage: { id: "agent", name: "agent" }, @@ -119,6 +122,7 @@ describe("RunBilling", () => { total_usd_micros: 240000, }), runtime_secs: 42, + state: "succeeded", }, ], totals: { @@ -155,11 +159,61 @@ describe("RunBilling", () => { expect(textFromInstance(byModelFooterCells[1])).toBe("1"); }); - test("keeps the empty state for runs with no completed stages", () => { + test("keeps the empty state for runs with no stages", () => { const renderer = renderBilling(billing()); const text = textFromNode(renderer.toJSON()); - expect(text).toContain("No completed stages yet"); - expect(text).toContain("Stages will appear once the run produces completed nodes."); + expect(text).toContain("No stages yet"); + expect(text).toContain("Stages will appear as soon as the run starts executing."); + }); + + test("renders an in-flight row with live runtime and includes its elapsed time in the footer", () => { + const originalNow = Date.now; + // Pin "now" to 30s after the in-flight row started. + const startedAt = "2026-04-29T12:00:00.000Z"; + const fakeNow = new Date("2026-04-29T12:00:30.000Z").getTime(); + Date.now = () => fakeNow; + + try { + const renderer = renderBilling( + billing({ + stages: [ + { + stage: { id: "in-flight", name: "in-flight" }, + model: null, + // Server reports 0 runtime / no billing; the row is still being executed. + billing: zeroBilling(), + runtime_secs: 0, + started_at: startedAt, + state: "running", + }, + ], + // Server total is 0 because the in-flight row hasn't been finalized. + totals: { + runtime_secs: 0, + ...zeroBilling(), + }, + }), + ); + + const text = textFromNode(renderer.toJSON()); + // Empty-state must NOT show — the table should appear as soon as the + // first stage starts. + expect(text).not.toContain("No stages yet"); + expect(text).toContain("in-flight"); + + // Both the row's runtime cell and the footer total should reflect + // ~30s elapsed since started_at. + expect(text).toContain("30s"); + + const footers = renderer.root.findAll((node) => node.type === "tfoot"); + const footerCells = footers[0].findAll((node) => node.type === "td"); + // The Run time column in the footer is index 3 (Total / [empty Model] / + // Tokens / Run time / Billing). + const footerRuntime = textFromInstance(footerCells[3]); + expect(footerRuntime).toContain("30s"); + } finally { + Date.now = originalNow; + } }); }); diff --git a/apps/fabro-web/app/routes/run-billing.tsx b/apps/fabro-web/app/routes/run-billing.tsx index 6bb6ab763..be0d52386 100644 --- a/apps/fabro-web/app/routes/run-billing.tsx +++ b/apps/fabro-web/app/routes/run-billing.tsx @@ -1,7 +1,11 @@ +import { useMemo } from "react"; + import { EmptyState } from "../components/state"; import { formatDurationSecs } from "../lib/format"; import { useRunBilling } from "../lib/queries"; -import type { RunBilling } from "@qltysh/fabro-api-client"; +import { IN_FLIGHT_STAGE_STATES } from "../lib/stage-sidebar"; +import { useTickingNow } from "../lib/time"; +import type { RunBilling, RunBillingStage } from "@qltysh/fabro-api-client"; const EMPTY_VALUE = "—"; @@ -14,78 +18,103 @@ function formatUsdMicros(usdMicros?: number | null) { return usdMicros == null ? EMPTY_VALUE : `$${(usdMicros / 1_000_000).toFixed(2)}`; } -function mapBilling(billing: RunBilling | undefined) { - if (!billing) { - return { - stages: [], - totalRuntime: formatDurationSecs(0), - totalUsdMicros: undefined, - totalInput: null, - totalOutput: null, - modelBreakdown: [], - modelStageCount: 0, - }; - } +function isInFlight(stage: RunBillingStage): boolean { + return stage.state != null && IN_FLIGHT_STAGE_STATES.has(stage.state); +} - const stages = billing.stages.map((stage) => { - const hasModel = stage.model != null; - return { - stage: stage.stage.name, - model: stage.model?.id ?? null, - inputTokens: hasModel ? stage.billing.input_tokens : null, - outputTokens: hasModel - ? stage.billing.output_tokens + stage.billing.reasoning_tokens - : null, - runtime: formatDurationSecs(stage.runtime_secs), - totalUsdMicros: stage.billing.total_usd_micros, - }; - }); - const totalRuntime = formatDurationSecs(billing.totals.runtime_secs); - const hasLlmStages = billing.by_model.length > 0; - const totalInput = hasLlmStages ? billing.totals.input_tokens : null; - const totalOutput = hasLlmStages - ? billing.totals.output_tokens + billing.totals.reasoning_tokens - : null; - const totalUsdMicros = billing.totals.total_usd_micros; - const modelBreakdown = billing.by_model - .map((entry) => ({ - model: entry.model.id, - stages: entry.stages, - inputTokens: entry.billing.input_tokens, - outputTokens: entry.billing.output_tokens + entry.billing.reasoning_tokens, - totalUsdMicros: entry.billing.total_usd_micros, - })) - .sort((a, b) => (b.totalUsdMicros ?? -1) - (a.totalUsdMicros ?? -1)); - const modelStageCount = modelBreakdown.reduce((sum, row) => sum + row.stages, 0); +interface MappedStageRow { + stage: string; + model: string | null; + inputTokens: number | null; + outputTokens: number | null; + runtimeSecs: number; + totalUsdMicros: number | null | undefined; +} + +function liveRuntimeSecs(stage: RunBillingStage, now: number): number { + if (stage.started_at) { + const startedMs = new Date(stage.started_at).getTime(); + if (Number.isFinite(startedMs)) { + return Math.max(0, (now - startedMs) / 1000); + } + } + return stage.runtime_secs; +} + +function mapStageRow(stage: RunBillingStage, runtimeSecs: number): MappedStageRow { + const hasModel = stage.model != null; return { - stages, - totalRuntime, - totalUsdMicros, - totalInput, - totalOutput, - modelBreakdown, - modelStageCount, + stage: stage.stage.name, + model: stage.model?.id ?? null, + inputTokens: hasModel ? stage.billing.input_tokens : null, + outputTokens: hasModel + ? stage.billing.output_tokens + stage.billing.reasoning_tokens + : null, + runtimeSecs, + totalUsdMicros: stage.billing.total_usd_micros, }; } export default function RunBilling({ params }: { params: { id: string } }) { const billingQuery = useRunBilling(params.id); - const { - stages, - totalRuntime, - totalUsdMicros, - totalInput, - totalOutput, - modelBreakdown, - modelStageCount, - } = mapBilling(billingQuery.data); + const billing = billingQuery.data; + const hasInFlight = billing?.stages.some(isInFlight) ?? false; - if (!stages.length) { + // Tick once per second only while a stage is in-flight. + const now = useTickingNow(hasInFlight); + + // Completed rows don't depend on `now`; memoize them by `billing` so we + // don't reallocate them every tick. + const completedRows = useMemo(() => { + if (!billing) return []; + return billing.stages.map((stage) => mapStageRow(stage, stage.runtime_secs)); + }, [billing]); + + // The model breakdown is server-derived and stable across ticks too. + const modelBreakdown = useMemo(() => { + if (!billing) return []; + return billing.by_model + .map((entry) => ({ + model: entry.model.id, + stages: entry.stages, + inputTokens: entry.billing.input_tokens, + outputTokens: entry.billing.output_tokens + entry.billing.reasoning_tokens, + totalUsdMicros: entry.billing.total_usd_micros, + })) + .sort((a, b) => (b.totalUsdMicros ?? -1) - (a.totalUsdMicros ?? -1)); + }, [billing]); + + // Re-derive only the in-flight rows on each tick; everything else stays put. + const rows = useMemo(() => { + if (!billing) return []; + if (!hasInFlight) return completedRows; + return billing.stages.map((stage, idx) => + isInFlight(stage) + ? mapStageRow(stage, liveRuntimeSecs(stage, now)) + : completedRows[idx], + ); + }, [billing, completedRows, hasInFlight, now]); + + // While ticking, sum the displayed row runtimes so the footer updates in + // lock-step. Otherwise trust the server's authoritative total. + const totalRuntimeSecs = hasInFlight + ? rows.reduce((sum, row) => sum + row.runtimeSecs, 0) + : (billing?.totals.runtime_secs ?? 0); + + const hasLlmStages = (billing?.by_model.length ?? 0) > 0; + const totalInput = hasLlmStages ? (billing?.totals.input_tokens ?? null) : null; + const totalOutput = hasLlmStages && billing + ? billing.totals.output_tokens + billing.totals.reasoning_tokens + : null; + const totalUsdMicros = billing?.totals.total_usd_micros; + const modelStageCount = modelBreakdown.reduce((sum, row) => sum + row.stages, 0); + + if (!rows.length) { return (
); @@ -105,7 +134,7 @@ export default function RunBilling({ params }: { params: { id: string } }) { - {stages.map((row) => ( + {rows.map((row) => ( {row.stage} @@ -115,7 +144,9 @@ export default function RunBilling({ params }: { params: { id: string } }) { {formatTokens(row.inputTokens)} /{" "} {formatTokens(row.outputTokens)} - {row.runtime} + + {formatDurationSecs(row.runtimeSecs)} + {formatUsdMicros(row.totalUsdMicros)} @@ -131,7 +162,7 @@ export default function RunBilling({ params }: { params: { id: string } }) { {formatTokens(totalOutput)} - {totalRuntime} + {formatDurationSecs(totalRuntimeSecs)} {formatUsdMicros(totalUsdMicros)} diff --git a/apps/fabro-web/app/routes/run-stages.tsx b/apps/fabro-web/app/routes/run-stages.tsx index a8fc69168..2b01fc5f3 100644 --- a/apps/fabro-web/app/routes/run-stages.tsx +++ b/apps/fabro-web/app/routes/run-stages.tsx @@ -40,6 +40,7 @@ import type { Stage } from "../components/stage-sidebar"; import { EmptyState } from "../components/state"; import { CopyButton } from "../components/ui"; import { formatDurationSecs } from "../lib/format"; +import { useTickingNow } from "../lib/time"; import { fetchRunCommandLog, useRunStageEvents, useRunStages } from "../lib/queries"; import { STAGE_ACTIVITY_EVENT_TYPES, type StageActivityEventType } from "../lib/run-events"; import { ACTIVE_STAGE_STATES, formatStageLabel, mapRunStagesToSidebarStages } from "../lib/stage-sidebar"; @@ -563,7 +564,6 @@ function RunningStageDuration({ const [startedAt, setStartedAt] = useState(() => isRunning ? Date.now() : null, ); - const [, setTick] = useState(0); useEffect(() => { setStartedAt((current) => { @@ -572,14 +572,10 @@ function RunningStageDuration({ }); }, [isRunning]); - useEffect(() => { - if (!isRunning) return; - const interval = setInterval(() => setTick((tick) => tick + 1), 1000); - return () => clearInterval(interval); - }, [isRunning]); + const now = useTickingNow(isRunning); if (isRunning && startedAt) { - return formatDurationSecs(Math.floor((Date.now() - startedAt) / 1000)); + return formatDurationSecs(Math.floor((now - startedAt) / 1000)); } return duration; } diff --git a/docs/public/api-reference/fabro-api.yaml b/docs/public/api-reference/fabro-api.yaml index 4319f6774..0b499e7fa 100644 --- a/docs/public/api-reference/fabro-api.yaml +++ b/docs/public/api-reference/fabro-api.yaml @@ -5314,6 +5314,20 @@ components: oneOf: - $ref: "#/components/schemas/CommandTermination" - type: "null" + started_at: + type: ["string", "null"] + format: date-time + description: Wall-clock time the latest attempt of this stage started, if known. + duration_ms: + type: ["integer", "null"] + format: uint64 + minimum: 0 + description: Wall-clock duration of the stage's latest terminal attempt, if known. + state: + oneOf: + - $ref: "#/components/schemas/StageState" + - type: "null" + description: Lifecycle state of the stage projection. InterviewOption: description: Option stored with an interview question in the event log. @@ -6333,6 +6347,11 @@ components: minimum: 1 description: 1-based visit count; bumped each time the workflow re-enters this node. example: 2 + started_at: + type: ["string", "null"] + format: date-time + description: Wall-clock time the latest attempt of this stage started, if known. + example: "2026-04-29T12:34:56Z" # ── File Diff Schemas ────────────────────────────────────────────── @@ -6526,6 +6545,16 @@ components: type: number description: Wall-clock runtime in seconds, summed across every visit of this node. example: 154.0 + started_at: + type: ["string", "null"] + format: date-time + description: Wall-clock time the latest attempt of this stage started, if known. + example: "2026-04-29T12:34:56Z" + state: + oneOf: + - $ref: "#/components/schemas/StageState" + - type: "null" + description: Lifecycle state of the stage. Use to detect in-flight rows for client-side runtime ticking. RunBillingTotals: description: Aggregate billing totals across all stages of a run. diff --git a/lib/crates/fabro-api/tests/run_billing_stage_round_trip.rs b/lib/crates/fabro-api/tests/run_billing_stage_round_trip.rs index c160a840a..52fb4ece1 100644 --- a/lib/crates/fabro-api/tests/run_billing_stage_round_trip.rs +++ b/lib/crates/fabro-api/tests/run_billing_stage_round_trip.rs @@ -1,4 +1,5 @@ use fabro_api::types::RunBillingStage; +use fabro_types::StageState; use serde_json::json; #[test] @@ -28,3 +29,59 @@ fn run_billing_stage_model_accepts_required_null() { assert!(encoded.get("model").is_some()); assert!(encoded["model"].is_null()); } + +#[test] +fn run_billing_stage_round_trips_terminal_row_with_started_at_and_state() { + let value = json!({ + "stage": { + "id": "build", + "name": "build" + }, + "model": { "id": "claude-sonnet-4-5" }, + "billing": { + "input_tokens": 12, + "output_tokens": 34, + "total_tokens": 46, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "runtime_secs": 5.5, + "started_at": "2026-04-29T12:34:56Z", + "state": "succeeded" + }); + + let stage: RunBillingStage = + serde_json::from_value(value.clone()).expect("terminal stage row should deserialize"); + assert!(stage.started_at.is_some()); + assert_eq!(stage.state, Some(StageState::Succeeded)); + assert_eq!(serde_json::to_value(stage).unwrap(), value); +} + +#[test] +fn run_billing_stage_round_trips_in_flight_row() { + let value = json!({ + "stage": { + "id": "build", + "name": "build" + }, + "model": null, + "billing": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "runtime_secs": 1.25, + "started_at": "2026-04-29T12:34:56Z", + "state": "running" + }); + + let stage: RunBillingStage = + serde_json::from_value(value.clone()).expect("in-flight stage row should deserialize"); + assert!(stage.model.is_none()); + assert_eq!(stage.state, Some(StageState::Running)); + assert_eq!(serde_json::to_value(stage).unwrap(), value); +} diff --git a/lib/crates/fabro-api/tests/stage_projection_round_trip.rs b/lib/crates/fabro-api/tests/stage_projection_round_trip.rs index 0523197cf..ad4dcb579 100644 --- a/lib/crates/fabro-api/tests/stage_projection_round_trip.rs +++ b/lib/crates/fabro-api/tests/stage_projection_round_trip.rs @@ -28,7 +28,10 @@ fn stage_projection_round_trips_representative_json() { "parallel_results": [{ "branch": 0, "status": "succeeded" }], "stdout": "ok", "stderr": "", - "termination": "exited" + "termination": "exited", + "started_at": "2026-04-29T12:34:00Z", + "duration_ms": 56000, + "state": "succeeded" }); let state: StageProjection = serde_json::from_value(value.clone()).unwrap(); diff --git a/lib/crates/fabro-server/src/demo/mod.rs b/lib/crates/fabro-server/src/demo/mod.rs index c900c8b91..539731c82 100644 --- a/lib/crates/fabro-server/src/demo/mod.rs +++ b/lib/crates/fabro-server/src/demo/mod.rs @@ -1214,30 +1214,35 @@ mod runs { "Detect Drift", StageState::Succeeded, Some(72.0), + None, ), run_stage_from_stage_id( &StageId::new("propose-changes", 1), "Propose Changes", StageState::Succeeded, Some(154.0), + None, ), run_stage_from_stage_id( &StageId::new("review-changes", 1), "Review Changes", StageState::Succeeded, Some(45.0), + None, ), run_stage_from_stage_id( &StageId::new("apply-changes", 1), "Apply Changes", StageState::Succeeded, Some(118.0), + None, ), run_stage_from_stage_id( &StageId::new("apply-changes", 2), "Apply Changes", StageState::Running, None, + None, ), ] } @@ -1374,6 +1379,8 @@ mod runs { total_usd_micros: Some(480_000), }, runtime_secs: 72.0, + started_at: None, + state: Some(StageState::Succeeded), }, RunBillingStage { stage: BillingStageRef { @@ -1393,6 +1400,8 @@ mod runs { total_usd_micros: Some(720_000), }, runtime_secs: 154.0, + started_at: None, + state: Some(StageState::Succeeded), }, RunBillingStage { stage: BillingStageRef { @@ -1412,6 +1421,8 @@ mod runs { total_usd_micros: Some(190_000), }, runtime_secs: 45.0, + started_at: None, + state: Some(StageState::Succeeded), }, RunBillingStage { stage: BillingStageRef { @@ -1431,6 +1442,8 @@ mod runs { total_usd_micros: Some(870_000), }, runtime_secs: 118.0, + started_at: None, + state: Some(StageState::Running), }, ], totals: RunBillingTotals { diff --git a/lib/crates/fabro-server/src/server.rs b/lib/crates/fabro-server/src/server.rs index 965989c8c..276374d2f 100644 --- a/lib/crates/fabro-server/src/server.rs +++ b/lib/crates/fabro-server/src/server.rs @@ -570,6 +570,7 @@ pub(crate) fn run_stage_from_stage_id( name: impl Into, status: StageState, duration_secs: Option, + started_at: Option>, ) -> RunStage { RunStage { id: stage_id.to_string(), @@ -579,6 +580,7 @@ pub(crate) fn run_stage_from_stage_id( node_id: stage_id.node_id().to_string(), visit: std::num::NonZeroU32::new(stage_id.visit()) .expect("StageId stores a non-zero visit"), + started_at, } } diff --git a/lib/crates/fabro-server/src/server/handler/billing.rs b/lib/crates/fabro-server/src/server/handler/billing.rs index 193ebd78d..5c000869c 100644 --- a/lib/crates/fabro-server/src/server/handler/billing.rs +++ b/lib/crates/fabro-server/src/server/handler/billing.rs @@ -1,13 +1,14 @@ +use std::collections::HashMap; use std::sync::Arc; -use fabro_store::RunProjectionReducer; -use fabro_types::{EventBody, RunProjection, StageId}; +use chrono::{DateTime, Utc}; +use fabro_types::{RunProjection, StageProjection, StageState}; use super::super::{ - ApiError, AppState, BillingByModel, BillingStageRef, EventEnvelope, HashMap, IntoResponse, - Json, ListResponse, ModelReference, PaginationParams, Path, Query, RequiredUser, Response, - Router, RunBilling, RunBillingStage, RunBillingTotals, RunId, StageState, State, StatusCode, - get, parse_run_id_path, run_stage_from_stage_id, + ApiError, AppState, BillingByModel, BillingStageRef, IntoResponse, Json, ListResponse, + ModelReference, PaginationParams, Path, Query, RequiredUser, Response, Router, RunBilling, + RunBillingStage, RunBillingTotals, RunId, State, StatusCode, get, parse_run_id_path, + run_stage_from_stage_id, }; pub(super) fn routes() -> Router> { @@ -16,41 +17,6 @@ pub(super) fn routes() -> Router> { .route("/runs/{id}/billing", get(get_run_billing)) } -/// Map a `stage.*` lifecycle event body to the [`StageState`] it implies. -/// Returns `None` for any other variant. -fn stage_state_from_lifecycle(body: &EventBody) -> Option { - match body { - EventBody::StageStarted(_) => Some(StageState::Running), - EventBody::StageRetrying(_) => Some(StageState::Retrying), - EventBody::StageFailed(props) => Some(if props.will_retry { - StageState::Retrying - } else { - StageState::Failed - }), - EventBody::StageCompleted(props) => Some(StageState::from(props.status)), - _ => None, - } -} - -/// Single-pass scan over `events` building the latest [`StageState`] for each -/// [`StageId`] from lifecycle events (started/retrying/completed/failed). Each -/// later lifecycle event overwrites earlier ones, leaving the latest as the -/// stored value — equivalent to "scan in reverse, take first match" but in O(E) -/// for the whole list rather than O(stages × events). -fn latest_stage_states(events: &[EventEnvelope]) -> HashMap { - let mut states = HashMap::new(); - for envelope in events { - let Some(stage_id) = envelope.event.stage_id.as_ref() else { - continue; - }; - let Some(state) = stage_state_from_lifecycle(&envelope.event.body) else { - continue; - }; - states.insert(stage_id.clone(), state); - } - states -} - async fn list_run_stages( _auth: RequiredUser, State(state): State>, @@ -62,42 +28,30 @@ async fn list_run_stages( Err(response) => return response, }; - let events = match state.store.open_run_reader(&id).await { - Ok(run_store) => run_store.list_events().await.unwrap_or_default(), - Err(_) => return ApiError::not_found("Run not found.").into_response(), + let Ok(run_store) = state.store.open_run_reader(&id).await else { + return ApiError::not_found("Run not found.").into_response(); }; - - let projection = match RunProjection::apply_events(&events) { - Ok(projection) => projection, + let projection = match run_store.state().await { + Ok(state) => state, Err(err) => { - tracing::warn!( - run_id = %id, - error = %err, - "Failed to build run projection; returning empty stages list", - ); - RunProjection::default() + return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) + .into_response(); } }; - let stage_durations = fabro_workflow::extract_stage_durations_by_stage_id(&events); - let lifecycle_states = latest_stage_states(&events); - let mut stages = Vec::new(); - for (stage_id, stage_projection) in projection.iter_stages() { - // Prefer the latest lifecycle event; fall back to the projection's - // stored completion (e.g. for runs recovered from snapshot only). - let status = lifecycle_states.get(stage_id).copied().unwrap_or_else(|| { - stage_projection - .completion - .as_ref() - .map_or(StageState::Pending, |c| StageState::from(c.outcome)) - }); - stages.push(run_stage_from_stage_id( - stage_id, - stage_id.node_id().to_string(), - status, - stage_durations.get(stage_id).map(|ms| *ms as f64 / 1000.0), - )); - } + let now = Utc::now(); + let stages = projection + .iter_stages() + .map(|(stage_id, stage)| { + run_stage_from_stage_id( + stage_id, + stage_id.node_id().to_string(), + stage.effective_state(), + stage.runtime_secs(now), + stage.started_at, + ) + }) + .collect::>(); (StatusCode::OK, Json(ListResponse::new(stages))).into_response() } @@ -121,6 +75,7 @@ async fn get_run_billing( .into_response(); } }; + let rollup = fabro_workflow::billing_rollup_from_projection(&projection); let by_model = rollup .by_model @@ -133,20 +88,33 @@ async fn get_run_billing( stages: model.stages, }) .collect::>(); - let stages = rollup + + let rollup_by_node = rollup .stages .iter() - .map(|stage| RunBillingStage { - billing: stage.billing.clone(), - model: stage - .model_id - .as_ref() - .map(|id| ModelReference { id: id.clone() }), - runtime_secs: stage.duration_ms as f64 / 1000.0, - stage: BillingStageRef { - id: stage.node_id.clone(), - name: stage.node_id.clone(), - }, + .map(|stage| (stage.node_id.as_str(), stage)) + .collect::>(); + let live_rows = live_billing_rows(&projection, Utc::now()); + let runtime_secs = live_rows.iter().map(|row| row.runtime_secs).sum::(); + let stages = live_rows + .into_iter() + .map(|row| { + let rollup_stage = rollup_by_node.get(row.node_id.as_str()); + RunBillingStage { + billing: rollup_stage + .map(|stage| stage.billing.clone()) + .unwrap_or_default(), + model: rollup_stage + .and_then(|stage| stage.model_id.as_ref()) + .map(|id| ModelReference { id: id.clone() }), + runtime_secs: row.runtime_secs, + stage: BillingStageRef { + id: row.node_id.clone(), + name: row.node_id, + }, + started_at: row.started_at, + state: row.state, + } }) .collect::>(); @@ -154,16 +122,80 @@ async fn get_run_billing( by_model, stages, totals: RunBillingTotals { - cache_read_tokens: rollup.totals.cache_read_tokens, + cache_read_tokens: rollup.totals.cache_read_tokens, cache_write_tokens: rollup.totals.cache_write_tokens, - input_tokens: rollup.totals.input_tokens, - output_tokens: rollup.totals.output_tokens, - reasoning_tokens: rollup.totals.reasoning_tokens, - runtime_secs: rollup.runtime_ms as f64 / 1000.0, - total_tokens: rollup.totals.total_tokens, - total_usd_micros: rollup.totals.total_usd_micros, + input_tokens: rollup.totals.input_tokens, + output_tokens: rollup.totals.output_tokens, + reasoning_tokens: rollup.totals.reasoning_tokens, + runtime_secs, + total_tokens: rollup.totals.total_tokens, + total_usd_micros: rollup.totals.total_usd_micros, }, }; (StatusCode::OK, Json(response)).into_response() } + +struct LiveBillingRow { + node_id: String, + runtime_secs: f64, + started_at: Option>, + state: Option, + latest_visit: u32, +} + +fn live_billing_rows(projection: &RunProjection, now: DateTime) -> Vec { + let mut row_indices = HashMap::::new(); + let mut rows = Vec::::new(); + + for (stage_id, stage) in projection.iter_stages() { + let node_id = stage_id.node_id(); + if is_exit_stage(projection, node_id) || !stage_has_billing_row(stage) { + continue; + } + + let index = *row_indices.entry(node_id.to_string()).or_insert_with(|| { + let index = rows.len(); + rows.push(LiveBillingRow { + node_id: node_id.to_string(), + runtime_secs: 0.0, + started_at: None, + state: None, + latest_visit: 0, + }); + index + }); + let row = &mut rows[index]; + row.runtime_secs += billing_runtime_secs(stage, now).unwrap_or(0.0); + + if stage_id.visit() >= row.latest_visit { + row.latest_visit = stage_id.visit(); + row.started_at = stage.started_at; + row.state = Some(stage.effective_state()); + } + } + + rows +} + +fn billing_runtime_secs(stage: &StageProjection, now: DateTime) -> Option { + stage + .duration_ms + .map(|ms| ms as f64 / 1000.0) + .or_else(|| stage.runtime_secs(now)) +} + +fn stage_has_billing_row(stage: &StageProjection) -> bool { + stage.completion.is_some() + || stage.duration_ms.is_some() + || stage.usage.is_some() + || stage.started_at.is_some() + || stage.state.is_some() +} + +fn is_exit_stage(projection: &RunProjection, node_id: &str) -> bool { + projection + .spec() + .and_then(|spec| spec.graph().nodes.get(node_id)) + .is_some_and(|node| node.handler_type() == Some("exit")) +} diff --git a/lib/crates/fabro-server/src/server/tests.rs b/lib/crates/fabro-server/src/server/tests.rs index 08196dbdd..aadc30c8e 100644 --- a/lib/crates/fabro-server/src/server/tests.rs +++ b/lib/crates/fabro-server/src/server/tests.rs @@ -2878,6 +2878,198 @@ async fn list_run_stages_shows_retrying_when_failed_will_retry() { assert_eq!(stage_status(&body, "work@1"), "retrying"); } +#[tokio::test] +async fn run_billing_retried_node_then_succeeded_emits_one_row_with_final_attempt_duration() { + let state = test_app_state_with_isolated_storage(); + let app = crate::test_support::build_test_router(Arc::clone(&state)); + let run_id = RunId::new(); + + create_durable_run_with_events(&state, run_id, &[ + workflow_event::Event::RunSubmitted { + definition_blob: None, + }, + workflow_event::Event::RunStarting, + workflow_event::Event::RunRunning, + workflow_event::Event::StageStarted { + node_id: "work".to_string(), + name: "Work".to_string(), + index: 0, + handler_type: "command".to_string(), + attempt: 1, + max_attempts: 3, + }, + workflow_event::Event::StageFailed { + node_id: "work".to_string(), + name: "Work".to_string(), + index: 0, + failure: FailureDetail::new("transient", FailureCategory::TransientInfra), + will_retry: true, + duration_ms: 10, + billing: None, + actor: None, + }, + workflow_event::Event::StageRetrying { + node_id: "work".to_string(), + name: "Work".to_string(), + index: 0, + attempt: 2, + max_attempts: 3, + delay_ms: 0, + }, + workflow_event::Event::StageStarted { + node_id: "work".to_string(), + name: "Work".to_string(), + index: 0, + handler_type: "command".to_string(), + attempt: 2, + max_attempts: 3, + }, + workflow_event::Event::StageCompleted { + node_id: "work".to_string(), + name: "Work".to_string(), + index: 0, + duration_ms: 25, + status: "succeeded".to_string(), + preferred_label: None, + suggested_next_ids: Vec::new(), + billing: None, + failure: None, + notes: None, + files_touched: Vec::new(), + context_updates: None, + jump_to_node: None, + context_values: None, + node_visits: None, + loop_failure_signatures: None, + restart_failure_signatures: None, + response: None, + attempt: 2, + max_attempts: 3, + }, + ]) + .await; + + let response = app + .oneshot( + Request::builder() + .method("GET") + .uri(api(&format!("/runs/{run_id}/billing"))) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let body = response_json!(response, StatusCode::OK).await; + let stages = body["stages"].as_array().unwrap(); + assert_eq!(stages.len(), 1, "retry collapses to one row per node_id"); + let row = &stages[0]; + assert_eq!(row["stage"]["id"], "work"); + assert_eq!( + row["state"], "succeeded", + "final state mirrors the latest StageCompleted" + ); + let runtime = row["runtime_secs"].as_f64().unwrap(); + assert!( + (runtime - 0.025).abs() < f64::EPSILON, + "runtime should equal final attempt's 25ms, got {runtime}" + ); +} + +fn revisit_test_started(node_id: &str) -> workflow_event::Event { + workflow_event::Event::StageStarted { + node_id: node_id.to_string(), + name: node_id.to_string(), + index: 0, + handler_type: "command".to_string(), + attempt: 1, + max_attempts: 1, + } +} + +fn revisit_test_completed_with_visit( + node_id: &str, + duration_ms: u64, + visit: usize, +) -> workflow_event::Event { + let mut node_visits = std::collections::BTreeMap::new(); + node_visits.insert(node_id.to_string(), visit); + workflow_event::Event::StageCompleted { + node_id: node_id.to_string(), + name: node_id.to_string(), + index: 0, + duration_ms, + status: "succeeded".to_string(), + preferred_label: None, + suggested_next_ids: Vec::new(), + billing: None, + failure: None, + notes: None, + files_touched: Vec::new(), + context_updates: None, + jump_to_node: None, + context_values: None, + node_visits: Some(node_visits), + loop_failure_signatures: None, + restart_failure_signatures: None, + response: None, + attempt: 1, + max_attempts: 1, + } +} + +#[tokio::test] +async fn run_billing_revisited_node_collapses_to_two_rows_with_summed_visit_duration() { + let state = test_app_state_with_isolated_storage(); + let app = crate::test_support::build_test_router(Arc::clone(&state)); + let run_id = RunId::new(); + + create_durable_run_with_events(&state, run_id, &[ + workflow_event::Event::RunSubmitted { + definition_blob: None, + }, + workflow_event::Event::RunStarting, + workflow_event::Event::RunRunning, + // A → B → A loop. Per-visit `node_visits` payload steers the reducer + // to attribute each StageCompleted to the right visit. + revisit_test_started("a"), + revisit_test_completed_with_visit("a", 1, 1), + revisit_test_started("b"), + revisit_test_completed_with_visit("b", 2, 1), + revisit_test_started("a"), + revisit_test_completed_with_visit("a", 99, 2), + ]) + .await; + + let response = app + .oneshot( + Request::builder() + .method("GET") + .uri(api(&format!("/runs/{run_id}/billing"))) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let body = response_json!(response, StatusCode::OK).await; + let stages = body["stages"].as_array().unwrap(); + assert_eq!(stages.len(), 2, "two distinct node_ids → two rows"); + assert_eq!( + stages[0]["stage"]["id"], "a", + "A appeared first → A's row first" + ); + assert_eq!(stages[1]["stage"]["id"], "b"); + let a_runtime = stages[0]["runtime_secs"].as_f64().unwrap(); + assert!( + (a_runtime - 0.1).abs() < f64::EPSILON, + "A should sum both visit durations (1ms + 99ms), got {a_runtime}" + ); + let b_runtime = stages[1]["runtime_secs"].as_f64().unwrap(); + assert!( + (b_runtime - 0.002).abs() < f64::EPSILON, + "B should carry its single visit's duration (2ms), got {b_runtime}" + ); +} + async fn append_raw_run_event( state: &Arc, run_id: RunId, diff --git a/lib/crates/fabro-store/src/run_state.rs b/lib/crates/fabro-store/src/run_state.rs index af467f557..ae23ff9fd 100644 --- a/lib/crates/fabro-store/src/run_state.rs +++ b/lib/crates/fabro-store/src/run_state.rs @@ -10,7 +10,7 @@ use fabro_types::{ BilledModelUsage, Checkpoint, Conclusion, EventBody, FailureSignature, InterviewQuestionRecord, Outcome, PendingInterviewRecord, PullRequestRecord, RunControlAction, RunEvent, RunId, RunProjection, RunSpec, RunStatus, RunSummary, SandboxRecord, StageCompletion, StageId, - StageOutcome, StageProjection, StartRecord, TerminalStatus, first_event_seq, + StageOutcome, StageProjection, StageState, StartRecord, TerminalStatus, first_event_seq, }; use fabro_util::error::render_with_causes; use serde_json::Value; @@ -290,11 +290,18 @@ impl RunProjectionReducer for RunProjection { let Some(stage_id) = stored.stage_id.as_ref() else { return Ok(()); }; - self.stage_entry( + let stage = self.stage_entry( stage_id.node_id(), stage_id.visit(), first_event_seq(event.seq), ); + stage.begin_attempt(ts); + } + EventBody::StageRetrying(_) => { + let Some(stage) = stage_at_stored_or_current_visit(self, stored, event.seq) else { + return Ok(()); + }; + stage.state = Some(StageState::Retrying); } EventBody::StagePrompt(props) => { let Some(stage) = stage_at_stored_or_visit(self, stored, props.visit, event.seq) @@ -323,22 +330,25 @@ impl RunProjectionReducer for RunProjection { stage.completion = Some(completion); stage.duration_ms = Some(props.duration_ms); stage.usage.clone_from(&props.billing); + stage.state = Some(StageState::from(outcome.status)); } EventBody::StageFailed(props) => { let failure_reason = props.failure.as_ref().map(|detail| detail.message.clone()); let Some(stage) = stage_at_stored_or_current_visit(self, stored, event.seq) else { return Ok(()); }; + let outcome = StageOutcome::Failed { + retry_requested: props.will_retry, + }; stage.completion = Some(StageCompletion { - outcome: StageOutcome::Failed { - retry_requested: props.will_retry, - }, + outcome, notes: None, failure_reason, timestamp: ts, }); stage.duration_ms = Some(props.duration_ms); stage.usage.clone_from(&props.billing); + stage.state = Some(StageState::from(outcome)); } EventBody::AgentSessionStarted(props) => { let Some(stage) = stage_at_stored_or_visit(self, stored, props.visit, event.seq) @@ -670,12 +680,13 @@ mod tests { use fabro_types::run_event::{ CheckpointCompletedProps, InterviewCompletedProps, InterviewOption, InterviewStartedProps, RunControlEffectProps, StageCompletedProps, StageFailedProps, StagePromptProps, - StageStartedProps, + StageRetryingProps, StageStartedProps, }; use fabro_types::{ - BilledModelUsage, BlockedReason, Checkpoint, EventBody, FailureReason, Outcome, - QuestionType, RunBlobId, RunControlAction, RunEvent, RunStatus, StageOutcome, - SuccessReason, TerminalStatus, WorkflowSettings, first_event_seq, fixtures, + BilledModelUsage, BlockedReason, Checkpoint, EventBody, FailureCategory, FailureDetail, + FailureReason, Outcome, QuestionType, RunBlobId, RunControlAction, RunEvent, RunStatus, + StageOutcome, StageState, SuccessReason, TerminalStatus, WorkflowSettings, first_event_seq, + fixtures, }; use serde_json::json; @@ -1775,4 +1786,224 @@ mod tests { ); assert_eq!(state.status_updated_at, updated_at); } + + fn started_props() -> StageStartedProps { + StageStartedProps { + index: 0, + handler_type: "agent".to_string(), + attempt: 1, + max_attempts: 3, + } + } + + fn failed_props(duration_ms: u64, will_retry: bool) -> StageFailedProps { + StageFailedProps { + index: 0, + failure: Some(FailureDetail::new("boom", FailureCategory::TransientInfra)), + will_retry, + duration_ms, + billing: None, + } + } + + fn retrying_props() -> StageRetryingProps { + StageRetryingProps { + index: 0, + attempt: 2, + max_attempts: 3, + delay_ms: 0, + } + } + + fn completed_props(duration_ms: u64, status: StageOutcome) -> StageCompletedProps { + StageCompletedProps { + index: 0, + duration_ms, + status, + preferred_label: None, + suggested_next_ids: Vec::new(), + billing: None, + failure: None, + notes: None, + files_touched: Vec::new(), + context_updates: None, + jump_to_node: None, + context_values: None, + node_visits: None, + loop_failure_signatures: None, + restart_failure_signatures: None, + response: None, + attempt: 1, + max_attempts: 3, + } + } + + fn billed_usage() -> BilledModelUsage { + serde_json::from_value(json!({ + "input": { + "usage": { + "model": { + "provider": "openai", + "model_id": "gpt-test" + }, + "tokens": { + "input_tokens": 10, + "output_tokens": 5, + "reasoning_tokens": 2, + "cache_read_tokens": 3, + "cache_write_tokens": 4 + } + }, + "facts": { "provider": "open_ai" } + }, + "total_usd_micros": 123 + })) + .expect("billing fixture should deserialize") + } + + #[test] + fn stage_started_records_started_at_and_running_state() { + let mut state = RunProjection::default(); + let stage_id = StageId::new("build", 1); + + state + .apply_event(&test_stage_event( + 3, + EventBody::StageStarted(started_props()), + stage_id.clone(), + )) + .unwrap(); + + let stage = state.stage(&stage_id).unwrap(); + assert_eq!(stage.state, Some(StageState::Running)); + assert!(stage.started_at.is_some()); + assert_eq!(stage.effective_state(), StageState::Running); + } + + #[test] + fn stage_completed_records_duration_usage_and_terminal_state() { + let mut state = RunProjection::default(); + let stage_id = StageId::new("build", 1); + let usage = billed_usage(); + + state + .apply_event(&test_stage_event( + 1, + EventBody::StageStarted(started_props()), + stage_id.clone(), + )) + .unwrap(); + let mut props = completed_props(42, StageOutcome::Succeeded); + props.billing = Some(usage.clone()); + state + .apply_event(&test_event( + 2, + EventBody::StageCompleted(props), + Some("build"), + )) + .unwrap(); + + let stage = state.stage(&stage_id).unwrap(); + assert_eq!(stage.duration_ms, Some(42)); + assert_eq!(stage.usage.as_ref(), Some(&usage)); + assert_eq!(stage.state, Some(StageState::Succeeded)); + assert_eq!(stage.effective_state(), StageState::Succeeded); + } + + #[test] + fn stage_failed_records_duration_and_failed_state() { + let mut state = RunProjection::default(); + let stage_id = StageId::new("build", 1); + + state + .apply_event(&test_stage_event( + 1, + EventBody::StageStarted(started_props()), + stage_id.clone(), + )) + .unwrap(); + state + .apply_event(&test_event( + 2, + EventBody::StageFailed(failed_props(10, false)), + Some("build"), + )) + .unwrap(); + + let stage = state.stage(&stage_id).unwrap(); + assert_eq!(stage.duration_ms, Some(10)); + assert_eq!(stage.state, Some(StageState::Failed)); + } + + #[test] + fn stage_retrying_sets_retrying_state() { + let mut state = RunProjection::default(); + let stage_id = StageId::new("build", 1); + + state + .apply_event(&test_stage_event( + 1, + EventBody::StageStarted(started_props()), + stage_id.clone(), + )) + .unwrap(); + state + .apply_event(&test_event( + 2, + EventBody::StageFailed(failed_props(10, true)), + Some("build"), + )) + .unwrap(); + state + .apply_event(&test_event( + 3, + EventBody::StageRetrying(retrying_props()), + Some("build"), + )) + .unwrap(); + + let stage = state.stage(&stage_id).unwrap(); + assert_eq!(stage.state, Some(StageState::Retrying)); + } + + #[test] + fn stage_started_after_retrying_returns_to_running_and_resets_attempt_data() { + let mut state = RunProjection::default(); + let stage_id = StageId::new("build", 1); + + state + .apply_event(&test_stage_event( + 1, + EventBody::StageStarted(started_props()), + stage_id.clone(), + )) + .unwrap(); + state + .apply_event(&test_event( + 2, + EventBody::StageFailed(failed_props(10, true)), + Some("build"), + )) + .unwrap(); + state + .apply_event(&test_event( + 3, + EventBody::StageRetrying(retrying_props()), + Some("build"), + )) + .unwrap(); + state + .apply_event(&test_stage_event( + 4, + EventBody::StageStarted(started_props()), + stage_id.clone(), + )) + .unwrap(); + + let stage = state.stage(&stage_id).unwrap(); + assert_eq!(stage.state, Some(StageState::Running)); + // Prior attempt's terminal data must not leak into the new attempt. + assert!(stage.completion.is_none()); + assert_eq!(stage.duration_ms, None); + } } diff --git a/lib/crates/fabro-store/tests/serializable_projection.rs b/lib/crates/fabro-store/tests/serializable_projection.rs index 2b24cd54e..8809acda2 100644 --- a/lib/crates/fabro-store/tests/serializable_projection.rs +++ b/lib/crates/fabro-store/tests/serializable_projection.rs @@ -123,6 +123,10 @@ fn serializable_projection_round_trips_and_trims_bulky_node_fields() { let serialized = serde_json::to_value(SerializableProjection(&projection)) .expect("projection should serialize"); + assert!( + serialized["stages"]["build@2"].get("usage").is_none(), + "stage usage is server-internal and should not be serialized" + ); let round_tripped: RunProjection = serde_json::from_value(serialized).expect("serialized projection should deserialize"); let node = round_tripped.stage(&stage_id).expect("node should remain"); @@ -163,7 +167,7 @@ fn serializable_projection_round_trips_and_trims_bulky_node_fields() { Some(json!([{ "stage": "fanout@1" }])) ); assert_eq!(node.duration_ms, Some(1234)); - assert_eq!(node.usage, Some(sample_usage())); + assert_eq!(node.usage, None); } #[test] diff --git a/lib/crates/fabro-types/src/run_projection.rs b/lib/crates/fabro-types/src/run_projection.rs index 87321d348..90da9b59b 100644 --- a/lib/crates/fabro-types/src/run_projection.rs +++ b/lib/crates/fabro-types/src/run_projection.rs @@ -6,7 +6,7 @@ use chrono::{DateTime, Utc}; use crate::{ BilledModelUsage, Checkpoint, Conclusion, InterviewQuestionRecord, InvalidTransition, PullRequestRecord, Retro, RunControlAction, RunId, RunSpec, RunStatus, SandboxRecord, - StageCompletion, StageId, StartRecord, + StageCompletion, StageId, StageState, StartRecord, }; #[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] @@ -44,10 +44,6 @@ pub struct StageProjection { pub prompt: Option, pub response: Option, pub completion: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub duration_ms: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub usage: Option, pub provider_used: Option, pub diff: Option, pub script_invocation: Option, @@ -65,6 +61,17 @@ pub struct StageProjection { pub live_streaming: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub termination: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub started_at: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub duration_ms: Option, + /// Server-internal billing usage for the latest attempt; not part of the + /// wire contract because `BilledModelUsage` is not modeled in OpenAPI. + /// Read only in-process by the billing handler. + #[serde(skip)] + pub usage: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub state: Option, } /// Convert a 1-based event sequence number into the `NonZeroU32` form used for @@ -96,8 +103,55 @@ impl StageProjection { streams_separated: None, live_streaming: None, termination: None, + started_at: None, + state: None, } } + + /// Effective lifecycle state derived from stored event data. + /// + /// Falls back to deriving from `completion` for projections that predate + /// the stored `state` field, so old serialized projections still work + /// without a backfill. + #[must_use] + pub fn effective_state(&self) -> StageState { + self.state.unwrap_or_else(|| match &self.completion { + Some(completion) => StageState::from(completion.outcome), + None => StageState::Running, + }) + } + + /// Live wall-clock runtime in seconds. + /// + /// While the stage is non-terminal (`Pending`, `Running`, or `Retrying`), + /// this returns the elapsed time since `started_at` so the UI can tick + /// client-side. Once terminal, the stored `duration_ms` is returned. This + /// also handles retries safely: a new `StageStarted` resets the state + /// back to `Running` and keeps the live computation correct even if a + /// previous attempt left a stale `duration_ms`. + #[must_use] + pub fn runtime_secs(&self, now: DateTime) -> Option { + let state = self.effective_state(); + if matches!( + state, + StageState::Running | StageState::Retrying | StageState::Pending + ) { + return self.started_at.map(|started| { + now.signed_duration_since(started).num_milliseconds().max(0) as f64 / 1000.0 + }); + } + self.duration_ms.map(|ms| ms as f64 / 1000.0) + } + + /// Begin a new attempt (or visit) for this stage: clear every + /// per-attempt field so prior-attempt data does not leak, then record + /// `started_at` and `state = Running`. Preserves `first_event_seq` + /// (identity / sort key). + pub fn begin_attempt(&mut self, started_at: DateTime) { + *self = Self::new(self.first_event_seq); + self.started_at = Some(started_at); + self.state = Some(StageState::Running); + } } impl RunProjection { diff --git a/lib/packages/fabro-api-client/src/models/board-column-definition.ts b/lib/packages/fabro-api-client/src/models/board-column-definition.ts index 314cbcde1..20deba1f5 100644 --- a/lib/packages/fabro-api-client/src/models/board-column-definition.ts +++ b/lib/packages/fabro-api-client/src/models/board-column-definition.ts @@ -21,6 +21,3 @@ export interface BoardColumnDefinition { 'id': BoardColumn; 'name': string; } - - - diff --git a/lib/packages/fabro-api-client/src/models/run-billing-stage.ts b/lib/packages/fabro-api-client/src/models/run-billing-stage.ts index 5d376b7cc..acdf57c97 100644 --- a/lib/packages/fabro-api-client/src/models/run-billing-stage.ts +++ b/lib/packages/fabro-api-client/src/models/run-billing-stage.ts @@ -22,6 +22,9 @@ import type { BillingStageRef } from './billing-stage-ref'; // May contain unused imports in some cases // @ts-ignore import type { ModelReference } from './model-reference'; +// May contain unused imports in some cases +// @ts-ignore +import type { StageState } from './stage-state'; /** * Token counts and billed totals for one workflow node within a run. Rows are grouped by node; billing and runtime sum every visit of that node. @@ -34,5 +37,9 @@ export interface RunBillingStage { * Wall-clock runtime in seconds, summed across every visit of this node. */ 'runtime_secs': number; + /** + * Wall-clock time the latest attempt of this stage started, if known. + */ + 'started_at'?: string | null; + 'state'?: StageState | null; } - diff --git a/lib/packages/fabro-api-client/src/models/run-stage.ts b/lib/packages/fabro-api-client/src/models/run-stage.ts index ad01b9791..060a6166e 100644 --- a/lib/packages/fabro-api-client/src/models/run-stage.ts +++ b/lib/packages/fabro-api-client/src/models/run-stage.ts @@ -42,7 +42,8 @@ export interface RunStage { * 1-based visit count; bumped each time the workflow re-enters this node. */ 'visit': number; + /** + * Wall-clock time the latest attempt of this stage started, if known. + */ + 'started_at'?: string | null; } - - - diff --git a/lib/packages/fabro-api-client/src/models/stage-projection.ts b/lib/packages/fabro-api-client/src/models/stage-projection.ts index 41321ec78..139fdd437 100644 --- a/lib/packages/fabro-api-client/src/models/stage-projection.ts +++ b/lib/packages/fabro-api-client/src/models/stage-projection.ts @@ -19,6 +19,9 @@ import type { CommandTermination } from './command-termination'; // May contain unused imports in some cases // @ts-ignore import type { StageCompletion } from './stage-completion'; +// May contain unused imports in some cases +// @ts-ignore +import type { StageState } from './stage-state'; /** * Observable projection data for one workflow stage execution. @@ -52,7 +55,13 @@ export interface StageProjection { 'streams_separated'?: boolean | null; 'live_streaming'?: boolean | null; 'termination'?: CommandTermination | null; + /** + * Wall-clock time the latest attempt of this stage started, if known. + */ + 'started_at'?: string | null; + /** + * Wall-clock duration of the stage\'s latest terminal attempt, if known. + */ + 'duration_ms'?: number | null; + 'state'?: StageState | null; } - - - From 5e4035981fbdd05201382a1a6445ce8c38154eb6 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Mon, 4 May 2026 17:08:44 -0400 Subject: [PATCH 16/17] refactor(cli): format auth status timestamps to seconds precision Use to_rfc3339_opts with SecondsFormat::Secs so auth status output shows clean second-precision timestamps instead of nanoseconds. Co-Authored-By: Claude Opus 4.7 (1M context) --- lib/crates/fabro-cli/src/commands/auth/status.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/crates/fabro-cli/src/commands/auth/status.rs b/lib/crates/fabro-cli/src/commands/auth/status.rs index db30eb115..2002fa1cd 100644 --- a/lib/crates/fabro-cli/src/commands/auth/status.rs +++ b/lib/crates/fabro-cli/src/commands/auth/status.rs @@ -1,5 +1,5 @@ use anyhow::Result; -use chrono::{DateTime, Utc}; +use chrono::{DateTime, SecondsFormat, Utc}; use fabro_client::{AuthEntry, AuthStore, OAuthEntry}; use serde::Serialize; @@ -108,12 +108,12 @@ pub(super) fn status_command(args: &AuthStatusArgs, ctx: &CommandContext) -> Res fabro_util::printerr!( printer, " Access expires: {}", - access_token_expires_at.to_rfc3339() + access_token_expires_at.to_rfc3339_opts(SecondsFormat::Secs, true) ); fabro_util::printerr!( printer, " Refresh expires: {}", - refresh_token_expires_at.to_rfc3339() + refresh_token_expires_at.to_rfc3339_opts(SecondsFormat::Secs, true) ); } StatusRow::DevToken { @@ -122,7 +122,11 @@ pub(super) fn status_command(args: &AuthStatusArgs, ctx: &CommandContext) -> Res } => { fabro_util::printerr!(printer, "{server}"); fabro_util::printerr!(printer, " Auth: dev-token"); - fabro_util::printerr!(printer, " Logged in: {}", logged_in_at.to_rfc3339()); + fabro_util::printerr!( + printer, + " Logged in: {}", + logged_in_at.to_rfc3339_opts(SecondsFormat::Secs, true) + ); } } } From 7cec7825d9dc5aeb4a194cc6742ae9c228503dad Mon Sep 17 00:00:00 2001 From: "fabro-sh-0530[bot]" <281434857+fabro-sh-0530[bot]@users.noreply.github.com> Date: Tue, 5 May 2026 09:54:22 -0400 Subject: [PATCH 17/17] Cancel in-flight agent stages with CancellationToken (#211) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Run cancellation now reaches in-flight agent work instead of waiting for an agent stage to finish or recording cancellation as a failed stage. The workflow cancellation primitive is now `tokio_util::sync::CancellationToken`, with child tokens passed through setup, handlers, manager-loop child runs, sandbox streaming commands, CLI agent invocations, and API agent sessions. ### Plan Summary - Promote run cancellation to `CancellationToken` while keeping stall timeout separate. - Route CLI agents through cancellable sandbox streaming with optional timeouts. - Bridge run cancellation into API sessions and preserve `Error::Cancelled` propagation. - Add typed events/projections for CLI cancellation and timeout. ## Cancellation flow ```mermaid flowchart TB RunToken[Run CancellationToken] Executor[Core executor] Services[RunServices] Manager[Manager-loop child run] CLI[Agent CLI backend] API[Agent API backend] Sandbox[Sandbox streaming exec] Session[fabro-agent Session] RunToken --> Executor RunToken --> Services Services -- child_token --> Manager Services -- child_token --> CLI CLI -- child_token --> Sandbox Services --> API API -- bridge guard --> Session ``` ## What changed and why - `RunOptions`, `RunServices`, core `ExecutorOptions`, CLI/server run state, and detached-run guards now use `CancellationToken` instead of `Arc`. Dropping services or tokens still does not mean cancellation; only explicit `.cancel()` does. - Manager-loop child workflows are given child tokens so parent cancellation propagates down, while stop/max-cycle cancellation remains scoped to the child workflow. - Stall timeout remains intentionally separate as a stall token and still returns `Error::StallTimeout { node_id }`, not `Error::Cancelled`. - Agent, prompt, human, fan-in, and parallel handler paths now pass cancellation tokens through and avoid converting `Error::Cancelled` into normal failed outcomes. ## Agent backend behavior CLI-mode agents no longer launch detached `setsid` jobs with temp stdout/stderr/exit-code polling. They run through `Sandbox::exec_command_streaming` with a child token; a missing node timeout passes `None` to preserve the existing unbounded agent runtime, while explicit node timeouts still apply. Cancelled CLI runs emit `agent.cli.cancelled`, clean temp files, and return `Error::Cancelled`; timed-out CLI runs emit `agent.cli.timed_out` and return a handler timeout error; `agent.cli.completed` remains natural-exit only. API-mode agents install a per-invocation `SessionCancelBridgeGuard` after acquiring a fresh or cached session. The guard maps the run token into the session interrupt reason and session cancel token, and aborts stale bridge tasks before session replacement or cache reinsertion so reused sessions are not tied to old run tokens. `Session::initialize` now returns `Result`, and project-doc, skill, MCP, and environment discovery paths check cancellation and pass child tokens to sandbox commands. ## Sandbox and event model `Sandbox::exec_command_streaming` now accepts `Option` for timeout. Production streaming implementations use a pending future for `None` instead of a giant sleep, while the trait fallback maps `None` to `u64::MAX` only when delegating to non-streaming `exec_command`. The run event model now includes typed `agent.cli.cancelled` and `agent.cli.timed_out` payloads with stdout, stderr, and duration, plus conversion and projection support. OpenAPI/client regeneration was unnecessary because the API schema already models run events with a free event string and arbitrary properties; only Rust event types changed. ## Reviewer notes Expect signature churn around `Session::initialize`, `CodergenBackend::run`, `RunOptions.cancel_token`, `StartServices.cancel_token`, and `Sandbox::exec_command_streaming`. The main behavioral checks are that user cancellation reaches in-flight CLI/API work and that timeout/stall paths remain distinct from user cancellation. ### Fabro Details
Ran 9 stages in 117m 40s for $150.32 | Stage | Duration | Cost | Retries | |---|---|---|---| | start | 0s | – | 0 | | toolchain | 1s | – | 0 | | preflight_compile | 2m 8s | – | 0 | | preflight_lint | 2m 13s | – | 0 | | implement | 77m 12s | $56.78 | 0 | | simplify_opus | 18m 5s | $5.83 | 0 | | simplify_gpt | 15m 33s | $87.71 | 0 | | verify | 1m 48s | – | 0 | | fmt | 2s | – | 0 | | **Total** | **117m 40s** | **$150.32** | **0** |
Ran ImplementPlan.fabro (12 nodes and 15 edges) ```dot digraph ImplementPlan { graph [ goal="Implement and simplify", model_stylesheet=" * { model: claude-opus-4-7; } " ] rankdir=LR start [shape=Mdiamond, label="Start"] exit [shape=Msquare, label="Exit"] toolchain [label="Toolchain", shape=parallelogram, script="command -v cargo >/dev/null || { curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && sudo ln -sf $HOME/.cargo/bin/* /usr/local/bin/; }; cargo --version 2>&1", max_retries=0] preflight_compile [label="Preflight Compile", shape=parallelogram, script="cargo check -q --workspace 2>&1", max_retries=0] preflight_lint [label="Preflight Lint", shape=parallelogram, script="cargo +nightly-2026-04-14 clippy -q --workspace --all-targets -- -D warnings 2>&1", max_retries=0] fix_lints [label="Fix Lints", prompt="The preflight lint step failed. Read the build output from context and fix all clippy lint warnings.", max_visits=3] implement [label="Implement", prompt="Read the plan file referenced in the goal and implement every step. Make all the code changes described in the plan. Use red/green TDD."] simplify_opus [label="Simplify (Opus)", prompt="@prompts/simplify.md"] simplify_gpt [label="Simplify (GPT-55)", prompt="@prompts/simplify.md", model="gpt-55"] verify [label="Verify", shape=parallelogram, script="cargo +nightly-2026-04-14 clippy -q --workspace --all-targets -- -D warnings 2>&1 && cargo nextest run --cargo-quiet --workspace --status-level fail 2>&1 && cargo dev docs refresh 2>&1 && cargo dev docs check 2>&1", goal_gate=true, retry_target="fixup"] fixup [label="Fixup", prompt="The verify step failed. Read the build output from context and fix all clippy lint warnings, test failures, and generated docs errors.", max_visits=3] fmt [label="Format", shape=parallelogram, script="cargo +nightly-2026-04-14 fmt --all 2>&1", max_retries=0] start -> toolchain toolchain -> preflight_compile [condition="outcome=succeeded"] toolchain -> exit preflight_compile -> preflight_lint [condition="outcome=succeeded"] preflight_compile -> exit preflight_lint -> implement [condition="outcome=succeeded"] preflight_lint -> fix_lints fix_lints -> preflight_lint implement -> simplify_opus -> simplify_gpt -> verify verify -> fmt [condition="outcome=succeeded"] verify -> fixup fixup -> verify fmt -> exit } ```
⚒️ Generated with [Fabro](https://fabro.sh) --------- Co-authored-by: Fabro Co-authored-by: Bryan Helmkamp --- docs/public/reference/sdk.mdx | 4 +- lib/crates/fabro-agent/README.md | 4 +- lib/crates/fabro-agent/src/cli.rs | 2 +- lib/crates/fabro-agent/src/memory.rs | 100 ++- lib/crates/fabro-agent/src/session.rs | 212 ++++-- lib/crates/fabro-agent/src/skills.rs | 48 +- lib/crates/fabro-agent/src/subagent.rs | 2 +- lib/crates/fabro-agent/src/v4a_patch.rs | 2 +- .../fabro-agent/tests/it/parity_matrix.rs | 8 +- .../fabro-cli/src/commands/run/runner.rs | 42 +- .../fabro-cli/tests/it/workflow/real_cli.rs | 1 + lib/crates/fabro-core/src/executor.rs | 50 +- lib/crates/fabro-core/src/stall.rs | 37 +- lib/crates/fabro-retro/src/retro_agent.rs | 5 +- lib/crates/fabro-sandbox/src/daytona/mod.rs | 51 +- lib/crates/fabro-sandbox/src/docker.rs | 14 +- lib/crates/fabro-sandbox/src/local.rs | 8 +- lib/crates/fabro-sandbox/src/sandbox.rs | 23 +- lib/crates/fabro-sandbox/src/worktree.rs | 2 +- .../tests/daytona_streaming_live.rs | 11 +- .../fabro-sandbox/tests/docker_streaming.rs | 2 +- lib/crates/fabro-server/Cargo.toml | 3 +- lib/crates/fabro-server/src/server.rs | 17 +- .../src/server/handler/lifecycle.rs | 16 +- lib/crates/fabro-store/src/run_state.rs | 170 ++++- lib/crates/fabro-types/src/run_event/misc.rs | 14 + lib/crates/fabro-types/src/run_event/mod.rs | 6 + .../fabro-workflow/src/devcontainer_bridge.rs | 126 ++-- .../fabro-workflow/src/event/convert.rs | 62 ++ lib/crates/fabro-workflow/src/event/events.rs | 26 + lib/crates/fabro-workflow/src/event/names.rs | 2 + .../fabro-workflow/src/event/stored_fields.rs | 4 +- .../fabro-workflow/src/handler/agent.rs | 13 + .../fabro-workflow/src/handler/command.rs | 13 +- .../fabro-workflow/src/handler/fan_in.rs | 5 + .../fabro-workflow/src/handler/human.rs | 7 +- .../fabro-workflow/src/handler/llm/api.rs | 603 +++++++++++++++--- .../fabro-workflow/src/handler/llm/cli.rs | 595 +++++++++++++---- .../src/handler/manager_loop.rs | 17 +- lib/crates/fabro-workflow/src/handler/mod.rs | 1 - .../fabro-workflow/src/handler/parallel.rs | 9 +- .../fabro-workflow/src/handler/prompt.rs | 17 +- .../fabro-workflow/src/lifecycle/git.rs | 4 +- .../fabro-workflow/src/operations/fork.rs | 2 + .../fabro-workflow/src/operations/start.rs | 26 +- .../fabro-workflow/src/pipeline/execute.rs | 4 +- .../src/pipeline/execute/tests.rs | 20 +- .../fabro-workflow/src/pipeline/finalize.rs | 6 +- .../fabro-workflow/src/pipeline/initialize.rs | 27 +- .../fabro-workflow/src/pipeline/retro.rs | 6 +- lib/crates/fabro-workflow/src/run_metadata.rs | 2 +- lib/crates/fabro-workflow/src/run_options.rs | 8 +- lib/crates/fabro-workflow/src/services.rs | 63 +- .../tests/it/daytona_integration.rs | 16 +- .../tests/it/git_integration.rs | 3 +- .../fabro-workflow/tests/it/integration.rs | 435 +++++++------ 56 files changed, 2201 insertions(+), 775 deletions(-) diff --git a/docs/public/reference/sdk.mdx b/docs/public/reference/sdk.mdx index 4e907270b..c070e11cf 100644 --- a/docs/public/reference/sdk.mdx +++ b/docs/public/reference/sdk.mdx @@ -42,7 +42,7 @@ async fn main() -> Result<(), Box> { let config = SessionOptions::default(); let mut session = Session::new(client, profile, sandbox, config); - session.initialize().await; + session.initialize().await?; // Subscribe to events before sending input let mut events = session.subscribe(); @@ -936,4 +936,4 @@ Register it on the client: ```rust client.register_provider(Arc::new(MyProvider)).await?; -``` +``` \ No newline at end of file diff --git a/lib/crates/fabro-agent/README.md b/lib/crates/fabro-agent/README.md index 4c25523d9..ce1afce1b 100644 --- a/lib/crates/fabro-agent/README.md +++ b/lib/crates/fabro-agent/README.md @@ -140,7 +140,7 @@ let config = SessionConfig { // 5. Create and initialize the session let mut session = Session::new(client, profile, env, config, None); -session.initialize().await; +session.initialize().await?; // 6. Subscribe to events (for UI rendering) let mut rx = session.subscribe(); @@ -232,4 +232,4 @@ profile.register_subagent_tools(manager, factory, 0); - **Tool output truncation** -- Per-tool character and line limits with head/tail or tail-only truncation modes - **Environment variable filtering** -- `LocalSandbox` strips secrets (`*_API_KEY`, `*_SECRET`, `*_TOKEN`, `*_PASSWORD`, `*_CREDENTIAL`) from subprocess environments - **Command timeouts** -- Configurable per-command with process group cleanup (SIGTERM then SIGKILL) -- **Project doc discovery** -- Automatically discovers `AGENTS.md`, `CLAUDE.md`, `GEMINI.md`, or `.codex/instructions.md` based on provider, with a 32KB budget +- **Project doc discovery** -- Automatically discovers `AGENTS.md`, `CLAUDE.md`, `GEMINI.md`, or `.codex/instructions.md` based on provider, with a 32KB budget \ No newline at end of file diff --git a/lib/crates/fabro-agent/src/cli.rs b/lib/crates/fabro-agent/src/cli.rs index 1276bba26..3b216d284 100644 --- a/lib/crates/fabro-agent/src/cli.rs +++ b/lib/crates/fabro-agent/src/cli.rs @@ -724,7 +724,7 @@ pub async fn run_with_args_and_client( }); // Initialize and run - session.initialize().await; + session.initialize().await?; let result = session.process_input(&args.prompt).await; if matches!(output_format, OutputFormat::Text) { diff --git a/lib/crates/fabro-agent/src/memory.rs b/lib/crates/fabro-agent/src/memory.rs index 8c5554537..071435c30 100644 --- a/lib/crates/fabro-agent/src/memory.rs +++ b/lib/crates/fabro-agent/src/memory.rs @@ -1,8 +1,10 @@ use std::collections::HashSet; use fabro_model::Provider; +use tokio_util::sync::CancellationToken; use tracing::{debug, info, warn}; +use crate::error::{Error, InterruptReason}; use crate::sandbox::Sandbox; const BUDGET_BYTES: usize = 32768; @@ -12,7 +14,8 @@ pub async fn discover_memory( git_root: &str, working_dir: &str, provider: Provider, -) -> Vec { + cancel_token: &CancellationToken, +) -> Result, Error> { let directories = build_directory_walk(git_root, working_dir); let candidate_filenames: Vec<&str> = match provider { @@ -34,8 +37,15 @@ pub async fn discover_memory( for dir in &directories { for filename in &candidate_filenames { + if cancel_token.is_cancelled() { + return Err(Error::Interrupted(InterruptReason::Cancelled)); + } let path = format!("{dir}/{filename}"); - if let Ok(content) = env.read_file(&path, None, None).await { + let read_result = env.read_file(&path, None, None).await; + if cancel_token.is_cancelled() { + return Err(Error::Interrupted(InterruptReason::Cancelled)); + } + if let Ok(content) = read_result { if content.is_empty() { warn!(path = %path, "Project doc file empty, skipping"); continue; @@ -68,7 +78,7 @@ pub async fn discover_memory( let total_bytes: usize = results.iter().map(std::string::String::len).sum(); info!(files = results.len(), total_bytes, "Project docs loaded"); - results + Ok(results) } fn build_directory_walk(git_root: &str, working_dir: &str) -> Vec { @@ -117,6 +127,8 @@ mod tests { use std::collections::HashMap; use std::sync::Arc; + use tokio_util::sync::CancellationToken; + use super::*; use crate::sandbox::Sandbox; use crate::test_support::MockSandbox; @@ -129,7 +141,15 @@ mod tests { files, ..Default::default() }); - let docs = discover_memory(env.as_ref(), "/repo", "/repo", Provider::Anthropic).await; + let docs = discover_memory( + env.as_ref(), + "/repo", + "/repo", + Provider::Anthropic, + &CancellationToken::new(), + ) + .await + .unwrap(); assert_eq!(docs.len(), 1); assert_eq!(docs[0], "Agent instructions"); } @@ -146,8 +166,15 @@ mod tests { files: files.clone(), ..Default::default() }); - let anthropic_docs = - discover_memory(env.as_ref(), "/repo", "/repo", Provider::Anthropic).await; + let anthropic_docs = discover_memory( + env.as_ref(), + "/repo", + "/repo", + Provider::Anthropic, + &CancellationToken::new(), + ) + .await + .unwrap(); assert_eq!(anthropic_docs.len(), 2); assert_eq!(anthropic_docs[0], "agents"); assert_eq!(anthropic_docs[1], "claude"); @@ -156,7 +183,15 @@ mod tests { files: files.clone(), ..Default::default() }); - let openai_docs = discover_memory(env.as_ref(), "/repo", "/repo", Provider::OpenAi).await; + let openai_docs = discover_memory( + env.as_ref(), + "/repo", + "/repo", + Provider::OpenAi, + &CancellationToken::new(), + ) + .await + .unwrap(); assert_eq!(openai_docs.len(), 2); assert_eq!(openai_docs[0], "agents"); assert_eq!(openai_docs[1], "copilot"); @@ -165,7 +200,15 @@ mod tests { files, ..Default::default() }); - let gemini_docs = discover_memory(env.as_ref(), "/repo", "/repo", Provider::Gemini).await; + let gemini_docs = discover_memory( + env.as_ref(), + "/repo", + "/repo", + Provider::Gemini, + &CancellationToken::new(), + ) + .await + .unwrap(); assert_eq!(gemini_docs.len(), 2); assert_eq!(gemini_docs[0], "agents"); assert_eq!(gemini_docs[1], "gemini"); @@ -184,7 +227,15 @@ mod tests { files, ..Default::default() }); - let docs = discover_memory(env.as_ref(), "/repo", "/repo", Provider::Anthropic).await; + let docs = discover_memory( + env.as_ref(), + "/repo", + "/repo", + Provider::Anthropic, + &CancellationToken::new(), + ) + .await + .unwrap(); assert_eq!(docs.len(), 2); assert_eq!(docs[0], large_content); // Second doc should be truncated to fit remaining budget @@ -201,7 +252,15 @@ mod tests { files, ..Default::default() }); - let docs = discover_memory(env.as_ref(), "/repo", "/repo", Provider::Anthropic).await; + let docs = discover_memory( + env.as_ref(), + "/repo", + "/repo", + Provider::Anthropic, + &CancellationToken::new(), + ) + .await + .unwrap(); assert_eq!(docs.len(), 1); assert_eq!(docs[0], "shared instructions"); } @@ -215,7 +274,15 @@ mod tests { files, ..Default::default() }); - let docs = discover_memory(env.as_ref(), "/repo", "/repo/src", Provider::Anthropic).await; + let docs = discover_memory( + env.as_ref(), + "/repo", + "/repo/src", + Provider::Anthropic, + &CancellationToken::new(), + ) + .await + .unwrap(); assert_eq!(docs.len(), 1); assert_eq!(docs[0], "shared instructions"); } @@ -231,8 +298,15 @@ mod tests { files, ..Default::default() }); - let docs = - discover_memory(env.as_ref(), "/repo", "/repo/src/app", Provider::Anthropic).await; + let docs = discover_memory( + env.as_ref(), + "/repo", + "/repo/src/app", + Provider::Anthropic, + &CancellationToken::new(), + ) + .await + .unwrap(); assert_eq!(docs.len(), 3); assert_eq!(docs[0], "root agents"); assert_eq!(docs[1], "src agents"); diff --git a/lib/crates/fabro-agent/src/session.rs b/lib/crates/fabro-agent/src/session.rs index 14105009b..9e8d5ac4b 100644 --- a/lib/crates/fabro-agent/src/session.rs +++ b/lib/crates/fabro-agent/src/session.rs @@ -130,13 +130,24 @@ impl Session { /// Initialize session by discovering project docs and capturing environment /// context. Call before `process_input`. - pub async fn initialize(&mut self) { + /// + /// # Errors + /// + /// Returns `Error::Interrupted(InterruptReason::Cancelled)` if the + /// session's cancel token fires during initialization. + pub async fn initialize(&mut self) -> Result<(), Error> { + let cancel_token = self.cancel_token.clone(); + self.event_emitter .emit(self.id.clone(), AgentEvent::SessionStarted { provider: Some(self.provider_profile.provider().to_string()), model: Some(self.provider_profile.model().to_string()), }); + if cancel_token.is_cancelled() { + return Err(Error::Interrupted(InterruptReason::Cancelled)); + } + let doc_root = self .config .git_root @@ -147,8 +158,9 @@ impl Session { &doc_root, self.sandbox.working_directory(), self.provider_profile.provider(), + &cancel_token, ) - .await; + .await?; // Discover skills let skill_dirs = if let Some(dirs) = &self.config.skill_dirs { @@ -158,7 +170,7 @@ impl Session { let skills_str = skills_dir.to_string_lossy().to_string(); default_skill_dirs(Some(&skills_str), self.config.git_root.as_deref()) }; - self.skills = discover_skills(self.sandbox.as_ref(), &skill_dirs).await; + self.skills = discover_skills(self.sandbox.as_ref(), &skill_dirs, &cancel_token).await?; debug!(skill_count = self.skills.len(), "Skills discovered"); // Register use_skill tool when skills are available @@ -175,7 +187,7 @@ impl Session { if !self.config.mcp_servers.is_empty() { // Resolve Sandbox transports: start the server inside the sandbox, // then rewrite the config to Http using the sandbox's preview URL. - let mcp_servers = self.resolve_sandbox_mcp_servers().await; + let mcp_servers = self.resolve_sandbox_mcp_servers(&cancel_token).await?; let mut manager = McpConnectionManager::new(); let results = manager.start_servers(&mcp_servers).await; @@ -209,7 +221,7 @@ impl Session { } // Populate environment context - self.env_context = self.build_env_context().await; + self.env_context = self.build_env_context(&cancel_token).await?; debug!( is_git_repo = self.env_context.is_git_repo, model = %self.env_context.model, @@ -224,19 +236,30 @@ impl Session { self.config.user_instructions.as_deref(), &self.skills, ); + + Ok(()) } /// Resolve `McpTransport::Sandbox` configs by starting the MCP server /// inside the sandbox and rewriting the transport to `Http` with the /// sandbox's preview URL. - async fn resolve_sandbox_mcp_servers(&self) -> Vec { + async fn resolve_sandbox_mcp_servers( + &self, + cancel_token: &CancellationToken, + ) -> Result, Error> { let mut resolved = Vec::with_capacity(self.config.mcp_servers.len()); for config in &self.config.mcp_servers { + if cancel_token.is_cancelled() { + return Err(Error::Interrupted(InterruptReason::Cancelled)); + } match &config.transport { McpTransport::Sandbox { command, port, env } => { let port = *port; - match self.start_sandbox_mcp_server(command, port, env).await { + match self + .start_sandbox_mcp_server(command, port, env, cancel_token) + .await? + { Ok((url, headers)) => { info!( server = %config.name, @@ -268,17 +291,24 @@ impl Session { } } - resolved + Ok(resolved) } /// Start an MCP server inside the sandbox and return (url, headers) for /// HTTP connection. + /// + /// The outer `Result` surfaces fatal cancellation as + /// `Error::Interrupted(InterruptReason::Cancelled)` (the running MCP + /// process group is terminated before returning). The inner `Result` + /// captures non-fatal startup failures that the caller logs and turns + /// into an `McpServerFailed` event. async fn start_sandbox_mcp_server( &self, command: &[String], port: u16, env: &std::collections::HashMap, - ) -> Result<(String, std::collections::HashMap), String> { + cancel_token: &CancellationToken, + ) -> Result), String>, Error> { let sandbox = self.sandbox.as_ref(); let cmd_str = command @@ -296,27 +326,63 @@ impl Session { quoted = fabro_sandbox::shell_quote(&inner) ); let env_ref = if env.is_empty() { None } else { Some(env) }; - let launch_result = sandbox - .exec_command(&launch_script, 30_000, None, env_ref, None) - .await - .map_err(|e| format!("Failed to launch MCP server: {}", e.display_with_causes()))?; - let pid = launch_result.stdout.trim(); - info!(pid, port, "MCP server process launched in sandbox"); + if cancel_token.is_cancelled() { + return Err(Error::Interrupted(InterruptReason::Cancelled)); + } + let launch_result = match sandbox + .exec_command( + &launch_script, + 30_000, + None, + env_ref, + Some(cancel_token.child_token()), + ) + .await + { + Ok(result) => result, + Err(e) => { + if cancel_token.is_cancelled() { + return Err(Error::Interrupted(InterruptReason::Cancelled)); + } + return Ok(Err(format!( + "Failed to launch MCP server: {}", + e.display_with_causes() + ))); + } + }; + + let pid = launch_result.stdout.trim().to_string(); + info!(pid = %pid, port, "MCP server process launched in sandbox"); // Wait for the server to start listening on the port let poll_cmd = format!( "for i in $(seq 1 30); do ss -tln | grep -q ':{port} ' && echo ready && exit 0; sleep 1; done; echo timeout" ); let poll_result = sandbox - .exec_command(&poll_cmd, 60_000, None, None, None) - .await - .map_err(|e| { - format!( + .exec_command( + &poll_cmd, + 60_000, + None, + None, + Some(cancel_token.child_token()), + ) + .await; + + if cancel_token.is_cancelled() { + kill_mcp_pid(sandbox, &pid).await; + return Err(Error::Interrupted(InterruptReason::Cancelled)); + } + + let poll_result = match poll_result { + Ok(result) => result, + Err(e) => { + return Ok(Err(format!( "Failed to poll MCP server readiness: {}", e.display_with_causes() - ) - })?; + ))); + } + }; if poll_result.stdout.trim() != "ready" { // Grab stderr for debugging @@ -326,51 +392,80 @@ impl Session { 10_000, None, None, - None, + Some(cancel_token.child_token()), ) .await .map(|r| r.stdout) .unwrap_or_default(); - return Err(format!( + return Ok(Err(format!( "MCP server did not start listening on port {port} within 30s. stderr:\n{stderr}" - )); + ))); } // Get the preview URL for the port, or fall back to localhost for local // sandboxes - if let Some(url_and_headers) = sandbox - .get_preview_url(port) - .await - .map_err(|e| e.display_with_causes())? - { - Ok(url_and_headers) + let preview = match sandbox.get_preview_url(port).await { + Ok(p) => p, + Err(e) => return Ok(Err(e.display_with_causes())), + }; + + if cancel_token.is_cancelled() { + kill_mcp_pid(sandbox, &pid).await; + return Err(Error::Interrupted(InterruptReason::Cancelled)); + } + + if let Some(url_and_headers) = preview { + Ok(Ok(url_and_headers)) } else { info!(port, "No preview URL available, using localhost"); - Ok(( + Ok(Ok(( format!("http://localhost:{port}"), std::collections::HashMap::new(), - )) + ))) } } - async fn build_env_context(&self) -> EnvContext { + async fn build_env_context( + &self, + cancel_token: &CancellationToken, + ) -> Result { let today = chrono::Local::now().format("%Y-%m-%d").to_string(); let model_name = self.provider_profile.model().to_string(); + if cancel_token.is_cancelled() { + return Err(Error::Interrupted(InterruptReason::Cancelled)); + } + // Detect git info via sandbox let git_branch = self .sandbox - .exec_command("git rev-parse --abbrev-ref HEAD", 5000, None, None, None) + .exec_command( + "git rev-parse --abbrev-ref HEAD", + 5000, + None, + None, + Some(cancel_token.child_token()), + ) .await .ok() .filter(fabro_sandbox::ExecResult::is_success) .map(|r| r.stdout.trim().to_string()); + if cancel_token.is_cancelled() { + return Err(Error::Interrupted(InterruptReason::Cancelled)); + } + let is_git_repo = git_branch.is_some(); let git_status_short = if is_git_repo { self.sandbox - .exec_command("git status --short", 5000, None, None, None) + .exec_command( + "git status --short", + 5000, + None, + None, + Some(cancel_token.child_token()), + ) .await .ok() .filter(fabro_sandbox::ExecResult::is_success) @@ -380,9 +475,19 @@ impl Session { None }; + if cancel_token.is_cancelled() { + return Err(Error::Interrupted(InterruptReason::Cancelled)); + } + let git_recent_commits = if is_git_repo { self.sandbox - .exec_command("git log --oneline -10", 5000, None, None, None) + .exec_command( + "git log --oneline -10", + 5000, + None, + None, + Some(cancel_token.child_token()), + ) .await .ok() .filter(fabro_sandbox::ExecResult::is_success) @@ -392,7 +497,11 @@ impl Session { None }; - EnvContext { + if cancel_token.is_cancelled() { + return Err(Error::Interrupted(InterruptReason::Cancelled)); + } + + Ok(EnvContext { git_branch, is_git_repo, current_date: today, @@ -400,7 +509,7 @@ impl Session { knowledge_cutoff: self.provider_profile.knowledge_cutoff().unwrap_or_default(), git_status_short, git_recent_commits, - } + }) } #[must_use] @@ -1031,6 +1140,23 @@ const fn is_auth_error(err: &LlmError) -> bool { ) } +/// Best-effort kill of a sandbox MCP server process group. Used when +/// `start_sandbox_mcp_server` is cancelled after spawning a detached +/// `setsid` child but before reporting readiness. Errors from the sandbox +/// are logged and swallowed; the caller is already returning a Cancelled +/// error. +async fn kill_mcp_pid(sandbox: &dyn Sandbox, pid: &str) { + let pid = pid.trim(); + if pid.is_empty() { + return; + } + let script = + format!("kill -TERM -{pid} 2>/dev/null; sleep 1; kill -KILL -{pid} 2>/dev/null; true"); + if let Err(err) = sandbox.exec_command(&script, 5_000, None, None, None).await { + warn!(pid, error = %err.display_with_causes(), "Failed to kill MCP server process group during cancellation"); + } +} + #[cfg(test)] mod tests { use std::sync::Arc; @@ -1297,7 +1423,7 @@ mod tests { let mut session = make_session(vec![text_response("Hello")]).await; let mut rx = session.subscribe(); - session.initialize().await; + session.initialize().await.unwrap(); session.process_input("Hi").await.unwrap(); session.close(); @@ -1825,7 +1951,7 @@ mod tests { let mut session = make_session(responses).await; let mut rx = session.subscribe(); - session.initialize().await; + session.initialize().await.unwrap(); session.process_input("one").await.unwrap(); session.process_input("two").await.unwrap(); session.close(); @@ -1858,7 +1984,7 @@ mod tests { ..Default::default() }; let mut session = Session::new(client, profile, env, config, None); - session.initialize().await; + session.initialize().await.unwrap(); session.process_input("test").await.unwrap(); // Verify user instructions are included in the system prompt @@ -2647,7 +2773,7 @@ mod tests { let mut rx = session.subscribe(); // Initialize starts the MCP server and registers tools - session.initialize().await; + session.initialize().await.unwrap(); // Verify McpServerReady event was emitted let mut mcp_ready = false; @@ -2842,7 +2968,7 @@ mod tests { #[tokio::test] async fn process_input_emits_processing_end_on_idle_transition() { let mut session = make_session(vec![text_response("Hello")]).await; - session.initialize().await; + session.initialize().await.unwrap(); let mut rx = session.subscribe(); session.process_input("Hi").await.unwrap(); diff --git a/lib/crates/fabro-agent/src/skills.rs b/lib/crates/fabro-agent/src/skills.rs index a4d00226e..fe5e3445a 100644 --- a/lib/crates/fabro-agent/src/skills.rs +++ b/lib/crates/fabro-agent/src/skills.rs @@ -1,7 +1,9 @@ use std::sync::Arc; use fabro_llm::types::ToolDefinition; +use tokio_util::sync::CancellationToken; +use crate::error::{Error, InterruptReason}; use crate::sandbox::Sandbox; use crate::tool_registry::RegisteredTool; use crate::tools::required_str; @@ -224,17 +226,35 @@ pub fn default_skill_dirs(fabro_skills_dir: Option<&str>, git_root: Option<&str> dirs } -pub async fn discover_skills(env: &dyn Sandbox, dirs: &[String]) -> Vec { +pub async fn discover_skills( + env: &dyn Sandbox, + dirs: &[String], + cancel_token: &CancellationToken, +) -> Result, Error> { let mut skills_by_name: std::collections::HashMap = std::collections::HashMap::new(); for dir in dirs { - let Ok(paths) = env.glob("*/SKILL.md", Some(dir)).await else { + if cancel_token.is_cancelled() { + return Err(Error::Interrupted(InterruptReason::Cancelled)); + } + let glob_result = env.glob("*/SKILL.md", Some(dir)).await; + if cancel_token.is_cancelled() { + return Err(Error::Interrupted(InterruptReason::Cancelled)); + } + let Ok(paths) = glob_result else { continue; }; for path in paths { - let Ok(content) = env.read_file(&path, None, None).await else { + if cancel_token.is_cancelled() { + return Err(Error::Interrupted(InterruptReason::Cancelled)); + } + let read_result = env.read_file(&path, None, None).await; + if cancel_token.is_cancelled() { + return Err(Error::Interrupted(InterruptReason::Cancelled)); + } + let Ok(content) = read_result else { continue; }; @@ -246,7 +266,7 @@ pub async fn discover_skills(env: &dyn Sandbox, dirs: &[String]) -> Vec { let mut skills: Vec = skills_by_name.into_values().collect(); skills.sort_by(|a, b| a.name.cmp(&b.name)); - skills + Ok(skills) } #[cfg(test)] @@ -454,7 +474,9 @@ name: trimmed ..Default::default() }; - let skills = discover_skills(&env, &["/skills".into()]).await; + let skills = discover_skills(&env, &["/skills".into()], &CancellationToken::new()) + .await + .unwrap(); assert_eq!(skills.len(), 1); assert_eq!(skills[0].name, "commit"); assert_eq!(skills[0].description, "Make a commit"); @@ -477,7 +499,9 @@ name: trimmed ..Default::default() }; - let skills = discover_skills(&env, &["/skills".into()]).await; + let skills = discover_skills(&env, &["/skills".into()], &CancellationToken::new()) + .await + .unwrap(); assert_eq!(skills.len(), 1); assert_eq!(skills[0].name, "good"); } @@ -485,7 +509,9 @@ name: trimmed #[tokio::test] async fn discover_empty_dirs() { let env = MockSandbox::default(); - let skills = discover_skills(&env, &[]).await; + let skills = discover_skills(&env, &[], &CancellationToken::new()) + .await + .unwrap(); assert!(skills.is_empty()); } @@ -514,7 +540,13 @@ name: trimmed }; // discover_skills iterates dirs in order; later dirs override earlier names - let skills = discover_skills(&env, &["/global".into(), "/project".into()]).await; + let skills = discover_skills( + &env, + &["/global".into(), "/project".into()], + &CancellationToken::new(), + ) + .await + .unwrap(); assert_eq!(skills.len(), 1); assert_eq!(skills[0].description, "Project commit"); } diff --git a/lib/crates/fabro-agent/src/subagent.rs b/lib/crates/fabro-agent/src/subagent.rs index 0db683603..04a58d18b 100644 --- a/lib/crates/fabro-agent/src/subagent.rs +++ b/lib/crates/fabro-agent/src/subagent.rs @@ -111,7 +111,7 @@ impl SubAgentManager { let task_prompt_for_spawn = task_prompt.clone(); let task = tokio::spawn(async move { - session.initialize().await; + session.initialize().await?; session.process_input(&task_prompt_for_spawn).await?; let turns = session.history().turns(); let last_text = turns.iter().rev().find_map(|t| match t { diff --git a/lib/crates/fabro-agent/src/v4a_patch.rs b/lib/crates/fabro-agent/src/v4a_patch.rs index c96745d33..78aa5a1ca 100644 --- a/lib/crates/fabro-agent/src/v4a_patch.rs +++ b/lib/crates/fabro-agent/src/v4a_patch.rs @@ -1466,7 +1466,7 @@ def farewell(name): SessionOptions::default(), None, ); - session.initialize().await; + session.initialize().await.unwrap(); session .process_input("Update the greeting functions") .await diff --git a/lib/crates/fabro-agent/tests/it/parity_matrix.rs b/lib/crates/fabro-agent/tests/it/parity_matrix.rs index 8b8630e4d..78e37d741 100644 --- a/lib/crates/fabro-agent/tests/it/parity_matrix.rs +++ b/lib/crates/fabro-agent/tests/it/parity_matrix.rs @@ -170,7 +170,7 @@ macro_rules! provider_test { async fn [<$prefix _ $scenario>]() { let tmp = tempfile::tempdir().expect("failed to create tempdir"); let mut session = make_session($provider, $model, tmp.path(), None).await; - session.initialize().await; + session.initialize().await.unwrap(); [](&mut session, tmp.path()).await; } } @@ -195,7 +195,7 @@ macro_rules! openai_twin_provider_test { tmp.path(), Some(twin), ).await; - session.initialize().await; + session.initialize().await.unwrap(); [](&mut session, tmp.path()).await; } } @@ -670,7 +670,7 @@ macro_rules! reasoning_effort_tests { }; let mut session = make_session_with_config($provider, $model, tmp.path(), config, None).await; - session.initialize().await; + session.initialize().await.unwrap(); session .process_input("Say hello") .await @@ -749,7 +749,7 @@ macro_rules! loop_detection_tests { }; let mut session = make_session_with_config($provider, $model, tmp.path(), config, None).await; - session.initialize().await; + session.initialize().await.unwrap(); session .process_input("Repeatedly read the file /dev/null") .await diff --git a/lib/crates/fabro-cli/src/commands/run/runner.rs b/lib/crates/fabro-cli/src/commands/run/runner.rs index 288c45afd..36de6b516 100644 --- a/lib/crates/fabro-cli/src/commands/run/runner.rs +++ b/lib/crates/fabro-cli/src/commands/run/runner.rs @@ -8,7 +8,6 @@ use std::collections::HashMap; use std::io::{BufRead as StdBufRead, BufReader as StdBufReader}; use std::path::{Path, PathBuf}; use std::sync::Arc; -use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; use anyhow::{Context, Result, anyhow}; @@ -34,6 +33,7 @@ use fabro_workflow::runtime_store::{RunStoreBackend, RunStoreHandle}; use tokio::signal::unix::{SignalKind, signal}; use tokio::sync::{Mutex, RwLock as AsyncRwLock, mpsc}; use tokio::time::sleep; +use tokio_util::sync::CancellationToken; use crate::args::RunWorkerMode; use crate::server_client; @@ -86,10 +86,10 @@ pub(crate) async fn execute( worker_token.to_owned(), ))); let interviewer = Arc::new(ControlInterviewer::new()); - let cancel_token = Arc::new(AtomicBool::new(false)); - spawn_worker_control_stream(Arc::clone(&interviewer), Arc::clone(&cancel_token))?; + let cancel_token = CancellationToken::new(); + spawn_worker_control_stream(Arc::clone(&interviewer), cancel_token.clone())?; let run_control = RunControlState::new(); - install_signal_handlers(Arc::clone(&run_control), Arc::clone(&cancel_token))?; + install_signal_handlers(Arc::clone(&run_control), cancel_token.clone())?; let vault = load_worker_vault(storage_dir.as_deref())?; let github_app = { let vault_guard = match &vault { @@ -100,7 +100,7 @@ pub(crate) async fn execute( }; let services = StartServices { run_id, - cancel_token: Some(Arc::clone(&cancel_token)), + cancel_token: cancel_token.clone(), emitter: Arc::new(Emitter::new(run_id)), interviewer, run_store: run_store.clone(), @@ -162,7 +162,7 @@ enum WorkerControlStreamEvent { )] fn spawn_worker_control_stream( interviewer: Arc, - cancel_token: Arc, + cancel_token: CancellationToken, ) -> Result<()> { let (event_tx, event_rx) = mpsc::unbounded_channel(); tokio::spawn(handle_worker_control_stream_events( @@ -205,7 +205,7 @@ fn read_worker_control_stream_blocking( async fn handle_worker_control_stream_events( interviewer: Arc, - cancel_token: Arc, + cancel_token: CancellationToken, mut event_rx: mpsc::UnboundedReceiver, ) { while let Some(event) = event_rx.recv().await { @@ -225,7 +225,7 @@ async fn handle_worker_control_stream_events( async fn apply_worker_control_line( interviewer: &ControlInterviewer, - cancel_token: &AtomicBool, + cancel_token: &CancellationToken, line: &str, ) { if line.trim().is_empty() { @@ -243,7 +243,7 @@ async fn apply_worker_control_line( .await; } WorkerControlMessage::RunCancel => { - cancel_token.store(true, Ordering::SeqCst); + cancel_token.cancel(); interviewer.interrupt_all().await; } } @@ -561,7 +561,7 @@ fn clone_sandbox_requires_github_credentials(provider: &str) -> bool { fn install_signal_handlers( run_control: Arc, - cancel_token: Arc, + cancel_token: CancellationToken, ) -> Result<()> { #[cfg(unix)] { @@ -581,17 +581,17 @@ fn install_signal_handlers( }); let mut terminate = signal(SignalKind::terminate())?; - let terminate_cancel = Arc::clone(&cancel_token); + let terminate_cancel = cancel_token.clone(); tokio::spawn(async move { while terminate.recv().await.is_some() { - terminate_cancel.store(true, Ordering::SeqCst); + terminate_cancel.cancel(); } }); let mut interrupt = signal(SignalKind::interrupt())?; tokio::spawn(async move { while interrupt.recv().await.is_some() { - cancel_token.store(true, Ordering::SeqCst); + cancel_token.cancel(); } }); } @@ -606,7 +606,6 @@ fn install_signal_handlers( )] mod tests { use std::sync::Arc; - use std::sync::atomic::{AtomicBool, Ordering}; use chrono::Utc; use fabro_auth::{AuthCredential, AuthDetails}; @@ -623,6 +622,7 @@ mod tests { }; use fabro_vault::{SecretType, Vault}; use fabro_workflow::event::RunEventSink; + use tokio_util::sync::CancellationToken; use super::{ WorkerControlStreamEvent, WorkerTitlePhase, apply_worker_control_line, @@ -823,7 +823,7 @@ mod tests { #[tokio::test] async fn worker_control_line_routes_answer_by_question_id() { let interviewer = Arc::new(ControlInterviewer::new()); - let cancel_token = Arc::new(AtomicBool::new(false)); + let cancel_token = CancellationToken::new(); let mut question = Question::new("Approve?", QuestionType::YesNo); question.id = "q-1".to_string(); let ask_interviewer = Arc::clone(&interviewer); @@ -838,13 +838,13 @@ mod tests { let answer = answer_task.await.unwrap().answer; assert_eq!(answer.value, AnswerValue::Yes); - assert!(!cancel_token.load(Ordering::SeqCst)); + assert!(!cancel_token.is_cancelled()); } #[tokio::test] async fn worker_control_line_cancel_sets_cancel_token_and_interrupts_pending_interviews() { let interviewer = Arc::new(ControlInterviewer::new()); - let cancel_token = Arc::new(AtomicBool::new(false)); + let cancel_token = CancellationToken::new(); let mut question = Question::new("Approve?", QuestionType::YesNo); question.id = "q-1".to_string(); let ask_interviewer = Arc::clone(&interviewer); @@ -860,7 +860,7 @@ mod tests { let answer = answer_task.await.unwrap().answer; assert_eq!(answer.value, AnswerValue::Interrupted); - assert!(cancel_token.load(Ordering::SeqCst)); + assert!(cancel_token.is_cancelled()); } #[tokio::test] @@ -893,7 +893,7 @@ mod tests { #[tokio::test] async fn worker_control_event_loop_eof_interrupts_pending_interviews() { let interviewer = Arc::new(ControlInterviewer::new()); - let cancel_token = Arc::new(AtomicBool::new(false)); + let cancel_token = CancellationToken::new(); let mut question = Question::new("Approve?", QuestionType::YesNo); question.id = "q-1".to_string(); let ask_interviewer = Arc::clone(&interviewer); @@ -905,14 +905,14 @@ mod tests { handle_worker_control_stream_events( Arc::clone(&interviewer), - Arc::clone(&cancel_token), + cancel_token.clone(), event_rx, ) .await; let answer = answer_task.await.unwrap().answer; assert_eq!(answer.value, AnswerValue::Interrupted); - assert!(!cancel_token.load(Ordering::SeqCst)); + assert!(!cancel_token.is_cancelled()); } #[tokio::test] diff --git a/lib/crates/fabro-cli/tests/it/workflow/real_cli.rs b/lib/crates/fabro-cli/tests/it/workflow/real_cli.rs index 1454a6765..efa322731 100644 --- a/lib/crates/fabro-cli/tests/it/workflow/real_cli.rs +++ b/lib/crates/fabro-cli/tests/it/workflow/real_cli.rs @@ -34,6 +34,7 @@ async fn run_real_cli_test(provider: Provider, model: &str) { &emitter, &env, None, + tokio_util::sync::CancellationToken::new(), ) .await .unwrap_or_else(|_| panic!("CLI backend ({provider}/{model}) should succeed")); diff --git a/lib/crates/fabro-core/src/executor.rs b/lib/crates/fabro-core/src/executor.rs index ac08d4606..d9ca0d6cc 100644 --- a/lib/crates/fabro-core/src/executor.rs +++ b/lib/crates/fabro-core/src/executor.rs @@ -1,5 +1,6 @@ use std::sync::Arc; -use std::sync::atomic::{AtomicBool, Ordering}; +#[cfg(test)] +use std::sync::atomic::Ordering; use std::time::Instant; use tokio::time::sleep; @@ -18,7 +19,7 @@ use crate::state::ExecutionState; #[derive(Default)] pub struct ExecutorOptions { - pub cancel_token: Option>, + pub cancel_token: Option, pub stall_token: Option, pub max_node_visits: Option, } @@ -58,7 +59,7 @@ impl ExecutorBuilder { } #[must_use] - pub fn cancel_token(mut self, token: Arc) -> Self { + pub fn cancel_token(mut self, token: CancellationToken) -> Self { self.options.cancel_token = Some(token); self } @@ -95,7 +96,7 @@ impl Executor { loop { // Check cancellation if let Some(ref token) = self.options.cancel_token { - if token.load(Ordering::Relaxed) { + if token.is_cancelled() { state.cancelled = true; let outcome = Outcome::fail("run cancelled"); self.lifecycle.on_run_end(&outcome, &state).await; @@ -500,7 +501,8 @@ mod tests { #[tokio::test] async fn executor_builder_sets_cancel_token() { - let token = Arc::new(AtomicBool::new(true)); // already cancelled + let token = CancellationToken::new(); + token.cancel(); // already cancelled let g = linear_graph(&["start", "end"]); let state = ExecutionState::new(&g).unwrap(); let executor = @@ -511,6 +513,38 @@ mod tests { assert!(matches!(result, Err(Error::Cancelled))); } + #[tokio::test] + async fn executor_cancel_token_fired_during_run_returns_cancelled() { + // Cancel token fired by a handler during the first node; the executor + // checks cancellation at the next node boundary and returns Cancelled. + let token = CancellationToken::new(); + let token_clone = token.clone(); + + struct CancellingHandler(CancellationToken); + #[async_trait] + impl NodeHandler for CancellingHandler { + async fn execute( + &self, + _node: &TestNode, + _context: &Context, + _g: &TestGraph, + ) -> Result { + self.0.cancel(); + Ok(Outcome::success()) + } + } + + let g = linear_graph(&["start", "work", "end"]); + let state = ExecutionState::new(&g).unwrap(); + let executor = ExecutorBuilder::new( + Arc::new(CancellingHandler(token_clone)) as Arc> + ) + .cancel_token(token) + .build(); + let result = executor.run(&g, state).await; + assert!(matches!(result, Err(Error::Cancelled))); + } + // ---- Step 9: Terminal nodes, goal gates, visit limits ---- #[tokio::test] @@ -908,10 +942,10 @@ mod tests { #[tokio::test] async fn executor_cancellation_stops_run() { - let token = Arc::new(AtomicBool::new(false)); + let token = CancellationToken::new(); let token_clone = token.clone(); - struct CancellingHandler(Arc); + struct CancellingHandler(CancellationToken); #[async_trait] impl NodeHandler for CancellingHandler { async fn execute( @@ -921,7 +955,7 @@ mod tests { _g: &TestGraph, ) -> Result { // Cancel after first node - self.0.store(true, Ordering::Relaxed); + self.0.cancel(); Ok(Outcome::success()) } } diff --git a/lib/crates/fabro-core/src/stall.rs b/lib/crates/fabro-core/src/stall.rs index 8068900bc..2319cbbf8 100644 --- a/lib/crates/fabro-core/src/stall.rs +++ b/lib/crates/fabro-core/src/stall.rs @@ -5,6 +5,7 @@ use std::time::Duration; use tokio::sync::Notify; use tokio::task::JoinHandle; use tokio::time::sleep; +use tokio_util::sync::CancellationToken; /// Trait for receiving stall timeout notifications. pub trait ActivityMonitor: Send + Sync { @@ -16,11 +17,11 @@ pub trait ActivityMonitor: Send + Sync { /// Watches for inactivity and fires a stall timeout if no activity is /// reported within the configured duration. pub struct StallWatchdog { - timeout: Duration, - cancel_token: Arc, - activity: Arc, - shutdown: Arc, - monitor: Arc, + timeout: Duration, + stall_token: CancellationToken, + activity: Arc, + shutdown: Arc, + monitor: Arc, } /// Guard that resets the stall timer on activity. Drop to stop watching. @@ -33,12 +34,12 @@ pub struct StallGuard { impl StallWatchdog { pub fn new( timeout: Duration, - cancel_token: Arc, + stall_token: CancellationToken, monitor: Arc, ) -> Self { Self { timeout, - cancel_token, + stall_token, activity: Arc::new(Notify::new()), shutdown: Arc::new(AtomicBool::new(false)), monitor, @@ -51,7 +52,7 @@ impl StallWatchdog { let activity = self.activity.clone(); let shutdown = self.shutdown.clone(); let timeout = self.timeout; - let cancel_token = self.cancel_token; + let stall_token = self.stall_token; let monitor = self.monitor; let handle = tokio::spawn(async move { @@ -66,7 +67,7 @@ impl StallWatchdog { "Stall timeout: no activity detected" ); monitor.on_stall_timeout(timeout); - cancel_token.store(true, Ordering::Relaxed); + stall_token.cancel(); return; } () = activity.notified() => { @@ -136,7 +137,7 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn stall_watchdog_cancels_on_inactivity() { - let cancel = Arc::new(AtomicBool::new(false)); + let cancel = CancellationToken::new(); let monitor = TestMonitor::new(); let watchdog = StallWatchdog::new(Duration::from_millis(50), cancel.clone(), monitor.clone()); @@ -145,13 +146,13 @@ mod tests { // Wait for timeout to fire sleep(Duration::from_millis(100)).await; - assert!(cancel.load(Ordering::Relaxed)); + assert!(cancel.is_cancelled()); assert_eq!(monitor.stalls(), 1); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn stall_watchdog_resets_on_activity() { - let cancel = Arc::new(AtomicBool::new(false)); + let cancel = CancellationToken::new(); let monitor = TestMonitor::new(); let watchdog = StallWatchdog::new(Duration::from_millis(80), cancel.clone(), monitor.clone()); @@ -164,17 +165,17 @@ mod tests { // After another 50ms (100ms total, but only 50ms since activity), should not // have timed out sleep(Duration::from_millis(50)).await; - assert!(!cancel.load(Ordering::Relaxed)); + assert!(!cancel.is_cancelled()); // Wait long enough for timeout after last activity (80ms + margin) sleep(Duration::from_millis(60)).await; - assert!(cancel.load(Ordering::Relaxed)); + assert!(cancel.is_cancelled()); assert_eq!(monitor.stalls(), 1); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn stall_watchdog_clean_shutdown_on_success() { - let cancel = Arc::new(AtomicBool::new(false)); + let cancel = CancellationToken::new(); let monitor = TestMonitor::new(); let watchdog = StallWatchdog::new(Duration::from_millis(50), cancel.clone(), monitor.clone()); @@ -187,13 +188,13 @@ mod tests { sleep(Duration::from_millis(100)).await; // Should NOT have triggered - assert!(!cancel.load(Ordering::Relaxed)); + assert!(!cancel.is_cancelled()); assert_eq!(monitor.stalls(), 0); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn stall_guard_cleanup_on_drop() { - let cancel = Arc::new(AtomicBool::new(false)); + let cancel = CancellationToken::new(); let monitor = TestMonitor::new(); let watchdog = StallWatchdog::new(Duration::from_millis(50), cancel.clone(), monitor.clone()); @@ -206,6 +207,6 @@ mod tests { sleep(Duration::from_millis(150)).await; // Cancel should not be set - assert!(!cancel.load(Ordering::Relaxed)); + assert!(!cancel.is_cancelled()); } } diff --git a/lib/crates/fabro-retro/src/retro_agent.rs b/lib/crates/fabro-retro/src/retro_agent.rs index f6275d7d6..30d5dd2d6 100644 --- a/lib/crates/fabro-retro/src/retro_agent.rs +++ b/lib/crates/fabro-retro/src/retro_agent.rs @@ -204,7 +204,10 @@ pub async fn run_retro_agent( // Optionally forward agent events via the callback let event_forwarder_handle = event_callback.map(|cb| spawn_retro_event_forwarder(&session, cb)); - session.initialize().await; + session + .initialize() + .await + .context("Retro agent session initialization failed")?; let prompt = build_retro_prompt(RETRO_DATA_DIR); diff --git a/lib/crates/fabro-sandbox/src/daytona/mod.rs b/lib/crates/fabro-sandbox/src/daytona/mod.rs index 03b3ed2d9..8230bcaee 100644 --- a/lib/crates/fabro-sandbox/src/daytona/mod.rs +++ b/lib/crates/fabro-sandbox/src/daytona/mod.rs @@ -26,7 +26,7 @@ use tokio_util::sync::CancellationToken; use crate::clone_source::{self, CloneDecision, EmptyWorkspaceReason}; use crate::redact::redact_auth_url; -use crate::sandbox::resolve_path; +use crate::sandbox::{optional_timeout, resolve_path}; use crate::{ CommandOutputCallback, DirEntry, ExecResult, ExecStreamingResult, GrepOptions, Sandbox, SandboxEvent, SandboxEventCallback, format_lines_numbered, shell_quote, @@ -37,6 +37,9 @@ const DEFAULT_SNAPSHOT: &str = "daytona-medium"; pub const DEFAULT_DAYTONA_API_URL: &str = "https://app.daytona.io/api"; const FABRO_SANDBOX_USER_AGENT: &str = concat!("fabro-sandbox/", env!("CARGO_PKG_VERSION")); const DAYTONA_PROBE_TIMEOUT: Duration = Duration::from_secs(20); +/// Upper bound on `DaytonaSession::close` so a stalled Daytona REST call cannot +/// block cancellation/timeout paths from returning. +const DAYTONA_SESSION_CLOSE_TIMEOUT: Duration = Duration::from_secs(10); /// Permissions a Daytona API key needs for Fabro's snapshot and sandbox flow. pub const REQUIRED_DAYTONA_PERMISSIONS: &[Permissions] = &[ @@ -1307,7 +1310,7 @@ impl Sandbox for DaytonaSandbox { async fn exec_command_streaming( &self, command: &str, - timeout_ms: u64, + timeout_ms: Option, working_dir: Option<&str>, env_vars: Option<&HashMap>, cancel_token: Option, @@ -1397,7 +1400,7 @@ impl Sandbox for DaytonaSandbox { &session, &command_id, session_exec.exit_code, - Duration::from_millis(timeout_ms), + timeout_ms, cancel_token.unwrap_or_default(), &mut stream_task, ) @@ -1673,19 +1676,39 @@ impl DaytonaSession { } /// Idempotent: a second call after `active=false` is a no-op. + /// + /// `delete_session` is bounded by [`DAYTONA_SESSION_CLOSE_TIMEOUT`] so a + /// stalled Daytona REST call cannot block cancellation paths indefinitely. async fn close(&mut self, reason: &'static str) { if !self.active { return; } self.active = false; if let Some(svc) = self.process_svc.take() { - if let Err(err) = svc.delete_session(&self.session_id).await { - tracing::warn!( - error = %err, - session_id = %self.session_id, - reason, - "failed to delete Daytona session" - ); + match time::timeout( + DAYTONA_SESSION_CLOSE_TIMEOUT, + svc.delete_session(&self.session_id), + ) + .await + { + Ok(Ok(())) => {} + Ok(Err(err)) => { + tracing::warn!( + error = %err, + session_id = %self.session_id, + reason, + "failed to delete Daytona session" + ); + } + Err(_) => { + tracing::warn!( + session_id = %self.session_id, + reason, + timeout_ms = u64::try_from(DAYTONA_SESSION_CLOSE_TIMEOUT.as_millis()) + .unwrap_or(u64::MAX), + "timed out deleting Daytona session" + ); + } } } } @@ -1730,7 +1753,7 @@ async fn wait_for_completion( session: &DaytonaSession, command_id: &str, initial_exit_code: Option, - timeout: Duration, + timeout_ms: Option, cancel_token: CancellationToken, stream_task: &mut JoinHandle>, ) -> crate::Result { @@ -1742,8 +1765,8 @@ async fn wait_for_completion( }); } - let timeout_sleep = time::sleep(timeout); - tokio::pin!(timeout_sleep); + let timeout_future = optional_timeout(timeout_ms); + tokio::pin!(timeout_future); loop { tokio::select! { () = time::sleep(Duration::from_millis(250)) => { @@ -1765,7 +1788,7 @@ async fn wait_for_completion( }); } } - () = &mut timeout_sleep => { + () = &mut timeout_future => { return Ok(WaitOutcome { exit_code: None, termination: CommandTermination::TimedOut, diff --git a/lib/crates/fabro-sandbox/src/docker.rs b/lib/crates/fabro-sandbox/src/docker.rs index e6562e5d6..5e9dcce85 100644 --- a/lib/crates/fabro-sandbox/src/docker.rs +++ b/lib/crates/fabro-sandbox/src/docker.rs @@ -2,7 +2,7 @@ use std::collections::HashMap; use std::fmt::Write as _; use std::io::Cursor; use std::sync::atomic::{AtomicU64, Ordering}; -use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; use async_trait::async_trait; use bollard::Docker; @@ -24,7 +24,7 @@ use tokio_util::sync::CancellationToken; use crate::clone_source::{self, CloneDecision, EmptyWorkspaceReason}; use crate::redact::redact_auth_url; -use crate::sandbox::resolve_path; +use crate::sandbox::{optional_timeout, resolve_path}; use crate::{ CommandOutputCallback, DirEntry, ExecResult, ExecStreamingResult, GrepOptions, Sandbox, SandboxEvent, SandboxEventCallback, format_lines_numbered, shell_quote, @@ -362,7 +362,7 @@ impl DockerSandbox { async fn docker_exec_shell_streaming( &self, command: &str, - timeout_ms: u64, + timeout_ms: Option, working_dir: Option<&str>, env_vars: Option<&HashMap>, cancel_token: Option, @@ -380,7 +380,8 @@ impl DockerSandbox { controlled_command, ]; - let timeout_duration = Duration::from_millis(timeout_ms); + let timeout_future = optional_timeout(timeout_ms); + tokio::pin!(timeout_future); let token = cancel_token.unwrap_or_default(); let container_id = self.container_id()?.to_string(); @@ -399,7 +400,7 @@ impl DockerSandbox { joined .map_err(|e| crate::Error::context("Docker exec stream task failed", e))?? } - () = time::sleep(timeout_duration) => { + () = &mut timeout_future => { termination = CommandTermination::TimedOut; self.request_docker_exec_stop(&stop_file).await?; output_task @@ -1192,7 +1193,7 @@ impl Sandbox for DockerSandbox { async fn exec_command_streaming( &self, command: &str, - timeout_ms: u64, + timeout_ms: Option, working_dir: Option<&str>, env_vars: Option<&HashMap>, cancel_token: Option, @@ -1539,6 +1540,7 @@ mod tests { reason = "unit test reads an in-memory tar entry synchronously" )] use std::io::Read as _; + use std::time::Duration; use tokio::process::Command; diff --git a/lib/crates/fabro-sandbox/src/local.rs b/lib/crates/fabro-sandbox/src/local.rs index 66d084b5f..a8b49e215 100644 --- a/lib/crates/fabro-sandbox/src/local.rs +++ b/lib/crates/fabro-sandbox/src/local.rs @@ -10,6 +10,7 @@ use tokio::task::spawn_blocking; use tokio::{fs, time}; use tokio_util::sync::CancellationToken; +use crate::sandbox::optional_timeout; use crate::{ CommandOutputCallback, DirEntry, ExecResult, ExecStreamingResult, GrepOptions, Sandbox, SandboxEvent, SandboxEventCallback, format_lines_numbered, @@ -330,7 +331,7 @@ impl Sandbox for LocalSandbox { async fn exec_command_streaming( &self, command: &str, - timeout_ms: u64, + timeout_ms: Option, working_dir: Option<&str>, env_vars: Option<&std::collections::HashMap>, cancel_token: Option, @@ -370,7 +371,8 @@ impl Sandbox for LocalSandbox { .spawn() .map_err(|e| crate::Error::context("Failed to spawn command", e))?; - let timeout_duration = std::time::Duration::from_millis(timeout_ms); + let timeout_future = optional_timeout(timeout_ms); + tokio::pin!(timeout_future); let token = cancel_token.unwrap_or_default(); let stdout_pipe = child.stdout.take(); @@ -390,7 +392,7 @@ impl Sandbox for LocalSandbox { .map_err(|e| crate::Error::context("Failed to wait for process", e))?; (CommandTermination::Exited, status.code()) } - () = time::sleep(timeout_duration) => { + () = &mut timeout_future => { sigterm_then_kill(&mut child).await; (CommandTermination::TimedOut, None) } diff --git a/lib/crates/fabro-sandbox/src/sandbox.rs b/lib/crates/fabro-sandbox/src/sandbox.rs index e30012969..bfd31d91b 100644 --- a/lib/crates/fabro-sandbox/src/sandbox.rs +++ b/lib/crates/fabro-sandbox/src/sandbox.rs @@ -17,6 +17,16 @@ const GIT: &str = "git -c maintenance.auto=0 -c gc.auto=0"; pub const DEFAULT_EXEC_OUTPUT_TAIL_BYTES: usize = 8 * 1024; +/// Sleep for `timeout_ms` if `Some`, otherwise never resolves. Used by +/// streaming `exec_command` impls to model "no timeout" without scheduling a +/// `Duration::from_millis(u64::MAX)` sleep. +pub(crate) async fn optional_timeout(timeout_ms: Option) { + match timeout_ms { + Some(ms) => time::sleep(Duration::from_millis(ms)).await, + None => std::future::pending::<()>().await, + } +} + /// Information returned when a sandbox sets up git for a workflow run. #[derive(Debug, Clone)] pub struct GitRunInfo { @@ -93,7 +103,7 @@ macro_rules! delegate_sandbox { async fn exec_command_streaming( &self, command: &str, - timeout_ms: u64, + timeout_ms: Option, working_dir: Option<&str>, env_vars: Option<&std::collections::HashMap>, cancel_token: Option, @@ -607,14 +617,21 @@ pub trait Sandbox: Send + Sync { async fn exec_command_streaming( &self, command: &str, - timeout_ms: u64, + timeout_ms: Option, working_dir: Option<&str>, env_vars: Option<&std::collections::HashMap>, cancel_token: Option, output_callback: CommandOutputCallback, ) -> crate::Result { + let fallback_timeout_ms = timeout_ms.unwrap_or(u64::MAX); let result = self - .exec_command(command, timeout_ms, working_dir, env_vars, cancel_token) + .exec_command( + command, + fallback_timeout_ms, + working_dir, + env_vars, + cancel_token, + ) .await?; if !result.stdout.is_empty() { output_callback( diff --git a/lib/crates/fabro-sandbox/src/worktree.rs b/lib/crates/fabro-sandbox/src/worktree.rs index f412c87d4..d7941dde0 100644 --- a/lib/crates/fabro-sandbox/src/worktree.rs +++ b/lib/crates/fabro-sandbox/src/worktree.rs @@ -236,7 +236,7 @@ impl Sandbox for WorktreeSandbox { async fn exec_command_streaming( &self, command: &str, - timeout_ms: u64, + timeout_ms: Option, working_dir: Option<&str>, env_vars: Option<&HashMap>, cancel_token: Option, diff --git a/lib/crates/fabro-sandbox/tests/daytona_streaming_live.rs b/lib/crates/fabro-sandbox/tests/daytona_streaming_live.rs index 10c385ae9..518d6bbbd 100644 --- a/lib/crates/fabro-sandbox/tests/daytona_streaming_live.rs +++ b/lib/crates/fabro-sandbox/tests/daytona_streaming_live.rs @@ -63,7 +63,7 @@ mod daytona_streaming_live { sandbox_for_exec .exec_command_streaming( "printf 'live-out\\n'; printf 'live-err\\n' >&2; sleep 30", - 60_000, + Some(60_000), None, None, Some(cancel_for_exec), @@ -186,7 +186,14 @@ mod daytona_streaming_live { let chunks = Arc::new(Mutex::new(Vec::new())); let callback = capture_callback(Arc::clone(&chunks)); let result = sandbox - .exec_command_streaming(command, timeout_ms, None, None, cancel_token, callback) + .exec_command_streaming( + command, + Some(timeout_ms), + None, + None, + cancel_token, + callback, + ) .await?; let chunks = chunks.lock().await.clone(); diff --git a/lib/crates/fabro-sandbox/tests/docker_streaming.rs b/lib/crates/fabro-sandbox/tests/docker_streaming.rs index 437c3fce1..848f149fb 100644 --- a/lib/crates/fabro-sandbox/tests/docker_streaming.rs +++ b/lib/crates/fabro-sandbox/tests/docker_streaming.rs @@ -49,7 +49,7 @@ async fn streaming_timeout_terminates_docker_exec_before_returning() { let result = sandbox .exec_command_streaming( &format!("trap '' HUP TERM; echo start; sleep 5 # {marker}"), - 200, + Some(200), None, None, None, diff --git a/lib/crates/fabro-server/Cargo.toml b/lib/crates/fabro-server/Cargo.toml index 11451e094..3bfa871a7 100644 --- a/lib/crates/fabro-server/Cargo.toml +++ b/lib/crates/fabro-server/Cargo.toml @@ -56,6 +56,7 @@ globset.workspace = true tower = "0.5" tower-http = { version = "0.6", features = ["trace"] } tokio-stream = { workspace = true, features = ["sync"] } +tokio-util.workspace = true base64.workspace = true jsonwebtoken.workspace = true hkdf.workspace = true @@ -108,4 +109,4 @@ tokio-util.workspace = true fabro-macros = { path = "../fabro-macros" } fabro-sandbox = { path = "../fabro-sandbox", features = ["test-support"] } fabro-test = { workspace = true } -fabro-types = { path = "../fabro-types", features = ["test-support"] } +fabro-types = { path = "../fabro-types", features = ["test-support"] } \ No newline at end of file diff --git a/lib/crates/fabro-server/src/server.rs b/lib/crates/fabro-server/src/server.rs index 276374d2f..c0508a8e3 100644 --- a/lib/crates/fabro-server/src/server.rs +++ b/lib/crates/fabro-server/src/server.rs @@ -111,6 +111,7 @@ use tokio::task::spawn_blocking; use tokio::time::{sleep, timeout}; use tokio_stream::StreamExt; use tokio_stream::wrappers::{BroadcastStream, UnboundedReceiverStream}; +use tokio_util::sync::CancellationToken; use tower::{ServiceExt, service_fn}; use tracing::{Instrument, debug, error, info, warn}; use ulid::Ulid; @@ -198,7 +199,7 @@ struct ManagedRun { event_tx: Option>, checkpoint: Option, cancel_tx: Option>, - cancel_token: Option>, + cancel_token: Option, worker_pid: Option, worker_pgid: Option, run_dir: Option, @@ -1522,7 +1523,7 @@ async fn delete_run_internal( if let Some(mut managed_run) = managed_run { if let Some(token) = &managed_run.cancel_token { - token.store(true, Ordering::SeqCst); + token.cancel(); } if let Some(answer_transport) = managed_run.answer_transport.clone() { let _ = answer_transport.cancel_run().await; @@ -2615,12 +2616,12 @@ async fn execute_run_in_process(state: Arc, run_id: RunId) { }; let (cancel_tx, cancel_rx) = oneshot::channel::<()>(); - let cancel_token = Arc::new(AtomicBool::new(false)); + let cancel_token = CancellationToken::new(); let (event_tx, _) = broadcast::channel(256); managed_run.status = RunStatus::Starting; managed_run.cancel_tx = Some(cancel_tx); - managed_run.cancel_token = Some(Arc::clone(&cancel_token)); + managed_run.cancel_token = Some(cancel_token.clone()); managed_run.event_tx = Some(event_tx); ( @@ -2713,7 +2714,7 @@ async fn execute_run_in_process(state: Arc, run_id: RunId) { }; let server_settings = state.server_settings(); let github_settings = &server_settings.server.integrations.github; - if cancel_token.load(Ordering::SeqCst) { + if cancel_token.is_cancelled() { finish_cancelled_run_before_execution(&state, run_id).await; return; } @@ -2748,7 +2749,7 @@ async fn execute_run_in_process(state: Arc, run_id: RunId) { let github_app = match github_app_result { Ok(github_app) => github_app, Err(e) => { - if cancel_token.load(Ordering::SeqCst) { + if cancel_token.is_cancelled() { finish_cancelled_run_before_execution(&state, run_id).await; return; } @@ -2775,7 +2776,7 @@ async fn execute_run_in_process(state: Arc, run_id: RunId) { .collect(); let services = operations::StartServices { run_id, - cancel_token: Some(Arc::clone(&cancel_token)), + cancel_token: cancel_token.clone(), emitter: Arc::clone(&emitter), interviewer: Arc::clone(&interview_runtime), run_store: run_store.clone().into(), @@ -2799,7 +2800,7 @@ async fn execute_run_in_process(state: Arc, run_id: RunId) { let result = tokio::select! { result = execution => ExecutionResult::Completed(Box::new(result)), _ = cancel_rx => { - cancel_token.store(true, Ordering::SeqCst); + cancel_token.cancel(); ExecutionResult::CancelledBySignal } }; diff --git a/lib/crates/fabro-server/src/server/handler/lifecycle.rs b/lib/crates/fabro-server/src/server/handler/lifecycle.rs index 1201bf8f7..50d556c66 100644 --- a/lib/crates/fabro-server/src/server/handler/lifecycle.rs +++ b/lib/crates/fabro-server/src/server/handler/lifecycle.rs @@ -1,13 +1,13 @@ use std::sync::Arc; use super::super::{ - ApiError, AppState, FailureReason, ForkRequest, ForkResponse, IntoResponse, Json, Ordering, - Path, Principal, RequiredUser, Response, RewindRequest, RewindResponse, Router, - RunAnswerTransport, RunControlAction, RunExecutionMode, RunId, RunStatus, RunStatusResponse, - StartRunRequest, State, StatusCode, Storage, TimelineEntryResponse, WORKER_CANCEL_GRACE, - WorkflowError, append_control_request, get, load_pending_control, managed_run, operations, - parse_run_id_path, persist_cancelled_run_status, post, reject_if_archived, sleep, - update_live_run_from_event, workflow_event, + ApiError, AppState, FailureReason, ForkRequest, ForkResponse, IntoResponse, Json, Path, + Principal, RequiredUser, Response, RewindRequest, RewindResponse, Router, RunAnswerTransport, + RunControlAction, RunExecutionMode, RunId, RunStatus, RunStatusResponse, StartRunRequest, + State, StatusCode, Storage, TimelineEntryResponse, WORKER_CANCEL_GRACE, WorkflowError, + append_control_request, get, load_pending_control, managed_run, operations, parse_run_id_path, + persist_cancelled_run_status, post, reject_if_archived, sleep, update_live_run_from_event, + workflow_event, }; pub(super) fn routes() -> Router> { @@ -249,7 +249,7 @@ async fn cancel_run( } if let Some(token) = &cancel_token { - token.store(true, Ordering::SeqCst); + token.cancel(); } let sent_cancel_signal = if let Some(cancel_tx) = cancel_tx { let _ = cancel_tx.send(()); diff --git a/lib/crates/fabro-store/src/run_state.rs b/lib/crates/fabro-store/src/run_state.rs index ae23ff9fd..16b70e0cc 100644 --- a/lib/crates/fabro-store/src/run_state.rs +++ b/lib/crates/fabro-store/src/run_state.rs @@ -7,10 +7,11 @@ use fabro_types::run_event::{ RunFailedProps, StageCompletedProps, StagePromptProps, }; use fabro_types::{ - BilledModelUsage, Checkpoint, Conclusion, EventBody, FailureSignature, InterviewQuestionRecord, - Outcome, PendingInterviewRecord, PullRequestRecord, RunControlAction, RunEvent, RunId, - RunProjection, RunSpec, RunStatus, RunSummary, SandboxRecord, StageCompletion, StageId, - StageOutcome, StageProjection, StageState, StartRecord, TerminalStatus, first_event_seq, + BilledModelUsage, Checkpoint, CommandTermination, Conclusion, EventBody, FailureSignature, + InterviewQuestionRecord, Outcome, PendingInterviewRecord, PullRequestRecord, RunControlAction, + RunEvent, RunId, RunProjection, RunSpec, RunStatus, RunSummary, SandboxRecord, StageCompletion, + StageId, StageOutcome, StageProjection, StageState, StartRecord, TerminalStatus, + first_event_seq, }; use fabro_util::error::render_with_causes; use serde_json::Value; @@ -389,6 +390,42 @@ impl RunProjectionReducer for RunProjection { stage.termination = Some(props.termination); stage.script_timing = Some(script_timing); } + EventBody::AgentCliCompleted(props) => { + let Some(stage) = stage_at_current_visit(self, stored, event.seq) else { + return Ok(()); + }; + apply_agent_cli_terminal( + stage, + props, + &props.stdout, + &props.stderr, + CommandTermination::Exited, + )?; + } + EventBody::AgentCliCancelled(props) => { + let Some(stage) = stage_at_current_visit(self, stored, event.seq) else { + return Ok(()); + }; + apply_agent_cli_terminal( + stage, + props, + &props.stdout, + &props.stderr, + CommandTermination::Cancelled, + )?; + } + EventBody::AgentCliTimedOut(props) => { + let Some(stage) = stage_at_current_visit(self, stored, event.seq) else { + return Ok(()); + }; + apply_agent_cli_terminal( + stage, + props, + &props.stdout, + &props.stderr, + CommandTermination::TimedOut, + )?; + } EventBody::ParallelCompleted(props) => { let parallel_results = serde_json::to_value(&props.results).map_err(|err| { Error::InvalidEvent(format!("invalid parallel.completed payload: {err}")) @@ -671,6 +708,22 @@ fn provider_used_from_agent_cli_started(props: &AgentCliStartedProps) -> Value { Value::Object(provider_used) } +fn apply_agent_cli_terminal( + stage: &mut StageProjection, + props: &impl serde::Serialize, + stdout: &str, + stderr: &str, + termination: CommandTermination, +) -> Result<()> { + let script_timing = serde_json::to_value(props) + .map_err(|err| Error::InvalidEvent(format!("invalid agent.cli terminal payload: {err}")))?; + stage.stdout = Some(stdout.to_string()); + stage.stderr = Some(stderr.to_string()); + stage.termination = Some(termination); + stage.script_timing = Some(script_timing); + Ok(()) +} + #[cfg(test)] mod tests { use std::collections::{BTreeMap, HashMap}; @@ -678,15 +731,16 @@ mod tests { use chrono::Utc; use fabro_types::run_event::run::RunFailedProps; use fabro_types::run_event::{ + AgentCliCancelledProps, AgentCliCompletedProps, AgentCliTimedOutProps, CheckpointCompletedProps, InterviewCompletedProps, InterviewOption, InterviewStartedProps, RunControlEffectProps, StageCompletedProps, StageFailedProps, StagePromptProps, StageRetryingProps, StageStartedProps, }; use fabro_types::{ - BilledModelUsage, BlockedReason, Checkpoint, EventBody, FailureCategory, FailureDetail, - FailureReason, Outcome, QuestionType, RunBlobId, RunControlAction, RunEvent, RunStatus, - StageOutcome, StageState, SuccessReason, TerminalStatus, WorkflowSettings, first_event_seq, - fixtures, + BilledModelUsage, BlockedReason, Checkpoint, CommandTermination, EventBody, + FailureCategory, FailureDetail, FailureReason, Outcome, QuestionType, RunBlobId, + RunControlAction, RunEvent, RunStatus, StageOutcome, StageState, SuccessReason, + TerminalStatus, WorkflowSettings, first_event_seq, fixtures, }; use serde_json::json; @@ -955,6 +1009,106 @@ mod tests { assert_eq!(stage.prompt.as_deref(), Some("prompt")); } + fn start_stage(state: &mut RunProjection, stage_id: &StageId) { + state + .apply_event(&test_stage_event( + 3, + EventBody::StageStarted(StageStartedProps { + index: 0, + handler_type: "agent".to_string(), + attempt: 1, + max_attempts: 1, + }), + stage_id.clone(), + )) + .unwrap(); + } + + #[test] + fn agent_cli_completed_updates_stage_output_projection() { + let mut state = RunProjection::default(); + let stage_id = StageId::new("code", 1); + start_stage(&mut state, &stage_id); + + state + .apply_event(&test_stage_event( + 4, + EventBody::AgentCliCompleted(AgentCliCompletedProps { + stdout: "done".to_string(), + stderr: "warn".to_string(), + exit_code: 0, + duration_ms: 42, + }), + stage_id.clone(), + )) + .unwrap(); + + let stage = state.stage(&stage_id).unwrap(); + assert_eq!(stage.stdout.as_deref(), Some("done")); + assert_eq!(stage.stderr.as_deref(), Some("warn")); + assert_eq!(stage.termination, Some(CommandTermination::Exited)); + assert_eq!( + stage.script_timing.as_ref().unwrap()["duration_ms"], + serde_json::json!(42) + ); + } + + #[test] + fn agent_cli_cancelled_updates_stage_output_projection() { + let mut state = RunProjection::default(); + let stage_id = StageId::new("code", 1); + start_stage(&mut state, &stage_id); + + state + .apply_event(&test_stage_event( + 4, + EventBody::AgentCliCancelled(AgentCliCancelledProps { + stdout: "partial".to_string(), + stderr: "cancelled".to_string(), + duration_ms: 7, + }), + stage_id.clone(), + )) + .unwrap(); + + let stage = state.stage(&stage_id).unwrap(); + assert_eq!(stage.stdout.as_deref(), Some("partial")); + assert_eq!(stage.stderr.as_deref(), Some("cancelled")); + assert_eq!(stage.termination, Some(CommandTermination::Cancelled)); + assert_eq!( + stage.script_timing.as_ref().unwrap()["duration_ms"], + serde_json::json!(7) + ); + } + + #[test] + fn agent_cli_timed_out_updates_stage_output_projection() { + let mut state = RunProjection::default(); + let stage_id = StageId::new("code", 1); + start_stage(&mut state, &stage_id); + + state + .apply_event(&test_stage_event( + 4, + EventBody::AgentCliTimedOut(AgentCliTimedOutProps { + stdout: "partial".to_string(), + stderr: "timeout".to_string(), + duration_ms: 600, + }), + stage_id.clone(), + )) + .unwrap(); + + let stage = state.stage(&stage_id).unwrap(); + assert_eq!(stage.stdout.as_deref(), Some("partial")); + assert_eq!(stage.stderr.as_deref(), Some("timeout")); + assert_eq!(stage.termination, Some(CommandTermination::TimedOut)); + assert_eq!( + stage.script_timing.as_ref().unwrap()["duration_ms"], + serde_json::json!(600) + ); + } + #[test] fn stage_completed_event_captures_duration_and_usage_per_visit() { let mut state = RunProjection::default(); diff --git a/lib/crates/fabro-types/src/run_event/misc.rs b/lib/crates/fabro-types/src/run_event/misc.rs index 1570050a4..5123d4c8b 100644 --- a/lib/crates/fabro-types/src/run_event/misc.rs +++ b/lib/crates/fabro-types/src/run_event/misc.rs @@ -239,6 +239,20 @@ pub struct AgentCliCompletedProps { pub duration_ms: u64, } +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct AgentCliCancelledProps { + pub stdout: String, + pub stderr: String, + pub duration_ms: u64, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct AgentCliTimedOutProps { + pub stdout: String, + pub stderr: String, + pub duration_ms: u64, +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct PullRequestCreatedProps { pub pr_url: String, diff --git a/lib/crates/fabro-types/src/run_event/mod.rs b/lib/crates/fabro-types/src/run_event/mod.rs index 647efc381..f84d943c1 100644 --- a/lib/crates/fabro-types/src/run_event/mod.rs +++ b/lib/crates/fabro-types/src/run_event/mod.rs @@ -252,6 +252,10 @@ pub enum EventBody { AgentCliStarted(AgentCliStartedProps), #[serde(rename = "agent.cli.completed")] AgentCliCompleted(AgentCliCompletedProps), + #[serde(rename = "agent.cli.cancelled")] + AgentCliCancelled(AgentCliCancelledProps), + #[serde(rename = "agent.cli.timed_out")] + AgentCliTimedOut(AgentCliTimedOutProps), #[serde(rename = "pull_request.created")] PullRequestCreated(PullRequestCreatedProps), #[serde(rename = "pull_request.failed")] @@ -433,6 +437,8 @@ impl EventBody { Self::CommandCompleted(_) => "command.completed", Self::AgentCliStarted(_) => "agent.cli.started", Self::AgentCliCompleted(_) => "agent.cli.completed", + Self::AgentCliCancelled(_) => "agent.cli.cancelled", + Self::AgentCliTimedOut(_) => "agent.cli.timed_out", Self::PullRequestCreated(_) => "pull_request.created", Self::PullRequestFailed(_) => "pull_request.failed", Self::DevcontainerResolved(_) => "devcontainer.resolved", diff --git a/lib/crates/fabro-workflow/src/devcontainer_bridge.rs b/lib/crates/fabro-workflow/src/devcontainer_bridge.rs index 8c530a177..37e2cf255 100644 --- a/lib/crates/fabro-workflow/src/devcontainer_bridge.rs +++ b/lib/crates/fabro-workflow/src/devcontainer_bridge.rs @@ -1,5 +1,3 @@ -use std::sync::Arc; -use std::sync::atomic::AtomicBool; use std::time::Instant; use fabro_agent::sandbox::Sandbox; @@ -7,10 +5,10 @@ use fabro_devcontainer::DevcontainerSpec; use fabro_sandbox::daytona::{DaytonaSnapshotConfig, DockerfileSource}; use futures::future::try_join_all; use sha2::{Digest, Sha256}; +use tokio_util::sync::CancellationToken; use crate::error::Error; use crate::event::{Emitter, Event}; -use crate::handler::sandbox_cancel_token; /// Compute a deterministic snapshot name from Dockerfile content. pub fn snapshot_name_for_dockerfile(dockerfile: &str) -> String { @@ -39,7 +37,7 @@ pub async fn run_devcontainer_lifecycle( phase: &str, commands: &[fabro_devcontainer::Command], timeout_ms: u64, - cancel_requested: Option>, + cancel_token: CancellationToken, ) -> Result<(), Error> { if commands.is_empty() { return Ok(()); @@ -61,7 +59,7 @@ pub async fn run_devcontainer_lifecycle( &format!("sh -c {}", shlex::try_quote(s).unwrap_or_else(|_| s.into())), index, timeout_ms, - cancel_requested.clone(), + cancel_token.clone(), ) .await?; } @@ -78,7 +76,7 @@ pub async fn run_devcontainer_lifecycle( &joined, index, timeout_ms, - cancel_requested.clone(), + cancel_token.clone(), ) .await?; } @@ -92,7 +90,7 @@ pub async fn run_devcontainer_lifecycle( ); let phase = phase.to_string(); let name = name.clone(); - let cancel_requested = cancel_requested.clone(); + let cancel_token = cancel_token.clone(); async move { let cmd_start = Instant::now(); emitter.emit(&Event::DevcontainerLifecycleCommandStarted { @@ -100,14 +98,14 @@ pub async fn run_devcontainer_lifecycle( command: name.clone(), index, }); - let cancel_token = sandbox_cancel_token(cancel_requested); + let child_token = cancel_token.child_token(); let result = sandbox .exec_command( &command, timeout_ms, None, None, - cancel_token.clone(), + Some(child_token.clone()), ) .await .map_err(|e| { @@ -115,12 +113,10 @@ pub async fn run_devcontainer_lifecycle( "Devcontainer {phase} parallel command '{name}' failed: {e}" )) })?; - if let Some(token) = &cancel_token { - if token.is_cancelled() { - return Err(Error::Cancelled); - } - token.cancel(); + if cancel_token.is_cancelled() { + return Err(Error::Cancelled); } + child_token.cancel(); let cmd_duration = crate::millis_u64(cmd_start.elapsed()); if !result.is_success() { let exit_code = result.display_exit_code(); @@ -175,7 +171,7 @@ async fn run_single_lifecycle_command( command: &str, index: usize, timeout_ms: u64, - cancel_requested: Option>, + cancel_token: CancellationToken, ) -> Result<(), Error> { emitter.emit(&Event::DevcontainerLifecycleCommandStarted { phase: phase.to_string(), @@ -183,19 +179,17 @@ async fn run_single_lifecycle_command( index, }); let cmd_start = Instant::now(); - let cancel_token = sandbox_cancel_token(cancel_requested); + let child_token = cancel_token.child_token(); let result = sandbox - .exec_command(command, timeout_ms, None, None, cancel_token.clone()) + .exec_command(command, timeout_ms, None, None, Some(child_token.clone())) .await .map_err(|e| { Error::engine_with_source(format!("Devcontainer {phase} command failed"), &e) })?; - if let Some(token) = &cancel_token { - if token.is_cancelled() { - return Err(Error::Cancelled); - } - token.cancel(); + if cancel_token.is_cancelled() { + return Err(Error::Cancelled); } + child_token.cancel(); let cmd_duration = crate::millis_u64(cmd_start.elapsed()); if !result.is_success() { let exit_code = result.display_exit_code(); @@ -227,7 +221,6 @@ async fn run_single_lifecycle_command( #[cfg(test)] mod tests { use std::collections::HashMap; - use std::sync::atomic::AtomicBool; use std::sync::{Arc, Mutex}; use async_trait::async_trait; @@ -437,9 +430,16 @@ mod tests { let sandbox = TestSandbox::new(); let emitter = Emitter::default(); let commands = vec![fabro_devcontainer::Command::Shell("echo hi".to_string())]; - run_devcontainer_lifecycle(&sandbox, &emitter, "on_create", &commands, 300_000, None) - .await - .unwrap(); + run_devcontainer_lifecycle( + &sandbox, + &emitter, + "on_create", + &commands, + 300_000, + CancellationToken::new(), + ) + .await + .unwrap(); let captured = sandbox.captured_commands(); assert_eq!(captured.len(), 1); assert!(captured[0].contains("echo hi"), "command: {}", captured[0]); @@ -453,9 +453,16 @@ mod tests { "echo".to_string(), "hi".to_string(), ])]; - run_devcontainer_lifecycle(&sandbox, &emitter, "on_create", &commands, 300_000, None) - .await - .unwrap(); + run_devcontainer_lifecycle( + &sandbox, + &emitter, + "on_create", + &commands, + 300_000, + CancellationToken::new(), + ) + .await + .unwrap(); let captured = sandbox.captured_commands(); assert_eq!(captured.len(), 1); assert!( @@ -475,9 +482,16 @@ mod tests { }); let sandbox = TestSandbox::new(); let commands = vec![fabro_devcontainer::Command::Shell("echo hi".to_string())]; - run_devcontainer_lifecycle(&sandbox, &emitter, "on_create", &commands, 300_000, None) - .await - .unwrap(); + run_devcontainer_lifecycle( + &sandbox, + &emitter, + "on_create", + &commands, + 300_000, + CancellationToken::new(), + ) + .await + .unwrap(); let events = events.lock().unwrap(); let started = events[0].properties().unwrap(); assert_eq!(events[0].event_name(), "devcontainer.lifecycle.started"); @@ -515,9 +529,15 @@ mod tests { }); let sandbox = TestSandbox::with_exit_code(1); let commands = vec![fabro_devcontainer::Command::Shell("false".to_string())]; - let result = - run_devcontainer_lifecycle(&sandbox, &emitter, "on_create", &commands, 300_000, None) - .await; + let result = run_devcontainer_lifecycle( + &sandbox, + &emitter, + "on_create", + &commands, + 300_000, + CancellationToken::new(), + ) + .await; assert!(result.is_err()); let events = events.lock().unwrap(); let failed = events @@ -550,9 +570,16 @@ mod tests { events_clone.lock().unwrap().push(event.clone()); }); let sandbox = TestSandbox::new(); - run_devcontainer_lifecycle(&sandbox, &emitter, "on_create", &[], 300_000, None) - .await - .unwrap(); + run_devcontainer_lifecycle( + &sandbox, + &emitter, + "on_create", + &[], + 300_000, + CancellationToken::new(), + ) + .await + .unwrap(); assert!(events.lock().unwrap().is_empty()); } @@ -564,9 +591,16 @@ mod tests { map.insert("install".to_string(), "npm install".to_string()); map.insert("build".to_string(), "npm run build".to_string()); let commands = vec![fabro_devcontainer::Command::Parallel(map)]; - run_devcontainer_lifecycle(&sandbox, &emitter, "post_create", &commands, 300_000, None) - .await - .unwrap(); + run_devcontainer_lifecycle( + &sandbox, + &emitter, + "post_create", + &commands, + 300_000, + CancellationToken::new(), + ) + .await + .unwrap(); let captured = sandbox.captured_commands(); assert_eq!(captured.len(), 2); } @@ -576,7 +610,8 @@ mod tests { let sandbox = TestSandbox::waiting_for_cancel(); let emitter = Emitter::default(); let commands = vec![fabro_devcontainer::Command::Shell("sleep 5".to_string())]; - let cancel_requested = Arc::new(AtomicBool::new(true)); + let cancel_token = CancellationToken::new(); + cancel_token.cancel(); let result = run_devcontainer_lifecycle( &sandbox, @@ -584,7 +619,7 @@ mod tests { "on_create", &commands, 300_000, - Some(cancel_requested), + cancel_token, ) .await; @@ -600,7 +635,8 @@ mod tests { map.insert("install".to_string(), "sleep 5".to_string()); map.insert("build".to_string(), "sleep 5".to_string()); let commands = vec![fabro_devcontainer::Command::Parallel(map)]; - let cancel_requested = Arc::new(AtomicBool::new(true)); + let cancel_token = CancellationToken::new(); + cancel_token.cancel(); let result = run_devcontainer_lifecycle( &sandbox, @@ -608,7 +644,7 @@ mod tests { "post_create", &commands, 300_000, - Some(cancel_requested), + cancel_token, ) .await; diff --git a/lib/crates/fabro-workflow/src/event/convert.rs b/lib/crates/fabro-workflow/src/event/convert.rs index 255d235ea..4a5737fd5 100644 --- a/lib/crates/fabro-workflow/src/event/convert.rs +++ b/lib/crates/fabro-workflow/src/event/convert.rs @@ -1008,6 +1008,26 @@ fn event_body_from_event(event: &Event) -> EventBody { exit_code: *exit_code, duration_ms: *duration_ms, }), + Event::AgentCliCancelled { + stdout, + stderr, + duration_ms, + .. + } => EventBody::AgentCliCancelled(fabro_types::AgentCliCancelledProps { + stdout: stdout.clone(), + stderr: stderr.clone(), + duration_ms: *duration_ms, + }), + Event::AgentCliTimedOut { + stdout, + stderr, + duration_ms, + .. + } => EventBody::AgentCliTimedOut(fabro_types::AgentCliTimedOutProps { + stdout: stdout.clone(), + stderr: stderr.clone(), + duration_ms: *duration_ms, + }), Event::PullRequestCreated { pr_url, pr_number, @@ -1836,6 +1856,48 @@ mod tests { }); } + #[test] + fn agent_cli_cancelled_maps_to_event_body_with_node_id() { + let stored = to_run_event(&fixtures::RUN_1, &Event::AgentCliCancelled { + node_id: "code".to_string(), + stdout: "out".to_string(), + stderr: "err".to_string(), + duration_ms: 42, + }); + + assert_eq!(stored.event_name(), "agent.cli.cancelled"); + assert_eq!(stored.node_id.as_deref(), Some("code")); + match &stored.body { + EventBody::AgentCliCancelled(props) => { + assert_eq!(props.stdout, "out"); + assert_eq!(props.stderr, "err"); + assert_eq!(props.duration_ms, 42); + } + other => panic!("expected AgentCliCancelled, got {other:?}"), + } + } + + #[test] + fn agent_cli_timed_out_maps_to_event_body_with_node_id() { + let stored = to_run_event(&fixtures::RUN_1, &Event::AgentCliTimedOut { + node_id: "code".to_string(), + stdout: "out".to_string(), + stderr: "err".to_string(), + duration_ms: 99, + }); + + assert_eq!(stored.event_name(), "agent.cli.timed_out"); + assert_eq!(stored.node_id.as_deref(), Some("code")); + match &stored.body { + EventBody::AgentCliTimedOut(props) => { + assert_eq!(props.stdout, "out"); + assert_eq!(props.stderr, "err"); + assert_eq!(props.duration_ms, 99); + } + other => panic!("expected AgentCliTimedOut, got {other:?}"), + } + } + #[test] fn stall_watchdog_timeout_populates_watchdog_actor() { let stored = to_run_event(&fixtures::RUN_1, &Event::StallWatchdogTimeout { diff --git a/lib/crates/fabro-workflow/src/event/events.rs b/lib/crates/fabro-workflow/src/event/events.rs index c753e20c6..82cddd44f 100644 --- a/lib/crates/fabro-workflow/src/event/events.rs +++ b/lib/crates/fabro-workflow/src/event/events.rs @@ -531,6 +531,18 @@ pub enum Event { exit_code: i32, duration_ms: u64, }, + AgentCliCancelled { + node_id: String, + stdout: String, + stderr: String, + duration_ms: u64, + }, + AgentCliTimedOut { + node_id: String, + stdout: String, + stderr: String, + duration_ms: u64, + }, PullRequestCreated { pr_url: String, pr_number: u64, @@ -1248,6 +1260,20 @@ impl Event { } => { debug!(node_id, exit_code, duration_ms, "Agent CLI completed"); } + Self::AgentCliCancelled { + node_id, + duration_ms, + .. + } => { + debug!(node_id, duration_ms, "Agent CLI cancelled"); + } + Self::AgentCliTimedOut { + node_id, + duration_ms, + .. + } => { + debug!(node_id, duration_ms, "Agent CLI timed out"); + } Self::PullRequestCreated { pr_url, pr_number, diff --git a/lib/crates/fabro-workflow/src/event/names.rs b/lib/crates/fabro-workflow/src/event/names.rs index e176b2474..5c69b52fb 100644 --- a/lib/crates/fabro-workflow/src/event/names.rs +++ b/lib/crates/fabro-workflow/src/event/names.rs @@ -116,6 +116,8 @@ pub fn event_name(event: &Event) -> &'static str { Event::CommandCompleted { .. } => "command.completed", Event::AgentCliStarted { .. } => "agent.cli.started", Event::AgentCliCompleted { .. } => "agent.cli.completed", + Event::AgentCliCancelled { .. } => "agent.cli.cancelled", + Event::AgentCliTimedOut { .. } => "agent.cli.timed_out", Event::PullRequestCreated { .. } => "pull_request.created", Event::PullRequestFailed { .. } => "pull_request.failed", Event::DevcontainerResolved { .. } => "devcontainer.resolved", diff --git a/lib/crates/fabro-workflow/src/event/stored_fields.rs b/lib/crates/fabro-workflow/src/event/stored_fields.rs index 3c9fd5872..c0b15d307 100644 --- a/lib/crates/fabro-workflow/src/event/stored_fields.rs +++ b/lib/crates/fabro-workflow/src/event/stored_fields.rs @@ -116,7 +116,9 @@ fn stored_event_fields_for_variant(event: &Event) -> StoredEventFields { | Event::CommandStarted { node_id, .. } | Event::CommandCompleted { node_id, .. } | Event::AgentCliStarted { node_id, .. } - | Event::AgentCliCompleted { node_id, .. } => node_stored_fields(Some(node_id.clone())), + | Event::AgentCliCompleted { node_id, .. } + | Event::AgentCliCancelled { node_id, .. } + | Event::AgentCliTimedOut { node_id, .. } => node_stored_fields(Some(node_id.clone())), Event::Agent { stage, visit, diff --git a/lib/crates/fabro-workflow/src/handler/agent.rs b/lib/crates/fabro-workflow/src/handler/agent.rs index 7e3a4be0c..d66ef8ed6 100644 --- a/lib/crates/fabro-workflow/src/handler/agent.rs +++ b/lib/crates/fabro-workflow/src/handler/agent.rs @@ -7,6 +7,7 @@ use fabro_agent::Sandbox; use fabro_graphviz::graph::{Graph, Node}; use fabro_template::{TemplateContext, render as render_template}; use fabro_types::RunId; +use tokio_util::sync::CancellationToken; use super::{EngineServices, Handler}; use crate::context::{Context, WorkflowContext, keys}; @@ -44,6 +45,7 @@ pub trait CodergenBackend: Send + Sync { emitter: &Arc, sandbox: &Arc, tool_hooks: Option>, + cancel_token: CancellationToken, ) -> Result; /// Run a single LLM call with no tools (one_shot mode). @@ -299,6 +301,7 @@ impl Handler for AgentHandler { &services.run.emitter, &services.run.sandbox, tool_hooks, + services.run.cancel_token(), ) .await; match result { @@ -309,6 +312,7 @@ impl Handler for AgentHandler { files_touched, last_file_touched, }) => (text, usage, files_touched, last_file_touched), + Err(Error::Cancelled) => return Err(Error::Cancelled), Err(e) if e.is_retryable() => { return Err(e); } @@ -617,6 +621,7 @@ mod tests { _emitter: &Arc, _sandbox: &Arc, _tool_hooks: Option>, + _cancel_token: CancellationToken, ) -> Result { Ok(CodergenResult::Text { text: @@ -677,6 +682,7 @@ mod tests { _emitter: &Arc, _sandbox: &Arc, _tool_hooks: Option>, + _cancel_token: CancellationToken, ) -> Result { Ok(CodergenResult::Text { text: "Done writing results.".to_string(), @@ -738,6 +744,7 @@ mod tests { emitter: &Arc, _sandbox: &Arc, _tool_hooks: Option>, + _cancel_token: CancellationToken, ) -> Result { let scope = StageScope::for_handler(context, &node.id); emitter.emit_scoped( @@ -849,6 +856,7 @@ mod tests { _emitter: &Arc, _sandbox: &Arc, _tool_hooks: Option>, + _cancel_token: CancellationToken, ) -> Result { *self.captured_thread_id.lock().unwrap() = Some(thread_id.map(String::from)); Ok(CodergenResult::Text { @@ -901,6 +909,7 @@ mod tests { _emitter: &Arc, _sandbox: &Arc, _tool_hooks: Option>, + _cancel_token: CancellationToken, ) -> Result { *self.captured_thread_id.lock().unwrap() = Some(thread_id.map(String::from)); Ok(CodergenResult::Text { @@ -948,6 +957,7 @@ mod tests { _emitter: &Arc, _sandbox: &Arc, _tool_hooks: Option>, + _cancel_token: CancellationToken, ) -> Result { Err(Error::handler("Request timed out".to_string())) } @@ -1095,6 +1105,7 @@ Some text in between. _emitter: &Arc, _sandbox: &Arc, _tool_hooks: Option>, + _cancel_token: CancellationToken, ) -> Result { Err(Error::Validation("bad config".to_string())) } @@ -1135,6 +1146,7 @@ Some text in between. _emitter: &Arc, _sandbox: &Arc, _tool_hooks: Option>, + _cancel_token: CancellationToken, ) -> Result { *self.captured_prompt.lock().unwrap() = Some(prompt.to_string()); Ok(CodergenResult::Text { @@ -1204,6 +1216,7 @@ Some text in between. _emitter: &Arc, _sandbox: &Arc, _tool_hooks: Option>, + _cancel_token: CancellationToken, ) -> Result { *self.captured_prompt.lock().unwrap() = Some(prompt.to_string()); Ok(CodergenResult::Text { diff --git a/lib/crates/fabro-workflow/src/handler/command.rs b/lib/crates/fabro-workflow/src/handler/command.rs index 4ce355536..dbb8fa933 100644 --- a/lib/crates/fabro-workflow/src/handler/command.rs +++ b/lib/crates/fabro-workflow/src/handler/command.rs @@ -109,7 +109,7 @@ impl Handler for CommandHandler { } else { Some(&services.env) }; - let cancel_token = services.run.sandbox_cancel_token(); + let cancel_token = services.run.cancel_token().child_token(); let stage_id = stage_scope.stage_id(); let recorder = CommandLogRecorder::create(run_dir, &stage_id).await?; let output_callback: CommandOutputCallback = { @@ -130,16 +130,14 @@ impl Handler for CommandHandler { .sandbox .exec_command_streaming( &command, - timeout_ms, + Some(timeout_ms), None, env_vars, - cancel_token.clone(), + Some(cancel_token.clone()), output_callback, ) .await; - if let Some(token) = cancel_token { - token.cancel(); - } + cancel_token.cancel(); let streaming = match result { Ok(streaming) => streaming, Err(err) => { @@ -237,7 +235,6 @@ fn tail_bytes(text: &str, max_bytes: usize) -> String { #[cfg(test)] mod tests { use std::sync::Arc; - use std::sync::atomic::AtomicBool; use std::time::Duration; use bytes::Bytes; @@ -1126,7 +1123,7 @@ mod tests { let mut services = make_spy_services(spy.clone()); services.run = services .run - .with_cancel_requested(Some(Arc::new(AtomicBool::new(false)))); + .with_cancel_token(tokio_util::sync::CancellationToken::new()); handler .execute(&node, &context, &graph, run_dir.path(), &services) diff --git a/lib/crates/fabro-workflow/src/handler/fan_in.rs b/lib/crates/fabro-workflow/src/handler/fan_in.rs index 48c484726..65c838731 100644 --- a/lib/crates/fabro-workflow/src/handler/fan_in.rs +++ b/lib/crates/fabro-workflow/src/handler/fan_in.rs @@ -4,6 +4,7 @@ use std::sync::Arc; use async_trait::async_trait; use fabro_agent::Sandbox; use fabro_graphviz::graph::{Graph, Node}; +use tokio_util::sync::CancellationToken; use super::agent::{CodergenBackend, CodergenResult}; use super::{EngineServices, Handler}; @@ -86,6 +87,7 @@ impl Handler for FanInHandler { &node.id, &services.run.emitter, &services.run.sandbox, + services.run.cancel_token(), ) .await? } else { @@ -223,6 +225,7 @@ async fn llm_evaluate( node_id: &str, emitter: &Arc, sandbox: &Arc, + cancel_token: CancellationToken, ) -> Result { let results_text = serde_json::to_string_pretty(results).unwrap_or_else(|_| results.to_string()); @@ -259,6 +262,7 @@ async fn llm_evaluate( emitter, sandbox, None, + cancel_token, ) .await { @@ -474,6 +478,7 @@ mod tests { _emitter: &Arc, _sandbox: &Arc, _tool_hooks: Option>, + _cancel_token: CancellationToken, ) -> Result { // Return text that contains the ID "branch_b" Ok(CodergenResult::Text { diff --git a/lib/crates/fabro-workflow/src/handler/human.rs b/lib/crates/fabro-workflow/src/handler/human.rs index 5e3f7bb9d..fd0df72cc 100644 --- a/lib/crates/fabro-workflow/src/handler/human.rs +++ b/lib/crates/fabro-workflow/src/handler/human.rs @@ -325,12 +325,7 @@ impl Handler for HumanHandler { // 5. Handle unanswered / interrupted interview sessions. if answer.value == AnswerValue::Interrupted { - if services - .run - .cancel_requested - .as_ref() - .is_some_and(|flag| flag.load(Ordering::SeqCst)) - { + if services.run.cancel_token().is_cancelled() { return Err(Error::Cancelled); } self.emit( diff --git a/lib/crates/fabro-workflow/src/handler/llm/api.rs b/lib/crates/fabro-workflow/src/handler/llm/api.rs index 9247c19ec..cb99d2344 100644 --- a/lib/crates/fabro-workflow/src/handler/llm/api.rs +++ b/lib/crates/fabro-workflow/src/handler/llm/api.rs @@ -14,6 +14,8 @@ use fabro_llm::types::{Message, Request, TokenCounts}; use fabro_mcp::config::McpServerSettings; use fabro_model::{FallbackTarget, Provider}; use tokio::sync::Mutex as TokioMutex; +use tokio::task::JoinHandle; +use tokio_util::sync::CancellationToken; use super::super::agent::{CodergenBackend, CodergenResult}; use crate::context::keys::Fidelity; @@ -22,6 +24,98 @@ use crate::error::Error; use crate::event::{Emitter, Event, StageScope}; use crate::outcome::billed_model_usage_from_llm; +/// Spawn a task that, when the run-level token cancels, sets the agent +/// `Session`'s interrupt reason to `Cancelled` and cancels the session token. +/// +/// Factored out of `SessionCancelBridgeGuard::replace` so it can be unit-tested +/// without constructing a real `Session`. +fn spawn_bridge_task( + run_token: CancellationToken, + interrupt_reason: Arc>>, + session_token: CancellationToken, +) -> JoinHandle<()> { + tokio::spawn(async move { + run_token.cancelled().await; + { + let mut guard = interrupt_reason + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if guard.is_none() { + *guard = Some(fabro_agent::InterruptReason::Cancelled); + } + } + session_token.cancel(); + }) +} + +/// Per-invocation guard that maps a run-level `CancellationToken` to an agent +/// `Session`'s interrupt reason and cancel token. +/// +/// Dropping the guard aborts the spawned bridge task so a still-cached session +/// (after success) is not left wired to a stale run token. +struct SessionCancelBridgeGuard { + handle: Option>, +} + +impl SessionCancelBridgeGuard { + fn new() -> Self { + Self { handle: None } + } + + fn replace(&mut self, run_token: CancellationToken, session: &Session) { + self.abort(); + self.handle = Some(spawn_bridge_task( + run_token, + session.interrupt_reason_handle(), + session.cancel_token(), + )); + } + + fn abort(&mut self) { + if let Some(handle) = self.handle.take() { + handle.abort(); + } + } +} + +impl Drop for SessionCancelBridgeGuard { + fn drop(&mut self) { + self.abort(); + } +} + +/// Classification of an `fabro_agent::Error` for the API backend's `run` path. +enum AgentApiErrorDisposition { + /// Session was interrupted via cancellation; surface as `Error::Cancelled`. + Cancelled, + /// Underlying LLM error eligible for provider failover. + FailoverEligible(fabro_llm::Error), + /// Terminal error; abort the invocation with this workflow `Error`. + Terminal(Error), +} + +fn classify_agent_error(err: fabro_agent::Error, allow_failover: bool) -> AgentApiErrorDisposition { + match err { + fabro_agent::Error::Interrupted(fabro_agent::InterruptReason::Cancelled) => { + AgentApiErrorDisposition::Cancelled + } + fabro_agent::Error::Interrupted(fabro_agent::InterruptReason::WallClockTimeout) => { + AgentApiErrorDisposition::Terminal(Error::Precondition( + "Agent session hit its wall-clock timeout".to_string(), + )) + } + fabro_agent::Error::Llm(err) if allow_failover && err.failover_eligible() => { + AgentApiErrorDisposition::FailoverEligible(err) + } + fabro_agent::Error::Llm(err) => AgentApiErrorDisposition::Terminal(Error::Llm(err)), + other @ (fabro_agent::Error::SessionClosed + | fabro_agent::Error::InvalidState(_) + | fabro_agent::Error::ToolExecution(_)) => AgentApiErrorDisposition::Terminal( + Error::Precondition(format!("Agent session failed: {other}")), + ), + } +} + fn build_profile(model: &str, provider: Provider) -> Box { match provider { Provider::OpenAi => Box::new(OpenAiProfile::new(model)), @@ -430,6 +524,7 @@ impl CodergenBackend for AgentApiBackend { emitter: &Arc, sandbox: &Arc, tool_hooks: Option>, + cancel_token: CancellationToken, ) -> Result { let actual_model = node.model().unwrap_or(&self.model).to_string(); let _actual_provider = node @@ -444,25 +539,36 @@ impl CodergenBackend for AgentApiBackend { None }; - // Take a cached session if reusing, otherwise create a new one. + let mut bridge = SessionCancelBridgeGuard::new(); + + // Take a cached session if reusing, otherwise create a new one. Cancel + // checks bracket `Client::from_source(...)` so cancellation arriving + // during credential refresh is not lost. + if cancel_token.is_cancelled() { + return Err(Error::Cancelled); + } let (mut session, is_reused) = if let Some(ref key) = reuse_key { let existing = self.sessions.lock().unwrap().remove(key); if let Some(s) = existing { (s, true) } else { - ( - self.create_session(node, sandbox, tool_hooks.clone()) - .await?, - false, - ) + let created = self.create_session(node, sandbox, tool_hooks.clone()).await; + if cancel_token.is_cancelled() { + return Err(Error::Cancelled); + } + (created?, false) } } else { - ( - self.create_session(node, sandbox, tool_hooks.clone()) - .await?, - false, - ) + let created = self.create_session(node, sandbox, tool_hooks.clone()).await; + if cancel_token.is_cancelled() { + return Err(Error::Cancelled); + } + (created?, false) }; + if cancel_token.is_cancelled() { + return Err(Error::Cancelled); + } + bridge.replace(cancel_token.clone(), &session); tracing::info!( node = %node.id, @@ -491,99 +597,157 @@ impl CodergenBackend for AgentApiBackend { // Record turn count before processing so we only aggregate new usage. let turns_before = session.history().turns().len(); - if !is_reused { - session.initialize().await; - } - - let result = session.process_input(prompt).await; - - // On failover-eligible error, try fallback providers. - let result = match result { - Ok(()) => Ok(()), - Err(fabro_agent::Error::Llm(ref sdk_err)) - if sdk_err.failover_eligible() && !self.fallback_chain.is_empty() => - { - let error_msg = sdk_err.to_string(); - let from_provider = self.provider.to_string(); - let from_model = self.model.clone(); - - let mut last_err = Error::Llm(sdk_err.clone()); - let mut succeeded = false; - - for target in &self.fallback_chain { - emitter.emit_scoped( - &Event::Failover { - stage: node.id.clone(), - from_provider: from_provider.clone(), - from_model: from_model.clone(), - to_provider: target.provider.clone(), - to_model: target.model.clone(), - error: error_msg.clone(), - }, - &stage_scope, - ); - - let target_provider: Provider = match target.provider.parse() { - Ok(p) => p, - Err(_) => continue, - }; - - let new_session = match Self::create_session_for( - &target.model, - target_provider, - node, - sandbox, - self.source.as_ref(), - &self.env, - tool_hooks.clone(), - self.mcp_servers.clone(), - ) - .await - { - Ok(s) => s, - Err(e) => { - last_err = e; - continue; - } - }; - session = new_session; - - // Re-subscribe to forward events + track files from the new session - spawn_event_forwarder( - &session, - node.id.clone(), - stage_scope.clone(), - Arc::clone(emitter), - Arc::clone(&file_tracking), - ); - - session.initialize().await; - match session.process_input(prompt).await { - Ok(()) => { - succeeded = true; - break; - } - Err(fabro_agent::Error::Llm(err)) if err.failover_eligible() => { - last_err = Error::Llm(err); - } - Err(fabro_agent::Error::Llm(err)) => return Err(Error::Llm(err)), - Err(fabro_agent::Error::Interrupted(_)) => { - return Err(Error::Cancelled); - } - Err(other) => { - return Err(Error::handler(format!("Agent session failed: {other}"))); - } + let allow_failover_primary = !self.fallback_chain.is_empty(); + let init_result = if is_reused { + Ok(()) + } else { + match session.initialize().await { + Ok(()) => Ok(()), + Err(err) => match classify_agent_error(err, allow_failover_primary) { + AgentApiErrorDisposition::Cancelled => { + bridge.abort(); + return Err(Error::Cancelled); } - } - - if succeeded { Ok(()) } else { Err(last_err) } + AgentApiErrorDisposition::Terminal(err) => { + bridge.abort(); + return Err(err); + } + AgentApiErrorDisposition::FailoverEligible(sdk_err) => { + Err(fabro_agent::Error::Llm(sdk_err)) + } + }, } - Err(fabro_agent::Error::Llm(sdk_err)) => Err(Error::Llm(sdk_err)), - Err(fabro_agent::Error::Interrupted(_)) => Err(Error::Cancelled), - Err(other) => Err(Error::handler(format!("Agent session failed: {other}"))), }; - // On error, drop the session (don't cache failed state). + // If initialize failed with a failover-eligible error, treat as a + // process_input failover trigger; otherwise run process_input. + let result = match init_result { + Ok(()) => session.process_input(prompt).await, + Err(err) => Err(err), + }; + + // On failover-eligible error, try fallback providers. + let result: Result<(), Error> = match result { + Ok(()) => Ok(()), + Err(err) => match classify_agent_error(err, allow_failover_primary) { + AgentApiErrorDisposition::Cancelled => { + bridge.abort(); + return Err(Error::Cancelled); + } + AgentApiErrorDisposition::Terminal(err) => { + bridge.abort(); + return Err(err); + } + AgentApiErrorDisposition::FailoverEligible(sdk_err) => { + let error_msg = sdk_err.to_string(); + let from_provider = self.provider.to_string(); + let from_model = self.model.clone(); + + let mut last_err = Error::Llm(sdk_err); + let mut succeeded = false; + + for (index, target) in self.fallback_chain.iter().enumerate() { + emitter.emit_scoped( + &Event::Failover { + stage: node.id.clone(), + from_provider: from_provider.clone(), + from_model: from_model.clone(), + to_provider: target.provider.clone(), + to_model: target.model.clone(), + error: error_msg.clone(), + }, + &stage_scope, + ); + + let target_provider: Provider = match target.provider.parse() { + Ok(p) => p, + Err(_) => continue, + }; + + // Detach the bridge from the failing session before + // refreshing credentials and building a new one. + bridge.abort(); + if cancel_token.is_cancelled() { + return Err(Error::Cancelled); + } + let new_session_result = Self::create_session_for( + &target.model, + target_provider, + node, + sandbox, + self.source.as_ref(), + &self.env, + tool_hooks.clone(), + self.mcp_servers.clone(), + ) + .await; + if cancel_token.is_cancelled() { + return Err(Error::Cancelled); + } + let new_session = match new_session_result { + Ok(s) => s, + Err(e) => { + last_err = e; + continue; + } + }; + session = new_session; + bridge.replace(cancel_token.clone(), &session); + + // Re-subscribe to forward events + track files from the new session + spawn_event_forwarder( + &session, + node.id.clone(), + stage_scope.clone(), + Arc::clone(emitter), + Arc::clone(&file_tracking), + ); + + let allow_failover_next = index + 1 < self.fallback_chain.len(); + if let Err(err) = session.initialize().await { + match classify_agent_error(err, allow_failover_next) { + AgentApiErrorDisposition::Cancelled => { + bridge.abort(); + return Err(Error::Cancelled); + } + AgentApiErrorDisposition::Terminal(err) => { + bridge.abort(); + return Err(err); + } + AgentApiErrorDisposition::FailoverEligible(sdk_err) => { + last_err = Error::Llm(sdk_err); + continue; + } + } + } + match session.process_input(prompt).await { + Ok(()) => { + succeeded = true; + break; + } + Err(err) => match classify_agent_error(err, allow_failover_next) { + AgentApiErrorDisposition::Cancelled => { + bridge.abort(); + return Err(Error::Cancelled); + } + AgentApiErrorDisposition::Terminal(err) => { + bridge.abort(); + return Err(err); + } + AgentApiErrorDisposition::FailoverEligible(sdk_err) => { + last_err = Error::Llm(sdk_err); + } + }, + } + } + + if succeeded { Ok(()) } else { Err(last_err) } + } + }, + }; + + // On error, drop the session (don't cache failed state). The bridge's + // `Drop` will abort the spawned task on early return. result?; // Aggregate token usage only from new turns (prevents double-counting on @@ -626,8 +790,10 @@ impl CodergenBackend for AgentApiBackend { (v, s.last.clone()) }; - // Cache session back for reuse on success. + // Cache session back for reuse on success. Detach the bridge first so + // the cached session is not left wired to this run's cancel token. if let Some(key) = reuse_key { + bridge.abort(); self.sessions.lock().unwrap().insert(key, session); } @@ -644,6 +810,7 @@ impl CodergenBackend for AgentApiBackend { mod tests { use fabro_agent::subagent::SessionFactory; use fabro_auth::{AuthCredential, AuthDetails, VaultCredentialSource}; + use fabro_llm::{Error as LlmError, ProviderErrorDetail, ProviderErrorKind}; use fabro_vault::{SecretType, Vault}; use tokio::sync::RwLock as AsyncRwLock; @@ -824,4 +991,238 @@ mod tests { assert_eq!(client.provider_names(), vec!["anthropic"]); } + + // --- Bridge guard tests --- + + fn failover_eligible_llm_error() -> LlmError { + LlmError::Network { + message: "boom".into(), + source: None, + } + } + + fn non_failover_llm_error() -> LlmError { + LlmError::Provider { + kind: ProviderErrorKind::Authentication, + detail: Box::new(ProviderErrorDetail { + message: "bad key".into(), + provider: "openai".into(), + status_code: Some(401), + error_code: None, + retry_after: None, + raw: None, + }), + } + } + + #[tokio::test] + async fn spawn_bridge_task_sets_cancelled_and_cancels_session_token() { + let run_token = CancellationToken::new(); + let interrupt_reason = Arc::new(Mutex::new(None)); + let session_token = CancellationToken::new(); + + let handle = spawn_bridge_task( + run_token.clone(), + Arc::clone(&interrupt_reason), + session_token.clone(), + ); + + assert!(!session_token.is_cancelled()); + assert!(interrupt_reason.lock().unwrap().is_none()); + + run_token.cancel(); + handle.await.unwrap(); + + assert!(session_token.is_cancelled()); + assert_eq!( + *interrupt_reason.lock().unwrap(), + Some(fabro_agent::InterruptReason::Cancelled) + ); + } + + #[tokio::test] + async fn spawn_bridge_task_preserves_existing_interrupt_reason() { + let run_token = CancellationToken::new(); + let interrupt_reason = Arc::new(Mutex::new(Some( + fabro_agent::InterruptReason::WallClockTimeout, + ))); + let session_token = CancellationToken::new(); + + let handle = spawn_bridge_task( + run_token.clone(), + Arc::clone(&interrupt_reason), + session_token.clone(), + ); + run_token.cancel(); + handle.await.unwrap(); + + // Existing reason wins; the bridge does not overwrite a wall-clock + // timeout already recorded by the session. + assert_eq!( + *interrupt_reason.lock().unwrap(), + Some(fabro_agent::InterruptReason::WallClockTimeout) + ); + assert!(session_token.is_cancelled()); + } + + #[tokio::test] + async fn bridge_guard_drop_aborts_pending_task() { + let run_token = CancellationToken::new(); + let interrupt_reason = Arc::new(Mutex::new(None)); + let session_token = CancellationToken::new(); + + { + let mut guard = SessionCancelBridgeGuard::new(); + guard.handle = Some(spawn_bridge_task( + run_token.clone(), + Arc::clone(&interrupt_reason), + session_token.clone(), + )); + // guard dropped here + } + + // Trigger the run token after the guard has been dropped. The aborted + // task must not write to interrupt_reason or cancel session_token. + run_token.cancel(); + // Yield enough times for any errant task to run. + for _ in 0..10 { + tokio::task::yield_now().await; + } + + assert!(interrupt_reason.lock().unwrap().is_none()); + assert!(!session_token.is_cancelled()); + } + + #[tokio::test] + async fn bridge_guard_replace_aborts_prior_task() { + // First (prior) bridge wiring. + let prior_run_token = CancellationToken::new(); + let prior_interrupt_reason = Arc::new(Mutex::new(None)); + let prior_session_token = CancellationToken::new(); + + // Second (replacement) bridge wiring. + let new_run_token = CancellationToken::new(); + let new_interrupt_reason = Arc::new(Mutex::new(None)); + let new_session_token = CancellationToken::new(); + + let mut guard = SessionCancelBridgeGuard::new(); + guard.handle = Some(spawn_bridge_task( + prior_run_token.clone(), + Arc::clone(&prior_interrupt_reason), + prior_session_token.clone(), + )); + + // Replace with a new task pointing at different handles. + guard.handle = { + // Manually mirror `replace` semantics: abort then install. + if let Some(h) = guard.handle.take() { + h.abort(); + } + Some(spawn_bridge_task( + new_run_token.clone(), + Arc::clone(&new_interrupt_reason), + new_session_token.clone(), + )) + }; + + // Cancelling the prior run token must not affect anything because the + // prior task was aborted by `replace`. + prior_run_token.cancel(); + for _ in 0..10 { + tokio::task::yield_now().await; + } + assert!(prior_interrupt_reason.lock().unwrap().is_none()); + assert!(!prior_session_token.is_cancelled()); + + // The replacement task must still be alive and react to its own token. + new_run_token.cancel(); + guard.handle.take().unwrap().await.unwrap(); + assert_eq!( + *new_interrupt_reason.lock().unwrap(), + Some(fabro_agent::InterruptReason::Cancelled) + ); + assert!(new_session_token.is_cancelled()); + } + + // --- classify_agent_error tests --- + + #[test] + fn classify_interrupted_cancelled_is_cancelled() { + let err = fabro_agent::Error::Interrupted(fabro_agent::InterruptReason::Cancelled); + assert!(matches!( + classify_agent_error(err, true), + AgentApiErrorDisposition::Cancelled + )); + } + + #[test] + fn classify_interrupted_wall_clock_is_terminal_precondition() { + let err = fabro_agent::Error::Interrupted(fabro_agent::InterruptReason::WallClockTimeout); + match classify_agent_error(err, true) { + AgentApiErrorDisposition::Terminal(Error::Precondition(msg)) => { + assert!(msg.contains("wall-clock")); + } + _ => panic!("expected Terminal(Error::Precondition) for WallClockTimeout"), + } + } + + #[test] + fn classify_failover_eligible_llm_returns_failover_when_allowed() { + let err = fabro_agent::Error::Llm(failover_eligible_llm_error()); + assert!(matches!( + classify_agent_error(err, true), + AgentApiErrorDisposition::FailoverEligible(_) + )); + } + + #[test] + fn classify_failover_eligible_llm_returns_terminal_when_not_allowed() { + let err = fabro_agent::Error::Llm(failover_eligible_llm_error()); + match classify_agent_error(err, false) { + AgentApiErrorDisposition::Terminal(Error::Llm(_)) => {} + _ => panic!("expected Terminal(Error::Llm) when failover disallowed"), + } + } + + #[test] + fn classify_non_failover_eligible_llm_is_terminal_llm() { + let err = fabro_agent::Error::Llm(non_failover_llm_error()); + match classify_agent_error(err, true) { + AgentApiErrorDisposition::Terminal(Error::Llm(_)) => {} + _ => panic!("expected Terminal(Error::Llm) for non-failover-eligible LLM error"), + } + } + + #[test] + fn classify_session_closed_is_terminal_precondition() { + let err = fabro_agent::Error::SessionClosed; + match classify_agent_error(err, true) { + AgentApiErrorDisposition::Terminal(Error::Precondition(message)) => { + assert!(message.contains("Agent session failed")); + } + _ => panic!("expected Terminal(Error::Precondition) for SessionClosed"), + } + } + + #[test] + fn classify_invalid_state_is_terminal_precondition() { + let err = fabro_agent::Error::InvalidState("oops".into()); + match classify_agent_error(err, true) { + AgentApiErrorDisposition::Terminal(Error::Precondition(message)) => { + assert!(message.contains("Agent session failed")); + } + _ => panic!("expected Terminal(Error::Precondition) for InvalidState"), + } + } + + #[test] + fn classify_tool_execution_is_terminal_precondition() { + let err = fabro_agent::Error::ToolExecution("tool blew up".into()); + match classify_agent_error(err, true) { + AgentApiErrorDisposition::Terminal(Error::Precondition(message)) => { + assert!(message.contains("Agent session failed")); + } + _ => panic!("expected Terminal(Error::Precondition) for ToolExecution"), + } + } } diff --git a/lib/crates/fabro-workflow/src/handler/llm/cli.rs b/lib/crates/fabro-workflow/src/handler/llm/cli.rs index 7fcf60ed8..90832063a 100644 --- a/lib/crates/fabro-workflow/src/handler/llm/cli.rs +++ b/lib/crates/fabro-workflow/src/handler/llm/cli.rs @@ -1,16 +1,37 @@ use std::collections::HashMap; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use async_trait::async_trait; -use fabro_agent::Sandbox; -use fabro_agent::sandbox::ExecResult; +use fabro_agent::{Sandbox, shell_quote}; use fabro_auth::{CliAgentKind, CredentialResolver, CredentialUsage, ResolvedCredential}; use fabro_graphviz::graph::Node; use fabro_llm::types::TokenCounts; use fabro_model::Provider; -use fabro_types::CommandTermination; +use fabro_types::{CommandOutputStream, CommandTermination}; use fabro_util::time::elapsed_ms; -use tokio::time::sleep; +use tokio_util::sync::CancellationToken; + +/// Returns up to the last `n` characters of `s`, preserving char boundaries. +fn tail_chars(s: &str, n: usize) -> String { + let total = s.chars().count(); + if total <= n { + return s.to_string(); + } + s.chars().skip(total - n).collect() +} + +/// Build a "\nstdout: " detail string for CLI failure +/// messages, falling back to the original command when both streams are empty. +fn cli_failure_detail(stdout: &str, stderr: &str, command: &str) -> String { + let stderr_tail = tail_chars(stderr, 500); + let stdout_tail = tail_chars(stdout, 500); + match (stderr_tail.is_empty(), stdout_tail.is_empty()) { + (false, false) => format!("{stderr_tail}\nstdout: {stdout_tail}"), + (false, true) => stderr_tail, + (true, false) => format!("stdout: {stdout_tail}"), + (true, true) => format!("command: {command}"), + } +} use super::super::agent::{CodergenBackend, CodergenResult}; use crate::context::Context; @@ -66,6 +87,7 @@ async fn ensure_cli( provider: Provider, sandbox: &Arc, emitter: &Arc, + cancel_token: &CancellationToken, ) -> Result<(), Error> { let start = std::time::Instant::now(); let cli_name = cli.name(); @@ -84,7 +106,7 @@ async fn ensure_cli( 30_000, None, None, - None, + Some(cancel_token.child_token()), ) .await .map_err(|e| { @@ -112,7 +134,13 @@ async fn ensure_cli( cli.npm_package() ); let install_result = sandbox - .exec_command(&install_cmd, 180_000, None, None, None) + .exec_command( + &install_cmd, + 180_000, + None, + None, + Some(cancel_token.child_token()), + ) .await .map_err(|e| Error::handler_with_source(format!("Failed to install {cli_name}"), &e))?; @@ -161,9 +189,11 @@ pub fn is_cli_only_model(model: &str) -> bool { /// is piped into the command's stdin via `cat`. #[must_use] pub fn cli_command_for_provider(provider: Provider, model: &str, prompt_file: &str) -> String { + let prompt_file = shell_quote(prompt_file); let model_flag = if model.is_empty() { String::new() } else { + let model = shell_quote(model); match provider { Provider::OpenAi | Provider::Gemini @@ -362,14 +392,6 @@ pub fn parse_cli_response(provider: Provider, output: &str) -> Option String { - shlex::try_quote(val).map_or_else( - |_| format!("'{}'", val.replace('\'', "'\\''")), - std::borrow::Cow::into_owned, - ) -} - /// CLI backend that invokes external CLI tools (claude, codex, gemini) via /// `exec_command()`. pub struct AgentCliBackend { @@ -477,6 +499,7 @@ impl CodergenBackend for AgentCliBackend { emitter: &Arc, sandbox: &Arc, _tool_hooks: Option>, + cancel_token: CancellationToken, ) -> Result { // 1. Snapshot git state before the CLI run let files_before = self.detect_changed_files(sandbox).await; @@ -485,9 +508,6 @@ impl CodergenBackend for AgentCliBackend { let run_id = uuid::Uuid::new_v4().to_string(); let tmp_prefix = format!("/tmp/fabro_cli_{run_id}"); let prompt_path = format!("{tmp_prefix}_prompt.txt"); - let stdout_path = format!("{tmp_prefix}_stdout.log"); - let stderr_path = format!("{tmp_prefix}_stderr.log"); - let exit_code_path = format!("{tmp_prefix}_exit_code"); let env_path = format!("{tmp_prefix}_env.sh"); sandbox @@ -504,7 +524,7 @@ impl CodergenBackend for AgentCliBackend { // Ensure the CLI tool is installed in the sandbox let cli = AgentCli::for_provider(provider); - ensure_cli(cli, provider, sandbox, emitter).await?; + ensure_cli(cli, provider, sandbox, emitter, &cancel_token).await?; let command = cli_command_for_provider(provider, model, &prompt_path); let stage_scope = StageScope::for_handler(context, &node.id); @@ -521,11 +541,8 @@ impl CodergenBackend for AgentCliBackend { ); // Forward provider API key and custom env vars so the CLI tool can - // authenticate. Build a HashMap to pass via exec_command's env_vars - // parameter — this prepends `export` statements directly into the - // base64-encoded command, avoiding filesystem-to-process race - // conditions that can occur when writing an env file via the fs API and - // sourcing it via the process API. + // authenticate. Resolve credentials and run any pre-login command + // before the main CLI invocation. let cli_agent = match cli { AgentCli::Claude => CliAgentKind::Claude, AgentCli::Codex => CliAgentKind::Codex, @@ -541,7 +558,13 @@ impl CodergenBackend for AgentCliBackend { }; if let Some(login_cmd) = &cli_credential.login_command { let login_result = sandbox - .exec_command(login_cmd, 30_000, None, None, None) + .exec_command( + login_cmd, + 30_000, + None, + None, + Some(cancel_token.child_token()), + ) .await .map_err(|e| Error::handler_with_source("codex login failed", &e))?; if !login_result.is_success() { @@ -566,127 +589,183 @@ impl CodergenBackend for AgentCliBackend { launch_env.insert(name.clone(), val.clone()); } - // Also write env file as fallback for commands that source it (e.g. ensure_cli - // PATH) + // Write env file so the inner shell that runs the CLI command picks up + // PATH and provider env vars; we still pass `launch_env` to + // `exec_command_streaming` for parity. let mut env_lines: Vec = vec!["export PATH=\"$HOME/.local/bin:$PATH\"".to_string()]; env_lines.extend( launch_env .iter() .map(|(k, v)| format!("export {k}={}", shell_quote(v))), ); - { - sandbox - .write_file(&env_path, &env_lines.join("\n")) - .await - .map_err(|e| Error::handler_with_source("Failed to write env file", &e))?; - } + sandbox + .write_file(&env_path, &env_lines.join("\n")) + .await + .map_err(|e| Error::handler_with_source("Failed to write env file", &e))?; - // 3a. Disable auto-stop so the sandbox stays alive during long CLI runs + // Disable auto-stop so the sandbox stays alive during long CLI runs. if let Err(e) = sandbox.set_autostop_interval(0).await { tracing::warn!("Failed to disable sandbox auto-stop: {e}"); } - // 3b. Launch CLI command in background (env file is always written) - let inner_command = format!(". {env_path} && {command}"); - // Use setsid (if available) to create a new session so the child process is - // fully detached from the shell. Without this, Daytona's POST /process/execute - // blocks until ALL descendant processes exit, causing a 60s HTTP timeout. - // $SID is empty on macOS (where setsid doesn't exist but isn't needed since - // the local exec implementation doesn't wait for grandchildren). - let bg_command = format!( - "SID=$(command -v setsid || true)\n$SID sh -c '{inner_command} > {stdout_path} 2>{stderr_path}; echo $? > {exit_code_path}' /dev/null 2>&1 &\necho $!" - ); - let launch_start = std::time::Instant::now(); + // Stream the CLI command directly: the previous detached `setsid &` + // launcher could not be cancelled mid-flight. By running through + // `exec_command_streaming` the run-level cancel token (and node + // timeout, when set) terminate the CLI and its descendants. + let outer_command = format!(". {} && {command}", shell_quote(&env_path)); + // Use a synchronous Mutex: each callback invocation only does a short + // `extend_from_slice` with no awaits while the lock is held, so an + // async Mutex would just add per-chunk scheduling overhead. + let stdout_buffer: Arc>> = Arc::new(Mutex::new(Vec::new())); + let stderr_buffer: Arc>> = Arc::new(Mutex::new(Vec::new())); + let stdout_buf_cb = Arc::clone(&stdout_buffer); + let stderr_buf_cb = Arc::clone(&stderr_buffer); + let emitter_for_callback = Arc::clone(emitter); + let output_callback: fabro_agent::CommandOutputCallback = Arc::new(move |stream, bytes| { + let stdout_buf = Arc::clone(&stdout_buf_cb); + let stderr_buf = Arc::clone(&stderr_buf_cb); + let emitter = Arc::clone(&emitter_for_callback); + Box::pin(async move { + // Touch the stall watchdog whenever the CLI emits output + // so long-running invocations don't trip stall timeout. + emitter.touch(); + let buf = match stream { + CommandOutputStream::Stdout => stdout_buf, + CommandOutputStream::Stderr => stderr_buf, + }; + buf.lock() + .expect("CLI output buffer mutex poisoned") + .extend_from_slice(&bytes); + Ok(()) + }) + }); let launch_env_ref = if launch_env.is_empty() { None } else { Some(&launch_env) }; - let launch_result = sandbox - .exec_command(&bg_command, 30_000, None, launch_env_ref, None) - .await - .map_err(|e| Error::handler_with_source("Failed to launch CLI command", &e))?; - let pid = launch_result.stdout.trim(); - tracing::info!(pid, "CLI process launched in background"); + let timeout_ms = node.timeout().map(crate::millis_u64); + let invocation_token = cancel_token.child_token(); + let launch_start = std::time::Instant::now(); + let streaming_result = sandbox + .exec_command_streaming( + &outer_command, + timeout_ms, + None, + launch_env_ref, + Some(invocation_token.clone()), + output_callback, + ) + .await; - // 3c. Poll for completion - let poll_command = - format!("[ -f {exit_code_path} ] && cat {exit_code_path} || echo running"); - let poll_interval = self.poll_interval; - let exit_code: i32 = loop { - sleep(poll_interval).await; - emitter.touch(); // keep the stall watchdog alive while polling - let poll_result = sandbox - .exec_command(&poll_command, 30_000, None, None, None) - .await - .map_err(|e| Error::handler_with_source("Failed to poll CLI command", &e))?; - let status = poll_result.stdout.trim(); - - if status != "running" { - break status.parse::().unwrap_or(-1); + let cleanup_temp_files = || { + let sandbox = Arc::clone(sandbox); + let cleanup_cmd = format!("rm -f {}_*", shell_quote(&tmp_prefix)); + async move { + let _ = sandbox + .exec_command(&cleanup_cmd, 30_000, None, None, None) + .await; } }; - // 3d. Read results - let duration_ms = u64::try_from(launch_start.elapsed().as_millis()).unwrap_or(u64::MAX); - let stdout_result = sandbox - .exec_command(&format!("cat {stdout_path}"), 60_000, None, None, None) - .await - .map_err(|e| Error::handler_with_source("Failed to read stdout", &e))?; - let stderr_result = sandbox - .exec_command(&format!("cat {stderr_path}"), 60_000, None, None, None) - .await - .map_err(|e| Error::handler_with_source("Failed to read stderr", &e))?; - - let result = ExecResult { - stdout: stdout_result.stdout, - stderr: stderr_result.stdout, - exit_code: Some(exit_code), - termination: CommandTermination::Exited, - duration_ms, + let streaming = match streaming_result { + Ok(streaming) => streaming, + Err(err) => { + cleanup_temp_files().await; + return Err(Error::handler_with_source( + "Failed to run CLI command", + &err, + )); + } }; - emitter.emit_scoped( - &Event::AgentCliCompleted { - node_id: node.id.clone(), - stdout: result.stdout.clone(), - stderr: result.stderr.clone(), - exit_code: result.exit_code.unwrap_or(-1), - duration_ms: result.duration_ms, - }, - &stage_scope, - ); + let result = streaming.result; + // Prefer the buffered streaming output (live chunks); fall back to the + // result struct for sandboxes that bundle output at the end. + let buffered_stdout = { + let buf = stdout_buffer + .lock() + .expect("CLI stdout buffer mutex poisoned"); + String::from_utf8_lossy(&buf).into_owned() + }; + let buffered_stderr = { + let buf = stderr_buffer + .lock() + .expect("CLI stderr buffer mutex poisoned"); + String::from_utf8_lossy(&buf).into_owned() + }; + let stdout = if buffered_stdout.is_empty() { + result.stdout.clone() + } else { + buffered_stdout + }; + let stderr = if buffered_stderr.is_empty() { + result.stderr.clone() + } else { + buffered_stderr + }; + let duration_ms = elapsed_ms(launch_start); - // 3e. Cleanup temp files - let _ = sandbox - .exec_command(&format!("rm -f {tmp_prefix}_*"), 30_000, None, None, None) - .await; + match result.termination { + CommandTermination::Cancelled => { + emitter.emit_scoped( + &Event::AgentCliCancelled { + node_id: node.id.clone(), + stdout: stdout.clone(), + stderr: stderr.clone(), + duration_ms, + }, + &stage_scope, + ); + cleanup_temp_files().await; + return Err(Error::Cancelled); + } + CommandTermination::TimedOut => { + emitter.emit_scoped( + &Event::AgentCliTimedOut { + node_id: node.id.clone(), + stdout: stdout.clone(), + stderr: stderr.clone(), + duration_ms, + }, + &stage_scope, + ); + cleanup_temp_files().await; + let detail = cli_failure_detail(&stdout, &stderr, &command); + return Err(Error::handler(format!( + "CLI command timed out after {duration_ms} ms: {detail}" + ))); + } + CommandTermination::Exited => { + emitter.emit_scoped( + &Event::AgentCliCompleted { + node_id: node.id.clone(), + stdout: stdout.clone(), + stderr: stderr.clone(), + exit_code: result.exit_code.unwrap_or(-1), + duration_ms, + }, + &stage_scope, + ); + } + } - if !result.is_success() { - let tail = |s: &str, n: usize| -> String { - s.chars() - .rev() - .take(n) - .collect::>() - .into_iter() - .rev() - .collect() - }; - let stderr_tail = tail(&result.stderr, 500); - let stdout_tail = tail(&result.stdout, 500); - let detail = match (stderr_tail.is_empty(), stdout_tail.is_empty()) { - (false, false) => format!("{stderr_tail}\nstdout: {stdout_tail}"), - (false, true) => stderr_tail, - (true, false) => format!("stdout: {stdout_tail}"), - (true, true) => format!("command: {command}"), - }; + // Cleanup temp files (Exited path). + cleanup_temp_files().await; + + let exited_success = + result.termination == CommandTermination::Exited && result.exit_code == Some(0); + if !exited_success { + let detail = cli_failure_detail(&stdout, &stderr, &command); return Err(Error::handler(format!( "CLI command exited with code {}: {detail}", - result.display_exit_code(), + result + .exit_code + .map_or_else(|| "".to_string(), |c| c.to_string()), ))); } // 4. Parse the CLI output - let parsed = parse_cli_response(provider, &result.stdout) + let parsed = parse_cli_response(provider, &stdout) .ok_or_else(|| Error::handler("Failed to parse CLI output".to_string()))?; // 5. Detect changed files @@ -700,10 +779,7 @@ impl CodergenBackend for AgentCliBackend { let last_file_touched = if files_touched.is_empty() { None } else { - let quoted_files: Vec = files_touched - .iter() - .filter_map(|f| shlex::try_quote(f).ok().map(std::borrow::Cow::into_owned)) - .collect(); + let quoted_files: Vec = files_touched.iter().map(|f| shell_quote(f)).collect(); let cmd = format!("ls -t {} | head -1", quoted_files.join(" ")); if let Ok(result) = sandbox.exec_command(&cmd, 5_000, None, None, None).await { let trimmed = result.stdout.trim().to_string(); @@ -789,17 +865,32 @@ impl CodergenBackend for BackendRouter { emitter: &Arc, sandbox: &Arc, tool_hooks: Option>, + cancel_token: CancellationToken, ) -> Result { if self.should_use_cli(node) { self.cli_backend .run( - node, prompt, context, thread_id, emitter, sandbox, tool_hooks, + node, + prompt, + context, + thread_id, + emitter, + sandbox, + tool_hooks, + cancel_token, ) .await } else { self.api_backend .run( - node, prompt, context, thread_id, emitter, sandbox, tool_hooks, + node, + prompt, + context, + thread_id, + emitter, + sandbox, + tool_hooks, + cancel_token, ) .await } @@ -824,6 +915,7 @@ impl CodergenBackend for BackendRouter { mod tests { use std::path::Path; + use fabro_agent::sandbox::ExecResult; use fabro_graphviz::graph::AttrValue; use super::*; @@ -1003,7 +1095,14 @@ mod tests { )); let emitter = Arc::new(Emitter::default()); - let result = ensure_cli(AgentCli::Claude, Provider::Anthropic, &sandbox, &emitter).await; + let result = ensure_cli( + AgentCli::Claude, + Provider::Anthropic, + &sandbox, + &emitter, + &CancellationToken::new(), + ) + .await; assert!(result.is_ok()); let commands = commands.lock().unwrap(); @@ -1024,7 +1123,14 @@ mod tests { )); let emitter = Arc::new(Emitter::default()); - let result = ensure_cli(AgentCli::Claude, Provider::Anthropic, &sandbox, &emitter).await; + let result = ensure_cli( + AgentCli::Claude, + Provider::Anthropic, + &sandbox, + &emitter, + &CancellationToken::new(), + ) + .await; assert!(result.is_ok()); let commands = commands.lock().unwrap(); @@ -1049,7 +1155,14 @@ mod tests { move |event| events.lock().unwrap().push(event.clone()) }); - let result = ensure_cli(AgentCli::Claude, Provider::Anthropic, &sandbox, &emitter).await; + let result = ensure_cli( + AgentCli::Claude, + Provider::Anthropic, + &sandbox, + &emitter, + &CancellationToken::new(), + ) + .await; assert!(result.is_err()); let error = result.unwrap_err().to_string(); assert!(error.contains("install exited with code 1")); @@ -1280,6 +1393,7 @@ mod tests { _emitter: &Arc, _sandbox: &Arc, _tool_hooks: Option>, + _cancel_token: CancellationToken, ) -> Result { Ok(CodergenResult::Text { text: "stub".to_string(), @@ -1289,4 +1403,233 @@ mod tests { }) } } + + /// Sandbox stub whose `exec_command_streaming` returns a configurable + /// `CommandTermination` so we can exercise the cancel/timeout paths in + /// `AgentCliBackend::run` without spawning real processes. + struct StreamingCliMock { + commands: Arc>>, + termination: CommandTermination, + exit_code: Option, + } + + #[async_trait] + impl Sandbox for StreamingCliMock { + async fn read_file( + &self, + _path: &str, + _offset: Option, + _limit: Option, + ) -> fabro_sandbox::Result { + Ok(String::new()) + } + async fn write_file(&self, _path: &str, _content: &str) -> fabro_sandbox::Result<()> { + Ok(()) + } + async fn delete_file(&self, _path: &str) -> fabro_sandbox::Result<()> { + Ok(()) + } + async fn file_exists(&self, _path: &str) -> fabro_sandbox::Result { + Ok(false) + } + async fn list_directory( + &self, + _path: &str, + _depth: Option, + ) -> fabro_sandbox::Result> { + Ok(vec![]) + } + async fn exec_command( + &self, + command: &str, + _timeout_ms: u64, + _working_dir: Option<&str>, + _env_vars: Option<&std::collections::HashMap>, + _cancel_token: Option, + ) -> fabro_sandbox::Result { + self.commands.lock().unwrap().push(command.to_string()); + // Default: success for git/version/cat/rm/ls. + if command.contains("--version") { + return Ok(ok_result()); + } + Ok(ExecResult { + stdout: String::new(), + stderr: String::new(), + exit_code: Some(0), + termination: CommandTermination::Exited, + duration_ms: 1, + }) + } + async fn exec_command_streaming( + &self, + command: &str, + _timeout_ms: Option, + _working_dir: Option<&str>, + _env_vars: Option<&std::collections::HashMap>, + _cancel_token: Option, + _output_callback: fabro_agent::CommandOutputCallback, + ) -> fabro_sandbox::Result { + self.commands.lock().unwrap().push(command.to_string()); + Ok(fabro_sandbox::ExecStreamingResult { + result: ExecResult { + stdout: String::new(), + stderr: String::new(), + exit_code: self.exit_code, + termination: self.termination, + duration_ms: 5, + }, + streams_separated: true, + live_streaming: true, + }) + } + async fn grep( + &self, + _pattern: &str, + _path: &str, + _options: &fabro_agent::sandbox::GrepOptions, + ) -> fabro_sandbox::Result> { + Ok(vec![]) + } + async fn glob( + &self, + _pattern: &str, + _path: Option<&str>, + ) -> fabro_sandbox::Result> { + Ok(vec![]) + } + async fn download_file_to_local(&self, _: &str, _: &Path) -> fabro_sandbox::Result<()> { + Ok(()) + } + async fn upload_file_from_local(&self, _: &Path, _: &str) -> fabro_sandbox::Result<()> { + Ok(()) + } + async fn initialize(&self) -> fabro_sandbox::Result<()> { + Ok(()) + } + async fn cleanup(&self) -> fabro_sandbox::Result<()> { + Ok(()) + } + fn working_directory(&self) -> &str { + "/workspace" + } + fn platform(&self) -> &str { + "linux" + } + fn os_version(&self) -> String { + "Ubuntu 22.04".into() + } + async fn set_autostop_interval(&self, _minutes: i32) -> fabro_sandbox::Result<()> { + Ok(()) + } + } + + fn collect_events(emitter: &Arc) -> Arc>> { + let events = Arc::new(Mutex::new(Vec::new())); + let events_clone = Arc::clone(&events); + emitter.on_event(move |event| events_clone.lock().unwrap().push(event.clone())); + events + } + + #[tokio::test] + async fn agent_cli_backend_run_emits_cancelled_event_and_returns_cancelled() { + let commands = Arc::new(Mutex::new(Vec::new())); + let sandbox: Arc = Arc::new(StreamingCliMock { + commands: Arc::clone(&commands), + termination: CommandTermination::Cancelled, + exit_code: None, + }); + let backend = AgentCliBackend::new_from_env("claude-opus-4-6".into(), Provider::Anthropic); + let node = Node::new("step"); + let context = Context::new(); + let emitter = Arc::new(Emitter::default()); + let events = collect_events(&emitter); + + let result = backend + .run( + &node, + "Do something", + &context, + None, + &emitter, + &sandbox, + None, + CancellationToken::new(), + ) + .await; + + let Err(err) = result else { + panic!("cancelled streaming should bubble Error::Cancelled"); + }; + assert!(matches!(err, Error::Cancelled)); + + let events = events.lock().unwrap(); + let names: Vec = events + .iter() + .map(|e| e.body.event_name().to_string()) + .collect(); + assert!( + names.iter().any(|n| n == "agent.cli.cancelled"), + "expected agent.cli.cancelled, got events: {names:?}" + ); + assert!( + !names.iter().any(|n| n == "agent.cli.completed"), + "should not emit agent.cli.completed on cancellation" + ); + // Cleanup `rm -f` ran. + let cmds = commands.lock().unwrap(); + assert!( + cmds.iter().any(|c| c.starts_with("rm -f /tmp/fabro_cli_")), + "expected temp cleanup, got commands: {cmds:?}" + ); + } + + #[tokio::test] + async fn agent_cli_backend_run_emits_timed_out_event_and_returns_handler_error() { + let commands = Arc::new(Mutex::new(Vec::new())); + let sandbox: Arc = Arc::new(StreamingCliMock { + commands: Arc::clone(&commands), + termination: CommandTermination::TimedOut, + exit_code: None, + }); + let backend = AgentCliBackend::new_from_env("claude-opus-4-6".into(), Provider::Anthropic); + let node = Node::new("step"); + let context = Context::new(); + let emitter = Arc::new(Emitter::default()); + let events = collect_events(&emitter); + + let result = backend + .run( + &node, + "Do something slow", + &context, + None, + &emitter, + &sandbox, + None, + CancellationToken::new(), + ) + .await; + + let Err(err) = result else { + panic!("timeout streaming should produce a handler error"); + }; + assert!( + matches!(err, Error::Handler { .. }), + "expected handler error on timeout, got {err:?}" + ); + + let events = events.lock().unwrap(); + let names: Vec = events + .iter() + .map(|e| e.body.event_name().to_string()) + .collect(); + assert!( + names.iter().any(|n| n == "agent.cli.timed_out"), + "expected agent.cli.timed_out, got events: {names:?}" + ); + assert!( + !names.iter().any(|n| n == "agent.cli.completed"), + "should not emit agent.cli.completed on timeout" + ); + } } diff --git a/lib/crates/fabro-workflow/src/handler/manager_loop.rs b/lib/crates/fabro-workflow/src/handler/manager_loop.rs index 86e340626..8cc366796 100644 --- a/lib/crates/fabro-workflow/src/handler/manager_loop.rs +++ b/lib/crates/fabro-workflow/src/handler/manager_loop.rs @@ -1,7 +1,6 @@ use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::sync::Arc; -use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; use async_trait::async_trait; @@ -197,13 +196,12 @@ impl Handler for SubWorkflowHandler { let child_logs = run_dir.join(format!("stages/{}@{visit}/child", node.id)); let _ = fs::create_dir_all(&child_logs).await; - let cancel_token = Arc::new(AtomicBool::new(false)); - let child_cancel = Arc::clone(&cancel_token); + let child_run_token = services.run.cancel_token().child_token(); let child_run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: child_logs, - cancel_token: Some(cancel_token), + cancel_token: child_run_token.clone(), // Child workflows are part of the parent run's event stream. run_id: services.run.emitter.run_id(), labels: HashMap::new(), @@ -246,11 +244,14 @@ impl Handler for SubWorkflowHandler { .map_err(|err| Error::engine(err.to_string()))?; let artifact_store = ArtifactStore::new(object_store, "artifacts"); - // Spawn child engine + // Spawn child engine. Child runs receive a derived cancel token from + // the parent run; parent cancellation propagates parent-to-child via + // `child_token()`, but child cancellation does not cancel the parent. + let child_run_token_for_services = child_run_token.clone(); let mut child_handle = tokio::spawn(async move { let child_run = parent_run .with_run_store(run_store.into()) - .with_cancel_requested(None); + .with_cancel_token(child_run_token_for_services); let initialized = Initialized { graph: child_graph, source: String::new(), @@ -319,7 +320,7 @@ impl Handler for SubWorkflowHandler { if !stop_condition.is_empty() { let dummy_outcome = Outcome::success(); if evaluate_condition(stop_condition, &dummy_outcome, context) { - child_cancel.store(true, Ordering::Relaxed); + child_run_token.cancel(); // Give child a moment to wind down let _ = timeout( Duration::from_millis(100), @@ -337,7 +338,7 @@ impl Handler for SubWorkflowHandler { } // Max cycles exceeded — cancel child - child_cancel.store(true, Ordering::Relaxed); + child_run_token.cancel(); let _ = timeout(Duration::from_millis(100), &mut child_handle).await; Ok(Outcome::fail_classify(format!( diff --git a/lib/crates/fabro-workflow/src/handler/mod.rs b/lib/crates/fabro-workflow/src/handler/mod.rs index edd5b3f3a..8dc13ae6e 100644 --- a/lib/crates/fabro-workflow/src/handler/mod.rs +++ b/lib/crates/fabro-workflow/src/handler/mod.rs @@ -23,7 +23,6 @@ use fabro_interview::Interviewer; use crate::context::Context; use crate::error::Error; use crate::outcome::{Outcome, OutcomeExt}; -pub(crate) use crate::services::sandbox_cancel_token; pub use crate::services::{EngineServices, RunServices}; /// The handler interface for node execution. diff --git a/lib/crates/fabro-workflow/src/handler/parallel.rs b/lib/crates/fabro-workflow/src/handler/parallel.rs index c58104f83..dcf6de747 100644 --- a/lib/crates/fabro-workflow/src/handler/parallel.rs +++ b/lib/crates/fabro-workflow/src/handler/parallel.rs @@ -464,11 +464,18 @@ impl Handler for ParallelHandler { // Collect results let mut results: Vec = Vec::new(); - for handle in handles { + let mut handles = handles.into_iter(); + while let Some(handle) = handles.next() { match handle.await { Ok(Ok(result)) => { results.push(result); } + Ok(Err(Error::Cancelled)) => { + for handle in handles { + handle.abort(); + } + return Err(Error::Cancelled); + } Ok(Err(e)) => { results.push(BranchResult { id: String::new(), diff --git a/lib/crates/fabro-workflow/src/handler/prompt.rs b/lib/crates/fabro-workflow/src/handler/prompt.rs index caad1e058..50dc04e06 100644 --- a/lib/crates/fabro-workflow/src/handler/prompt.rs +++ b/lib/crates/fabro-workflow/src/handler/prompt.rs @@ -66,13 +66,21 @@ impl Handler for PromptHandler { .provider() .and_then(|s| s.parse::().ok()) .unwrap_or(services.run.provider); - let docs = fabro_agent::discover_memory( + let docs = match fabro_agent::discover_memory( &*services.run.sandbox, working_dir, working_dir, provider, + &services.run.cancel_token(), ) - .await; + .await + { + Ok(docs) => docs, + Err(fabro_agent::Error::Interrupted(fabro_agent::InterruptReason::Cancelled)) => { + return Err(Error::Cancelled); + } + Err(_) => Vec::new(), + }; if docs.is_empty() { None @@ -121,6 +129,7 @@ impl Handler for PromptHandler { files_touched, .. }) => (text, usage, files_touched), + Err(Error::Cancelled) => return Err(Error::Cancelled), Err(e) if e.is_retryable() => { return Err(e); } @@ -191,6 +200,7 @@ mod tests { use fabro_types::fixtures; use object_store::memory::InMemory; use tempfile::TempDir; + use tokio_util::sync::CancellationToken; use super::*; use crate::event::Emitter; @@ -277,6 +287,7 @@ mod tests { _emitter: &Arc, _sandbox: &Arc, _tool_hooks: Option>, + _cancel_token: CancellationToken, ) -> Result { panic!("run() should not be called for prompt handler"); } @@ -339,6 +350,7 @@ mod tests { _emitter: &Arc, _sandbox: &Arc, _tool_hooks: Option>, + _cancel_token: CancellationToken, ) -> Result { panic!("run() should not be called for prompt handler"); } @@ -398,6 +410,7 @@ mod tests { _emitter: &Arc, _sandbox: &Arc, _tool_hooks: Option>, + _cancel_token: CancellationToken, ) -> Result { panic!("run() should not be called for prompt handler"); } diff --git a/lib/crates/fabro-workflow/src/lifecycle/git.rs b/lib/crates/fabro-workflow/src/lifecycle/git.rs index 2f9d5bfdb..840894756 100644 --- a/lib/crates/fabro-workflow/src/lifecycle/git.rs +++ b/lib/crates/fabro-workflow/src/lifecycle/git.rs @@ -614,7 +614,7 @@ mod tests { Arc::new(RunOptions { settings: WorkflowSettings::default(), run_dir: run_dir.to_path_buf(), - cancel_token: None, + cancel_token: tokio_util::sync::CancellationToken::new(), run_id: fixtures::RUN_1, labels: HashMap::new(), workflow_slug: Some("metadata".to_string()), @@ -1011,7 +1011,7 @@ mod tests { repo_dir.path().to_path_buf(), )), None, - None, + tokio_util::sync::CancellationToken::new(), fabro_model::Provider::Anthropic, Arc::new(fabro_auth::EnvCredentialSource::new()), Arc::new(SandboxGitRuntime::new()), diff --git a/lib/crates/fabro-workflow/src/operations/fork.rs b/lib/crates/fabro-workflow/src/operations/fork.rs index 8213ab0ca..f2a88c899 100644 --- a/lib/crates/fabro-workflow/src/operations/fork.rs +++ b/lib/crates/fabro-workflow/src/operations/fork.rs @@ -250,6 +250,8 @@ fn replay_event_for_fork_projection(body: &EventBody) -> bool { | EventBody::InterviewInterrupted(_) | EventBody::AgentSessionStarted(_) | EventBody::AgentCliStarted(_) + | EventBody::AgentCliCancelled(_) + | EventBody::AgentCliTimedOut(_) | EventBody::CommandStarted(_) | EventBody::CommandCompleted(_) | EventBody::ParallelCompleted(_) diff --git a/lib/crates/fabro-workflow/src/operations/start.rs b/lib/crates/fabro-workflow/src/operations/start.rs index 4e623bcec..d519e7fba 100644 --- a/lib/crates/fabro-workflow/src/operations/start.rs +++ b/lib/crates/fabro-workflow/src/operations/start.rs @@ -1,6 +1,5 @@ use std::collections::HashMap; use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; @@ -30,6 +29,7 @@ use fabro_types::settings::run::{ use fabro_vault::Vault; use tokio::runtime::Handle; use tokio::sync::RwLock as AsyncRwLock; +use tokio_util::sync::CancellationToken; use crate::ManifestPath; use crate::artifact_upload::ArtifactSink; @@ -54,7 +54,7 @@ use crate::runtime_store::RunStoreHandle; use crate::workflow_bundle::{RunDefinition, WorkflowBundle}; struct RunSession { - cancel_token: Option>, + cancel_token: CancellationToken, emitter: Arc, sandbox: SandboxSpec, llm: LlmSpec, @@ -86,7 +86,7 @@ struct RunSession { pub struct StartServices { pub run_id: RunId, - pub cancel_token: Option>, + pub cancel_token: CancellationToken, pub emitter: Arc, pub interviewer: Arc, pub run_store: RunStoreHandle, @@ -831,7 +831,7 @@ impl RunSession { struct DetachedRunBootstrapGuard { run_id: RunId, event_sink: RunEventSink, - cancel_token: Option>, + cancel_token: CancellationToken, active: bool, } @@ -840,7 +840,7 @@ impl DetachedRunBootstrapGuard { run_id: RunId, _run_dir: &Path, event_sink: RunEventSink, - cancel_token: Option>, + cancel_token: CancellationToken, ) -> Self { Self { run_id, @@ -858,10 +858,7 @@ impl DetachedRunBootstrapGuard { impl Drop for DetachedRunBootstrapGuard { fn drop(&mut self) { if self.active { - let cancelled = self - .cancel_token - .as_ref() - .is_some_and(|token| token.load(Ordering::SeqCst)); + let cancelled = self.cancel_token.is_cancelled(); let reason = if cancelled { FailureReason::Cancelled } else { @@ -891,12 +888,12 @@ const POSTRUN_CANCELLED_MESSAGE: &str = "Run cancelled before post-run finalizat struct DetachedRunCompletionGuard { event_sink: RunEventSink, run_id: RunId, - cancel_token: Option>, + cancel_token: CancellationToken, active: bool, } impl DetachedRunCompletionGuard { - fn arm(run_id: RunId, event_sink: RunEventSink, cancel_token: Option>) -> Self { + fn arm(run_id: RunId, event_sink: RunEventSink, cancel_token: CancellationToken) -> Self { Self { event_sink, run_id, @@ -916,10 +913,7 @@ impl Drop for DetachedRunCompletionGuard { return; } - let cancelled = self - .cancel_token - .as_ref() - .is_some_and(|token| token.load(Ordering::SeqCst)); + let cancelled = self.cancel_token.is_cancelled(); let reason = if cancelled { FailureReason::Cancelled } else { @@ -1108,7 +1102,7 @@ mod tests { ) -> StartServices { StartServices { run_id: fixtures::RUN_1, - cancel_token: None, + cancel_token: CancellationToken::new(), emitter, interviewer: Arc::new(fabro_interview::AutoApproveInterviewer::engine()), run_store: store.open_run(&fixtures::RUN_1).await.unwrap().into(), diff --git a/lib/crates/fabro-workflow/src/pipeline/execute.rs b/lib/crates/fabro-workflow/src/pipeline/execute.rs index f7912548a..57ed86cfc 100644 --- a/lib/crates/fabro-workflow/src/pipeline/execute.rs +++ b/lib/crates/fabro-workflow/src/pipeline/execute.rs @@ -243,9 +243,7 @@ pub async fn execute(init: Initialized) -> Executed { let mut builder = ExecutorBuilder::new(handler as Arc>) .lifecycle(Box::new(lifecycle)); - if let Some(ref cancel) = run_options.cancel_token { - builder = builder.cancel_token(cancel.clone()); - } + builder = builder.cancel_token(run_options.cancel_token.clone()); if let Some(token) = stall_token.clone() { builder = builder.stall_token(token); } diff --git a/lib/crates/fabro-workflow/src/pipeline/execute/tests.rs b/lib/crates/fabro-workflow/src/pipeline/execute/tests.rs index ff70db28c..31bd95a08 100644 --- a/lib/crates/fabro-workflow/src/pipeline/execute/tests.rs +++ b/lib/crates/fabro-workflow/src/pipeline/execute/tests.rs @@ -7,7 +7,7 @@ use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::sync::Arc; -use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; +use std::sync::atomic::{AtomicU32, Ordering}; use std::time::Duration; use async_trait::async_trait; @@ -91,7 +91,7 @@ fn test_emitter_arc(label: &str) -> Arc { fn test_run_options(run_dir: &Path, run_id: &str) -> RunOptions { RunOptions { run_dir: run_dir.to_path_buf(), - cancel_token: None, + cancel_token: tokio_util::sync::CancellationToken::new(), run_id: test_run_id(run_id), settings: WorkflowSettings::default(), git: None, @@ -864,16 +864,16 @@ async fn execute_cancelled_mid_run() { g.edges.push(Edge::new("start", "work")); g.edges.push(Edge::new("work", "exit")); - let cancel_token = Arc::new(AtomicBool::new(false)); - let cancel_token_clone = Arc::clone(&cancel_token); + let cancel_token = tokio_util::sync::CancellationToken::new(); + let cancel_token_clone = cancel_token.clone(); let mut registry = make_registry(); registry.register("slow", Box::new(SlowHandler { sleep_ms: 200 })); let mut run_options = test_run_options(dir.path(), "test-run"); - run_options.cancel_token = Some(cancel_token); + run_options.cancel_token = cancel_token; tokio::spawn(async move { tokio::time::sleep(Duration::from_millis(50)).await; - cancel_token_clone.store(true, Ordering::Relaxed); + cancel_token_clone.cancel(); }); let result = run_graph( @@ -901,16 +901,16 @@ async fn execute_cancelled_mid_run_persists_cancelled_status() { g.edges.push(Edge::new("start", "work")); g.edges.push(Edge::new("work", "exit")); - let cancel_token = Arc::new(AtomicBool::new(false)); - let cancel_token_clone = Arc::clone(&cancel_token); + let cancel_token = tokio_util::sync::CancellationToken::new(); + let cancel_token_clone = cancel_token.clone(); let mut registry = make_registry(); registry.register("slow", Box::new(SlowHandler { sleep_ms: 200 })); let mut run_options = test_run_options(dir.path(), "test-run"); - run_options.cancel_token = Some(cancel_token); + run_options.cancel_token = cancel_token; tokio::spawn(async move { tokio::time::sleep(Duration::from_millis(50)).await; - cancel_token_clone.store(true, Ordering::Relaxed); + cancel_token_clone.cancel(); }); let executed = execute_test_run_with_options(run_options, g, Some(Arc::new(registry))).await; diff --git a/lib/crates/fabro-workflow/src/pipeline/finalize.rs b/lib/crates/fabro-workflow/src/pipeline/finalize.rs index cb58bd345..a1cc84d19 100644 --- a/lib/crates/fabro-workflow/src/pipeline/finalize.rs +++ b/lib/crates/fabro-workflow/src/pipeline/finalize.rs @@ -640,7 +640,7 @@ mod tests { RunOptions { settings: WorkflowSettings::default(), run_dir: run_dir.to_path_buf(), - cancel_token: None, + cancel_token: tokio_util::sync::CancellationToken::new(), run_id: test_run_id(), labels: HashMap::new(), workflow_slug: None, @@ -917,7 +917,7 @@ mod tests { emitter, sandbox, None, - None, + tokio_util::sync::CancellationToken::new(), fabro_model::Provider::Anthropic, Arc::new(fabro_auth::EnvCredentialSource::new()), Arc::new(SandboxGitRuntime::new()), @@ -943,7 +943,7 @@ mod tests { std::env::current_dir().unwrap(), )), None, - None, + tokio_util::sync::CancellationToken::new(), fabro_model::Provider::Anthropic, Arc::new(fabro_auth::EnvCredentialSource::new()), Arc::new(SandboxGitRuntime::new()), diff --git a/lib/crates/fabro-workflow/src/pipeline/initialize.rs b/lib/crates/fabro-workflow/src/pipeline/initialize.rs index 788b86881..81042bab9 100644 --- a/lib/crates/fabro-workflow/src/pipeline/initialize.rs +++ b/lib/crates/fabro-workflow/src/pipeline/initialize.rs @@ -30,7 +30,7 @@ use crate::error::Error; use crate::event::{Emitter, Event, RunNoticeCode, RunNoticeLevel}; use crate::git::RUN_BRANCH_PREFIX; use crate::handler::llm::{AgentApiBackend, AgentCliBackend, BackendRouter}; -use crate::handler::{HandlerRegistry, default_registry, sandbox_cancel_token}; +use crate::handler::{HandlerRegistry, default_registry}; use crate::run_metadata::{ RunMetadataRuntime, build_metadata_writer, metadata_branch_name, mint_token, }; @@ -665,23 +665,21 @@ pub async fn initialize( index, }); let cmd_start = Instant::now(); - let cancel_token = sandbox_cancel_token(options.run_options.cancel_token.clone()); + let cancel_token = options.run_options.cancel_token.child_token(); let result = sandbox .exec_command( command, options.lifecycle.setup_command_timeout_ms, None, None, - cancel_token.clone(), + Some(cancel_token.clone()), ) .await .map_err(|e| Error::engine_with_source("Setup command failed", &e))?; - if let Some(token) = &cancel_token { - if token.is_cancelled() { - return Err(Error::Cancelled); - } - token.cancel(); + if options.run_options.cancel_token.is_cancelled() { + return Err(Error::Cancelled); } + cancel_token.cancel(); let duration_ms = crate::millis_u64(cmd_start.elapsed()); if !result.is_success() { let exit_code = result.display_exit_code(); @@ -781,7 +779,6 @@ pub async fn initialize( mod tests { use std::collections::HashMap; use std::sync::Arc; - use std::sync::atomic::AtomicBool; use std::time::Duration; use fabro_auth::{AuthCredential, AuthDetails}; @@ -874,7 +871,7 @@ mod tests { RunOptions { settings: WorkflowSettings::default(), run_dir: run_dir.to_path_buf(), - cancel_token: None, + cancel_token: tokio_util::sync::CancellationToken::new(), run_id: test_run_id(), labels: HashMap::new(), workflow_slug: None, @@ -1296,9 +1293,10 @@ mod tests { std::fs::create_dir_all(&run_dir).unwrap(); let (graph, source) = simple_graph(); let persisted = test_persisted(graph, source, &run_dir); - let cancel_token = Arc::new(AtomicBool::new(true)); + let cancel_token = tokio_util::sync::CancellationToken::new(); + cancel_token.cancel(); let mut run_options = test_settings(&run_dir); - run_options.cancel_token = Some(cancel_token); + run_options.cancel_token = cancel_token; let result = initialize(persisted, InitOptions { run_id: test_run_id(), @@ -1357,9 +1355,10 @@ mod tests { std::fs::create_dir_all(&run_dir).unwrap(); let (graph, source) = simple_graph(); let persisted = test_persisted(graph, source, &run_dir); - let cancel_token = Arc::new(AtomicBool::new(true)); + let cancel_token = tokio_util::sync::CancellationToken::new(); + cancel_token.cancel(); let mut run_options = test_settings(&run_dir); - run_options.cancel_token = Some(cancel_token); + run_options.cancel_token = cancel_token; let result = initialize(persisted, InitOptions { run_id: test_run_id(), diff --git a/lib/crates/fabro-workflow/src/pipeline/retro.rs b/lib/crates/fabro-workflow/src/pipeline/retro.rs index d4c492373..2749cf023 100644 --- a/lib/crates/fabro-workflow/src/pipeline/retro.rs +++ b/lib/crates/fabro-workflow/src/pipeline/retro.rs @@ -304,7 +304,7 @@ mod tests { RunOptions { settings: WorkflowSettings::default(), run_dir: run_dir.to_path_buf(), - cancel_token: None, + cancel_token: tokio_util::sync::CancellationToken::new(), run_id: test_run_id(), labels: HashMap::new(), workflow_slug: None, @@ -340,7 +340,7 @@ mod tests { Arc::clone(&emitter), Arc::clone(&sandbox), None, - None, + tokio_util::sync::CancellationToken::new(), fabro_llm::Provider::Anthropic, test_llm_source(), Arc::new(crate::sandbox_git_runtime::SandboxGitRuntime::new()), @@ -395,7 +395,7 @@ mod tests { std::env::current_dir().unwrap(), )), None, - None, + tokio_util::sync::CancellationToken::new(), fabro_llm::Provider::Anthropic, test_llm_source(), Arc::new(crate::sandbox_git_runtime::SandboxGitRuntime::new()), diff --git a/lib/crates/fabro-workflow/src/run_metadata.rs b/lib/crates/fabro-workflow/src/run_metadata.rs index bcfde102b..de27f743e 100644 --- a/lib/crates/fabro-workflow/src/run_metadata.rs +++ b/lib/crates/fabro-workflow/src/run_metadata.rs @@ -642,7 +642,7 @@ mod tests { RunOptions { settings: WorkflowSettings::default(), run_dir: tempfile::tempdir().unwrap().path().to_path_buf(), - cancel_token: None, + cancel_token: tokio_util::sync::CancellationToken::new(), run_id: fabro_types::fixtures::RUN_1, labels: HashMap::new(), workflow_slug: Some("metadata".to_string()), diff --git a/lib/crates/fabro-workflow/src/run_options.rs b/lib/crates/fabro-workflow/src/run_options.rs index 705dc6a65..6a9d9f65a 100644 --- a/lib/crates/fabro-workflow/src/run_options.rs +++ b/lib/crates/fabro-workflow/src/run_options.rs @@ -1,10 +1,9 @@ use std::collections::HashMap; use std::path::PathBuf; -use std::sync::Arc; -use std::sync::atomic::AtomicBool; use fabro_types::settings::run::RunMode; use fabro_types::{ForkSourceRef, GitContext, RunId, WorkflowSettings}; +use tokio_util::sync::CancellationToken; use crate::git::{GitAuthor, git_author_from_settings}; @@ -21,7 +20,10 @@ pub struct GitCheckpointOptions { pub struct RunOptions { pub settings: WorkflowSettings, pub run_dir: PathBuf, - pub cancel_token: Option>, + /// Cancellation token for this run. Cancelling this token cancels the + /// run and propagates to handlers, sandbox commands, and child runs. + /// Default constructors should use `CancellationToken::new()`. + pub cancel_token: CancellationToken, /// Unique identifier for this workflow run. pub run_id: RunId, /// User-defined key-value labels for this run. diff --git a/lib/crates/fabro-workflow/src/services.rs b/lib/crates/fabro-workflow/src/services.rs index ae3912790..5ad0b983b 100644 --- a/lib/crates/fabro-workflow/src/services.rs +++ b/lib/crates/fabro-workflow/src/services.rs @@ -2,7 +2,7 @@ use std::collections::HashMap; #[cfg(test)] use std::path::PathBuf; use std::sync::Arc; -use std::sync::atomic::{AtomicBool, Ordering}; +#[cfg(test)] use std::time::Duration; use fabro_agent::Sandbox; @@ -11,7 +11,6 @@ use fabro_auth::CredentialSource; use fabro_auth::ResolvedCredentials; use fabro_hooks::{HookContext, HookDecision, HookRunner}; use fabro_model::Provider; -use tokio::time; use tokio_util::sync::CancellationToken; use crate::ManifestPath; @@ -24,13 +23,20 @@ use crate::sandbox_git_runtime::SandboxGitRuntime; use crate::workflow_bundle::WorkflowBundle; /// Services shared across workflow phases. +/// +/// Production construction is expected to happen from pipeline initialization +/// with the run's root cancellation token. Use +/// [`RunServices::with_cancel_token`] only with the same root token or a +/// `child_token()` derived from it. The token semantically means "cancel this +/// run or child run," not a generic shutdown signal — dropping a `RunServices` +/// does NOT count as cancellation. #[derive(Clone)] pub struct RunServices { pub run_store: RunStoreHandle, pub emitter: Arc, pub sandbox: Arc, pub hook_runner: Option>, - pub cancel_requested: Option>, + pub(crate) cancel_token: CancellationToken, pub provider: Provider, pub llm_source: Arc, pub(crate) sandbox_git: Arc, @@ -45,7 +51,7 @@ impl RunServices { emitter: Arc, sandbox: Arc, hook_runner: Option>, - cancel_requested: Option>, + cancel_token: CancellationToken, provider: Provider, llm_source: Arc, sandbox_git: Arc, @@ -57,7 +63,7 @@ impl RunServices { emitter, sandbox, hook_runner, - cancel_requested, + cancel_token, provider, llm_source, sandbox_git, @@ -66,10 +72,11 @@ impl RunServices { }) } - /// Bridge the core executor's atomic cancel flag to sandbox command - /// cancellation. - pub fn sandbox_cancel_token(&self) -> Option { - sandbox_cancel_token(self.cancel_requested.clone()) + /// The run-level cancellation token. Cancel this to terminate the run. + /// Derive child tokens via `cancel_token().child_token()` for sandbox + /// command invocations. + pub fn cancel_token(&self) -> CancellationToken { + self.cancel_token.clone() } /// Run lifecycle hooks and return the merged decision. @@ -107,13 +114,15 @@ impl RunServices { }) } + /// Replace the cancellation token. Use only with the same root token or + /// a child derived from it via `child_token()`. #[must_use] - pub fn with_cancel_requested( + pub(crate) fn with_cancel_token( self: &Arc, - cancel_requested: Option>, + cancel_token: CancellationToken, ) -> Arc { Arc::new(Self { - cancel_requested, + cancel_token, ..self.as_ref().clone() }) } @@ -209,7 +218,7 @@ impl EngineServices { std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")), )), None, - None, + CancellationToken::new(), Provider::Anthropic, Arc::new(StubCredentialSource), Arc::new(SandboxGitRuntime::new()), @@ -227,34 +236,6 @@ impl EngineServices { } } -pub(crate) fn sandbox_cancel_token( - cancel_requested: Option>, -) -> Option { - let cancel_requested = cancel_requested?; - let token = CancellationToken::new(); - - if cancel_requested.load(Ordering::Relaxed) { - token.cancel(); - return Some(token); - } - - let token_clone = token.clone(); - tokio::spawn(async move { - loop { - if token_clone.is_cancelled() { - return; - } - if cancel_requested.load(Ordering::Relaxed) { - token_clone.cancel(); - return; - } - time::sleep(Duration::from_millis(10)).await; - } - }); - - Some(token) -} - #[cfg(test)] mod tests { use super::EngineServices; diff --git a/lib/crates/fabro-workflow/tests/it/daytona_integration.rs b/lib/crates/fabro-workflow/tests/it/daytona_integration.rs index b8188a539..ef32bbc84 100644 --- a/lib/crates/fabro-workflow/tests/it/daytona_integration.rs +++ b/lib/crates/fabro-workflow/tests/it/daytona_integration.rs @@ -41,6 +41,7 @@ use fabro_workflow::records::Checkpoint; use fabro_workflow::run_options::{GitCheckpointOptions, RunOptions}; use fabro_workflow::test_support::{WorkflowRunner, test_store_dir}; use object_store::local::LocalFileSystem; +use tokio_util::sync::CancellationToken; use ulid::Ulid; fn test_run_id(label: &str) -> RunId { @@ -249,7 +250,7 @@ async fn daytona_exec_command_cancelled() { let env = create_env_with_github_app(Some(creds)).await; env.initialize().await.unwrap(); - let token = tokio_util::sync::CancellationToken::new(); + let token = CancellationToken::new(); let token_clone = token.clone(); // Cancel the token shortly after starting @@ -513,7 +514,7 @@ async fn daytona_pipeline_artifact_offload_and_sync() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -698,7 +699,7 @@ async fn daytona_git_checkpoint_remote_emits_events() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("git-cp-test"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -871,7 +872,7 @@ async fn daytona_parallel_git_branching_e2e() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: run_tmp.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id, labels: std::collections::HashMap::new(), workflow_slug: None, @@ -1078,6 +1079,7 @@ async fn run_daytona_cli_test(provider: Provider, model: &str, install_command: &emitter, &env, None, + CancellationToken::new(), ) .await; @@ -1209,7 +1211,7 @@ async fn daytona_git_checkpoint_with_shadow_branch() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id, labels: std::collections::HashMap::new(), workflow_slug: None, @@ -1366,7 +1368,7 @@ async fn daytona_asset_collection() { ..WorkflowSettings::default() }, run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("artifact-test-daytona"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -1634,7 +1636,7 @@ async fn daytona_git_push_run_branch_to_origin() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id, labels: std::collections::HashMap::new(), workflow_slug: None, diff --git a/lib/crates/fabro-workflow/tests/it/git_integration.rs b/lib/crates/fabro-workflow/tests/it/git_integration.rs index 611b2f055..8ff8f342d 100644 --- a/lib/crates/fabro-workflow/tests/it/git_integration.rs +++ b/lib/crates/fabro-workflow/tests/it/git_integration.rs @@ -21,6 +21,7 @@ use fabro_workflow::handler::exit::ExitHandler; use fabro_workflow::handler::start::StartHandler; use fabro_workflow::run_options::{GitCheckpointOptions, RunOptions}; use fabro_workflow::test_support::run_graph; +use tokio_util::sync::CancellationToken; fn assert_success(output: &Output, context: &str) { assert!( @@ -154,7 +155,7 @@ fn make_registry() -> HandlerRegistry { fn test_run_options(run_dir: &Path) -> RunOptions { RunOptions { run_dir: run_dir.to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: fixtures::RUN_2, settings: WorkflowSettings::default(), git: None, diff --git a/lib/crates/fabro-workflow/tests/it/integration.rs b/lib/crates/fabro-workflow/tests/it/integration.rs index b89fc7f5a..cdffc7837 100644 --- a/lib/crates/fabro-workflow/tests/it/integration.rs +++ b/lib/crates/fabro-workflow/tests/it/integration.rs @@ -54,6 +54,7 @@ use fabro_workflow::test_support::{WorkflowRunner, run_graph_with_hooks, test_st use fabro_workflow::transforms::stylesheet::{apply_stylesheet, parse_stylesheet}; use fabro_workflow::transforms::{StylesheetApplicationTransform, TemplateTransform, Transform}; use object_store::local::LocalFileSystem; +use tokio_util::sync::CancellationToken; use ulid::Ulid; fn local_env() -> Arc { @@ -416,7 +417,7 @@ async fn end_to_end_linear_pipeline() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -549,7 +550,7 @@ async fn end_to_end_branching_pipeline() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -669,7 +670,7 @@ async fn end_to_end_human_gate_pipeline() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -765,7 +766,7 @@ async fn human_gate_interrupted_input_fails_closed_without_fail_route() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -878,7 +879,7 @@ async fn human_gate_interrupted_input_routes_via_outcome_fail_condition() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -992,7 +993,7 @@ async fn goal_gate_routes_to_retry_target_on_failure() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -1115,7 +1116,7 @@ async fn goal_gate_routes_to_retry_target_when_present() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -1430,7 +1431,7 @@ async fn retry_on_failure_then_succeed() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -1505,7 +1506,7 @@ async fn pipeline_with_many_nodes() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -1593,6 +1594,7 @@ impl CodergenBackend for MockCodergenBackend { _emitter: &Arc, _sandbox: &Arc, _tool_hooks: Option>, + _cancel_token: tokio_util::sync::CancellationToken, ) -> Result { Ok(CodergenResult::Text { text: format!( @@ -1852,7 +1854,7 @@ async fn smoke_test_with_mock_codergen_backend() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -1954,7 +1956,7 @@ async fn end_to_end_parallel_fan_out_fan_in() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -2067,7 +2069,7 @@ async fn resume_from_checkpoint_completes_pipeline() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -2166,7 +2168,7 @@ async fn resume_from_checkpoint_preserves_goal_gate_outcomes() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -2209,7 +2211,7 @@ async fn graph_goal_in_context() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -2248,7 +2250,7 @@ async fn event_streaming_lifecycle() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -2328,7 +2330,7 @@ async fn context_flow_between_stages() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -2384,7 +2386,7 @@ async fn tool_handler_e2e() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -2455,7 +2457,7 @@ async fn auto_approve_interviewer_e2e() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -2495,7 +2497,7 @@ async fn codergen_without_backend_simulated() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -2600,7 +2602,7 @@ async fn branching_loop_back_on_failure() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -2686,7 +2688,7 @@ async fn human_gate_loops_back() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -2751,7 +2753,7 @@ async fn scenario_ship_a_feature() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -2839,7 +2841,7 @@ async fn scenario_parallel_expert_review() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -2926,7 +2928,7 @@ async fn scenario_node_retries_on_retry_status() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -2991,7 +2993,7 @@ async fn scenario_loop_restart_resets_context() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -3059,7 +3061,7 @@ async fn scenario_bug_triage_router() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -3121,7 +3123,7 @@ async fn scenario_crash_recovery() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -3231,7 +3233,7 @@ async fn manager_loop_stop_condition_satisfied_e2e() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -3313,7 +3315,7 @@ async fn manager_loop_max_cycles_exceeded_e2e() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -3456,7 +3458,7 @@ async fn conditional_branching_success_fail_paths() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -3512,7 +3514,7 @@ async fn edge_selection_condition_match_wins_over_weight() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -3562,7 +3564,7 @@ async fn edge_selection_weight_breaks_ties() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -3604,7 +3606,7 @@ async fn edge_selection_lexical_tiebreak() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -3665,7 +3667,7 @@ async fn context_updates_visible_across_nodes() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -3712,7 +3714,7 @@ async fn stylesheet_applies_model_override() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -3768,7 +3770,7 @@ async fn custom_handler_registration_and_execution() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -3846,7 +3848,7 @@ async fn integration_smoke_plan_implement_review_done() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -3938,7 +3940,7 @@ async fn manager_loop_runs_child_engine_e2e() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -4072,7 +4074,7 @@ async fn manager_loop_context_flows_e2e() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -4148,7 +4150,7 @@ async fn manager_loop_child_dotfile_e2e() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -4252,7 +4254,7 @@ async fn import_e2e_through_engine() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -4407,7 +4409,7 @@ async fn fidelity_default_is_compact() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -4464,7 +4466,7 @@ async fn fidelity_graph_default_applied() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -4517,7 +4519,7 @@ async fn fidelity_node_overrides_graph_default() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -4576,7 +4578,7 @@ async fn fidelity_edge_overrides_node_and_graph() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -4625,7 +4627,7 @@ async fn fidelity_full_produces_empty_preamble() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -4684,7 +4686,7 @@ async fn fidelity_truncate_preamble_minimal() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -4756,7 +4758,7 @@ async fn fidelity_summary_low_mode() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -4823,7 +4825,7 @@ async fn fidelity_summary_medium_mode() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -4890,7 +4892,7 @@ async fn fidelity_summary_high_mode() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -4950,7 +4952,7 @@ async fn fidelity_full_sets_thread_id_in_context() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -5021,7 +5023,7 @@ async fn fidelity_full_nodes_share_thread_id() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -5102,7 +5104,7 @@ async fn fidelity_resume_degrades_full_to_summary_high() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -5199,7 +5201,7 @@ async fn fidelity_resume_degrade_only_affects_first_hop() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -5283,7 +5285,7 @@ async fn fidelity_resume_no_degrade_when_not_full() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -5325,7 +5327,7 @@ async fn fidelity_stored_in_checkpoint_context() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -5418,7 +5420,7 @@ async fn fidelity_precedence_multi_node_pipeline() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -5486,7 +5488,7 @@ async fn fidelity_compact_preamble_includes_completed_stages_and_context() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -5561,7 +5563,7 @@ async fn fidelity_summary_low_excludes_context_values_in_pipeline() { let run_options_low = RunOptions { settings: WorkflowSettings::default(), run_dir: dir_low.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -5628,7 +5630,7 @@ async fn fidelity_summary_low_excludes_context_values_in_pipeline() { let run_options_med = RunOptions { settings: WorkflowSettings::default(), run_dir: dir_med.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -5700,7 +5702,7 @@ async fn fidelity_thread_id_fallback_to_previous_node_in_pipeline() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -5754,7 +5756,7 @@ async fn fidelity_thread_id_from_node_class_in_pipeline() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -5811,7 +5813,7 @@ async fn fidelity_edge_thread_id_override_in_pipeline() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -5869,7 +5871,7 @@ async fn fidelity_full_without_explicit_thread_id_uses_previous_node() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -5937,7 +5939,7 @@ async fn fidelity_from_parsed_dot_pipeline() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -5986,7 +5988,7 @@ async fn fidelity_checkpoint_roundtrip_preserves_fidelity() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -6059,7 +6061,7 @@ async fn fidelity_node_thread_id_overrides_edge_thread_id_in_pipeline() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -6146,7 +6148,7 @@ async fn fidelity_resume_preserves_context_values_across_checkpoint() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -6194,6 +6196,7 @@ mod real_llm { use fabro_workflow::context::Context; use fabro_workflow::error::Error; use fabro_workflow::handler::agent::{AgentHandler, CodergenBackend, CodergenResult}; + use tokio_util::sync::CancellationToken; struct LlmCodergenBackend { client: Arc, @@ -6212,6 +6215,7 @@ mod real_llm { _emitter: &Arc, _sandbox: &Arc, _tool_hooks: Option>, + _cancel_token: tokio_util::sync::CancellationToken, ) -> Result { self.complete(prompt).await } @@ -6384,7 +6388,7 @@ mod real_llm { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -6493,7 +6497,7 @@ mod real_llm { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -6626,7 +6630,7 @@ mod real_llm { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -6727,7 +6731,7 @@ mod real_llm { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -6871,7 +6875,7 @@ async fn workflow_run_with_vault_only_openai_codex_builds_pr_body() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("vault-only-openai-codex-pr-body"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -6985,7 +6989,7 @@ async fn human_gate_freeform_only_routes_text() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -7116,7 +7120,7 @@ async fn human_gate_freeform_with_fixed_choice_match() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -7233,7 +7237,7 @@ async fn human_gate_freeform_fallback_on_unmatched_text() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -7361,7 +7365,7 @@ async fn human_gate_freeform_sets_allow_freeform_on_question() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -7470,7 +7474,7 @@ async fn human_gate_without_freeform_sets_allow_freeform_false() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -7774,7 +7778,7 @@ fn make_run_options(dir: &std::path::Path) -> RunOptions { RunOptions { settings: WorkflowSettings::default(), run_dir: dir.to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("hook-test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -8713,7 +8717,7 @@ async fn run_fidelity_prompt_pipeline(fidelity: &str) -> String { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -8915,7 +8919,7 @@ async fn large_context_values_are_offloaded_to_artifact_store() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -9120,7 +9124,7 @@ async fn artifact_pointers_rewritten_for_remote_sandbox() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -9208,7 +9212,7 @@ async fn downstream_local_execution_materializes_blob_refs_to_runtime_files() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -9296,7 +9300,7 @@ async fn downstream_remote_execution_materializes_blob_refs_to_sandbox_files() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -9427,7 +9431,7 @@ async fn node_dir_uses_visit_count_on_revisit() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -9576,46 +9580,11 @@ impl fabro_agent::Sandbox for CliTestEnv { }); } - // Background launch: return PID - if command.contains("echo $!") { + // CLI version check during ensure_cli — return success so install path + // is skipped. + if command.contains("--version") { return Ok(fabro_agent::ExecResult { - stdout: "12345\n".into(), - stderr: String::new(), - exit_code: Some(0), - - termination: CommandTermination::Exited, - duration_ms: 1, - }); - } - - // Poll for completion: return exit code 0 immediately - if command.contains("exit_code") && command.contains("echo running") { - return Ok(fabro_agent::ExecResult { - stdout: "0\n".into(), - stderr: String::new(), - exit_code: Some(0), - - termination: CommandTermination::Exited, - duration_ms: 1, - }); - } - - // Read stdout file - if command.starts_with("cat") && command.contains("stdout.log") { - return Ok(fabro_agent::ExecResult { - stdout: self.cli_stdout.clone(), - stderr: String::new(), - exit_code: Some(0), - - termination: CommandTermination::Exited, - duration_ms: 1, - }); - } - - // Read stderr file - if command.starts_with("cat") && command.contains("stderr.log") { - return Ok(fabro_agent::ExecResult { - stdout: String::new(), + stdout: "1.0.0\n".into(), stderr: String::new(), exit_code: Some(0), @@ -9636,7 +9605,21 @@ impl fabro_agent::Sandbox for CliTestEnv { }); } - // Fallback + // ls -t for last_file_touched + if command.starts_with("ls -t ") { + return Ok(fabro_agent::ExecResult { + stdout: String::new(), + stderr: String::new(), + exit_code: Some(0), + + termination: CommandTermination::Exited, + duration_ms: 1, + }); + } + + // Fallback: this is the streaming CLI invocation. The default trait + // implementation of `exec_command_streaming` delegates to this path + // and replays output through the streaming callback. Ok(fabro_agent::ExecResult { stdout: self.cli_stdout.clone(), stderr: String::new(), @@ -9724,6 +9707,7 @@ async fn cli_backend_run_writes_prompt_and_calls_exec() { &emitter, &env, None, + CancellationToken::new(), ) .await .expect("CLI backend should succeed"); @@ -9741,20 +9725,21 @@ async fn cli_backend_run_writes_prompt_and_calls_exec() { ); assert_eq!(prompt_file.1, "Fix the authentication bug"); - // Verify the CLI command was called (now wrapped in background launch) + // Verify the CLI command was streamed (env file is sourced, then `cat + // | claude -p ...` runs as the inner shell command). let commands = test_env.recorded_commands(); let cli_cmd = commands .iter() - .find(|c| c.contains("claude") && c.contains("echo $!")) - .expect("should launch claude CLI in background"); + .find(|c| c.contains("claude") && c.contains("_prompt.txt")) + .expect("should run claude CLI command"); assert!(cli_cmd.contains("-p"), "should use pipe mode"); assert!( cli_cmd.contains("claude-opus-4-6"), "should use correct model" ); assert!( - cli_cmd.contains("_prompt.txt"), - "should reference prompt file" + cli_cmd.contains(". /tmp/fabro_cli_") && cli_cmd.contains("_env.sh"), + "should source the env file before invoking the CLI: {cli_cmd}" ); // Verify parsed response @@ -9796,6 +9781,7 @@ async fn cli_backend_run_detects_changed_files() { &emitter, &env, None, + CancellationToken::new(), ) .await .expect("CLI backend should succeed"); @@ -9821,16 +9807,25 @@ async fn cli_backend_run_with_codex_provider() { let emitter = Arc::new(Emitter::default()); let result = backend - .run(&node, "Build the API", &context, None, &emitter, &env, None) + .run( + &node, + "Build the API", + &context, + None, + &emitter, + &env, + None, + CancellationToken::new(), + ) .await .expect("CLI backend should succeed"); - // Verify codex command was called (now wrapped in background launch) + // Verify codex command was streamed. let commands = test_env.recorded_commands(); let cli_cmd = commands .iter() - .find(|c| c.contains("codex") && c.contains("echo $!")) - .expect("should launch codex CLI in background"); + .find(|c| c.contains("codex") && c.contains("_prompt.txt")) + .expect("should run codex CLI command"); assert!(cli_cmd.contains("exec --json"), "should use exec mode"); assert!( cli_cmd.contains("gpt-5.3-codex"), @@ -9898,10 +9893,10 @@ async fn cli_backend_run_fails_on_nonzero_exit() { duration_ms: 0, }); } - // Background launch: return PID - if command.contains("echo $!") { + // CLI version check during ensure_cli — pretend already installed. + if command.contains("--version") { return Ok(fabro_agent::ExecResult { - stdout: "12345\n".into(), + stdout: "1.0.0\n".into(), stderr: String::new(), exit_code: Some(0), @@ -9909,24 +9904,12 @@ async fn cli_backend_run_fails_on_nonzero_exit() { duration_ms: 0, }); } - // Poll: return non-zero exit code - if command.contains("exit_code") && command.contains("echo running") { + // The streaming CLI invocation: return non-zero exit with stderr. + if command.contains("claude") || command.contains("codex") { return Ok(fabro_agent::ExecResult { - stdout: "127\n".into(), - stderr: String::new(), - exit_code: Some(0), - - termination: CommandTermination::Exited, - duration_ms: 0, - }); - } - // Read stderr file - if command.starts_with("cat") && command.contains("stderr.log") { - return Ok(fabro_agent::ExecResult { - stdout: "command not found: claude".into(), - stderr: String::new(), - exit_code: Some(0), - + stdout: String::new(), + stderr: "command not found: claude".into(), + exit_code: Some(127), termination: CommandTermination::Exited, duration_ms: 0, }); @@ -10000,6 +9983,7 @@ async fn cli_backend_run_fails_on_nonzero_exit() { &emitter, &failing_env, None, + CancellationToken::new(), ) .await; @@ -10029,7 +10013,16 @@ async fn cli_backend_run_fails_on_unparseable_output() { let emitter = Arc::new(Emitter::default()); let result = backend - .run(&node, "do something", &context, None, &emitter, &env, None) + .run( + &node, + "do something", + &context, + None, + &emitter, + &env, + None, + CancellationToken::new(), + ) .await; let err = match result { @@ -10062,14 +10055,23 @@ async fn cli_backend_run_uses_node_model_override() { let emitter = Arc::new(Emitter::default()); backend - .run(&node, "test", &context, None, &emitter, &env, None) + .run( + &node, + "test", + &context, + None, + &emitter, + &env, + None, + CancellationToken::new(), + ) .await .expect("should succeed"); let commands = test_env.recorded_commands(); let cli_cmd = commands .iter() - .find(|c| c.contains("claude") && c.contains("echo $!")) + .find(|c| c.contains("claude") && c.contains("_prompt.txt")) .unwrap(); assert!( cli_cmd.contains("claude-sonnet-4-5"), @@ -10103,14 +10105,23 @@ async fn cli_backend_run_uses_node_provider_override() { let emitter = Arc::new(Emitter::default()); backend - .run(&node, "test", &context, None, &emitter, &env, None) + .run( + &node, + "test", + &context, + None, + &emitter, + &env, + None, + CancellationToken::new(), + ) .await .expect("should succeed"); let commands = test_env.recorded_commands(); let cli_cmd = commands .iter() - .find(|c| c.contains("codex") && c.contains("echo $!")) + .find(|c| c.contains("codex") && c.contains("_prompt.txt")) .expect("should launch codex based on provider override"); assert!(cli_cmd.contains("gpt-5.3-codex")); } @@ -10128,7 +10139,16 @@ async fn cli_backend_run_returns_text_and_usage() { let emitter = Arc::new(Emitter::default()); let result = backend - .run(&node, "test", &context, None, &emitter, &env, None) + .run( + &node, + "test", + &context, + None, + &emitter, + &env, + None, + CancellationToken::new(), + ) .await .expect("should succeed"); @@ -10168,7 +10188,16 @@ async fn backend_router_delegates_to_cli_for_cli_node() { let emitter = Arc::new(Emitter::default()); let result = router - .run(&node, "Fix the bug", &context, None, &emitter, &env, None) + .run( + &node, + "Fix the bug", + &context, + None, + &emitter, + &env, + None, + CancellationToken::new(), + ) .await .expect("router should succeed"); @@ -10202,7 +10231,16 @@ async fn backend_router_delegates_to_api_for_normal_node() { let emitter = Arc::new(Emitter::default()); let result = router - .run(&node, "Plan the work", &context, None, &emitter, &env, None) + .run( + &node, + "Plan the work", + &context, + None, + &emitter, + &env, + None, + CancellationToken::new(), + ) .await .expect("router should succeed"); @@ -10239,7 +10277,16 @@ async fn backend_router_delegates_to_cli_for_backend_attr() { let emitter = Arc::new(Emitter::default()); let result = router - .run(&node, "Build it", &context, None, &emitter, &env, None) + .run( + &node, + "Build it", + &context, + None, + &emitter, + &env, + None, + CancellationToken::new(), + ) .await .expect("router should succeed"); @@ -10332,7 +10379,7 @@ async fn full_pipeline_with_cli_backend_node() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -10451,7 +10498,7 @@ async fn stylesheet_backend_property_routes_to_cli() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -10648,7 +10695,7 @@ async fn git_checkpoint_host_emits_events_and_diff_patch() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: run_dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-docker"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -10814,7 +10861,7 @@ async fn git_checkpoint_host_skips_metadata_branch_without_writer_prereqs() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: run_dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id, labels: std::collections::HashMap::new(), workflow_slug: None, @@ -11005,7 +11052,7 @@ async fn parallel_git_branching_host_e2e() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: run_dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id, labels: std::collections::HashMap::new(), workflow_slug: None, @@ -11255,7 +11302,7 @@ async fn git_checkpoint_host_skips_empty_diff_patch() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: run_dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("empty-diff"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -11626,7 +11673,7 @@ async fn e2e_circuit_breaker_deterministic_self_loop() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("e2e-circuit-breaker"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -11673,7 +11720,7 @@ async fn e2e_circuit_breaker_custom_limit() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("e2e-custom-limit"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -11713,7 +11760,7 @@ async fn e2e_circuit_breaker_ignores_transient_failures() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("e2e-transient-no-breaker"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -11760,7 +11807,7 @@ async fn e2e_circuit_breaker_different_reasons_separate_counters() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("e2e-varying-reasons"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -11800,7 +11847,7 @@ async fn e2e_circuit_breaker_loop_restart() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("e2e-restart-breaker"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -11863,7 +11910,7 @@ async fn e2e_failure_signature_persisted_in_context() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("e2e-sig-context"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -11927,7 +11974,7 @@ async fn e2e_failure_signature_hint_overrides_reason_in_context() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("e2e-sig-hint"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -11985,7 +12032,7 @@ async fn e2e_signature_maps_persist_in_checkpoint() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("e2e-sig-persist"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -12113,7 +12160,7 @@ async fn e2e_circuit_breaker_emits_events_before_abort() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("e2e-events"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -12180,7 +12227,7 @@ async fn e2e_circuit_breaker_does_not_fire_below_limit() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("e2e-below-limit"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -12276,7 +12323,7 @@ async fn e2e_circuit_breaker_multi_stage_impl_verify_cycle() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("e2e-impl-verify-cycle"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -12374,7 +12421,7 @@ async fn e2e_loop_restart_blocked_for_deterministic_failure() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("e2e-restart-blocked-det"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -12414,7 +12461,7 @@ async fn e2e_loop_restart_blocked_for_structural_failure() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("e2e-restart-blocked-struct"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -12454,7 +12501,7 @@ async fn e2e_loop_restart_blocked_for_budget_exhausted_failure() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("e2e-restart-blocked-budget"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -12494,7 +12541,7 @@ async fn e2e_loop_restart_blocked_for_canceled_failure() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("e2e-restart-blocked-canceled"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -12531,7 +12578,7 @@ async fn e2e_loop_restart_blocked_for_compilation_loop_failure() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("e2e-restart-blocked-comploop"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -12572,7 +12619,7 @@ async fn e2e_loop_restart_allowed_for_transient_infra() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("e2e-restart-allowed-transient"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -12680,7 +12727,7 @@ async fn e2e_stall_watchdog_triggers_from_dot_parsed_pipeline() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("stall-e2e"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -12736,7 +12783,7 @@ async fn e2e_stall_watchdog_kept_alive_by_handler_events() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("stall-alive-e2e"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -12782,7 +12829,7 @@ async fn e2e_stall_watchdog_disabled_with_zero_timeout() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("stall-disabled-e2e"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -12848,7 +12895,7 @@ async fn e2e_stall_watchdog_with_explicit_timeout_override() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("stall-override-e2e"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -12990,7 +13037,7 @@ async fn asset_collection_local_sandbox_success() { ..WorkflowSettings::default() }, run_dir: run_dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("artifact-test-local"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -13126,7 +13173,7 @@ async fn asset_collection_local_sandbox_on_failure() { ..WorkflowSettings::default() }, run_dir: run_dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("artifact-test-fail"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -13236,7 +13283,7 @@ async fn asset_collection_docker_sandbox() { ..WorkflowSettings::default() }, run_dir: run_dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("artifact-test-docker"), labels: std::collections::HashMap::new(), workflow_slug: None, @@ -13308,7 +13355,7 @@ async fn wait_timer_e2e() { let run_options = RunOptions { settings: WorkflowSettings::default(), run_dir: dir.path().to_path_buf(), - cancel_token: None, + cancel_token: CancellationToken::new(), run_id: test_run_id("test-run"), labels: std::collections::HashMap::new(), workflow_slug: None,