From f39e512990469ab9a84f41aa59f93ddc3e864692 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Mon, 4 May 2026 14:13:01 -0400 Subject: [PATCH 01/16] feat(web): split Queued column out of Initializing on the run board Submitted and Queued lifecycle statuses now live in a dedicated Queued column rendered to the left of Initializing; Starting stays in Initializing. The column is omitted from the board when it has no items so day-to-day boards stay compact. Co-Authored-By: Claude Opus 4.7 (1M context) --- apps/fabro-web/app/data/runs.ts | 4 ++- apps/fabro-web/app/routes/runs.test.tsx | 1 + apps/fabro-web/app/routes/runs.tsx | 9 ++++-- docs/public/api-reference/fabro-api.yaml | 1 + lib/crates/fabro-server/src/demo/mod.rs | 30 +++++++++++++++++-- .../fabro-server/src/server/handler/runs.rs | 4 ++- lib/crates/fabro-server/src/server/tests.rs | 4 +-- .../src/models/board-column.ts | 1 + 8 files changed, 46 insertions(+), 8 deletions(-) diff --git a/apps/fabro-web/app/data/runs.ts b/apps/fabro-web/app/data/runs.ts index 3922b7cd3..77d8a4711 100644 --- a/apps/fabro-web/app/data/runs.ts +++ b/apps/fabro-web/app/data/runs.ts @@ -37,9 +37,10 @@ export interface RunItem { sourceDirectory?: string; } -export type ColumnStatus = "initializing" | "running" | "blocked" | "succeeded" | "failed"; +export type ColumnStatus = "queued" | "initializing" | "running" | "blocked" | "succeeded" | "failed"; export const columnStatusDisplay: Record = { + queued: { label: "Queued", dot: "bg-fg-muted", text: "text-fg-muted" }, initializing: { label: "Initializing", dot: "bg-amber", text: "text-amber" }, running: { label: "Running", dot: "bg-teal-500", text: "text-teal-500" }, blocked: { label: "Blocked", dot: "bg-amber", text: "text-amber" }, @@ -113,6 +114,7 @@ export function columnForStatus(status: ApiRunStatus | null | undefined): Column switch (status?.kind) { case "submitted": case "queued": + return "queued"; case "starting": return "initializing"; case "running": diff --git a/apps/fabro-web/app/routes/runs.test.tsx b/apps/fabro-web/app/routes/runs.test.tsx index 948900769..3cd9dec0a 100644 --- a/apps/fabro-web/app/routes/runs.test.tsx +++ b/apps/fabro-web/app/routes/runs.test.tsx @@ -25,6 +25,7 @@ describe("runs route board mapping", () => { test("keeps blocked runs in the blocked lane and preserves question text", () => { const columns = buildBoardColumns({ columns: [ + { id: "queued", name: "Queued" }, { id: "initializing", name: "Initializing" }, { id: "running", name: "Running" }, { id: "blocked", name: "Blocked" }, diff --git a/apps/fabro-web/app/routes/runs.tsx b/apps/fabro-web/app/routes/runs.tsx index f46c19396..90ef09405 100644 --- a/apps/fabro-web/app/routes/runs.tsx +++ b/apps/fabro-web/app/routes/runs.tsx @@ -38,6 +38,7 @@ interface ColumnStyle { } const columnStyles: Record = { + queued: { iconType: "branch", actions: [] }, initializing: { iconType: "branch", actions: [] }, running: { iconType: "branch", actions: ["Watch", "Steer"] }, blocked: { iconType: "branch", actions: ["Answer Question"] }, @@ -65,6 +66,7 @@ type Column = { }; const SKELETON_STATUSES: ColumnStatus[] = [ + "queued", "initializing", "running", "blocked", @@ -754,6 +756,9 @@ export default function Runs() { (sum, col) => sum + col.items.length, 0, ); + const visibleColumns = filteredColumns.filter( + (col) => col.id !== "queued" || col.items.length > 0, + ); return ( @@ -815,7 +820,7 @@ export default function Runs() { {view === "columns" ? ( <>
- {filteredColumns.map((col) => ( + {visibleColumns.map((col) => (
@@ -838,7 +843,7 @@ export default function Runs() { ) : ( <>
- {filteredColumns.map((col) => { + {visibleColumns.map((col) => { const isCollapsed = collapsed.has(col.id); return (
diff --git a/docs/public/api-reference/fabro-api.yaml b/docs/public/api-reference/fabro-api.yaml index d1b8bc7f3..d10db4a47 100644 --- a/docs/public/api-reference/fabro-api.yaml +++ b/docs/public/api-reference/fabro-api.yaml @@ -5656,6 +5656,7 @@ components: description: Board column status for a run in the list view. type: string enum: + - queued - initializing - running - blocked diff --git a/lib/crates/fabro-server/src/demo/mod.rs b/lib/crates/fabro-server/src/demo/mod.rs index c9751cf83..787fb1ae4 100644 --- a/lib/crates/fabro-server/src/demo/mod.rs +++ b/lib/crates/fabro-server/src/demo/mod.rs @@ -795,8 +795,8 @@ mod runs { .collect() } - fn demo_run_ids() -> &'static [RunId; 6] { - static IDS: OnceLock<[RunId; 6]> = OnceLock::new(); + fn demo_run_ids() -> &'static [RunId; 7] { + static IDS: OnceLock<[RunId; 7]> = OnceLock::new(); IDS.get_or_init(|| { [ RunId::with_timestamp(ts("2026-03-06T14:30:00Z"), 1), @@ -805,6 +805,7 @@ mod runs { RunId::with_timestamp(ts("2026-03-04T10:00:00Z"), 4), RunId::with_timestamp(ts("2026-03-03T16:45:00Z"), 5), RunId::with_timestamp(ts("2026-02-28T14:00:00Z"), 6), + RunId::with_timestamp(ts("2026-03-06T14:35:00Z"), 7), ] }) } @@ -973,6 +974,10 @@ mod runs { pub(super) fn columns() -> Vec { vec![ + BoardColumnDefinition { + id: "queued".into(), + name: "Queued".into(), + }, BoardColumnDefinition { id: "initializing".into(), name: "Initializing".into(), @@ -1082,6 +1087,20 @@ mod runs { Some(720000), &[("release", "preview")], ), + summary( + 7, + "api-server", + "implement", + "Implement", + "Add audit log retention policy", + "queued", + "2026-03-06T14:35:00Z", + None, + None, + None, + None, + &[("owner", "platform")], + ), ] } @@ -1150,6 +1169,13 @@ mod runs { None, None, ), + board_item( + take_summary(&mut summaries, demo_run_id(7)), + BoardColumn::Queued, + None, + None, + None, + ), ] } diff --git a/lib/crates/fabro-server/src/server/handler/runs.rs b/lib/crates/fabro-server/src/server/handler/runs.rs index ff03b55e5..eb48c7c4f 100644 --- a/lib/crates/fabro-server/src/server/handler/runs.rs +++ b/lib/crates/fabro-server/src/server/handler/runs.rs @@ -84,7 +84,8 @@ impl ListRunsParams { fn board_column(status: RunStatus) -> Option<&'static str> { match status { - RunStatus::Submitted | RunStatus::Queued | RunStatus::Starting => Some("initializing"), + RunStatus::Submitted | RunStatus::Queued => Some("queued"), + RunStatus::Starting => Some("initializing"), RunStatus::Running | RunStatus::Paused { .. } => Some("running"), RunStatus::Blocked { .. } => Some("blocked"), RunStatus::Succeeded { .. } => Some("succeeded"), @@ -95,6 +96,7 @@ fn board_column(status: RunStatus) -> Option<&'static str> { pub(crate) fn board_columns() -> serde_json::Value { serde_json::json!([ + {"id": "queued", "name": "Queued"}, {"id": "initializing", "name": "Initializing"}, {"id": "running", "name": "Running"}, {"id": "blocked", "name": "Blocked"}, diff --git a/lib/crates/fabro-server/src/server/tests.rs b/lib/crates/fabro-server/src/server/tests.rs index 6078c362f..5480fef0f 100644 --- a/lib/crates/fabro-server/src/server/tests.rs +++ b/lib/crates/fabro-server/src/server/tests.rs @@ -6125,7 +6125,7 @@ async fn pause_run_sets_pending_control_on_board_response() { assert_eq!(body["pending_control"].as_str(), Some("pause")); // Verify the run appears on the board (store has Submitted status → - // "initializing" column) + // "queued" column) let req = Request::builder() .method("GET") .uri(api("/boards/runs")) @@ -6140,7 +6140,7 @@ async fn pause_run_sets_pending_control_on_board_response() { .find(|item| item["run_id"].as_str() == Some(run_id_str.as_str())) .expect("board item should exist"); assert!(item["status"].is_object()); - assert_eq!(item["column"].as_str(), Some("initializing")); + assert_eq!(item["column"].as_str(), Some("queued")); assert_eq!(item["pending_control"].as_str(), Some("pause")); } diff --git a/lib/packages/fabro-api-client/src/models/board-column.ts b/lib/packages/fabro-api-client/src/models/board-column.ts index 9d97dc120..66c24c107 100644 --- a/lib/packages/fabro-api-client/src/models/board-column.ts +++ b/lib/packages/fabro-api-client/src/models/board-column.ts @@ -19,6 +19,7 @@ */ export const BoardColumn = { + QUEUED: 'queued', INITIALIZING: 'initializing', RUNNING: 'running', BLOCKED: 'blocked', From ade721ae65e8b7cf0bb0b93e4bb9e6f13a78397b Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Mon, 4 May 2026 14:54:39 -0400 Subject: [PATCH 02/16] feat(web): coordinate SSE subscriptions across tabs Elect a single browser tab to own the global attach stream and broadcast run events to sibling tabs. Keep the existing per-tab EventSource path as the fallback when cross-tab coordination is unavailable. --- apps/fabro-web/app/lib/board-events.test.tsx | 84 +- apps/fabro-web/app/lib/board-events.ts | 41 +- apps/fabro-web/app/lib/cross-tab-sse.test.ts | 385 ++++++ apps/fabro-web/app/lib/cross-tab-sse.ts | 1175 ++++++++++++++++++ apps/fabro-web/app/lib/run-events.test.tsx | 128 +- apps/fabro-web/app/lib/run-events.ts | 75 +- 6 files changed, 1852 insertions(+), 36 deletions(-) create mode 100644 apps/fabro-web/app/lib/cross-tab-sse.test.ts create mode 100644 apps/fabro-web/app/lib/cross-tab-sse.ts diff --git a/apps/fabro-web/app/lib/board-events.test.tsx b/apps/fabro-web/app/lib/board-events.test.tsx index 853d300b9..9b398fdc1 100644 --- a/apps/fabro-web/app/lib/board-events.test.tsx +++ b/apps/fabro-web/app/lib/board-events.test.tsx @@ -4,6 +4,10 @@ import { shouldRefreshBoardForEvent, subscribeToBoardEvents, } from "./board-events"; +import { + createCrossTabSseCoordinator, + type BroadcastChannelLike, +} from "./cross-tab-sse"; import { queryKeys } from "./query-keys"; type MessageHandler = ((event: { data: string }) => void) | null; @@ -21,6 +25,14 @@ class FakeEventSource { } } +class FakeBroadcastChannel implements BroadcastChannelLike { + onmessage: ((event: { data: unknown }) => void) | null = null; + + postMessage() {} + + close() {} +} + describe("shouldRefreshBoardForEvent", () => { test("refreshes board for run and interview status changes only", () => { expect(shouldRefreshBoardForEvent("run.running")).toBe(true); @@ -31,10 +43,11 @@ describe("shouldRefreshBoardForEvent", () => { }); describe("subscribeToBoardEvents", () => { - test("shares one source and invalidates the board runs key", () => { + test("coordinated mode shares one global source and invalidates the board runs key", async () => { const source = new FakeEventSource(); const created: string[] = []; const keys: string[] = []; + const coordinator = createCoordinator(); const mutate = (key: string) => { keys.push(key); return Promise.resolve(); @@ -43,10 +56,13 @@ describe("subscribeToBoardEvents", () => { const firstCleanup = subscribeToBoardEvents(mutate, (url) => { created.push(url); return source; - }, { debounceMs: 0 }); + }, { debounceMs: 0, coordinator }); const secondCleanup = subscribeToBoardEvents(mutate, () => { throw new Error("source should be reused"); - }, { debounceMs: 0 }); + }, { debounceMs: 0, coordinator }); + + await waitFor(() => created.length === 1); + keys.length = 0; source.emit({ event: "run.running" }); @@ -57,5 +73,67 @@ describe("subscribeToBoardEvents", () => { expect(source.closed).toBe(false); secondCleanup(); expect(source.closed).toBe(true); + coordinator.close(); + }); + + test("fallback mode preserves the existing shared board EventSource", () => { + const source = new FakeEventSource(); + const created: string[] = []; + const keys: string[] = []; + const coordinator = createFallbackCoordinator(); + const mutate = (key: string) => { + keys.push(key); + return Promise.resolve(); + }; + + const firstCleanup = subscribeToBoardEvents(mutate, (url) => { + created.push(url); + return source; + }, { debounceMs: 0, coordinator }); + const secondCleanup = subscribeToBoardEvents(mutate, () => { + throw new Error("source should be reused"); + }, { debounceMs: 0, coordinator }); + + source.emit({ event: "run.running" }); + + expect(created).toEqual(["/api/v1/attach"]); + expect(keys).toEqual([queryKeys.boards.runs()]); + + firstCleanup(); + expect(source.closed).toBe(false); + secondCleanup(); + expect(source.closed).toBe(true); + coordinator.close(); }); }); + +function createCoordinator() { + return createCrossTabSseCoordinator({ + tabId: "board-test", + channelFactory: () => new FakeBroadcastChannel(), + addVisibilityChangeListener: () => () => {}, + addPagehideListener: () => () => {}, + timing: { + heartbeatMs: 10, + leaderStaleMs: 50, + electionJitterMs: 0, + }, + }); +} + +function createFallbackCoordinator() { + return createCrossTabSseCoordinator({ + channelFactory: () => { + throw new Error("BroadcastChannel unavailable"); + }, + }); +} + +async function waitFor(condition: () => boolean, timeoutMs = 200) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (condition()) return; + await new Promise((resolve) => setTimeout(resolve, 2)); + } + throw new Error("condition did not become true before timeout"); +} diff --git a/apps/fabro-web/app/lib/board-events.ts b/apps/fabro-web/app/lib/board-events.ts index ef30009b1..a182d9ad2 100644 --- a/apps/fabro-web/app/lib/board-events.ts +++ b/apps/fabro-web/app/lib/board-events.ts @@ -1,6 +1,10 @@ import { useEffect } from "react"; import { useSWRConfig } from "swr"; +import { + subscribeToCrossTabSse, + type CrossTabSseCoordinator, +} from "./cross-tab-sse"; import { queryKeys } from "./query-keys"; import { createBrowserEventSource, @@ -11,6 +15,11 @@ import { type SharedEventSubscription, } from "./sse"; +interface BoardEventOptions { + debounceMs?: number; + coordinator?: CrossTabSseCoordinator; +} + const BOARD_STATUS_EVENTS = new Set([ "run.submitted", "run.queued", @@ -41,23 +50,37 @@ export function shouldRefreshBoardForEvent(event: string) { export function subscribeToBoardEvents( mutate: MutateFn, eventSourceFactory: (url: string) => EventSourceLike = createBrowserEventSource, - { debounceMs = 500 }: { debounceMs?: number } = {}, + { debounceMs = 500, coordinator }: BoardEventOptions = {}, ): () => void { - return subscribeToSharedEventSource({ - subscriptions, + return subscribeToCrossTabSse({ + coordinator, subscriptionKey: BOARD_SUBSCRIPTION_KEY, - url: queryKeys.system.attach(), mutate, eventSourceFactory, debounceMs, - resolveInvalidation: (payload) => ({ - keys: payload.event && shouldRefreshBoardForEvent(payload.event) - ? [queryKeys.boards.runs()] - : [], - }), + resyncKeys: () => [queryKeys.boards.runs()], + resolveInvalidation: boardInvalidation, + fallbackSubscribe: () => + subscribeToSharedEventSource({ + subscriptions, + subscriptionKey: BOARD_SUBSCRIPTION_KEY, + url: queryKeys.system.attach(), + mutate, + eventSourceFactory, + debounceMs, + resolveInvalidation: boardInvalidation, + }), }); } +function boardInvalidation(payload: EventPayload) { + return { + keys: payload.event && shouldRefreshBoardForEvent(payload.event) + ? [queryKeys.boards.runs()] + : [], + }; +} + export function useBoardEvents() { const { mutate } = useSWRConfig(); diff --git a/apps/fabro-web/app/lib/cross-tab-sse.test.ts b/apps/fabro-web/app/lib/cross-tab-sse.test.ts new file mode 100644 index 000000000..32cff001c --- /dev/null +++ b/apps/fabro-web/app/lib/cross-tab-sse.test.ts @@ -0,0 +1,385 @@ +import { afterEach, describe, expect, test } from "bun:test"; + +import { + CROSS_TAB_SSE_CHANNEL, + createCrossTabSseCoordinator, + subscribeToCrossTabSse, + type BroadcastChannelLike, + type CrossTabSseCoordinator, + type CrossTabSseMessage, +} from "./cross-tab-sse"; +import type { EventPayload, MutateFn } from "./sse"; + +type MessageHandler = ((event: { data: string }) => void) | null; +type TabVisibility = "visible" | "hidden"; + +const TEST_TIMING = { + heartbeatMs: 10, + leaderStaleMs: 35, + electionJitterMs: 5, +}; + +class FakeEventSource { + onmessage: MessageHandler = null; + closed = false; + + constructor( + readonly url: string, + readonly owner: string, + ) {} + + emit(payload: unknown) { + this.onmessage?.({ data: JSON.stringify(payload) }); + } + + close() { + this.closed = true; + } +} + +class FakeBroadcastChannel implements BroadcastChannelLike { + static channels = new Set(); + static muted = false; + + onmessage: ((event: { data: unknown }) => void) | null = null; + closed = false; + + constructor(readonly name: string) { + FakeBroadcastChannel.channels.add(this); + } + + postMessage(message: CrossTabSseMessage) { + if (FakeBroadcastChannel.muted) return; + const recipients = [...FakeBroadcastChannel.channels].filter( + (channel) => channel !== this && !channel.closed && channel.name === this.name, + ); + queueMicrotask(() => { + for (const channel of recipients) { + if (channel.closed) continue; + channel.onmessage?.({ data: { ...message } }); + } + }); + } + + close() { + this.closed = true; + FakeBroadcastChannel.channels.delete(this); + } + + static reset() { + for (const channel of FakeBroadcastChannel.channels) { + channel.closed = true; + } + FakeBroadcastChannel.channels.clear(); + FakeBroadcastChannel.muted = false; + } +} + +class Harness { + readonly sources: FakeEventSource[] = []; + readonly coordinators = new Map(); + readonly visibility = new Map(); + readonly visibilityHandlers = new Map void>(); + now = 1000; + + createTab(tabId: string, visibility: TabVisibility = "visible") { + this.visibility.set(tabId, visibility); + const coordinator = createCrossTabSseCoordinator({ + tabId, + channelFactory: (name) => new FakeBroadcastChannel(name), + eventSourceFactory: (url) => { + const source = new FakeEventSource(url, tabId); + this.sources.push(source); + return source; + }, + getVisibility: () => this.visibility.get(tabId) ?? "visible", + addVisibilityChangeListener: (handler) => { + this.visibilityHandlers.set(tabId, handler); + return () => this.visibilityHandlers.delete(tabId); + }, + addPagehideListener: () => () => {}, + now: () => this.now, + timing: TEST_TIMING, + }); + this.coordinators.set(tabId, coordinator); + return coordinator; + } + + setVisibility(tabId: string, visibility: TabVisibility) { + this.visibility.set(tabId, visibility); + this.visibilityHandlers.get(tabId)?.(); + } + + openSources() { + return this.sources.filter((source) => !source.closed); + } + + close() { + for (const coordinator of this.coordinators.values()) { + coordinator.close(); + } + } +} + +const harnesses: Harness[] = []; + +afterEach(() => { + for (const harness of harnesses.splice(0)) { + harness.close(); + } + FakeBroadcastChannel.reset(); +}); + +describe("subscribeToCrossTabSse", () => { + test("opens one leader-owned global EventSource and keeps followers passive", async () => { + const harness = newHarness(); + const cleanups = ["a", "b", "c"].map((tabId) => { + const coordinator = harness.createTab(tabId); + return subscribeForRunEvent(coordinator, []); + }); + + await waitFor(() => harness.openSources().length === 1); + + expect(harness.openSources().map((source) => source.url)).toEqual(["/api/v1/attach"]); + expect([...FakeBroadcastChannel.channels].every((channel) => channel.name === CROSS_TAB_SSE_CHANNEL)).toBe(true); + + cleanups.forEach((cleanup) => cleanup()); + }); + + test("leader broadcasts events to all local subscribers", async () => { + const harness = newHarness(); + const keysByTab = new Map(); + + for (const tabId of ["a", "b", "c"]) { + keysByTab.set(tabId, []); + subscribeForRunEvent(harness.createTab(tabId), keysByTab.get(tabId)!); + } + + await waitFor(() => harness.openSources().length === 1); + clearRecordedKeys(keysByTab); + + harness.openSources()[0].emit(runEvent({ id: "evt-1", runId: "run-1", seq: 1 })); + await waitFor(() => [...keysByTab.values()].every((keys) => keys.length === 1)); + + expect(keysByTab.get("a")).toEqual(["event"]); + expect(keysByTab.get("b")).toEqual(["event"]); + expect(keysByTab.get("c")).toEqual(["event"]); + }); + + test("dedupes duplicate event ids until TTL or max-size eviction", async () => { + const harness = newHarness(); + const keys: string[] = []; + subscribeForRunEvent(harness.createTab("a"), keys); + + await waitFor(() => harness.openSources().length === 1); + keys.length = 0; + + const source = harness.openSources()[0]; + source.emit(runEvent({ id: "evt-dup", runId: "run-1", seq: 1 })); + source.emit(runEvent({ id: "evt-dup", runId: "run-1", seq: 1 })); + + expect(keys).toEqual(["event"]); + + harness.now += 5 * 60 * 1000 + 1; + source.emit(runEvent({ id: "evt-dup", runId: "run-1", seq: 1 })); + expect(keys).toEqual(["event", "event"]); + + keys.length = 0; + for (let i = 0; i < 1001; i += 1) { + source.emit(runEvent({ id: `evt-${i}`, runId: "run-1", seq: i + 2 })); + } + source.emit(runEvent({ id: "evt-0", runId: "run-1", seq: 2 })); + expect(keys).toHaveLength(1002); + }); + + test("visible followers take over from a fresh hidden leader and resync", async () => { + const harness = newHarness(); + const hiddenKeys: string[] = []; + const visibleKeys: string[] = []; + + subscribeForRunEvent(harness.createTab("z", "hidden"), hiddenKeys); + await waitFor(() => harness.openSources().length === 1); + const hiddenSource = harness.openSources()[0]; + + subscribeForRunEvent(harness.createTab("a", "visible"), visibleKeys); + + await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "a"); + + expect(hiddenSource.closed).toBe(true); + expect(visibleKeys).toContain("resync"); + }); + + test("visible candidates racing for the same hidden leader resolve lexically", async () => { + const harness = newHarness(); + + subscribeForRunEvent(harness.createTab("z", "hidden"), []); + await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "z"); + + subscribeForRunEvent(harness.createTab("b", "visible"), []); + subscribeForRunEvent(harness.createTab("a", "visible"), []); + + await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "a"); + }); + + test("a lower lexical follower does not preempt a fresh visible leader", async () => { + const harness = newHarness(); + + subscribeForRunEvent(harness.createTab("z", "visible"), []); + await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "z"); + + subscribeForRunEvent(harness.createTab("a", "visible"), []); + await sleep(TEST_TIMING.electionJitterMs * 4); + + expect(harness.openSources().map((source) => source.owner)).toEqual(["z"]); + }); + + test("stale leader detection opens a new leader source and resyncs followers", async () => { + const harness = newHarness(); + const followerKeys: string[] = []; + + subscribeForRunEvent(harness.createTab("a"), []); + subscribeForRunEvent(harness.createTab("b"), followerKeys); + await waitFor(() => harness.openSources().length === 1); + + const staleLeader = harness.openSources()[0]; + harness.coordinators.get(staleLeader.owner)?.close(); + harness.now += TEST_TIMING.leaderStaleMs + TEST_TIMING.heartbeatMs + 1; + + await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner !== staleLeader.owner); + + expect(followerKeys).toContain("resync"); + }); + + test("same-generation split brain converges to the higher-priority visible leader", async () => { + const harness = newHarness(); + FakeBroadcastChannel.muted = true; + + subscribeForRunEvent(harness.createTab("b"), []); + subscribeForRunEvent(harness.createTab("a"), []); + await waitFor(() => harness.openSources().length === 2); + + FakeBroadcastChannel.muted = false; + await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "a"); + }); + + test("old leader events are ignored after takeover", async () => { + const harness = newHarness(); + const keys: string[] = []; + + subscribeForRunEvent(harness.createTab("z", "hidden"), []); + await waitFor(() => harness.openSources().length === 1); + const oldSource = harness.openSources()[0]; + + subscribeForRunEvent(harness.createTab("a", "visible"), keys); + await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "a"); + keys.length = 0; + + oldSource.emit(runEvent({ id: "evt-old", runId: "run-1", seq: 1 })); + expect(keys).toEqual([]); + }); + + test("last unsubscribe closes the leader source and releases leadership", async () => { + const harness = newHarness(); + const cleanup = subscribeForRunEvent(harness.createTab("a"), []); + + await waitFor(() => harness.openSources().length === 1); + const source = harness.openSources()[0]; + + cleanup(); + + expect(source.closed).toBe(true); + expect(harness.openSources()).toEqual([]); + }); + + test("missing BroadcastChannel uses subscriber fallback", () => { + const coordinator = createCrossTabSseCoordinator({ + channelFactory: () => { + throw new Error("no channel"); + }, + }); + let fallbackStarted = 0; + let fallbackStopped = 0; + + const cleanup = subscribeToCrossTabSse({ + coordinator, + subscriptionKey: "fallback", + mutate: (() => Promise.resolve()) as MutateFn, + resolveInvalidation: () => ({ keys: [] }), + resyncKeys: () => [], + fallbackSubscribe: () => { + fallbackStarted += 1; + return () => { + fallbackStopped += 1; + }; + }, + debounceMs: 0, + }); + + cleanup(); + + expect(fallbackStarted).toBe(1); + expect(fallbackStopped).toBe(1); + }); +}); + +function newHarness() { + const harness = new Harness(); + harnesses.push(harness); + return harness; +} + +function subscribeForRunEvent(coordinator: CrossTabSseCoordinator, keys: string[]) { + return subscribeToCrossTabSse({ + coordinator, + subscriptionKey: "run-feed", + mutate: ((key: string) => { + keys.push(key); + return Promise.resolve(); + }) as MutateFn, + resolveInvalidation: (payload) => ({ + keys: payload.event === "run.running" ? ["event"] : [], + }), + resyncKeys: () => ["resync"], + fallbackSubscribe: () => { + throw new Error("fallback should not be used"); + }, + debounceMs: 0, + }); +} + +function runEvent({ + id, + runId, + seq, +}: { + id: string; + runId: string; + seq: number; +}) { + return { + id, + seq, + run_id: runId, + event: "run.running", + ts: "2026-05-04T12:00:00.000Z", + }; +} + +async function waitFor(condition: () => boolean, timeoutMs = 500) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (condition()) return; + await sleep(2); + } + throw new Error("condition did not become true before timeout"); +} + +function sleep(ms: number) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function clearRecordedKeys(keysByTab: Map) { + for (const keys of keysByTab.values()) { + keys.length = 0; + } +} diff --git a/apps/fabro-web/app/lib/cross-tab-sse.ts b/apps/fabro-web/app/lib/cross-tab-sse.ts new file mode 100644 index 000000000..255c4c7ce --- /dev/null +++ b/apps/fabro-web/app/lib/cross-tab-sse.ts @@ -0,0 +1,1175 @@ +import { queryKeys } from "./query-keys"; +import { + createBrowserEventSource, + type EventInvalidation, + type EventPayload, + type EventSourceLike, + type MutateFn, +} from "./sse"; +import { isRecord } from "./unknown"; + +export const CROSS_TAB_SSE_CHANNEL = "fabro:sse:v1"; +export const HEARTBEAT_MS = 1000; +export const LEADER_STALE_MS = 4000; +export const ELECTION_JITTER_MS = 150; + +const MESSAGE_VERSION = 1 as const; +const EVENT_DEDUPE_TTL_MS = 5 * 60 * 1000; +const EVENT_DEDUPE_MAX = 1000; + +type TabVisibility = "visible" | "hidden"; +type CandidateReason = "hidden-leader" | "stale-leader" | "release" | "no-leader"; + +interface BaseMessage { + type: string; + version: typeof MESSAGE_VERSION; + tabId: string; + sentAt: number; +} + +interface HelloMessage extends BaseMessage { + type: "hello"; +} + +interface HeartbeatMessage extends BaseMessage { + type: "heartbeat"; + leaderId: string; + generation: number; + visibility: TabVisibility; +} + +interface CandidateMessage extends BaseMessage { + type: "candidate"; + candidateId: string; + candidateGeneration: number; + visibility: TabVisibility; + observedLeaderId: string | null; + observedGeneration: number; + reason: CandidateReason; +} + +interface LeaderChangedMessage extends BaseMessage { + type: "leader-changed"; + leaderId: string; + generation: number; + visibility: TabVisibility; +} + +interface ReleaseMessage extends BaseMessage { + type: "release"; + leaderId: string; + generation: number; +} + +interface ResyncMessage extends BaseMessage { + type: "resync"; + leaderId: string | null; + generation: number; + reason: CandidateReason; +} + +interface EventMessage extends BaseMessage { + type: "event"; + leaderId: string; + generation: number; + payload: EventPayload; +} + +export type CrossTabSseMessage = + | HelloMessage + | HeartbeatMessage + | CandidateMessage + | LeaderChangedMessage + | ReleaseMessage + | ResyncMessage + | EventMessage; + +export interface BroadcastChannelLike { + onmessage: ((event: { data: unknown }) => void) | null; + postMessage(message: CrossTabSseMessage): void; + close(): void; +} + +interface TimingOptions { + heartbeatMs: number; + leaderStaleMs: number; + electionJitterMs: number; +} + +export interface CrossTabSseCoordinatorOptions { + tabId?: string; + channelFactory?: (name: string) => BroadcastChannelLike; + eventSourceFactory?: (url: string) => EventSourceLike; + getVisibility?: () => TabVisibility; + addVisibilityChangeListener?: (handler: () => void) => () => void; + addPagehideListener?: (handler: () => void) => () => void; + now?: () => number; + timing?: Partial; +} + +interface SubscribeOptions { + subscriptionKey: string; + mutate: MutateFn; + resolveInvalidation: (payload: TPayload) => EventInvalidation; + resyncKeys: () => string[]; + fallbackSubscribe: () => () => void; + eventSourceFactory?: (url: string) => EventSourceLike; + debounceMs?: number; +} + +export interface SubscribeToCrossTabSseOptions + extends SubscribeOptions { + coordinator?: CrossTabSseCoordinator; +} + +interface FallbackEntry { + count: number; + subscribe: () => () => void; + cleanup?: () => void; +} + +interface LocalSubscription { + refcount: number; + mutators: Map; + fallbacks: Map; + pendingKeys: Set; + debounceTimer: ReturnType | null; + debounceMs: number; + resolveInvalidation: (payload: EventPayload) => EventInvalidation; + resyncKeys: () => string[]; +} + +interface LeaderState { + leaderId: string; + generation: number; + visibility: TabVisibility; + lastSeen: number; +} + +class RecentEventCache { + private readonly seen = new Map(); + + constructor( + private readonly maxSize: number, + private readonly ttlMs: number, + ) {} + + remember(key: string | undefined, now: number): boolean { + if (!key) return true; + this.prune(now); + if (this.seen.has(key)) return false; + + this.seen.set(key, now); + while (this.seen.size > this.maxSize) { + const oldest = this.seen.keys().next().value; + if (oldest === undefined) break; + this.seen.delete(oldest); + } + return true; + } + + private prune(now: number) { + for (const [key, seenAt] of this.seen) { + if (now - seenAt > this.ttlMs) { + this.seen.delete(key); + } + } + } +} + +export class CrossTabSseCoordinator { + readonly tabId: string; + + private readonly channelFactory: (name: string) => BroadcastChannelLike; + private readonly getVisibility: () => TabVisibility; + private readonly addVisibilityChangeListener: (handler: () => void) => () => void; + private readonly addPagehideListener: (handler: () => void) => () => void; + private readonly now: () => number; + private readonly timing: TimingOptions; + private readonly recentEvents = new RecentEventCache(EVENT_DEDUPE_MAX, EVENT_DEDUPE_TTL_MS); + private readonly subscriptions = new Map(); + private readonly candidates = new Map(); + + private sourceFactory: (url: string) => EventSourceLike; + private channel: BroadcastChannelLike | null = null; + private source: EventSourceLike | null = null; + private initialized = false; + private coordinationUnavailable = false; + private fallbackMode = false; + private degradingToFallback = false; + private isLeader = false; + private leader: LeaderState | null = null; + private generation = 0; + private ownCandidate: CandidateMessage | null = null; + private candidateTimer: ReturnType | null = null; + private noLeaderTimer: ReturnType | null = null; + private heartbeatTimer: ReturnType | null = null; + private leaderCheckTimer: ReturnType | null = null; + private removeVisibilityListener: (() => void) | null = null; + private removePagehideListener: (() => void) | null = null; + private sourceFactoryLocked = false; + + constructor(options: CrossTabSseCoordinatorOptions = {}) { + this.tabId = options.tabId ?? createTabId(); + this.channelFactory = options.channelFactory ?? createBrowserBroadcastChannel; + this.sourceFactory = options.eventSourceFactory ?? createBrowserEventSource; + this.getVisibility = options.getVisibility ?? getBrowserVisibility; + this.addVisibilityChangeListener = + options.addVisibilityChangeListener ?? addBrowserVisibilityChangeListener; + this.addPagehideListener = options.addPagehideListener ?? addBrowserPagehideListener; + this.now = options.now ?? Date.now; + this.timing = { + heartbeatMs: options.timing?.heartbeatMs ?? HEARTBEAT_MS, + leaderStaleMs: options.timing?.leaderStaleMs ?? LEADER_STALE_MS, + electionJitterMs: options.timing?.electionJitterMs ?? ELECTION_JITTER_MS, + }; + } + + subscribe(options: SubscribeOptions): () => void { + if (options.eventSourceFactory && !this.source && !this.sourceFactoryLocked) { + this.sourceFactory = options.eventSourceFactory; + this.sourceFactoryLocked = true; + } + + if (this.coordinationUnavailable) { + return options.fallbackSubscribe(); + } + + if (!this.initialized && !this.initialize()) { + this.coordinationUnavailable = true; + return options.fallbackSubscribe(); + } + + const subscription = this.addLocalSubscription(options); + if (this.fallbackMode) { + this.startFallbacksFor(subscription); + } else { + this.ensureLeadershipProgress(); + } + + let active = true; + return () => { + if (!active) return; + active = false; + this.removeLocalSubscription(options.subscriptionKey, options.mutate); + }; + } + + close() { + this.releaseLeadership({ broadcast: false, resync: false }); + this.clearCandidate(); + this.clearNoLeaderTimer(); + this.shutdownTimersAndChannel(); + this.closeFallbacks(); + this.subscriptions.clear(); + this.leader = null; + this.initialized = false; + this.fallbackMode = false; + this.sourceFactoryLocked = false; + } + + private initialize(): boolean { + try { + this.channel = this.channelFactory(CROSS_TAB_SSE_CHANNEL); + } catch { + this.channel = null; + return false; + } + + this.channel.onmessage = (event) => this.handleMessage(event.data); + this.initialized = true; + this.removeVisibilityListener = this.addVisibilityChangeListener(() => { + this.handleVisibilityChange(); + }); + this.removePagehideListener = this.addPagehideListener(() => { + this.handlePagehide(); + }); + this.leaderCheckTimer = setInterval(() => { + this.checkLeaderFreshness(); + }, this.timing.heartbeatMs); + + return this.post({ type: "hello", version: MESSAGE_VERSION, tabId: this.tabId, sentAt: this.now() }); + } + + private addLocalSubscription( + options: SubscribeOptions, + ): LocalSubscription { + let subscription = this.subscriptions.get(options.subscriptionKey); + if (!subscription) { + subscription = { + refcount: 0, + mutators: new Map(), + fallbacks: new Map(), + pendingKeys: new Set(), + debounceTimer: null, + debounceMs: options.debounceMs ?? 300, + resolveInvalidation: options.resolveInvalidation as (payload: EventPayload) => EventInvalidation, + resyncKeys: options.resyncKeys, + }; + this.subscriptions.set(options.subscriptionKey, subscription); + } else { + subscription.resolveInvalidation = + options.resolveInvalidation as (payload: EventPayload) => EventInvalidation; + subscription.resyncKeys = options.resyncKeys; + subscription.debounceMs = options.debounceMs ?? subscription.debounceMs; + } + + subscription.refcount += 1; + subscription.mutators.set( + options.mutate, + (subscription.mutators.get(options.mutate) ?? 0) + 1, + ); + + const fallback = subscription.fallbacks.get(options.mutate); + if (fallback) { + fallback.count += 1; + fallback.subscribe = options.fallbackSubscribe; + } else { + subscription.fallbacks.set(options.mutate, { + count: 1, + subscribe: options.fallbackSubscribe, + }); + } + + return subscription; + } + + private removeLocalSubscription(subscriptionKey: string, mutate: MutateFn) { + const subscription = this.subscriptions.get(subscriptionKey); + if (!subscription) return; + + const mutateCount = subscription.mutators.get(mutate) ?? 0; + if (mutateCount <= 1) { + subscription.mutators.delete(mutate); + } else { + subscription.mutators.set(mutate, mutateCount - 1); + } + + const fallback = subscription.fallbacks.get(mutate); + if (fallback) { + fallback.count -= 1; + if (fallback.count <= 0) { + fallback.cleanup?.(); + subscription.fallbacks.delete(mutate); + } + } + + subscription.refcount -= 1; + if (subscription.refcount <= 0) { + if (subscription.debounceTimer) { + clearTimeout(subscription.debounceTimer); + } + this.subscriptions.delete(subscriptionKey); + } + + if (this.subscriptions.size === 0) { + this.releaseLeadership({ broadcast: true, resync: false }); + this.clearCandidate(); + this.clearNoLeaderTimer(); + this.shutdownTimersAndChannel(); + this.initialized = false; + this.fallbackMode = false; + this.sourceFactoryLocked = false; + } + } + + private handleMessage(data: unknown) { + const message = parseMessage(data); + if (!message || message.tabId === this.tabId) return; + + switch (message.type) { + case "hello": + if (this.isLeader) this.sendHeartbeat(); + break; + case "heartbeat": + this.handleLeaderAnnouncement(message, { resyncOnChange: true }); + break; + case "candidate": + this.handleCandidate(message); + break; + case "leader-changed": + this.handleLeaderAnnouncement(message, { resyncOnChange: true }); + break; + case "release": + this.handleRelease(message); + break; + case "resync": + this.handleResync(message); + break; + case "event": + this.handleBroadcastEvent(message); + break; + } + } + + private handleLeaderAnnouncement( + message: HeartbeatMessage | LeaderChangedMessage, + { resyncOnChange }: { resyncOnChange: boolean }, + ) { + const incoming: LeaderState = { + leaderId: message.leaderId, + generation: message.generation, + visibility: message.visibility, + lastSeen: this.now(), + }; + + if (this.isLeader && incoming.leaderId !== this.tabId) { + const own: LeaderState = { + leaderId: this.tabId, + generation: this.generation, + visibility: this.currentVisibility(), + lastSeen: this.now(), + }; + if ( + incoming.generation > own.generation || + (incoming.generation === own.generation && leaderHasHigherPriority(incoming, own)) + ) { + this.releaseLeadership({ broadcast: false, resync: true }); + } else { + return; + } + } + + const previous = this.leader; + if (this.ownCandidate && incoming.generation === this.ownCandidate.candidateGeneration) { + const sawIncomingCandidate = this.candidates.has( + `${incoming.generation}:${incoming.leaderId}`, + ); + const shouldAcceptFreshVisibleLeader = + this.ownCandidate.reason === "no-leader" && + incoming.visibility === "visible" && + !sawIncomingCandidate; + + if ( + !shouldAcceptFreshVisibleLeader && + !leaderHasHigherPriority(incoming, leaderStateForCandidate(this.ownCandidate, this.now())) + ) { + return; + } + } + + if (!this.shouldAcceptLeader(incoming)) return; + + this.leader = incoming; + this.clearNoLeaderTimer(); + this.generation = Math.max(this.generation, incoming.generation); + if (this.ownCandidate && incoming.generation >= this.ownCandidate.candidateGeneration) { + this.clearCandidate(); + } + + const changed = + !previous || + previous.leaderId !== incoming.leaderId || + previous.generation !== incoming.generation; + + if (changed && resyncOnChange) { + this.resyncAll(); + } + + if (incoming.visibility === "hidden" && this.currentVisibility() === "visible") { + this.enterCandidacy("hidden-leader", incoming); + } + } + + private handleCandidate(message: CandidateMessage) { + this.candidates.set(candidateKey(message), message); + + if ( + this.isLeader && + message.observedLeaderId === this.tabId && + message.observedGeneration >= this.generation + ) { + this.releaseLeadership({ broadcast: true, resync: true }); + } + + if ( + this.ownCandidate && + message.candidateGeneration === this.ownCandidate.candidateGeneration && + candidateHasHigherPriority(message, this.ownCandidate) + ) { + this.clearCandidate(); + } + } + + private handleRelease(message: ReleaseMessage) { + const current = this.leader; + if ( + current && + message.leaderId === current.leaderId && + message.generation >= current.generation + ) { + this.leader = null; + this.generation = Math.max(this.generation, message.generation); + this.enterCandidacy("release", { + leaderId: message.leaderId, + generation: message.generation, + visibility: "hidden", + lastSeen: this.now(), + }); + } + } + + private handleResync(message: ResyncMessage) { + if (this.leader && message.generation < this.leader.generation) return; + this.resyncAll(); + } + + private handleBroadcastEvent(message: EventMessage) { + if (!this.isCurrentLeader(message.leaderId, message.generation)) return; + if (!this.recentEvents.remember(eventDedupeKey(message.payload), this.now())) return; + this.dispatchPayload(message.payload); + } + + private ensureLeadershipProgress() { + if (this.subscriptions.size === 0 || this.isLeader || this.ownCandidate) return; + + if (!this.leader) { + this.scheduleNoLeaderCandidacy(); + return; + } + + if (this.leader.visibility === "hidden" && this.currentVisibility() === "visible") { + this.enterCandidacy("hidden-leader", this.leader); + } + } + + private checkLeaderFreshness() { + if (this.subscriptions.size === 0 || this.fallbackMode) return; + if (this.isLeader) return; + + const current = this.leader; + if (!current) { + this.scheduleNoLeaderCandidacy(); + return; + } + + if (this.now() - current.lastSeen > this.timing.leaderStaleMs) { + this.leader = null; + this.generation = Math.max(this.generation, current.generation); + this.resyncAll(); + this.enterCandidacy("stale-leader", current); + return; + } + + if (current.visibility === "hidden" && this.currentVisibility() === "visible") { + this.enterCandidacy("hidden-leader", current); + } + } + + private enterCandidacy(reason: CandidateReason, observedLeader: LeaderState | null = this.leader) { + if (this.subscriptions.size === 0 || this.fallbackMode) return; + + if ( + reason === "no-leader" && + this.leader && + this.leader.visibility === "visible" && + this.now() - this.leader.lastSeen <= this.timing.leaderStaleMs + ) { + return; + } + + const observedGeneration = observedLeader?.generation ?? this.generation; + const candidateGeneration = observedGeneration + 1; + if ( + this.ownCandidate && + this.ownCandidate.candidateGeneration >= candidateGeneration + ) { + return; + } + + this.clearCandidate(); + this.clearNoLeaderTimer(); + const candidate: CandidateMessage = { + type: "candidate", + version: MESSAGE_VERSION, + tabId: this.tabId, + sentAt: this.now(), + candidateId: this.tabId, + candidateGeneration, + visibility: this.currentVisibility(), + observedLeaderId: observedLeader?.leaderId ?? null, + observedGeneration, + reason, + }; + + this.ownCandidate = candidate; + this.candidates.set(candidateKey(candidate), candidate); + this.post(candidate); + this.candidateTimer = setTimeout(() => { + this.completeCandidacy(candidate); + }, this.timing.electionJitterMs); + } + + private completeCandidacy(candidate: CandidateMessage) { + if (this.ownCandidate !== candidate || this.fallbackMode) return; + + for (const other of this.candidates.values()) { + if ( + other.candidateGeneration === candidate.candidateGeneration && + candidateHasHigherPriority(other, candidate) + ) { + this.clearCandidate(); + return; + } + } + + if ( + this.leader && + this.now() - this.leader.lastSeen <= this.timing.leaderStaleMs + ) { + if (this.leader.generation > candidate.candidateGeneration) { + this.clearCandidate(); + return; + } + if ( + this.leader.generation === candidate.candidateGeneration && + leaderHasHigherPriority(this.leader, leaderStateForCandidate(candidate, this.now())) + ) { + this.clearCandidate(); + return; + } + } + + this.becomeLeader(candidate.candidateGeneration); + } + + private becomeLeader(generation: number) { + this.clearCandidate(); + this.closeSource(); + this.isLeader = true; + this.generation = generation; + this.leader = { + leaderId: this.tabId, + generation, + visibility: this.currentVisibility(), + lastSeen: this.now(), + }; + + const source = this.sourceFactory(queryKeys.system.attach()); + this.source = source; + source.onmessage = (message) => { + this.handleLeaderEventSourceMessage(message.data); + }; + + this.post({ + type: "leader-changed", + version: MESSAGE_VERSION, + tabId: this.tabId, + sentAt: this.now(), + leaderId: this.tabId, + generation, + visibility: this.currentVisibility(), + }); + this.startHeartbeat(); + this.resyncAll(); + } + + private handleLeaderEventSourceMessage(data: string) { + if (!this.isLeader) return; + + let payload: EventPayload; + try { + payload = JSON.parse(data) as EventPayload; + } catch { + return; + } + + if (!this.recentEvents.remember(eventDedupeKey(payload), this.now())) return; + this.dispatchPayload(payload); + this.post({ + type: "event", + version: MESSAGE_VERSION, + tabId: this.tabId, + sentAt: this.now(), + leaderId: this.tabId, + generation: this.generation, + payload, + }); + } + + private dispatchPayload(payload: EventPayload) { + for (const subscription of this.subscriptions.values()) { + const invalidation = subscription.resolveInvalidation(payload); + this.queueInvalidations(subscription, invalidation.keys, { + immediate: invalidation.immediate, + }); + } + } + + private queueInvalidations( + subscription: LocalSubscription, + keys: string[], + { immediate = false }: { immediate?: boolean } = {}, + ) { + if (keys.length === 0) return; + for (const key of keys) { + subscription.pendingKeys.add(key); + } + + if (immediate || subscription.debounceMs <= 0) { + this.flushInvalidations(subscription); + return; + } + + if (subscription.debounceTimer) { + clearTimeout(subscription.debounceTimer); + } + subscription.debounceTimer = setTimeout(() => { + subscription.debounceTimer = null; + this.flushInvalidations(subscription); + }, subscription.debounceMs); + } + + private flushInvalidations(subscription: LocalSubscription) { + if (subscription.pendingKeys.size === 0) return; + const keys = [...subscription.pendingKeys]; + subscription.pendingKeys.clear(); + + for (const mutator of subscription.mutators.keys()) { + for (const key of keys) { + void mutator(key); + } + } + } + + private resyncAll() { + for (const subscription of this.subscriptions.values()) { + this.queueInvalidations(subscription, subscription.resyncKeys(), { immediate: true }); + } + } + + private handleVisibilityChange() { + if (this.isLeader) { + this.sendHeartbeat(); + } + + if (this.currentVisibility() === "visible") { + this.resyncAll(); + if (this.leader?.visibility === "hidden") { + this.enterCandidacy("hidden-leader", this.leader); + } + } + } + + private handlePagehide() { + this.releaseLeadership({ broadcast: true, resync: false }); + this.clearCandidate(); + this.clearNoLeaderTimer(); + } + + private startHeartbeat() { + if (this.heartbeatTimer) { + clearInterval(this.heartbeatTimer); + } + this.sendHeartbeat(); + this.heartbeatTimer = setInterval(() => { + this.sendHeartbeat(); + }, this.timing.heartbeatMs); + } + + private sendHeartbeat() { + if (!this.isLeader) return; + const visibility = this.currentVisibility(); + this.leader = { + leaderId: this.tabId, + generation: this.generation, + visibility, + lastSeen: this.now(), + }; + this.post({ + type: "heartbeat", + version: MESSAGE_VERSION, + tabId: this.tabId, + sentAt: this.now(), + leaderId: this.tabId, + generation: this.generation, + visibility, + }); + } + + private releaseLeadership({ + broadcast, + resync, + }: { + broadcast: boolean; + resync: boolean; + }) { + if (!this.isLeader && !this.source) return; + + const generation = this.generation; + this.closeSource(); + this.isLeader = false; + if (this.heartbeatTimer) { + clearInterval(this.heartbeatTimer); + this.heartbeatTimer = null; + } + this.leader = null; + + if (broadcast) { + this.post({ + type: "release", + version: MESSAGE_VERSION, + tabId: this.tabId, + sentAt: this.now(), + leaderId: this.tabId, + generation, + }); + this.post({ + type: "resync", + version: MESSAGE_VERSION, + tabId: this.tabId, + sentAt: this.now(), + leaderId: this.tabId, + generation, + reason: "release", + }); + } + if (resync) this.resyncAll(); + } + + private closeSource() { + if (!this.source) return; + this.source.close(); + this.source = null; + } + + private clearCandidate() { + if (this.candidateTimer) { + clearTimeout(this.candidateTimer); + this.candidateTimer = null; + } + this.ownCandidate = null; + } + + private scheduleNoLeaderCandidacy() { + if ( + this.noLeaderTimer || + this.ownCandidate || + this.isLeader || + this.leader || + this.subscriptions.size === 0 || + this.fallbackMode + ) { + return; + } + + this.noLeaderTimer = setTimeout(() => { + this.noLeaderTimer = null; + if (!this.leader && !this.isLeader) { + this.enterCandidacy("no-leader"); + } + }, this.timing.electionJitterMs); + } + + private clearNoLeaderTimer() { + if (!this.noLeaderTimer) return; + clearTimeout(this.noLeaderTimer); + this.noLeaderTimer = null; + } + + private shutdownTimersAndChannel() { + this.clearNoLeaderTimer(); + if (this.heartbeatTimer) { + clearInterval(this.heartbeatTimer); + this.heartbeatTimer = null; + } + if (this.leaderCheckTimer) { + clearInterval(this.leaderCheckTimer); + this.leaderCheckTimer = null; + } + this.removeVisibilityListener?.(); + this.removeVisibilityListener = null; + this.removePagehideListener?.(); + this.removePagehideListener = null; + if (this.channel) { + this.channel.close(); + this.channel = null; + } + } + + private post(message: CrossTabSseMessage): boolean { + if (!this.channel) return false; + try { + this.channel.postMessage(message); + return true; + } catch { + this.degradeToFallback(); + return false; + } + } + + private degradeToFallback() { + if (this.degradingToFallback || this.fallbackMode) return; + this.degradingToFallback = true; + this.releaseLeadership({ broadcast: false, resync: false }); + this.clearCandidate(); + this.shutdownTimersAndChannel(); + this.initialized = false; + this.fallbackMode = true; + this.coordinationUnavailable = true; + for (const subscription of this.subscriptions.values()) { + this.startFallbacksFor(subscription); + } + this.degradingToFallback = false; + } + + private startFallbacksFor(subscription: LocalSubscription) { + for (const fallback of subscription.fallbacks.values()) { + if (!fallback.cleanup) { + fallback.cleanup = fallback.subscribe(); + } + } + } + + private closeFallbacks() { + for (const subscription of this.subscriptions.values()) { + for (const fallback of subscription.fallbacks.values()) { + fallback.cleanup?.(); + fallback.cleanup = undefined; + } + } + } + + private shouldAcceptLeader(incoming: LeaderState): boolean { + if (!this.leader) return true; + if (incoming.generation > this.leader.generation) return true; + if (incoming.generation < this.leader.generation) return false; + if (incoming.leaderId === this.leader.leaderId) return true; + return leaderHasHigherPriority(incoming, this.leader); + } + + private isCurrentLeader(leaderId: string, generation: number): boolean { + return Boolean( + this.leader && + this.leader.leaderId === leaderId && + this.leader.generation === generation, + ); + } + + private currentVisibility(): TabVisibility { + return this.getVisibility() === "hidden" ? "hidden" : "visible"; + } +} + +const defaultCoordinator = new CrossTabSseCoordinator(); + +export function createCrossTabSseCoordinator(options: CrossTabSseCoordinatorOptions = {}) { + return new CrossTabSseCoordinator(options); +} + +export function subscribeToCrossTabSse({ + coordinator = defaultCoordinator, + ...options +}: SubscribeToCrossTabSseOptions): () => void { + return coordinator.subscribe(options); +} + +function createBrowserBroadcastChannel(name: string): BroadcastChannelLike { + if (typeof BroadcastChannel === "undefined") { + throw new Error("BroadcastChannel is unavailable"); + } + return new BroadcastChannel(name); +} + +function createTabId(): string { + if (typeof crypto !== "undefined" && typeof crypto.randomUUID === "function") { + return crypto.randomUUID(); + } + return `tab-${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`; +} + +function getBrowserVisibility(): TabVisibility { + if (typeof document === "undefined") return "visible"; + return document.visibilityState === "visible" ? "visible" : "hidden"; +} + +function addBrowserVisibilityChangeListener(handler: () => void): () => void { + if (typeof document === "undefined") return () => {}; + document.addEventListener("visibilitychange", handler); + return () => document.removeEventListener("visibilitychange", handler); +} + +function addBrowserPagehideListener(handler: () => void): () => void { + if (typeof window === "undefined") return () => {}; + window.addEventListener("pagehide", handler); + return () => window.removeEventListener("pagehide", handler); +} + +function parseMessage(data: unknown): CrossTabSseMessage | undefined { + if (!isRecord(data)) return undefined; + if (data.version !== MESSAGE_VERSION) return undefined; + const type = data.type; + const tabId = data.tabId; + const sentAt = data.sentAt; + if (typeof type !== "string") return undefined; + if (typeof tabId !== "string") return undefined; + if (typeof sentAt !== "number") return undefined; + const base = { tabId, sentAt }; + + switch (type) { + case "hello": + return baseMessage(base, "hello"); + case "heartbeat": { + const { leaderId, generation, visibility } = data; + if (typeof leaderId === "string" && typeof generation === "number" && isVisibility(visibility)) { + return { + ...baseMessage(base, "heartbeat"), + leaderId, + generation, + visibility, + }; + } + return undefined; + } + case "candidate": { + const { + candidateId, + candidateGeneration, + visibility, + observedLeaderId, + observedGeneration, + reason, + } = data; + if ( + typeof candidateId === "string" && + typeof candidateGeneration === "number" && + isVisibility(visibility) && + (typeof observedLeaderId === "string" || observedLeaderId === null) && + typeof observedGeneration === "number" && + isCandidateReason(reason) + ) { + const normalizedObservedLeaderId = + typeof observedLeaderId === "string" ? observedLeaderId : null; + return { + ...baseMessage(base, "candidate"), + candidateId, + candidateGeneration, + visibility, + observedLeaderId: normalizedObservedLeaderId, + observedGeneration, + reason, + }; + } + return undefined; + } + case "leader-changed": { + const { leaderId, generation, visibility } = data; + if (typeof leaderId === "string" && typeof generation === "number" && isVisibility(visibility)) { + return { + ...baseMessage(base, "leader-changed"), + leaderId, + generation, + visibility, + }; + } + return undefined; + } + case "release": { + const { leaderId, generation } = data; + if (typeof leaderId === "string" && typeof generation === "number") { + return { + ...baseMessage(base, "release"), + leaderId, + generation, + }; + } + return undefined; + } + case "resync": { + const { leaderId, generation, reason } = data; + if ( + (typeof leaderId === "string" || leaderId === null) && + typeof generation === "number" && + isCandidateReason(reason) + ) { + const normalizedLeaderId = typeof leaderId === "string" ? leaderId : null; + return { + ...baseMessage(base, "resync"), + leaderId: normalizedLeaderId, + generation, + reason, + }; + } + return undefined; + } + case "event": { + const { leaderId, generation, payload } = data; + if (typeof leaderId === "string" && typeof generation === "number" && isRecord(payload)) { + return { + ...baseMessage(base, "event"), + leaderId, + generation, + payload, + }; + } + return undefined; + } + default: + return undefined; + } +} + +function baseMessage( + data: { + tabId: string; + sentAt: number; + }, + type: TType, +) { + return { + type, + version: MESSAGE_VERSION, + tabId: data.tabId, + sentAt: data.sentAt, + }; +} + +function isVisibility(value: unknown): value is TabVisibility { + return value === "visible" || value === "hidden"; +} + +function isCandidateReason(value: unknown): value is CandidateReason { + return ( + value === "hidden-leader" || + value === "stale-leader" || + value === "release" || + value === "no-leader" + ); +} + +function candidateHasHigherPriority(candidate: CandidateMessage, other: CandidateMessage): boolean { + if (candidate.visibility !== other.visibility) return candidate.visibility === "visible"; + return candidate.candidateId < other.candidateId; +} + +function leaderHasHigherPriority(candidate: LeaderState, other: LeaderState): boolean { + if (candidate.visibility !== other.visibility) return candidate.visibility === "visible"; + return candidate.leaderId < other.leaderId; +} + +function leaderStateForCandidate(candidate: CandidateMessage, lastSeen: number): LeaderState { + return { + leaderId: candidate.candidateId, + generation: candidate.candidateGeneration, + visibility: candidate.visibility, + lastSeen, + }; +} + +function candidateKey(candidate: CandidateMessage): string { + return `${candidate.candidateGeneration}:${candidate.candidateId}`; +} + +function eventDedupeKey(payload: EventPayload): string | undefined { + if (typeof payload.id === "string" && payload.id.length > 0) { + return payload.id; + } + + const runId = typeof payload.run_id === "string" ? payload.run_id : undefined; + const seq = typeof payload.seq === "number" ? payload.seq : undefined; + const event = typeof payload.event === "string" ? payload.event : undefined; + if (runId && seq != null && event) { + return `${runId}:${seq}:${event}`; + } + return undefined; +} diff --git a/apps/fabro-web/app/lib/run-events.test.tsx b/apps/fabro-web/app/lib/run-events.test.tsx index 14acf4f23..db35a757d 100644 --- a/apps/fabro-web/app/lib/run-events.test.tsx +++ b/apps/fabro-web/app/lib/run-events.test.tsx @@ -4,6 +4,10 @@ import { queryKeysForRunEvent, subscribeToRunEvents, } from "./run-events"; +import { + createCrossTabSseCoordinator, + type BroadcastChannelLike, +} from "./cross-tab-sse"; import { queryKeys } from "./query-keys"; type MessageHandler = ((event: { data: string }) => void) | null; @@ -25,6 +29,14 @@ class FakeEventSource { } } +class FakeBroadcastChannel implements BroadcastChannelLike { + onmessage: ((event: { data: unknown }) => void) | null = null; + + postMessage() {} + + close() {} +} + describe("queryKeysForRunEvent", () => { test("terminal events invalidate run-scoped resources", () => { expect(queryKeysForRunEvent("run-1", "run.completed")).toEqual([ @@ -39,10 +51,74 @@ describe("queryKeysForRunEvent", () => { }); describe("subscribeToRunEvents", () => { - test("refcounts shared sources and keeps mutators active until final unsubscribe", () => { + test("coordinated mode uses the global attach stream and filters by run_id", async () => { const source = new FakeEventSource(); const created: string[] = []; const keys: string[] = []; + const coordinator = createCoordinator(); + + const cleanup = subscribeToRunEvents( + "run-coordinated", + (key) => { + keys.push(key); + return Promise.resolve(); + }, + (url) => { + created.push(url); + return source; + }, + { debounceMs: 0, coordinator }, + ); + + await waitFor(() => created.length === 1); + keys.length = 0; + + source.emit({ event: "checkpoint.completed", run_id: "other-run" }); + source.emit({ event: "checkpoint.completed", run_id: "run-coordinated" }); + + expect(created).toEqual(["/api/v1/attach"]); + expect(keys).toEqual([queryKeys.runs.files("run-coordinated")]); + + cleanup(); + coordinator.close(); + }); + + test("coordinated terminal events invalidate without closing the global stream", async () => { + const source = new FakeEventSource(); + const keys: string[] = []; + const coordinator = createCoordinator(); + const cleanup = subscribeToRunEvents( + "run-terminal", + (key) => { + keys.push(key); + return Promise.resolve(); + }, + () => source, + { debounceMs: 0, coordinator }, + ); + + await waitFor(() => source.onmessage !== null); + keys.length = 0; + + source.emit({ event: "run.failed", run_id: "run-terminal" }); + expect(source.closed).toBe(false); + expect(keys).toContain(queryKeys.runs.files("run-terminal")); + expect(keys).toContain(queryKeys.runs.billing("run-terminal")); + + keys.length = 0; + source.emit({ event: "run.archived", run_id: "run-terminal" }); + expect(source.closed).toBe(false); + expect(keys).toEqual([queryKeys.runs.detail("run-terminal")]); + + cleanup(); + coordinator.close(); + }); + + test("fallback refcounts run-scoped sources and keeps mutators active until final unsubscribe", () => { + const source = new FakeEventSource(); + const created: string[] = []; + const keys: string[] = []; + const coordinator = createFallbackCoordinator(); const mutate = (key: string) => { keys.push(key); return Promise.resolve(); @@ -51,10 +127,10 @@ describe("subscribeToRunEvents", () => { const firstCleanup = subscribeToRunEvents("run-refcount", mutate, (url) => { created.push(url); return source; - }, { debounceMs: 0 }); + }, { debounceMs: 0, coordinator }); const secondCleanup = subscribeToRunEvents("run-refcount", mutate, () => { throw new Error("source should be reused"); - }, { debounceMs: 0 }); + }, { debounceMs: 0, coordinator }); expect(created).toEqual(["/api/v1/runs/run-refcount/attach"]); @@ -66,11 +142,13 @@ describe("subscribeToRunEvents", () => { secondCleanup(); expect(source.closed).toBe(true); + coordinator.close(); }); - test("terminal events close the source after invalidating keys", () => { + test("fallback terminal events close the source after invalidating keys", () => { const source = new FakeEventSource(); const keys: string[] = []; + const coordinator = createFallbackCoordinator(); const cleanup = subscribeToRunEvents( "run-terminal", (key) => { @@ -78,7 +156,7 @@ describe("subscribeToRunEvents", () => { return Promise.resolve(); }, () => source, - { debounceMs: 0 }, + { debounceMs: 0, coordinator }, ); source.emit({ event: "run.failed" }); @@ -88,13 +166,15 @@ describe("subscribeToRunEvents", () => { expect(keys).toContain(queryKeys.runs.billing("run-terminal")); cleanup(); + coordinator.close(); }); - test("malformed events are ignored and StrictMode-style cleanup does not underflow", () => { + test("fallback malformed events are ignored and StrictMode-style cleanup does not underflow", () => { const firstSource = new FakeEventSource(); const secondSource = new FakeEventSource(); const sources = [firstSource, secondSource]; const keys: string[] = []; + const coordinator = createFallbackCoordinator(); const firstCleanup = subscribeToRunEvents( "run-strict", @@ -103,7 +183,7 @@ describe("subscribeToRunEvents", () => { return Promise.resolve(); }, () => sources.shift()!, - { debounceMs: 0 }, + { debounceMs: 0, coordinator }, ); firstSource.emitRaw("{broken"); firstCleanup(); @@ -115,12 +195,44 @@ describe("subscribeToRunEvents", () => { return Promise.resolve(); }, () => sources.shift()!, - { debounceMs: 0 }, + { debounceMs: 0, coordinator }, ); secondCleanup(); expect(keys).toEqual([]); expect(firstSource.closed).toBe(true); expect(secondSource.closed).toBe(true); + coordinator.close(); }); }); + +function createCoordinator() { + return createCrossTabSseCoordinator({ + tabId: "run-test", + channelFactory: () => new FakeBroadcastChannel(), + addVisibilityChangeListener: () => () => {}, + addPagehideListener: () => () => {}, + timing: { + heartbeatMs: 10, + leaderStaleMs: 50, + electionJitterMs: 0, + }, + }); +} + +function createFallbackCoordinator() { + return createCrossTabSseCoordinator({ + channelFactory: () => { + throw new Error("BroadcastChannel unavailable"); + }, + }); +} + +async function waitFor(condition: () => boolean, timeoutMs = 200) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (condition()) return; + await new Promise((resolve) => setTimeout(resolve, 2)); + } + throw new Error("condition did not become true before timeout"); +} diff --git a/apps/fabro-web/app/lib/run-events.ts b/apps/fabro-web/app/lib/run-events.ts index 924264af2..80394da48 100644 --- a/apps/fabro-web/app/lib/run-events.ts +++ b/apps/fabro-web/app/lib/run-events.ts @@ -1,6 +1,10 @@ import { useEffect } from "react"; import { useSWRConfig } from "swr"; +import { + subscribeToCrossTabSse, + type CrossTabSseCoordinator, +} from "./cross-tab-sse"; import { queryKeys } from "./query-keys"; import { createBrowserEventSource, @@ -13,10 +17,16 @@ import { interface RunEventPayload extends EventPayload { event?: string; + run_id?: string; node_id?: string; properties?: Record; } +interface RunEventOptions { + debounceMs?: number; + coordinator?: CrossTabSseCoordinator; +} + const subscriptions = new Map(); const TERMINAL_EVENTS = new Set(["run.completed", "run.failed"]); @@ -104,31 +114,64 @@ export function subscribeToRunEvents( runId: string, mutate: MutateFn, eventSourceFactory: (url: string) => EventSourceLike = createBrowserEventSource, - { debounceMs = 300 }: { debounceMs?: number } = {}, + { debounceMs = 300, coordinator }: RunEventOptions = {}, ): () => void { - return subscribeToSharedEventSource({ - subscriptions, - subscriptionKey: runId, - url: queryKeys.runs.attach(runId), + return subscribeToCrossTabSse({ + coordinator, + subscriptionKey: `run:${runId}`, mutate, eventSourceFactory, debounceMs, + resyncKeys: () => resyncKeysForRun(runId), resolveInvalidation: (payload) => { - const event = payload.event; - if (!event) return { keys: [] }; - - const stageId = stageIdFromPayload(payload); - const keys = queryKeysForRunEvent(runId, event, stageId); - const terminal = TERMINAL_EVENTS.has(event); - return { - keys, - close: terminal, - immediate: terminal, - }; + if (payload.run_id !== runId) return { keys: [] }; + return runInvalidation(runId, payload, { closeOnTerminal: false }); }, + fallbackSubscribe: () => + subscribeToSharedEventSource({ + subscriptions, + subscriptionKey: runId, + url: queryKeys.runs.attach(runId), + mutate, + eventSourceFactory, + debounceMs, + resolveInvalidation: (payload) => + runInvalidation(runId, payload, { closeOnTerminal: true }), + }), }); } +function runInvalidation( + runId: string, + payload: RunEventPayload, + { closeOnTerminal }: { closeOnTerminal: boolean }, +) { + const event = payload.event; + if (!event) return { keys: [] }; + + const stageId = stageIdFromPayload(payload); + const keys = queryKeysForRunEvent(runId, event, stageId); + const terminal = TERMINAL_EVENTS.has(event); + return { + keys, + close: closeOnTerminal && terminal, + immediate: terminal, + }; +} + +function resyncKeysForRun(runId: string) { + return [ + queryKeys.runs.detail(runId), + queryKeys.runs.files(runId), + queryKeys.runs.billing(runId), + queryKeys.runs.stages(runId), + queryKeys.runs.events(runId, 1000), + queryKeys.runs.graph(runId, "LR"), + queryKeys.runs.graph(runId, "TB"), + queryKeys.runs.questions(runId, 25, 0), + ]; +} + function stageIdFromPayload(payload: RunEventPayload): string | undefined { if (typeof payload.node_id === "string") return payload.node_id; const nodeId = payload.properties?.node_id; From 38726666afecdb2b928d0b5f4a5a5fe5e0410319 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Mon, 4 May 2026 15:24:25 -0400 Subject: [PATCH 03/16] fix(web): harden cross-tab SSE fallback Stop coordinated election and leadership work when BroadcastChannel posting fails, so tabs degrade cleanly to per-subscriber fallback without stale resync or heartbeat side effects. Expand election coverage for the edge cases called out in the coordination plan. --- apps/fabro-web/app/lib/cross-tab-sse.test.ts | 183 ++++++++++++++++++- apps/fabro-web/app/lib/cross-tab-sse.ts | 21 ++- 2 files changed, 189 insertions(+), 15 deletions(-) diff --git a/apps/fabro-web/app/lib/cross-tab-sse.test.ts b/apps/fabro-web/app/lib/cross-tab-sse.test.ts index 32cff001c..573ef6589 100644 --- a/apps/fabro-web/app/lib/cross-tab-sse.test.ts +++ b/apps/fabro-web/app/lib/cross-tab-sse.test.ts @@ -40,6 +40,7 @@ class FakeEventSource { class FakeBroadcastChannel implements BroadcastChannelLike { static channels = new Set(); static muted = false; + static throwOnTypes = new Set(); onmessage: ((event: { data: unknown }) => void) | null = null; closed = false; @@ -49,6 +50,9 @@ class FakeBroadcastChannel implements BroadcastChannelLike { } postMessage(message: CrossTabSseMessage) { + if (FakeBroadcastChannel.throwOnTypes.has(message.type)) { + throw new Error(`postMessage failed for ${message.type}`); + } if (FakeBroadcastChannel.muted) return; const recipients = [...FakeBroadcastChannel.channels].filter( (channel) => channel !== this && !channel.closed && channel.name === this.name, @@ -61,6 +65,15 @@ class FakeBroadcastChannel implements BroadcastChannelLike { }); } + static broadcastExternal(message: CrossTabSseMessage) { + queueMicrotask(() => { + for (const channel of FakeBroadcastChannel.channels) { + if (channel.closed) continue; + channel.onmessage?.({ data: { ...message } }); + } + }); + } + close() { this.closed = true; FakeBroadcastChannel.channels.delete(this); @@ -72,6 +85,7 @@ class FakeBroadcastChannel implements BroadcastChannelLike { } FakeBroadcastChannel.channels.clear(); FakeBroadcastChannel.muted = false; + FakeBroadcastChannel.throwOnTypes.clear(); } } @@ -166,6 +180,40 @@ describe("subscribeToCrossTabSse", () => { expect(keysByTab.get("c")).toEqual(["event"]); }); + test("board and run subscriptions coexist on the same global stream", async () => { + const harness = newHarness(); + const coordinator = harness.createTab("a"); + const boardKeys: string[] = []; + const runKeys: string[] = []; + + subscribeForEvent(coordinator, { + subscriptionKey: "board", + keys: boardKeys, + resolveInvalidation: (payload) => ({ + keys: payload.event === "run.running" ? ["board"] : [], + }), + resyncKeys: () => ["board-resync"], + }); + subscribeForEvent(coordinator, { + subscriptionKey: "run:run-1", + keys: runKeys, + resolveInvalidation: (payload) => ({ + keys: payload.event === "run.running" && payload.run_id === "run-1" ? ["run"] : [], + }), + resyncKeys: () => ["run-resync"], + }); + + await waitFor(() => harness.openSources().length === 1); + boardKeys.length = 0; + runKeys.length = 0; + + harness.openSources()[0].emit(runEvent({ id: "evt-coexist", runId: "run-1", seq: 1 })); + + expect(boardKeys).toEqual(["board"]); + expect(runKeys).toEqual(["run"]); + expect(harness.openSources().map((source) => source.url)).toEqual(["/api/v1/attach"]); + }); + test("dedupes duplicate event ids until TTL or max-size eviction", async () => { const harness = newHarness(); const keys: string[] = []; @@ -218,7 +266,8 @@ describe("subscribeToCrossTabSse", () => { subscribeForRunEvent(harness.createTab("b", "visible"), []); subscribeForRunEvent(harness.createTab("a", "visible"), []); - await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "a"); + await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner !== "z"); + expect(harness.openSources().map((source) => source.owner)).toEqual(["a"]); }); test("a lower lexical follower does not preempt a fresh visible leader", async () => { @@ -250,6 +299,46 @@ describe("subscribeToCrossTabSse", () => { expect(followerKeys).toContain("resync"); }); + test("simultaneous stale leader elections resolve to the lexical winner", async () => { + const harness = newHarness(); + + subscribeForRunEvent(harness.createTab("z"), []); + await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "z"); + subscribeForRunEvent(harness.createTab("b"), []); + subscribeForRunEvent(harness.createTab("a"), []); + await sleep(TEST_TIMING.heartbeatMs * 2); + + harness.coordinators.get("z")?.close(); + harness.now += TEST_TIMING.leaderStaleMs + TEST_TIMING.heartbeatMs + 1; + + await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "a"); + }); + + test("hidden leader ignores candidates for old observed leadership", async () => { + const harness = newHarness(); + + subscribeForRunEvent(harness.createTab("z", "hidden"), []); + await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "z"); + const hiddenSource = harness.openSources()[0]; + + FakeBroadcastChannel.broadcastExternal({ + type: "candidate", + version: 1, + tabId: "ghost", + sentAt: harness.now, + candidateId: "ghost", + candidateGeneration: 1, + visibility: "visible", + observedLeaderId: "z", + observedGeneration: 0, + reason: "hidden-leader", + }); + await sleep(TEST_TIMING.electionJitterMs * 2); + + expect(hiddenSource.closed).toBe(false); + expect(harness.openSources().map((source) => source.owner)).toEqual(["z"]); + }); + test("same-generation split brain converges to the higher-priority visible leader", async () => { const harness = newHarness(); FakeBroadcastChannel.muted = true; @@ -278,6 +367,29 @@ describe("subscribeToCrossTabSse", () => { expect(keys).toEqual([]); }); + test("old leader heartbeats are ignored after takeover", async () => { + const harness = newHarness(); + + subscribeForRunEvent(harness.createTab("z", "hidden"), []); + await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "z"); + + subscribeForRunEvent(harness.createTab("a", "visible"), []); + await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "a"); + + FakeBroadcastChannel.broadcastExternal({ + type: "heartbeat", + version: 1, + tabId: "z", + sentAt: harness.now, + leaderId: "z", + generation: 1, + visibility: "hidden", + }); + await sleep(TEST_TIMING.heartbeatMs * 2); + + expect(harness.openSources().map((source) => source.owner)).toEqual(["a"]); + }); + test("last unsubscribe closes the leader source and releases leadership", async () => { const harness = newHarness(); const cleanup = subscribeForRunEvent(harness.createTab("a"), []); @@ -320,6 +432,41 @@ describe("subscribeToCrossTabSse", () => { expect(fallbackStarted).toBe(1); expect(fallbackStopped).toBe(1); }); + + test("postMessage failure after initialization degrades to fallback without coordinated resync", async () => { + const harness = newHarness(); + const coordinator = harness.createTab("a"); + const keys: string[] = []; + let fallbackStarted = 0; + let fallbackStopped = 0; + + FakeBroadcastChannel.throwOnTypes.add("leader-changed"); + const cleanup = subscribeToCrossTabSse({ + coordinator, + subscriptionKey: "throwing-channel", + mutate: ((key: string) => { + keys.push(key); + return Promise.resolve(); + }) as MutateFn, + resolveInvalidation: () => ({ keys: ["event"] }), + resyncKeys: () => ["resync"], + fallbackSubscribe: () => { + fallbackStarted += 1; + return () => { + fallbackStopped += 1; + }; + }, + debounceMs: 0, + }); + + await waitFor(() => fallbackStarted === 1); + + expect(harness.openSources()).toEqual([]); + expect(keys).toEqual([]); + + cleanup(); + expect(fallbackStopped).toBe(1); + }); }); function newHarness() { @@ -329,17 +476,39 @@ function newHarness() { } function subscribeForRunEvent(coordinator: CrossTabSseCoordinator, keys: string[]) { - return subscribeToCrossTabSse({ - coordinator, + return subscribeForEvent(coordinator, { subscriptionKey: "run-feed", - mutate: ((key: string) => { - keys.push(key); - return Promise.resolve(); - }) as MutateFn, + keys, resolveInvalidation: (payload) => ({ keys: payload.event === "run.running" ? ["event"] : [], }), resyncKeys: () => ["resync"], + }); +} + +function subscribeForEvent( + coordinator: CrossTabSseCoordinator, + { + subscriptionKey, + keys, + resolveInvalidation, + resyncKeys, + }: { + subscriptionKey: string; + keys: string[]; + resolveInvalidation: (payload: EventPayload) => { keys: string[] }; + resyncKeys: () => string[]; + }, +) { + return subscribeToCrossTabSse({ + coordinator, + subscriptionKey, + mutate: ((key: string) => { + keys.push(key); + return Promise.resolve(); + }) as MutateFn, + resolveInvalidation, + resyncKeys, fallbackSubscribe: () => { throw new Error("fallback should not be used"); }, diff --git a/apps/fabro-web/app/lib/cross-tab-sse.ts b/apps/fabro-web/app/lib/cross-tab-sse.ts index 255c4c7ce..0ddb6e220 100644 --- a/apps/fabro-web/app/lib/cross-tab-sse.ts +++ b/apps/fabro-web/app/lib/cross-tab-sse.ts @@ -594,7 +594,7 @@ export class CrossTabSseCoordinator { this.ownCandidate = candidate; this.candidates.set(candidateKey(candidate), candidate); - this.post(candidate); + if (!this.post(candidate)) return; this.candidateTimer = setTimeout(() => { this.completeCandidacy(candidate); }, this.timing.electionJitterMs); @@ -651,7 +651,7 @@ export class CrossTabSseCoordinator { this.handleLeaderEventSourceMessage(message.data); }; - this.post({ + const announced = this.post({ type: "leader-changed", version: MESSAGE_VERSION, tabId: this.tabId, @@ -660,6 +660,7 @@ export class CrossTabSseCoordinator { generation, visibility: this.currentVisibility(), }); + if (!announced) return; this.startHeartbeat(); this.resyncAll(); } @@ -761,14 +762,14 @@ export class CrossTabSseCoordinator { if (this.heartbeatTimer) { clearInterval(this.heartbeatTimer); } - this.sendHeartbeat(); + if (!this.sendHeartbeat()) return; this.heartbeatTimer = setInterval(() => { this.sendHeartbeat(); }, this.timing.heartbeatMs); } - private sendHeartbeat() { - if (!this.isLeader) return; + private sendHeartbeat(): boolean { + if (!this.isLeader) return false; const visibility = this.currentVisibility(); this.leader = { leaderId: this.tabId, @@ -776,7 +777,7 @@ export class CrossTabSseCoordinator { visibility, lastSeen: this.now(), }; - this.post({ + return this.post({ type: "heartbeat", version: MESSAGE_VERSION, tabId: this.tabId, @@ -806,7 +807,7 @@ export class CrossTabSseCoordinator { this.leader = null; if (broadcast) { - this.post({ + const released = this.post({ type: "release", version: MESSAGE_VERSION, tabId: this.tabId, @@ -814,6 +815,7 @@ export class CrossTabSseCoordinator { leaderId: this.tabId, generation, }); + if (!released) return; this.post({ type: "resync", version: MESSAGE_VERSION, @@ -888,7 +890,10 @@ export class CrossTabSseCoordinator { } private post(message: CrossTabSseMessage): boolean { - if (!this.channel) return false; + if (!this.channel) { + this.degradeToFallback(); + return false; + } try { this.channel.postMessage(message); return true; From 65298455545f3b43449e94e7b71e37b93535db9d Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Mon, 4 May 2026 15:29:42 -0400 Subject: [PATCH 04/16] fix(web): clean up cross-tab SSE lifecycle Prune stale election candidates as generations advance, reset coordination availability on explicit close, and keep fallback subscribers tracked so coordinator shutdown can clean them up consistently. --- apps/fabro-web/app/lib/cross-tab-sse.test.ts | 137 +++++++++++++++++++ apps/fabro-web/app/lib/cross-tab-sse.ts | 34 +++-- 2 files changed, 161 insertions(+), 10 deletions(-) diff --git a/apps/fabro-web/app/lib/cross-tab-sse.test.ts b/apps/fabro-web/app/lib/cross-tab-sse.test.ts index 573ef6589..f38c881fa 100644 --- a/apps/fabro-web/app/lib/cross-tab-sse.test.ts +++ b/apps/fabro-web/app/lib/cross-tab-sse.test.ts @@ -339,6 +339,32 @@ describe("subscribeToCrossTabSse", () => { expect(harness.openSources().map((source) => source.owner)).toEqual(["z"]); }); + test("prunes candidate records from older generations", async () => { + const harness = newHarness(); + subscribeForRunEvent(harness.createTab("z", "hidden"), []); + await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "z"); + + const coordinator = harness.createTab("a", "visible"); + subscribeForRunEvent(coordinator, []); + await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "a"); + + FakeBroadcastChannel.broadcastExternal({ + type: "candidate", + version: 1, + tabId: "old-candidate", + sentAt: harness.now, + candidateId: "old-candidate", + candidateGeneration: 1, + visibility: "visible", + observedLeaderId: "previous-leader", + observedGeneration: 0, + reason: "stale-leader", + }); + await sleep(TEST_TIMING.electionJitterMs * 2); + + expect(candidateGenerations(coordinator)).not.toContain(1); + }); + test("same-generation split brain converges to the higher-priority visible leader", async () => { const harness = newHarness(); FakeBroadcastChannel.muted = true; @@ -467,6 +493,89 @@ describe("subscribeToCrossTabSse", () => { cleanup(); expect(fallbackStopped).toBe(1); }); + + test("close resets coordination availability after an initial channel failure", async () => { + let channelUnavailable = true; + const sources: FakeEventSource[] = []; + const coordinator = createCrossTabSseCoordinator({ + tabId: "recovering", + channelFactory: (name) => { + if (channelUnavailable) throw new Error("channel unavailable"); + return new FakeBroadcastChannel(name); + }, + eventSourceFactory: (url) => { + const source = new FakeEventSource(url, "recovering"); + sources.push(source); + return source; + }, + addVisibilityChangeListener: () => () => {}, + addPagehideListener: () => () => {}, + timing: TEST_TIMING, + }); + let firstFallbackStarted = 0; + let secondFallbackStarted = 0; + + const firstCleanup = subscribeWithFallback(coordinator, { + fallbackSubscribe: () => { + firstFallbackStarted += 1; + return () => {}; + }, + }); + firstCleanup(); + coordinator.close(); + + channelUnavailable = false; + const secondCleanup = subscribeWithFallback(coordinator, { + fallbackSubscribe: () => { + secondFallbackStarted += 1; + return () => {}; + }, + }); + + await waitFor(() => sources.some((source) => !source.closed)); + + expect(firstFallbackStarted).toBe(1); + expect(secondFallbackStarted).toBe(0); + expect(sources.filter((source) => !source.closed).map((source) => source.url)).toEqual(["/api/v1/attach"]); + + secondCleanup(); + coordinator.close(); + }); + + test("close stops fallback subscriptions added after degradation", async () => { + const harness = newHarness(); + const coordinator = harness.createTab("a"); + let fallbackStarted = 0; + let fallbackStopped = 0; + + FakeBroadcastChannel.throwOnTypes.add("leader-changed"); + subscribeWithFallback(coordinator, { + subscriptionKey: "before-degrade", + fallbackSubscribe: () => { + fallbackStarted += 1; + return () => { + fallbackStopped += 1; + }; + }, + }); + await waitFor(() => fallbackStarted === 1); + + FakeBroadcastChannel.throwOnTypes.clear(); + subscribeWithFallback(coordinator, { + subscriptionKey: "after-degrade", + fallbackSubscribe: () => { + fallbackStarted += 1; + return () => { + fallbackStopped += 1; + }; + }, + }); + expect(fallbackStarted).toBe(2); + + coordinator.close(); + + expect(fallbackStopped).toBe(2); + }); }); function newHarness() { @@ -516,6 +625,34 @@ function subscribeForEvent( }); } +function subscribeWithFallback( + coordinator: CrossTabSseCoordinator, + { + subscriptionKey = "fallback-test", + fallbackSubscribe, + }: { + subscriptionKey?: string; + fallbackSubscribe: () => () => void; + }, +) { + return subscribeToCrossTabSse({ + coordinator, + subscriptionKey, + mutate: (() => Promise.resolve()) as MutateFn, + resolveInvalidation: () => ({ keys: [] }), + resyncKeys: () => [], + fallbackSubscribe, + debounceMs: 0, + }); +} + +function candidateGenerations(coordinator: CrossTabSseCoordinator): number[] { + const inspectable = coordinator as unknown as { + candidates: Map; + }; + return [...inspectable.candidates.values()].map((candidate) => candidate.candidateGeneration); +} + function runEvent({ id, runId, diff --git a/apps/fabro-web/app/lib/cross-tab-sse.ts b/apps/fabro-web/app/lib/cross-tab-sse.ts index 0ddb6e220..97ffeb9f7 100644 --- a/apps/fabro-web/app/lib/cross-tab-sse.ts +++ b/apps/fabro-web/app/lib/cross-tab-sse.ts @@ -231,17 +231,16 @@ export class CrossTabSseCoordinator { this.sourceFactoryLocked = true; } - if (this.coordinationUnavailable) { - return options.fallbackSubscribe(); - } - - if (!this.initialized && !this.initialize()) { - this.coordinationUnavailable = true; - return options.fallbackSubscribe(); - } - const subscription = this.addLocalSubscription(options); - if (this.fallbackMode) { + + if (this.coordinationUnavailable) { + this.fallbackMode = true; + this.startFallbacksFor(subscription); + } else if (!this.initialized && !this.initialize()) { + this.coordinationUnavailable = true; + this.fallbackMode = true; + this.startFallbacksFor(subscription); + } else if (this.fallbackMode) { this.startFallbacksFor(subscription); } else { this.ensureLeadershipProgress(); @@ -262,8 +261,10 @@ export class CrossTabSseCoordinator { this.shutdownTimersAndChannel(); this.closeFallbacks(); this.subscriptions.clear(); + this.candidates.clear(); this.leader = null; this.initialized = false; + this.coordinationUnavailable = false; this.fallbackMode = false; this.sourceFactoryLocked = false; } @@ -453,6 +454,7 @@ export class CrossTabSseCoordinator { this.leader = incoming; this.clearNoLeaderTimer(); this.generation = Math.max(this.generation, incoming.generation); + this.pruneStaleCandidates(); if (this.ownCandidate && incoming.generation >= this.ownCandidate.candidateGeneration) { this.clearCandidate(); } @@ -473,6 +475,7 @@ export class CrossTabSseCoordinator { private handleCandidate(message: CandidateMessage) { this.candidates.set(candidateKey(message), message); + this.pruneStaleCandidates(); if ( this.isLeader && @@ -500,6 +503,7 @@ export class CrossTabSseCoordinator { ) { this.leader = null; this.generation = Math.max(this.generation, message.generation); + this.pruneStaleCandidates(); this.enterCandidacy("release", { leaderId: message.leaderId, generation: message.generation, @@ -546,6 +550,7 @@ export class CrossTabSseCoordinator { if (this.now() - current.lastSeen > this.timing.leaderStaleMs) { this.leader = null; this.generation = Math.max(this.generation, current.generation); + this.pruneStaleCandidates(); this.resyncAll(); this.enterCandidacy("stale-leader", current); return; @@ -638,6 +643,7 @@ export class CrossTabSseCoordinator { this.closeSource(); this.isLeader = true; this.generation = generation; + this.pruneStaleCandidates(); this.leader = { leaderId: this.tabId, generation, @@ -843,6 +849,14 @@ export class CrossTabSseCoordinator { this.ownCandidate = null; } + private pruneStaleCandidates() { + for (const [key, candidate] of this.candidates) { + if (candidate.candidateGeneration < this.generation) { + this.candidates.delete(key); + } + } + } + private scheduleNoLeaderCandidacy() { if ( this.noLeaderTimer || From e4e51511e08c29a196146a92862f0b1a6457a3ce Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Mon, 4 May 2026 15:38:19 -0400 Subject: [PATCH 05/16] refactor(web): simplify cross-tab SSE message parsing and helpers Use unknown.ts helpers in parseMessage, factor out parseLeaderPair/Triple and per-variant parsers to remove repeated typeof guards. Extract leaderIsFresh() for the staleness check used in three places, and make RecentEventCache amortized O(1) by walking expired entries from the oldest instead of scanning the whole map per event. Drop the closeOnTerminal parameter in run-events; the fallback path computes close at its single call site. Co-Authored-By: Claude Opus 4.7 (1M context) --- apps/fabro-web/app/lib/cross-tab-sse.ts | 216 +++++++++++------------- apps/fabro-web/app/lib/run-events.ts | 22 +-- 2 files changed, 104 insertions(+), 134 deletions(-) diff --git a/apps/fabro-web/app/lib/cross-tab-sse.ts b/apps/fabro-web/app/lib/cross-tab-sse.ts index 97ffeb9f7..99281a8f0 100644 --- a/apps/fabro-web/app/lib/cross-tab-sse.ts +++ b/apps/fabro-web/app/lib/cross-tab-sse.ts @@ -6,7 +6,7 @@ import { type EventSourceLike, type MutateFn, } from "./sse"; -import { isRecord } from "./unknown"; +import { getNumber, getString, isRecord, type UnknownRecord } from "./unknown"; export const CROSS_TAB_SSE_CHANNEL = "fabro:sse:v1"; export const HEARTBEAT_MS = 1000; @@ -156,7 +156,7 @@ class RecentEventCache { remember(key: string | undefined, now: number): boolean { if (!key) return true; - this.prune(now); + this.evictExpired(now); if (this.seen.has(key)) return false; this.seen.set(key, now); @@ -168,11 +168,10 @@ class RecentEventCache { return true; } - private prune(now: number) { + private evictExpired(now: number) { for (const [key, seenAt] of this.seen) { - if (now - seenAt > this.ttlMs) { - this.seen.delete(key); - } + if (now - seenAt <= this.ttlMs) return; + this.seen.delete(key); } } } @@ -547,7 +546,7 @@ export class CrossTabSseCoordinator { return; } - if (this.now() - current.lastSeen > this.timing.leaderStaleMs) { + if (!this.leaderIsFresh(current)) { this.leader = null; this.generation = Math.max(this.generation, current.generation); this.pruneStaleCandidates(); @@ -568,7 +567,7 @@ export class CrossTabSseCoordinator { reason === "no-leader" && this.leader && this.leader.visibility === "visible" && - this.now() - this.leader.lastSeen <= this.timing.leaderStaleMs + this.leaderIsFresh(this.leader) ) { return; } @@ -618,10 +617,7 @@ export class CrossTabSseCoordinator { } } - if ( - this.leader && - this.now() - this.leader.lastSeen <= this.timing.leaderStaleMs - ) { + if (this.leader && this.leaderIsFresh(this.leader)) { if (this.leader.generation > candidate.candidateGeneration) { this.clearCandidate(); return; @@ -968,6 +964,10 @@ export class CrossTabSseCoordinator { private currentVisibility(): TabVisibility { return this.getVisibility() === "hidden" ? "hidden" : "visible"; } + + private leaderIsFresh(leader: LeaderState): boolean { + return this.now() - leader.lastSeen <= this.timing.leaderStaleMs; + } } const defaultCoordinator = new CrossTabSseCoordinator(); @@ -1017,132 +1017,108 @@ function addBrowserPagehideListener(handler: () => void): () => void { function parseMessage(data: unknown): CrossTabSseMessage | undefined { if (!isRecord(data)) return undefined; if (data.version !== MESSAGE_VERSION) return undefined; - const type = data.type; - const tabId = data.tabId; - const sentAt = data.sentAt; - if (typeof type !== "string") return undefined; - if (typeof tabId !== "string") return undefined; - if (typeof sentAt !== "number") return undefined; - const base = { tabId, sentAt }; + const type = getString(data, "type"); + const tabId = getString(data, "tabId"); + const sentAt = getNumber(data, "sentAt"); + if (!type || !tabId || sentAt === undefined) return undefined; + const base = { type, version: MESSAGE_VERSION, tabId, sentAt }; switch (type) { case "hello": - return baseMessage(base, "hello"); + return { ...base, type: "hello" }; case "heartbeat": { - const { leaderId, generation, visibility } = data; - if (typeof leaderId === "string" && typeof generation === "number" && isVisibility(visibility)) { - return { - ...baseMessage(base, "heartbeat"), - leaderId, - generation, - visibility, - }; - } - return undefined; - } - case "candidate": { - const { - candidateId, - candidateGeneration, - visibility, - observedLeaderId, - observedGeneration, - reason, - } = data; - if ( - typeof candidateId === "string" && - typeof candidateGeneration === "number" && - isVisibility(visibility) && - (typeof observedLeaderId === "string" || observedLeaderId === null) && - typeof observedGeneration === "number" && - isCandidateReason(reason) - ) { - const normalizedObservedLeaderId = - typeof observedLeaderId === "string" ? observedLeaderId : null; - return { - ...baseMessage(base, "candidate"), - candidateId, - candidateGeneration, - visibility, - observedLeaderId: normalizedObservedLeaderId, - observedGeneration, - reason, - }; - } - return undefined; + const triple = parseLeaderTriple(data); + return triple && { ...base, type: "heartbeat", ...triple }; } + case "candidate": + return parseCandidate(data, base); case "leader-changed": { - const { leaderId, generation, visibility } = data; - if (typeof leaderId === "string" && typeof generation === "number" && isVisibility(visibility)) { - return { - ...baseMessage(base, "leader-changed"), - leaderId, - generation, - visibility, - }; - } - return undefined; + const triple = parseLeaderTriple(data); + return triple && { ...base, type: "leader-changed", ...triple }; } case "release": { - const { leaderId, generation } = data; - if (typeof leaderId === "string" && typeof generation === "number") { - return { - ...baseMessage(base, "release"), - leaderId, - generation, - }; - } - return undefined; - } - case "resync": { - const { leaderId, generation, reason } = data; - if ( - (typeof leaderId === "string" || leaderId === null) && - typeof generation === "number" && - isCandidateReason(reason) - ) { - const normalizedLeaderId = typeof leaderId === "string" ? leaderId : null; - return { - ...baseMessage(base, "resync"), - leaderId: normalizedLeaderId, - generation, - reason, - }; - } - return undefined; - } - case "event": { - const { leaderId, generation, payload } = data; - if (typeof leaderId === "string" && typeof generation === "number" && isRecord(payload)) { - return { - ...baseMessage(base, "event"), - leaderId, - generation, - payload, - }; - } - return undefined; + const pair = parseLeaderPair(data); + return pair && { ...base, type: "release", ...pair }; } + case "resync": + return parseResync(data, base); + case "event": + return parseEvent(data, base); default: return undefined; } } -function baseMessage( - data: { - tabId: string; - sentAt: number; - }, - type: TType, -) { +interface ParsedBase { + version: typeof MESSAGE_VERSION; + tabId: string; + sentAt: number; +} + +function parseLeaderPair(data: unknown): { leaderId: string; generation: number } | undefined { + const leaderId = getString(data, "leaderId"); + const generation = getNumber(data, "generation"); + if (!leaderId || generation === undefined) return undefined; + return { leaderId, generation }; +} + +function parseLeaderTriple( + data: unknown, +): { leaderId: string; generation: number; visibility: TabVisibility } | undefined { + const pair = parseLeaderPair(data); + const visibility = getString(data, "visibility"); + if (!pair || !isVisibility(visibility)) return undefined; + return { ...pair, visibility }; +} + +function parseCandidate(data: UnknownRecord, base: ParsedBase): CandidateMessage | undefined { + const candidateId = getString(data, "candidateId"); + const candidateGeneration = getNumber(data, "candidateGeneration"); + const visibility = getString(data, "visibility"); + const observedGeneration = getNumber(data, "observedGeneration"); + const reason = getString(data, "reason"); + const observedLeaderRaw = data.observedLeaderId; + const observedLeaderId = + typeof observedLeaderRaw === "string" ? observedLeaderRaw : observedLeaderRaw === null ? null : undefined; + if ( + !candidateId || + candidateGeneration === undefined || + !isVisibility(visibility) || + observedLeaderId === undefined || + observedGeneration === undefined || + !isCandidateReason(reason) + ) { + return undefined; + } return { - type, - version: MESSAGE_VERSION, - tabId: data.tabId, - sentAt: data.sentAt, + ...base, + type: "candidate", + candidateId, + candidateGeneration, + visibility, + observedLeaderId, + observedGeneration, + reason, }; } +function parseResync(data: UnknownRecord, base: ParsedBase): ResyncMessage | undefined { + const generation = getNumber(data, "generation"); + const reason = getString(data, "reason"); + const leaderRaw = data.leaderId; + const leaderId = typeof leaderRaw === "string" ? leaderRaw : leaderRaw === null ? null : undefined; + if (generation === undefined || !isCandidateReason(reason) || leaderId === undefined) { + return undefined; + } + return { ...base, type: "resync", leaderId, generation, reason }; +} + +function parseEvent(data: UnknownRecord, base: ParsedBase): EventMessage | undefined { + const pair = parseLeaderPair(data); + if (!pair || !isRecord(data.payload)) return undefined; + return { ...base, type: "event", ...pair, payload: data.payload }; +} + function isVisibility(value: unknown): value is TabVisibility { return value === "visible" || value === "hidden"; } diff --git a/apps/fabro-web/app/lib/run-events.ts b/apps/fabro-web/app/lib/run-events.ts index 80394da48..f0a5f6a66 100644 --- a/apps/fabro-web/app/lib/run-events.ts +++ b/apps/fabro-web/app/lib/run-events.ts @@ -125,7 +125,7 @@ export function subscribeToRunEvents( resyncKeys: () => resyncKeysForRun(runId), resolveInvalidation: (payload) => { if (payload.run_id !== runId) return { keys: [] }; - return runInvalidation(runId, payload, { closeOnTerminal: false }); + return runInvalidation(runId, payload); }, fallbackSubscribe: () => subscribeToSharedEventSource({ @@ -135,28 +135,22 @@ export function subscribeToRunEvents( mutate, eventSourceFactory, debounceMs, - resolveInvalidation: (payload) => - runInvalidation(runId, payload, { closeOnTerminal: true }), + resolveInvalidation: (payload) => { + const result = runInvalidation(runId, payload); + return { ...result, close: result.immediate }; + }, }), }); } -function runInvalidation( - runId: string, - payload: RunEventPayload, - { closeOnTerminal }: { closeOnTerminal: boolean }, -) { +function runInvalidation(runId: string, payload: RunEventPayload) { const event = payload.event; - if (!event) return { keys: [] }; + if (!event) return { keys: [], immediate: false }; const stageId = stageIdFromPayload(payload); const keys = queryKeysForRunEvent(runId, event, stageId); const terminal = TERMINAL_EVENTS.has(event); - return { - keys, - close: closeOnTerminal && terminal, - immediate: terminal, - }; + return { keys, immediate: terminal }; } function resyncKeysForRun(runId: string) { From 63940fdddc1cce458f7498035e45ea3b2511ba3b Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Mon, 4 May 2026 15:52:18 -0400 Subject: [PATCH 06/16] fix(web): recover cross-tab SSE coordination after fallback Reset coordinator state when the last subscriber leaves, clear pending debounce timers on close, and keep coordinated EventSource construction owned by the coordinator while fallback subscriptions keep their local factories. --- apps/fabro-web/app/lib/board-events.test.tsx | 14 +++--- apps/fabro-web/app/lib/board-events.ts | 1 - apps/fabro-web/app/lib/cross-tab-sse.test.ts | 45 ++++++++++++++++++++ apps/fabro-web/app/lib/cross-tab-sse.ts | 41 ++++++++++-------- apps/fabro-web/app/lib/run-events.test.tsx | 16 ++++--- apps/fabro-web/app/lib/run-events.ts | 1 - 6 files changed, 87 insertions(+), 31 deletions(-) diff --git a/apps/fabro-web/app/lib/board-events.test.tsx b/apps/fabro-web/app/lib/board-events.test.tsx index 9b398fdc1..497bb6984 100644 --- a/apps/fabro-web/app/lib/board-events.test.tsx +++ b/apps/fabro-web/app/lib/board-events.test.tsx @@ -9,6 +9,7 @@ import { type BroadcastChannelLike, } from "./cross-tab-sse"; import { queryKeys } from "./query-keys"; +import type { EventSourceLike } from "./sse"; type MessageHandler = ((event: { data: string }) => void) | null; @@ -47,15 +48,17 @@ describe("subscribeToBoardEvents", () => { const source = new FakeEventSource(); const created: string[] = []; const keys: string[] = []; - const coordinator = createCoordinator(); + const coordinator = createCoordinator((url) => { + created.push(url); + return source; + }); const mutate = (key: string) => { keys.push(key); return Promise.resolve(); }; - const firstCleanup = subscribeToBoardEvents(mutate, (url) => { - created.push(url); - return source; + const firstCleanup = subscribeToBoardEvents(mutate, () => { + throw new Error("source should be created by coordinator"); }, { debounceMs: 0, coordinator }); const secondCleanup = subscribeToBoardEvents(mutate, () => { throw new Error("source should be reused"); @@ -107,10 +110,11 @@ describe("subscribeToBoardEvents", () => { }); }); -function createCoordinator() { +function createCoordinator(eventSourceFactory: (url: string) => EventSourceLike) { return createCrossTabSseCoordinator({ tabId: "board-test", channelFactory: () => new FakeBroadcastChannel(), + eventSourceFactory, addVisibilityChangeListener: () => () => {}, addPagehideListener: () => () => {}, timing: { diff --git a/apps/fabro-web/app/lib/board-events.ts b/apps/fabro-web/app/lib/board-events.ts index a182d9ad2..dc01765ee 100644 --- a/apps/fabro-web/app/lib/board-events.ts +++ b/apps/fabro-web/app/lib/board-events.ts @@ -56,7 +56,6 @@ export function subscribeToBoardEvents( coordinator, subscriptionKey: BOARD_SUBSCRIPTION_KEY, mutate, - eventSourceFactory, debounceMs, resyncKeys: () => [queryKeys.boards.runs()], resolveInvalidation: boardInvalidation, diff --git a/apps/fabro-web/app/lib/cross-tab-sse.test.ts b/apps/fabro-web/app/lib/cross-tab-sse.test.ts index f38c881fa..76fc0b16e 100644 --- a/apps/fabro-web/app/lib/cross-tab-sse.test.ts +++ b/apps/fabro-web/app/lib/cross-tab-sse.test.ts @@ -542,6 +542,51 @@ describe("subscribeToCrossTabSse", () => { coordinator.close(); }); + test("last unsubscribe retries coordination after an initial channel failure", async () => { + let channelUnavailable = true; + const sources: FakeEventSource[] = []; + const coordinator = createCrossTabSseCoordinator({ + tabId: "retry-after-unsubscribe", + channelFactory: (name) => { + if (channelUnavailable) throw new Error("channel unavailable"); + return new FakeBroadcastChannel(name); + }, + eventSourceFactory: (url) => { + const source = new FakeEventSource(url, "retry-after-unsubscribe"); + sources.push(source); + return source; + }, + addVisibilityChangeListener: () => () => {}, + addPagehideListener: () => () => {}, + timing: TEST_TIMING, + }); + let fallbackStarted = 0; + + const firstCleanup = subscribeWithFallback(coordinator, { + fallbackSubscribe: () => { + fallbackStarted += 1; + return () => {}; + }, + }); + firstCleanup(); + + channelUnavailable = false; + const secondCleanup = subscribeWithFallback(coordinator, { + fallbackSubscribe: () => { + fallbackStarted += 1; + return () => {}; + }, + }); + + await waitFor(() => sources.some((source) => !source.closed)); + + expect(fallbackStarted).toBe(1); + expect(sources.filter((source) => !source.closed).map((source) => source.url)).toEqual(["/api/v1/attach"]); + + secondCleanup(); + coordinator.close(); + }); + test("close stops fallback subscriptions added after degradation", async () => { const harness = newHarness(); const coordinator = harness.createTab("a"); diff --git a/apps/fabro-web/app/lib/cross-tab-sse.ts b/apps/fabro-web/app/lib/cross-tab-sse.ts index 99281a8f0..6975e6948 100644 --- a/apps/fabro-web/app/lib/cross-tab-sse.ts +++ b/apps/fabro-web/app/lib/cross-tab-sse.ts @@ -113,7 +113,6 @@ interface SubscribeOptions { resolveInvalidation: (payload: TPayload) => EventInvalidation; resyncKeys: () => string[]; fallbackSubscribe: () => () => void; - eventSourceFactory?: (url: string) => EventSourceLike; debounceMs?: number; } @@ -206,7 +205,6 @@ export class CrossTabSseCoordinator { private leaderCheckTimer: ReturnType | null = null; private removeVisibilityListener: (() => void) | null = null; private removePagehideListener: (() => void) | null = null; - private sourceFactoryLocked = false; constructor(options: CrossTabSseCoordinatorOptions = {}) { this.tabId = options.tabId ?? createTabId(); @@ -225,11 +223,6 @@ export class CrossTabSseCoordinator { } subscribe(options: SubscribeOptions): () => void { - if (options.eventSourceFactory && !this.source && !this.sourceFactoryLocked) { - this.sourceFactory = options.eventSourceFactory; - this.sourceFactoryLocked = true; - } - const subscription = this.addLocalSubscription(options); if (this.coordinationUnavailable) { @@ -259,13 +252,10 @@ export class CrossTabSseCoordinator { this.clearNoLeaderTimer(); this.shutdownTimersAndChannel(); this.closeFallbacks(); + this.clearSubscriptionTimers(); this.subscriptions.clear(); this.candidates.clear(); - this.leader = null; - this.initialized = false; - this.coordinationUnavailable = false; - this.fallbackMode = false; - this.sourceFactoryLocked = false; + this.resetIdleState(); } private initialize(): boolean { @@ -356,9 +346,7 @@ export class CrossTabSseCoordinator { subscription.refcount -= 1; if (subscription.refcount <= 0) { - if (subscription.debounceTimer) { - clearTimeout(subscription.debounceTimer); - } + this.clearSubscriptionTimer(subscription); this.subscriptions.delete(subscriptionKey); } @@ -367,9 +355,7 @@ export class CrossTabSseCoordinator { this.clearCandidate(); this.clearNoLeaderTimer(); this.shutdownTimersAndChannel(); - this.initialized = false; - this.fallbackMode = false; - this.sourceFactoryLocked = false; + this.resetIdleState(); } } @@ -945,6 +931,25 @@ export class CrossTabSseCoordinator { } } + private clearSubscriptionTimers() { + for (const subscription of this.subscriptions.values()) { + this.clearSubscriptionTimer(subscription); + } + } + + private clearSubscriptionTimer(subscription: LocalSubscription) { + if (!subscription.debounceTimer) return; + clearTimeout(subscription.debounceTimer); + subscription.debounceTimer = null; + } + + private resetIdleState() { + this.leader = null; + this.initialized = false; + this.coordinationUnavailable = false; + this.fallbackMode = false; + } + private shouldAcceptLeader(incoming: LeaderState): boolean { if (!this.leader) return true; if (incoming.generation > this.leader.generation) return true; diff --git a/apps/fabro-web/app/lib/run-events.test.tsx b/apps/fabro-web/app/lib/run-events.test.tsx index db35a757d..e9faa93fc 100644 --- a/apps/fabro-web/app/lib/run-events.test.tsx +++ b/apps/fabro-web/app/lib/run-events.test.tsx @@ -9,6 +9,7 @@ import { type BroadcastChannelLike, } from "./cross-tab-sse"; import { queryKeys } from "./query-keys"; +import type { EventSourceLike } from "./sse"; type MessageHandler = ((event: { data: string }) => void) | null; @@ -55,7 +56,10 @@ describe("subscribeToRunEvents", () => { const source = new FakeEventSource(); const created: string[] = []; const keys: string[] = []; - const coordinator = createCoordinator(); + const coordinator = createCoordinator((url) => { + created.push(url); + return source; + }); const cleanup = subscribeToRunEvents( "run-coordinated", @@ -63,9 +67,8 @@ describe("subscribeToRunEvents", () => { keys.push(key); return Promise.resolve(); }, - (url) => { - created.push(url); - return source; + () => { + throw new Error("source should be created by coordinator"); }, { debounceMs: 0, coordinator }, ); @@ -86,7 +89,7 @@ describe("subscribeToRunEvents", () => { test("coordinated terminal events invalidate without closing the global stream", async () => { const source = new FakeEventSource(); const keys: string[] = []; - const coordinator = createCoordinator(); + const coordinator = createCoordinator(() => source); const cleanup = subscribeToRunEvents( "run-terminal", (key) => { @@ -206,10 +209,11 @@ describe("subscribeToRunEvents", () => { }); }); -function createCoordinator() { +function createCoordinator(eventSourceFactory: (url: string) => EventSourceLike) { return createCrossTabSseCoordinator({ tabId: "run-test", channelFactory: () => new FakeBroadcastChannel(), + eventSourceFactory, addVisibilityChangeListener: () => () => {}, addPagehideListener: () => () => {}, timing: { diff --git a/apps/fabro-web/app/lib/run-events.ts b/apps/fabro-web/app/lib/run-events.ts index f0a5f6a66..1413d38d4 100644 --- a/apps/fabro-web/app/lib/run-events.ts +++ b/apps/fabro-web/app/lib/run-events.ts @@ -120,7 +120,6 @@ export function subscribeToRunEvents( coordinator, subscriptionKey: `run:${runId}`, mutate, - eventSourceFactory, debounceMs, resyncKeys: () => resyncKeysForRun(runId), resolveInvalidation: (payload) => { From b5b08e78d389a1746aa52490efe5d66f26a07eaa Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Mon, 4 May 2026 15:52:24 -0400 Subject: [PATCH 07/16] refactor(api): reuse board column contract across clients Make BoardColumnDefinition.id reference the existing BoardColumn schema and carry that typed contract through generated TypeScript, server responses, demo data, and the runs board UI. --- apps/fabro-web/app/data/runs.ts | 21 ++++++-- apps/fabro-web/app/lib/queries.ts | 9 ++-- apps/fabro-web/app/routes/runs.tsx | 17 ++---- docs/public/api-reference/fabro-api.yaml | 2 +- lib/crates/fabro-server/src/demo/mod.rs | 12 ++--- .../fabro-server/src/server/handler/runs.rs | 54 +++++++++++++------ lib/crates/fabro-server/src/server/tests.rs | 4 +- .../src/models/board-column-definition.ts | 6 ++- 8 files changed, 74 insertions(+), 51 deletions(-) diff --git a/apps/fabro-web/app/data/runs.ts b/apps/fabro-web/app/data/runs.ts index 77d8a4711..b397c5ecc 100644 --- a/apps/fabro-web/app/data/runs.ts +++ b/apps/fabro-web/app/data/runs.ts @@ -1,8 +1,10 @@ import { formatElapsedSecs, formatDurationSecs } from "../lib/format"; -import type { - RunListItem, - RunStatus as ApiRunStatus, - RunSummary, +import { + BoardColumn, + type BoardColumn as ApiBoardColumn, + type RunListItem, + type RunStatus as ApiRunStatus, + type RunSummary, } from "@qltysh/fabro-api-client"; export type CiStatus = "passing" | "failing" | "pending"; @@ -37,7 +39,16 @@ export interface RunItem { sourceDirectory?: string; } -export type ColumnStatus = "queued" | "initializing" | "running" | "blocked" | "succeeded" | "failed"; +export type ColumnStatus = ApiBoardColumn; + +export const columnStatuses = [ + BoardColumn.QUEUED, + BoardColumn.INITIALIZING, + BoardColumn.RUNNING, + BoardColumn.BLOCKED, + BoardColumn.SUCCEEDED, + BoardColumn.FAILED, +] as const satisfies readonly ColumnStatus[]; export const columnStatusDisplay: Record = { queued: { label: "Queued", dot: "bg-fg-muted", text: "text-fg-muted" }, diff --git a/apps/fabro-web/app/lib/queries.ts b/apps/fabro-web/app/lib/queries.ts index 4de23a73e..0da17785e 100644 --- a/apps/fabro-web/app/lib/queries.ts +++ b/apps/fabro-web/app/lib/queries.ts @@ -34,6 +34,9 @@ const immutableOptions: SWRConfiguration = { revalidateOnReconnect: false, }; +type BoardRunsEnvelope = PaginatedEnvelope & + Pick; + export function useAuthConfig() { return useSWR<{ methods: string[] }>(queryKeys.auth.config(), apiFetcher, immutableOptions); } @@ -61,11 +64,7 @@ export function useSystemInfo() { } export function useBoardsRuns() { - return useSWR< - PaginatedEnvelope & { - columns: { id: string; name: string }[]; - } - >(queryKeys.boards.runs(), apiPaginatedFetcher); + return useSWR(queryKeys.boards.runs(), apiPaginatedFetcher); } export function useRun(id: string | undefined) { diff --git a/apps/fabro-web/app/routes/runs.tsx b/apps/fabro-web/app/routes/runs.tsx index 90ef09405..78db03496 100644 --- a/apps/fabro-web/app/routes/runs.tsx +++ b/apps/fabro-web/app/routes/runs.tsx @@ -18,7 +18,7 @@ import { arrayMove, } from "@dnd-kit/sortable"; import { CSS } from "@dnd-kit/utilities"; -import { ciConfig, columnStatusDisplay, deriveCiStatus, mapRunListItem } from "../data/runs"; +import { ciConfig, columnStatusDisplay, columnStatuses, deriveCiStatus, mapRunListItem } from "../data/runs"; import type { CiStatus, CheckRun, CheckStatus, RunItem, RunWithStatus, ColumnStatus } from "../data/runs"; import { EmptyState } from "../components/state"; import { shouldRefreshBoardForEvent, useBoardEvents } from "../lib/board-events"; @@ -50,7 +50,7 @@ const defaultColumnStyle: ColumnStyle = { iconType: "branch", actions: [] }; const defaultColumnColors = { dot: "bg-fg-muted", text: "text-fg-muted" }; interface BoardRunsResponse { - columns: { id: string; name: string }[]; + columns: PaginatedBoardRunList["columns"]; data: PaginatedBoardRunList["data"]; meta: PaginatedBoardRunList["meta"]; } @@ -65,17 +65,8 @@ type Column = { items: RunItem[]; }; -const SKELETON_STATUSES: ColumnStatus[] = [ - "queued", - "initializing", - "running", - "blocked", - "succeeded", - "failed", -]; - function buildSkeletonColumns(): Column[] { - return SKELETON_STATUSES.map((id) => { + return columnStatuses.map((id) => { const colors = columnStatusDisplay[id]; return { id, @@ -100,7 +91,7 @@ export function buildBoardColumns(response: BoardRunsResponse): Column[] { } return response.columns.map((col) => { - const id = col.id as ColumnStatus; + const id = col.id; const colors = columnStatusDisplay[id] ?? defaultColumnColors; return { id, diff --git a/docs/public/api-reference/fabro-api.yaml b/docs/public/api-reference/fabro-api.yaml index d10db4a47..4137dfaad 100644 --- a/docs/public/api-reference/fabro-api.yaml +++ b/docs/public/api-reference/fabro-api.yaml @@ -5670,7 +5670,7 @@ components: - name properties: id: - type: string + $ref: "#/components/schemas/BoardColumn" name: type: string diff --git a/lib/crates/fabro-server/src/demo/mod.rs b/lib/crates/fabro-server/src/demo/mod.rs index 787fb1ae4..08423cd55 100644 --- a/lib/crates/fabro-server/src/demo/mod.rs +++ b/lib/crates/fabro-server/src/demo/mod.rs @@ -975,27 +975,27 @@ mod runs { pub(super) fn columns() -> Vec { vec![ BoardColumnDefinition { - id: "queued".into(), + id: BoardColumn::Queued, name: "Queued".into(), }, BoardColumnDefinition { - id: "initializing".into(), + id: BoardColumn::Initializing, name: "Initializing".into(), }, BoardColumnDefinition { - id: "running".into(), + id: BoardColumn::Running, name: "Running".into(), }, BoardColumnDefinition { - id: "blocked".into(), + id: BoardColumn::Blocked, name: "Blocked".into(), }, BoardColumnDefinition { - id: "succeeded".into(), + id: BoardColumn::Succeeded, name: "Succeeded".into(), }, BoardColumnDefinition { - id: "failed".into(), + id: BoardColumn::Failed, name: "Failed".into(), }, ] diff --git a/lib/crates/fabro-server/src/server/handler/runs.rs b/lib/crates/fabro-server/src/server/handler/runs.rs index eb48c7c4f..9f31597ad 100644 --- a/lib/crates/fabro-server/src/server/handler/runs.rs +++ b/lib/crates/fabro-server/src/server/handler/runs.rs @@ -10,7 +10,9 @@ use axum::{Json, Router}; use base64::Engine as _; use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; use bytes::Bytes; -use fabro_api::types::{RunManifest, RunStatusResponse, SubmitAnswerRequest}; +use fabro_api::types::{ + BoardColumn, BoardColumnDefinition, RunManifest, RunStatusResponse, SubmitAnswerRequest, +}; use fabro_config::Storage; use fabro_interview::AnswerSubmission; use fabro_types::{ @@ -82,27 +84,45 @@ impl ListRunsParams { } } -fn board_column(status: RunStatus) -> Option<&'static str> { +fn board_column(status: RunStatus) -> Option { match status { - RunStatus::Submitted | RunStatus::Queued => Some("queued"), - RunStatus::Starting => Some("initializing"), - RunStatus::Running | RunStatus::Paused { .. } => Some("running"), - RunStatus::Blocked { .. } => Some("blocked"), - RunStatus::Succeeded { .. } => Some("succeeded"), - RunStatus::Failed { .. } | RunStatus::Dead => Some("failed"), + RunStatus::Submitted | RunStatus::Queued => Some(BoardColumn::Queued), + RunStatus::Starting => Some(BoardColumn::Initializing), + RunStatus::Running | RunStatus::Paused { .. } => Some(BoardColumn::Running), + RunStatus::Blocked { .. } => Some(BoardColumn::Blocked), + RunStatus::Succeeded { .. } => Some(BoardColumn::Succeeded), + RunStatus::Failed { .. } | RunStatus::Dead => Some(BoardColumn::Failed), RunStatus::Removing | RunStatus::Archived { .. } => None, } } -pub(crate) fn board_columns() -> serde_json::Value { - serde_json::json!([ - {"id": "queued", "name": "Queued"}, - {"id": "initializing", "name": "Initializing"}, - {"id": "running", "name": "Running"}, - {"id": "blocked", "name": "Blocked"}, - {"id": "succeeded", "name": "Succeeded"}, - {"id": "failed", "name": "Failed"}, - ]) +pub(crate) fn board_columns() -> Vec { + vec![ + BoardColumnDefinition { + id: BoardColumn::Queued, + name: "Queued".into(), + }, + BoardColumnDefinition { + id: BoardColumn::Initializing, + name: "Initializing".into(), + }, + BoardColumnDefinition { + id: BoardColumn::Running, + name: "Running".into(), + }, + BoardColumnDefinition { + id: BoardColumn::Blocked, + name: "Blocked".into(), + }, + BoardColumnDefinition { + id: BoardColumn::Succeeded, + name: "Succeeded".into(), + }, + BoardColumnDefinition { + id: BoardColumn::Failed, + name: "Failed".into(), + }, + ] } async fn board_run_metadata( diff --git a/lib/crates/fabro-server/src/server/tests.rs b/lib/crates/fabro-server/src/server/tests.rs index 5480fef0f..74a18fc17 100644 --- a/lib/crates/fabro-server/src/server/tests.rs +++ b/lib/crates/fabro-server/src/server/tests.rs @@ -6615,8 +6615,8 @@ async fn queue_position_reported_for_queued_runs() { let first_run_id = create_and_start_run(&app, MINIMAL_DOT).await; let second_run_id = create_and_start_run(&app, MINIMAL_DOT).await; - // Queued runs are excluded from the board, so verify queue positions - // via the in-memory state directly. + // Queue position is tracked in memory even when queued runs are also + // visible on the board. let runs = state.runs.lock().expect("runs lock poisoned"); let positions = compute_queue_positions(&runs); let first_id = first_run_id.parse::().unwrap(); diff --git a/lib/packages/fabro-api-client/src/models/board-column-definition.ts b/lib/packages/fabro-api-client/src/models/board-column-definition.ts index f1730a16c..20deba1f5 100644 --- a/lib/packages/fabro-api-client/src/models/board-column-definition.ts +++ b/lib/packages/fabro-api-client/src/models/board-column-definition.ts @@ -13,9 +13,11 @@ */ +// May contain unused imports in some cases +// @ts-ignore +import type { BoardColumn } from './board-column'; export interface BoardColumnDefinition { - 'id': string; + 'id': BoardColumn; 'name': string; } - From 4b100d350d952e11cc5aff9c03423392408835d2 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Mon, 4 May 2026 16:09:12 -0400 Subject: [PATCH 08/16] chore: add plan --- .../2026-05-04-cross-tab-sse-coordination.md | 106 ++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 docs/superpowers/plans/2026-05-04-cross-tab-sse-coordination.md diff --git a/docs/superpowers/plans/2026-05-04-cross-tab-sse-coordination.md b/docs/superpowers/plans/2026-05-04-cross-tab-sse-coordination.md new file mode 100644 index 000000000..115ff9326 --- /dev/null +++ b/docs/superpowers/plans/2026-05-04-cross-tab-sse-coordination.md @@ -0,0 +1,106 @@ +# Cross-Tab SSE Coordination Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Coordinate Fabro web SSE subscriptions across tabs so one browser profile/origin opens at most one UI-owned `/api/v1/attach` EventSource in steady state when `BroadcastChannel` is available; brief overlap during election/takeover is tolerated and deduped. + +**Architecture:** Add a browser-side SSE coordinator that elects one tab as leader, has that leader own the global EventSource, and broadcasts parsed run events to follower tabs over `BroadcastChannel`. Existing board and run-detail invalidation logic becomes a consumer of that global event feed, with the current per-tab SSE behavior preserved as a compatibility fallback. + +**Tech Stack:** React, SWR, browser `BroadcastChannel`, browser `EventSource`, Bun tests, existing Fabro web API query keys. + +--- + +## Summary + +Build a browser-side SSE coordinator so Fabro web opens at most one `/api/v1/attach` EventSource per origin/browser profile in steady state when `BroadcastChannel` is available. Temporary duplicate leaders may exist during election/takeover, but event dedupe and generation checks make the overlap harmless and short-lived. The global stream becomes a shared cache-invalidation feed for both the runs board and run detail pages. No server API, OpenAPI, or Rust streaming contract changes are part of v1. + +This supersedes the earlier web SSE limitation documented in `docs/plans/2026-04-19-002-feat-web-ui-lifecycle-actions-plan.md`: the old shared hook was code reuse only; this plan adds actual socket deduplication. + +## Implementation Changes + +- Add `apps/fabro-web/app/lib/cross-tab-sse.ts`. + - Export `subscribeToCrossTabSse(...)` with the same invalidation style as `subscribeToSharedEventSource`, a `resyncKeys` callback for gap recovery, and a `fallbackSubscribe` callback used when cross-tab coordination is unavailable. + - Use `BroadcastChannel` name `fabro:sse:v1`. + - Generate `tabId` with `crypto.randomUUID()` and a safe random fallback. + - Open one leader-owned `EventSource` to `queryKeys.system.attach()` (`/api/v1/attach`) in steady state. + - Leader dispatches each parsed `EventEnvelope` locally and broadcasts it to follower tabs. + - Followers do not open EventSource while a valid visible leader heartbeat exists; lower lexical `tabId` does not preempt a healthy visible leader. + +- Implement leader election in the cross-tab module. + - Constants: `HEARTBEAT_MS = 1000`, `LEADER_STALE_MS = 4000`, `ELECTION_JITTER_MS = 150`. + - Messages: `hello`, `heartbeat`, `candidate`, `leader-changed`, `release`, `resync`, `event`. + - Define a typed message union. Every message includes `type`, `version: 1`, `tabId`, and `sentAt`. + - `heartbeat`: `{ type, version, tabId, sentAt, leaderId, generation, visibility }`. + - `candidate`: `{ type, version, tabId, sentAt, candidateId: tabId, candidateGeneration, visibility, observedLeaderId, observedGeneration, reason }`, where `reason` is `"hidden-leader" | "stale-leader" | "release" | "no-leader"`. + - `leader-changed`: `{ type, version, tabId, sentAt, leaderId, generation, visibility }`. + - `release`: `{ type, version, tabId, sentAt, leaderId, generation }`. + - `resync`: `{ type, version, tabId, sentAt, leaderId, generation, reason }`. + - `event`: `{ type, version, tabId, sentAt, leaderId, generation, payload }`. + - Use the candidacy phase for all leadership changes: hidden-leader takeover, stale-leader recovery, leader release, and no-leader startup. + - A candidate sets `candidateGeneration = observedGeneration + 1`, broadcasts `candidate`, waits jitter, and opens EventSource only if no higher-priority candidate for the same `candidateGeneration` appears. + - Candidate priority is election-scoped: visible candidates outrank hidden candidates; for equal visibility, lower lexical `candidateId` wins. This priority resolves elections and same-generation split brain only; it is not a reason to preempt a fresh visible leader. + - When a visible follower observes a fresh hidden leader heartbeat, it enters candidacy with `reason: "hidden-leader"`. + - When tabs detect a stale leader, leader release, or no known leader, they enter the same candidacy flow with the matching `reason`. + - If two visible candidates race for the same observed leader/generation, the lower lexical `candidateId` wins. + - Current leaders release when they observe a candidate whose `observedLeaderId` matches their `leaderId` and whose `observedGeneration` is current or newer. + - If same-generation split brain still occurs, lower-priority leaders release when they observe a same-generation higher-priority leader heartbeat or `leader-changed`. + - Each new leader uses `candidateGeneration`, broadcasts `leader-changed`, and followers ignore heartbeats/events from non-current leaders or stale generations. + - Hidden leader keeps the stream only when no visible candidate takes over. + - On `pagehide`/last local unsubscribe, a leader closes EventSource and broadcasts `release`. + - Brief split brain is tolerated; dedupe events by `payload.id`, falling back to `${run_id}:${seq}:${event}`. + - Keep dedupe bounded with a recent-event cache: max 1000 IDs and 5 minute TTL. Evict oldest entries when the max is exceeded and prune expired entries during event handling. Duplicate invalidations after eviction are acceptable; unbounded growth is not. + +- Migrate consumers. + - `apps/fabro-web/app/lib/board-events.ts`: subscribe through the cross-tab global stream; keep existing board event allowlist. + - `apps/fabro-web/app/lib/run-events.ts`: subscribe through the same global stream, filter by `payload.run_id === runId`, and reuse `queryKeysForRunEvent`. + - In coordinated mode, run detail pages stay subscribed while mounted, including terminal runs, so post-terminal archive/unarchive changes can reconcile live. + - Do not close the global stream on `run.completed` / `run.failed`; terminal events only invalidate run-scoped keys. + - Keep `subscribeToSharedEventSource` in `apps/fabro-web/app/lib/sse.ts` for fallback and existing local sharing behavior. + +- Gap and fallback behavior. + - If `BroadcastChannel` is unavailable or throws, call each subscriber's `fallbackSubscribe`. + - Board fallback uses the existing global `/api/v1/attach` path. + - Run detail fallback preserves the existing run-scoped `/api/v1/runs/:id/attach` path, so the old terminal-tab stale limitation remains only in fallback mode. + - Do not add replay to `/api/v1/attach`. + - On leader takeover, stale leader timeout, leader release, and new leader generation, broadcast `resync` or `leader-changed` so every tab with active local subscriptions runs its own `resyncKeys`. + - On `visibilitychange` back to visible without leadership change, run only that tab's local `resyncKeys`; do not broadcast cross-tab resync. + - Board `resyncKeys`: `queryKeys.boards.runs()`. + - Run `resyncKeys`: detail, files, billing, stages, events, LR graph, TB graph, and questions for that run. + +## Tests + +- Add `apps/fabro-web/app/lib/cross-tab-sse.test.ts` with fake `BroadcastChannel`, fake `EventSource`, and fake timers. + - One leader opens `/api/v1/attach`; followers open no EventSource. + - Leader broadcasts an event and all local subscribers receive invalidations. + - Run subscribers ignore events for other `run_id` values. + - Board and run subscriptions can coexist on the same global stream. + - Temporary duplicate leaders are allowed only during election/takeover and converge back to one leader. + - With fresh hidden-leader heartbeats, a visible tab broadcasts candidacy, hidden leader closes, visible tab opens `/api/v1/attach`, and followers resync. + - Two visible candidates racing for the same hidden leader resolve to the lower lexical `candidateId`. + - Two tabs detect the same stale leader simultaneously; only the winning candidate opens `/api/v1/attach` after jitter. + - Same-generation split brain converges to one leader by visibility, then lexical `tabId`. + - A fresh visible leader exists; a new visible follower with a lower lexical `tabId` joins and does not take leadership. + - A hidden leader does not release for a candidate that references an old `observedLeaderId` or stale `observedGeneration`. + - Stale heartbeat triggers takeover and every active tab calls its own `resyncKeys`. + - A follower tab calls its own board/run `resyncKeys` after another tab becomes leader. + - Duplicate event IDs are ignored. + - The recent-event dedupe cache evicts by TTL and max-size bound; duplicate invalidation may recur only after eviction. + - Stale heartbeat/event messages from an old leader/generation are ignored after takeover. + - Last unsubscribe closes leader EventSource and sends `release`. + - Missing/broken BroadcastChannel uses the per-tab fallback. + +- Update existing tests. + - `apps/fabro-web/app/lib/board-events.test.tsx`: assert coordinated mode uses `/api/v1/attach` once and fallback preserves current behavior. + - `apps/fabro-web/app/lib/run-events.test.tsx`: assert coordinated mode filters by `run_id`; fallback preserves current `/runs/:id/attach` behavior; terminal run events do not close the global coordinator; terminal detail tabs still receive archive/unarchive invalidations while mounted in coordinated mode. + +- Verification commands. + - `cd apps/fabro-web && bun test app/lib/cross-tab-sse.test.ts app/lib/board-events.test.tsx app/lib/run-events.test.tsx` + - `cd apps/fabro-web && bun run typecheck` + - Optional manual check: open 8 run-detail tabs for active runs; after election settles and with BroadcastChannel available, Chrome should show one active `/api/v1/attach` EventStream across the participating tabs and no UI-created `/api/v1/runs/:id/attach` streams. + +## Assumptions + +- Treat SSE as live cache invalidation, not an exact gapless event log. +- Preserve current run-specific attach endpoint for CLI, API clients, and fallback only. +- Do not touch unrelated dirty worktree files; intentionally replace or adapt the earlier interrupted SSE test edits as part of the new test suite. +- No docs or public API updates are required for v1 because this is an internal web transport change. From 7769c5cec1627d0733a08b46c65075952ccf25a1 Mon Sep 17 00:00:00 2001 From: "fabro-sh-0530[bot]" <281434857+fabro-sh-0530[bot]@users.noreply.github.com> Date: Mon, 4 May 2026 22:00:15 -0400 Subject: [PATCH 09/16] Generate Fabro PR titles and bodies with structured output (#208) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Summary Fabro now asks the LLM for a structured PR title and reviewer-sized body instead of deriving every title from the workflow goal. This ports the compound-engineering PR-writing recipe into the existing pull request pipeline while preserving Fabro's programmatically appended trailing sections. ### What changed - Replaced plain-text PR body generation with `generate_object` and a strict `{ title, body }` schema. - Added the sizing matrix, writing principles, visual-aid guidance, and duplicate-section guardrails to the PR prompt. - Added model-aware goal/plan/diff truncation caps, with unknown or smaller-context models using the conservative tier. - Kept goal-derived titles as a narrow fallback only when the LLM returns a usable body with an empty title. - Enforced a 72-character title cap across both LLM-generated and fallback titles. - Updated workflow, server, and integration tests for structured responses, fallback behavior, title truncation, and blank-body failures. ### Plan Summary - Move PR content generation to structured output. - Keep existing body assembly and appended sections intact. - Add coverage for title fallback and validation edge cases. ### Fabro Details
Ran 9 stages in 42m 43s for $44.59 | Stage | Duration | Cost | Retries | |---|---|---|---| | start | 0s | – | 0 | | toolchain | 1s | – | 0 | | preflight_compile | 2m 3s | – | 0 | | preflight_lint | 2m 13s | – | 0 | | implement | 15m 8s | $6.37 | 0 | | simplify_opus | 11m 16s | $3.53 | 0 | | simplify_gpt | 9m 5s | $34.69 | 0 | | verify | 2m 17s | – | 0 | | fmt | 2s | – | 0 | | **Total** | **42m 43s** | **$44.59** | **0** |
Ran ImplementPlan.fabro (12 nodes and 15 edges) ```dot digraph ImplementPlan { graph [ goal="Implement and simplify", model_stylesheet=" * { model: claude-opus-4-7; } " ] rankdir=LR start [shape=Mdiamond, label="Start"] exit [shape=Msquare, label="Exit"] toolchain [label="Toolchain", shape=parallelogram, script="command -v cargo >/dev/null || { curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && sudo ln -sf $HOME/.cargo/bin/* /usr/local/bin/; }; cargo --version 2>&1", max_retries=0] preflight_compile [label="Preflight Compile", shape=parallelogram, script="cargo check -q --workspace 2>&1", max_retries=0] preflight_lint [label="Preflight Lint", shape=parallelogram, script="cargo +nightly-2026-04-14 clippy -q --workspace --all-targets -- -D warnings 2>&1", max_retries=0] fix_lints [label="Fix Lints", prompt="The preflight lint step failed. Read the build output from context and fix all clippy lint warnings.", max_visits=3] implement [label="Implement", prompt="Read the plan file referenced in the goal and implement every step. Make all the code changes described in the plan. Use red/green TDD."] simplify_opus [label="Simplify (Opus)", prompt="@prompts/simplify.md"] simplify_gpt [label="Simplify (GPT-55)", prompt="@prompts/simplify.md", model="gpt-55"] verify [label="Verify", shape=parallelogram, script="cargo +nightly-2026-04-14 clippy -q --workspace --all-targets -- -D warnings 2>&1 && cargo nextest run --cargo-quiet --workspace --status-level fail 2>&1 && cargo dev docs refresh 2>&1 && cargo dev docs check 2>&1", goal_gate=true, retry_target="fixup"] fixup [label="Fixup", prompt="The verify step failed. Read the build output from context and fix all clippy lint warnings, test failures, and generated docs errors.", max_visits=3] fmt [label="Format", shape=parallelogram, script="cargo +nightly-2026-04-14 fmt --all 2>&1", max_retries=0] start -> toolchain toolchain -> preflight_compile [condition="outcome=succeeded"] toolchain -> exit preflight_compile -> preflight_lint [condition="outcome=succeeded"] preflight_compile -> exit preflight_lint -> implement [condition="outcome=succeeded"] preflight_lint -> fix_lints fix_lints -> preflight_lint implement -> simplify_opus -> simplify_gpt -> verify verify -> fmt [condition="outcome=succeeded"] verify -> fixup fixup -> verify fmt -> exit } ```
⚒️ Generated with [Fabro](https://fabro.sh) --------- Co-authored-by: Fabro --- lib/crates/fabro-server/src/server/tests.rs | 8 +- .../src/pipeline/pull_request.rs | 587 ++++++++++++++++-- .../fabro-workflow/tests/it/integration.rs | 11 +- 3 files changed, 547 insertions(+), 59 deletions(-) diff --git a/lib/crates/fabro-server/src/server/tests.rs b/lib/crates/fabro-server/src/server/tests.rs index 74a18fc17..5587e1952 100644 --- a/lib/crates/fabro-server/src/server/tests.rs +++ b/lib/crates/fabro-server/src/server/tests.rs @@ -3665,7 +3665,13 @@ async fn create_run_pull_request_creates_and_persists_record() { .header("authorization", "Bearer openai-key"); then.status(200) .header("content-type", "application/json") - .json_body(openai_responses_payload("Narrative from mock.")); + .json_body(openai_responses_payload( + &serde_json::to_string(&json!({ + "title": "Mock title", + "body": "Narrative from mock.", + })) + .unwrap(), + )); }) .await; let openai_base_url = llm.url("/v1"); diff --git a/lib/crates/fabro-workflow/src/pipeline/pull_request.rs b/lib/crates/fabro-workflow/src/pipeline/pull_request.rs index 5b9355ccf..8825f8eef 100644 --- a/lib/crates/fabro-workflow/src/pipeline/pull_request.rs +++ b/lib/crates/fabro-workflow/src/pipeline/pull_request.rs @@ -1,10 +1,11 @@ -use std::sync::Arc; +use std::sync::{Arc, LazyLock}; use fabro_auth::CredentialSource; use fabro_github::{self as github_app, ssh_url_to_https}; use fabro_graphviz::parser; use fabro_llm::client::Client; -use fabro_llm::generate::{GenerateParams, generate}; +use fabro_llm::generate::{GenerateParams, generate_object}; +use fabro_model::Catalog; use fabro_retro::retro::Retro; use fabro_store::RunProjection; use fabro_types::PullRequestRecord; @@ -18,17 +19,165 @@ use crate::outcome::{StageOutcome, format_cost as outcome_format_cost}; use crate::records::{Conclusion, RunSpec}; use crate::runtime_store::RunStoreHandle; +/// Maximum length of a PR title (Unicode scalar values). Single source of +/// truth — referenced by the structured-output schema, the system prompt, +/// and [`enforce_title_cap`]. +const PR_TITLE_MAX_CHARS: usize = 72; + +/// Structured output schema for the LLM-generated PR title and body. +/// +/// `title` is required but allows empty strings (the only signal that +/// triggers the deterministic title fallback in +/// [`maybe_open_pull_request`]). `body` requires `minLength: 1` because +/// there is no body fallback — an empty body is fatal. +static PR_CONTENT_SCHEMA: LazyLock = LazyLock::new(|| { + serde_json::json!({ + "type": "object", + "properties": { + "title": { "type": "string", "maxLength": PR_TITLE_MAX_CHARS }, + "body": { "type": "string", "minLength": 1 } + }, + "required": ["title", "body"], + "additionalProperties": false + }) +}); + +#[derive(Debug, serde::Deserialize)] +struct GeneratedPrContent { + title: String, + body: String, +} + +/// System prompt that instructs the LLM how to write a Fabro PR title and +/// body. The trailing programmatic sections (Plan `
`, Retro, +/// Fabro Details, footer) are appended after the LLM body — the prompt +/// explicitly forbids the LLM from duplicating them. +// +// The "max 72 characters" instruction must stay in sync with +// `PR_TITLE_MAX_CHARS` and the schema above; the prompt is advisory and +// `enforce_title_cap` is the actual enforcement. +const PR_BODY_SYSTEM_PROMPT: &str = "You are writing a pull request title and description for a code change produced by an AI workflow. + +OUTPUT FORMAT +Return a JSON object with exactly two fields: +- \"title\": a one-line title, max 72 characters, no trailing period. +- \"body\": the markdown body as described below. + +DO NOT INCLUDE in the body +- A `#` or `##` title heading at the top — the title goes in the `title` field. +- A \"Retro\" section, \"Fabro Details\" section, cost/duration table, or \"Generated with\" footer — those are appended programmatically after your output. +- The full plan text — the full plan is appended programmatically as a
block. +- Bare `#1`, `#2` list prefixes — GitHub auto-links those as issue references. Use plain `1.`, `2.` instead. +- A test plan unless the testing approach is non-obvious. + +SIZE THE BODY TO THE CHANGE +First classify along two axes from the diff: +- Size: how many files changed, how large the diff is. +- Complexity: trivial (rename / typo / dep bump / config) vs. design decisions / new patterns / cross-cutting concerns. + +Then write at the matching depth: + +| Profile | Body shape | +|---|---| +| Small + simple (typo, config, dep bump) | 1–2 sentences, no headers, total under ~300 characters | +| Small + non-trivial (targeted bugfix, behavioral change) | Short \"Problem / Fix\" narrative, 3–5 sentences. No headers unless two distinct concerns. | +| Medium feature or refactor | Summary paragraph, then a section explaining what changed and why. Call out design decisions. | +| Large or architecturally significant | Full narrative: problem context, approach chosen (and why), key decisions, migration/rollback notes if relevant. | +| Performance improvement | Include before/after measurements if available. A markdown table works well here. | + +Brevity matters for small changes. A 3-line bugfix with a 20-line description signals miscalibration. When in doubt, shorter is better — reviewers can read the diff. + +WRITING PRINCIPLES +- Lead with value: the first sentence tells the reviewer *why this PR exists*, not *what files changed*. +- Describe the net result, not the journey: skip intermediate failures, debugging steps, and refactors done during development. +- Trust the final diff: if the goal or plan disagree with the diff, the diff is authoritative. +- Explain the non-obvious: spend description space on what the diff doesn't show — why this approach, what was rejected, what to look at first. +- Use structure when it earns its keep: no empty sections, no template headers without content. +- If the body uses any `##` heading, the opening summary must also be under a heading (e.g. `## Summary`); otherwise a bare paragraph is fine. + +PLAN SUMMARY +The full plan is attached separately as a
block, so do not restate it. Include a brief `### Plan Summary` with bullet points only when the change is medium or larger in the sizing matrix above. Skip it for small changes. + +VISUAL AIDS +Include a visual aid only when a reviewer would struggle to reconstruct the mental model from prose alone — based on what changes structurally, not on PR size. Skip for trivial / mechanical changes, or when prose already communicates clearly. + +| PR changes... | Visual aid | +|---|---| +| 3+ interacting components or services | Mermaid component / interaction diagram | +| Multi-step workflow or pipeline with non-obvious sequencing | Mermaid flow diagram | +| 3+ behavioral modes or variants | Markdown comparison table | +| Before/after data or trade-offs | Markdown table | +| Data model changes with 3+ related entities | Mermaid ERD | + +Mermaid: prefer `TB` direction, ≤10 nodes typical. Place inline at the point of relevance, not in a separate \"Diagrams\" section."; + +/// Truncation budget for the LLM prompt's goal / plan / diff sections. +struct TruncationCaps { + goal: usize, + plan: usize, + diff: usize, +} + +/// Generous tier for models with ≥200k context windows. +const TRUNCATION_LARGE: TruncationCaps = TruncationCaps { + goal: 75_000, + plan: 75_000, + diff: 250_000, +}; + +/// Conservative tier (matches the pre-refactor values). Used for smaller +/// or unknown models. +const TRUNCATION_SMALL: TruncationCaps = TruncationCaps { + goal: 20_000, + plan: 20_000, + diff: 50_000, +}; + +/// Resolve truncation caps based on the model's context window. Unknown +/// models fall through to the conservative tier. +fn truncation_caps(model: &str) -> &'static TruncationCaps { + let large_enough = Catalog::builtin() + .get(model) + .is_some_and(|m| m.context_window() >= 200_000); + if large_enough { + &TRUNCATION_LARGE + } else { + &TRUNCATION_SMALL + } +} + +/// Truncate `s` to at most `max` Unicode scalar values without splitting a +/// UTF-8 sequence. +fn truncate_chars(s: &str, max: usize) -> &str { + s.char_indices() + .nth(max) + .map_or(s, |(boundary, _)| &s[..boundary]) +} + +/// Truncate `s` to at most `max` Unicode scalar values, replacing the +/// trailing char with `…` when truncation occurs. +fn truncate_with_ellipsis(s: &str, max: usize) -> String { + if s.chars().count() > max { + let truncated: String = s.chars().take(max - 1).collect(); + format!("{truncated}\u{2026}") + } else { + s.to_string() + } +} + +/// Cap a PR title at [`PR_TITLE_MAX_CHARS`]. +fn enforce_title_cap(title: &str) -> String { + truncate_with_ellipsis(title, PR_TITLE_MAX_CHARS) +} + /// Derive a PR title from the workflow goal. /// -/// Uses the first line, truncated to 120 characters for readability. +/// Uses the first line, truncated to 120 characters for readability. The +/// caller is expected to apply [`enforce_title_cap`] afterwards if a +/// stricter cap is required (the wider cap here is the legacy behaviour +/// for the deterministic fallback path). fn pr_title_from_goal(goal: &str) -> String { - let stripped = strip_goal_decoration(goal); - if stripped.chars().count() > 120 { - let truncated: String = stripped.chars().take(119).collect(); - format!("{truncated}…") - } else { - stripped.to_string() - } + truncate_with_ellipsis(strip_goal_decoration(goal), 120) } /// Truncate a PR body to fit GitHub's 65,536 character limit. @@ -286,8 +435,13 @@ async fn load_pull_request_diff(run_store: &RunStoreHandle) -> String { .unwrap_or_default() } -/// Build a complete PR body by combining LLM-generated narrative with -/// programmatic sections (plan, retro, fabro details). +/// Build a complete PR title and body by combining LLM-generated narrative +/// with programmatic sections (plan, retro, fabro details). +/// +/// Returns `(title, body)`. The title may be the empty string when the LLM +/// returned a usable body but no usable title — callers fall back to +/// [`pr_title_from_goal`] in that case. Every other generation failure is +/// surfaced as `Err`. pub async fn build_pr_body( diff: &str, goal: &str, @@ -295,7 +449,7 @@ pub async fn build_pr_body( run_store: &RunStoreHandle, llm_source: &dyn CredentialSource, conclusion: Option<&Conclusion>, -) -> Result { +) -> Result<(String, String), String> { let client = Client::from_source(llm_source) .await .map_err(|e| format!("Failed to create LLM client: {e}"))?; @@ -310,7 +464,7 @@ async fn build_pr_body_with_client( run_store: &RunStoreHandle, conclusion: Option<&Conclusion>, client: Arc, -) -> Result { +) -> Result<(String, String), String> { build_pr_body_with_client_and_state(diff, goal, model, run_store, conclusion, client, None) .await } @@ -323,7 +477,7 @@ async fn build_pr_body_with_source_and_state( llm_source: &dyn CredentialSource, conclusion: Option<&Conclusion>, run_state: Option<&fabro_store::RunProjection>, -) -> Result { +) -> Result<(String, String), String> { let client = Client::from_source(llm_source) .await .map_err(|e| format!("Failed to create LLM client: {e}"))?; @@ -348,7 +502,7 @@ async fn build_pr_body_with_client_and_state( conclusion: Option<&Conclusion>, client: Arc, run_state: Option<&fabro_store::RunProjection>, -) -> Result { +) -> Result<(String, String), String> { info!("Building PR body"); let loaded_run_state = if run_state.is_none() { @@ -369,45 +523,39 @@ async fn build_pr_body_with_client_and_state( let run_spec = run_state.and_then(|state| state.spec.clone()); let dot_source = run_state.and_then(|state| state.graph_source.clone()); - // Build LLM prompt - let system = if plan_text.is_some() { - "Write a PR description with: (1) 2-3 concise paragraphs explaining the change, then (2) a '### Plan Summary' section with bullet points summarizing the plan. Do not include a title. Do not include the full plan.".to_string() - } else { - "Write a concise PR description in 2-3 paragraphs explaining the change. Do not include a title.".to_string() - }; - - // Truncate diff to fit context windows (~50k chars) - let max_diff_len = 50_000; - let truncated_diff = if diff.len() > max_diff_len { - &diff[..diff.floor_char_boundary(max_diff_len)] - } else { - diff - }; + let caps = truncation_caps(model); + let truncated_goal = truncate_chars(goal, caps.goal); + let truncated_diff = truncate_chars(diff, caps.diff); let prompt = if let Some(ref plan) = plan_text { - // Truncate plan for LLM context (~20k chars) - let max_plan_len = 20_000; - let truncated_plan = if plan.len() > max_plan_len { - &plan[..plan.floor_char_boundary(max_plan_len)] - } else { - plan.as_str() - }; + let truncated_plan = truncate_chars(plan, caps.plan); format!( - "Goal: {goal}\n\nPlan:\n```\n{truncated_plan}\n```\n\nDiff:\n```\n{truncated_diff}\n```" + "Goal: {truncated_goal}\n\nPlan:\n```\n{truncated_plan}\n```\n\nDiff:\n```\n{truncated_diff}\n```" ) } else { - format!("Goal: {goal}\n\nDiff:\n```\n{truncated_diff}\n```") + format!("Goal: {truncated_goal}\n\nDiff:\n```\n{truncated_diff}\n```") }; let params = GenerateParams::new(model, client) - .system(system) + .system(PR_BODY_SYSTEM_PROMPT) .prompt(prompt); - let result = generate(params) + let result = generate_object(params, PR_CONTENT_SCHEMA.clone()) .await .map_err(|e| format!("LLM generation failed: {e}"))?; - let llm_output = result.response.text(); + let output = result + .output + .ok_or_else(|| "LLM generation returned no structured output".to_string())?; + let generated: GeneratedPrContent = serde_json::from_value(output) + .map_err(|e| format!("Failed to deserialize PR content: {e}"))?; + + if generated.body.trim().is_empty() { + return Err("LLM generated an empty PR body".to_string()); + } + + let title = enforce_title_cap(generated.title.trim()); + let llm_body = generated.body; let retro_section = retro.as_ref().map(format_retro_section).unwrap_or_default(); let arc_details_section = conclusion @@ -416,7 +564,7 @@ async fn build_pr_body_with_client_and_state( .unwrap_or_default(); let body = assemble_pr_body( - &llm_output, + &llm_body, plan_text.as_deref(), &retro_section, &arc_details_section, @@ -424,7 +572,7 @@ async fn build_pr_body_with_client_and_state( info!("PR body generated"); - Ok(body) + Ok((title, body)) } /// Auto-merge configuration for a pull request. @@ -465,7 +613,7 @@ pub async fn maybe_open_pull_request( let (owner, repo) = github_app::parse_github_owner_repo(&https_url).map_err(|err| format!("{err:#}"))?; - let body = build_pr_body_with_source_and_state( + let (llm_title, body) = build_pr_body_with_source_and_state( req.diff, req.goal, req.model, @@ -478,7 +626,12 @@ pub async fn maybe_open_pull_request( .map_err(|err| format!("{err:#}"))?; let body = truncate_pr_body(&body); - let title = pr_title_from_goal(req.goal); + let title = if llm_title.is_empty() { + pr_title_from_goal(req.goal) + } else { + llm_title + }; + let title = enforce_title_cap(&title); let created = github_app::create_pull_request( &req.github, @@ -782,6 +935,16 @@ mod tests { }) } + /// JSON string the MockProvider/openai mock returns to simulate the + /// structured-output response for `(title, body)`. + fn pr_content_json(title: &str, body: &str) -> String { + serde_json::to_string(&serde_json::json!({ + "title": title, + "body": body, + })) + .unwrap() + } + fn make_test_conclusion() -> Conclusion { Conclusion { timestamp: Utc::now(), @@ -1126,17 +1289,21 @@ mod tests { async fn build_pr_body_uses_in_memory_conclusion() { let store = test_store(); let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); - let body = build_pr_body_with_client( + let (title, body) = build_pr_body_with_client( "diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n", "Implement feature", "mock-model", &run_store.clone().into(), Some(&make_test_conclusion()), - explicit_client("mock", "Narrative from mock."), + explicit_client( + "mock", + &pr_content_json("Mock title", "Narrative from mock."), + ), ) .await .unwrap(); + assert_eq!(title, "Mock title"); assert!(body.contains("Narrative from mock.")); assert!(body.contains("### Fabro Details")); assert!(body.contains("Ran 3 stages in 2m 30s for $0.42")); @@ -1196,13 +1363,16 @@ mod tests { .await .unwrap(); - let body = build_pr_body_with_client( + let (_, body) = build_pr_body_with_client( "diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n", "Implement feature", "mock-model", &run_store.clone().into(), Some(&make_test_conclusion()), - explicit_client("mock", "Narrative from mock."), + explicit_client( + "mock", + &pr_content_json("Mock title", "Narrative from mock."), + ), ) .await .unwrap(); @@ -1283,13 +1453,16 @@ mod tests { .await .unwrap(); - let body = build_pr_body_with_client( + let (_, body) = build_pr_body_with_client( "diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n", "Implement feature", "mock-model", &run_store.clone().into(), Some(&make_test_conclusion()), - explicit_client("mock", "Narrative from mock."), + explicit_client( + "mock", + &pr_content_json("Mock title", "Narrative from mock."), + ), ) .await .unwrap(); @@ -1302,13 +1475,16 @@ mod tests { async fn build_pr_body_uses_explicit_llm_client() { let store = test_store(); let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); - let body = build_pr_body_with_client( + let (_, body) = build_pr_body_with_client( "diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n", "Implement feature", "gpt-5.4", &run_store.clone().into(), Some(&make_test_conclusion()), - explicit_client("openai", "Narrative from explicit client."), + explicit_client( + "openai", + &pr_content_json("Explicit title", "Narrative from explicit client."), + ), ) .await .unwrap(); @@ -1327,7 +1503,10 @@ mod tests { .header("authorization", "Bearer vault-openai-key"); then.status(200) .header("content-type", "application/json") - .json_body(openai_responses_payload("Narrative from vault source.")); + .json_body(openai_responses_payload(&pr_content_json( + "Vault title", + "Narrative from vault source.", + ))); }) .await; @@ -1355,7 +1534,7 @@ mod tests { let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); let run_store_handle: RunStoreHandle = run_store.into(); - let body = build_pr_body( + let (title, body) = build_pr_body( "diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n", "Implement feature", "gpt-5.4", @@ -1366,6 +1545,7 @@ mod tests { .await .unwrap(); + assert_eq!(title, "Vault title"); assert!(body.contains("Narrative from vault source.")); response_mock.assert_async().await; } @@ -1575,4 +1755,299 @@ mod tests { assert!(diff.contains("from_store")); } + + // ── Structured-output PR content tests ────────────────────────────── + + /// MockProvider returns an over-long title; builder must cap it at 72 + /// chars and end with `…`. Exercises [`enforce_title_cap`] inside + /// [`build_pr_body_with_client_and_state`]. + #[tokio::test] + async fn build_pr_body_truncates_long_title() { + let store = test_store(); + let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); + let long_title = "x".repeat(200); + let payload = pr_content_json(&long_title, "Body content."); + let (title, _) = build_pr_body_with_client( + "diff --git a/src/lib.rs b/src/lib.rs\n+fn x() {}\n", + "Implement feature", + "mock-model", + &run_store.clone().into(), + Some(&make_test_conclusion()), + explicit_client("mock", &payload), + ) + .await + .unwrap(); + + assert_eq!(title.chars().count(), 72); + assert!(title.ends_with('\u{2026}')); + } + + /// Empty bodies are fatal. Real providers may reject this via the + /// schema's `minLength`; the Rust-side trim check also catches it for + /// local/mock providers. + #[tokio::test] + async fn build_pr_body_returns_err_when_body_empty() { + let store = test_store(); + let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); + let payload = pr_content_json("Mock", ""); + let result = build_pr_body_with_client( + "diff --git a/src/lib.rs b/src/lib.rs\n+fn x() {}\n", + "Implement feature", + "mock-model", + &run_store.clone().into(), + Some(&make_test_conclusion()), + explicit_client("mock", &payload), + ) + .await; + + assert!(result.is_err(), "expected Err, got {result:?}"); + } + + /// Whitespace-only bodies pass schema validation but fail the + /// `body.trim().is_empty()` check inside the builder. + #[tokio::test] + async fn build_pr_body_returns_err_when_body_whitespace() { + let store = test_store(); + let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); + let payload = pr_content_json("Mock", " \n"); + let result = build_pr_body_with_client( + "diff --git a/src/lib.rs b/src/lib.rs\n+fn x() {}\n", + "Implement feature", + "mock-model", + &run_store.clone().into(), + Some(&make_test_conclusion()), + explicit_client("mock", &payload), + ) + .await; + + let err = result.expect_err("expected Err for whitespace-only body"); + assert!(err.contains("empty PR body"), "unexpected error: {err}"); + } + + // ── maybe_open_pull_request fallback tests ────────────────────────── + + /// Set of mock servers and credentials for the `maybe_open_pull_request` + /// fallback path. The builder's `Client::from_source` rebuilds the LLM + /// client from the credential source, so the in-process MockProvider + /// cannot intercept — we mock the OpenAI HTTP endpoint instead. + struct FallbackHarness { + _vault_dir: tempfile::TempDir, + // Held to keep the mock listener alive for the duration of the test; + // the test interacts with it via `Client::from_source` (which goes + // out via HTTP to the mock URL stored in `llm_source`). + openai_server: MockServer, + github_server: MockServer, + openai_mock_id: usize, + github_mock_id: usize, + llm_source: Arc, + creds: fabro_github::GitHubCredentials, + run_store: RunStoreHandle, + } + + impl FallbackHarness { + async fn assert_mocks_called_once(&self) { + httpmock::Mock::new(self.openai_mock_id, &self.openai_server) + .assert_async() + .await; + httpmock::Mock::new(self.github_mock_id, &self.github_server) + .assert_async() + .await; + } + } + + /// Stand up an OpenAI mock that returns the given structured-output + /// payload, a GitHub mock that accepts a PR creation, a vault-backed + /// credential source, and a run store seeded with a non-empty + /// `final_patch`. + async fn setup_fallback_test_harness(openai_payload_text: &str) -> FallbackHarness { + let openai_server = MockServer::start_async().await; + let openai_mock = openai_server + .mock_async(|when, then| { + when.method(POST) + .path("/v1/responses") + .header("authorization", "Bearer vault-openai-key"); + then.status(200) + .header("content-type", "application/json") + .json_body(openai_responses_payload(openai_payload_text)); + }) + .await; + + let github_server = MockServer::start_async().await; + let github_mock = github_server + .mock_async(|when, then| { + when.method(POST) + .path("/repos/owner/repo/pulls") + .header("authorization", "Bearer test-token"); + then.status(201) + .header("content-type", "application/json") + .json_body(serde_json::json!({ + "number": 1, + "html_url": "https://example.test/owner/repo/pull/1", + "node_id": "PR_kwTest1", + })); + }) + .await; + + let vault_dir = tempfile::tempdir().unwrap(); + let mut vault = Vault::load(vault_dir.path().join("secrets.json")).unwrap(); + vault + .set( + "openai_codex", + &serde_json::to_string(&openai_api_key_credential("vault-openai-key")).unwrap(), + SecretType::Credential, + None, + ) + .unwrap(); + let base_url = openai_server.url("/v1"); + let llm_source: Arc = + Arc::new(VaultCredentialSource::with_env_lookup( + Arc::new(AsyncRwLock::new(vault)), + move |name| match name { + "OPENAI_BASE_URL" => Some(base_url.clone()), + _ => None, + }, + )); + + let creds = fabro_github::GitHubCredentials::Token("test-token".to_string()); + + let store = test_store(); + let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); + // Seed a non-empty `final_patch` so `load_pull_request_diff` returns + // diff content and the early-return for empty diffs does not fire. + let run_spec = RunSpec { + run_id: fixtures::RUN_1, + settings: fabro_types::WorkflowSettings::default(), + graph: Graph::new("test"), + workflow_slug: None, + source_directory: None, + git: None, + labels: HashMap::new(), + provenance: None, + manifest_blob: None, + definition_blob: None, + fork_source_ref: None, + in_place: false, + }; + append_event(&run_store, &fixtures::RUN_1, &Event::RunCreated { + run_id: fixtures::RUN_1, + settings: serde_json::to_value(&run_spec.settings).unwrap(), + graph: serde_json::to_value(&run_spec.graph).unwrap(), + workflow_source: None, + workflow_config: None, + labels: run_spec.labels.clone().into_iter().collect(), + run_dir: "/tmp/x".to_string(), + source_directory: None, + workflow_slug: None, + db_prefix: None, + provenance: None, + manifest_blob: None, + git: None, + fork_source_ref: None, + in_place: false, + web_url: None, + }) + .await + .unwrap(); + append_event(&run_store, &fixtures::RUN_1, &Event::WorkflowRunCompleted { + duration_ms: 1, + artifact_count: 0, + status: "succeeded".to_string(), + reason: SuccessReason::Completed, + total_usd_micros: None, + final_git_commit_sha: None, + final_patch: Some( + "diff --git a/src/lib.rs b/src/lib.rs\n+fn from_store() {}\n".to_string(), + ), + billing: None, + }) + .await + .unwrap(); + + let openai_mock_id = openai_mock.id; + let github_mock_id = github_mock.id; + + FallbackHarness { + _vault_dir: vault_dir, + openai_server, + github_server, + openai_mock_id, + github_mock_id, + llm_source, + creds, + run_store: run_store.into(), + } + } + + /// LLM returns a usable body but an empty title; `maybe_open_pull_request` + /// must fall back to `pr_title_from_goal` (first line, decoration + /// stripped) and the PR creation must succeed with that title. + #[tokio::test] + async fn maybe_open_pull_request_falls_back_to_goal_title_when_llm_returns_empty_title() { + let payload = pr_content_json("", "Narrative."); + let harness = setup_fallback_test_harness(&payload).await; + + let github_base_url = harness.github_server.url(""); + let github = github_app::GitHubContext::new(&harness.creds, &github_base_url); + + let result = maybe_open_pull_request(OpenPullRequestRequest { + github, + origin_url: "https://github.com/owner/repo.git", + base_branch: "main", + head_branch: "fabro/run/123", + goal: "Fix telemetry leak\n\ndetails...", + diff: "diff --git a/src/lib.rs b/src/lib.rs\n+fn x() {}\n", + model: "gpt-5.4", + draft: false, + auto_merge: None, + run_store: &harness.run_store, + llm_source: harness.llm_source.as_ref(), + conclusion: None, + run_state: None, + }) + .await + .expect("PR creation should succeed"); + + let record = result.expect("PR record should be Some"); + assert_eq!(record.title, "Fix telemetry leak"); + harness.assert_mocks_called_once().await; + } + + /// LLM returns an empty title; the fallback path produces a long title + /// (close to `pr_title_from_goal`'s 120-char cap), and the unconditional + /// `enforce_title_cap` in `maybe_open_pull_request` must still bring it + /// down to 72 chars ending with `…`. + #[tokio::test] + async fn maybe_open_pull_request_caps_fallback_title_at_72_chars() { + let payload = pr_content_json("", "Narrative."); + let harness = setup_fallback_test_harness(&payload).await; + + let github_base_url = harness.github_server.url(""); + let github = github_app::GitHubContext::new(&harness.creds, &github_base_url); + + // Single ~200-char line, no `Plan:` / heading prefix, no newlines. + let goal = "x".repeat(200); + + let result = maybe_open_pull_request(OpenPullRequestRequest { + github, + origin_url: "https://github.com/owner/repo.git", + base_branch: "main", + head_branch: "fabro/run/123", + goal: &goal, + diff: "diff --git a/src/lib.rs b/src/lib.rs\n+fn x() {}\n", + model: "gpt-5.4", + draft: false, + auto_merge: None, + run_store: &harness.run_store, + llm_source: harness.llm_source.as_ref(), + conclusion: None, + run_state: None, + }) + .await + .expect("PR creation should succeed"); + + let record = result.expect("PR record should be Some"); + assert_eq!(record.title.chars().count(), 72); + assert!(record.title.ends_with('\u{2026}')); + harness.assert_mocks_called_once().await; + } } diff --git a/lib/crates/fabro-workflow/tests/it/integration.rs b/lib/crates/fabro-workflow/tests/it/integration.rs index 002d473da..742218c14 100644 --- a/lib/crates/fabro-workflow/tests/it/integration.rs +++ b/lib/crates/fabro-workflow/tests/it/integration.rs @@ -6809,7 +6809,13 @@ async fn workflow_run_with_vault_only_openai_codex_builds_pr_body() { .header("authorization", "Bearer vault-openai-key"); then.status(200) .header("content-type", "application/json") - .json_body(openai_responses_payload("Narrative from vault source.")); + .json_body(openai_responses_payload( + &serde_json::to_string(&serde_json::json!({ + "title": "Vault title", + "body": "Narrative from vault source.", + })) + .unwrap(), + )); }) .await; @@ -6890,7 +6896,7 @@ async fn workflow_run_with_vault_only_openai_codex_builds_pr_body() { let run_store = store.open_run_reader(&run_options.run_id).await.unwrap(); let run_store_handle: fabro_workflow::runtime_store::RunStoreHandle = run_store.into(); - let body = fabro_workflow::pull_request::build_pr_body( + let (title, body) = fabro_workflow::pull_request::build_pr_body( "diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n", "Implement feature", "gpt-5.4", @@ -6910,6 +6916,7 @@ async fn workflow_run_with_vault_only_openai_codex_builds_pr_body() { .await .expect("PR body should build from vault-only credentials"); + assert_eq!(title, "Vault title"); assert!(body.contains("Narrative from vault source.")); response_mock.assert_async().await; } From a2fbac1d60d76018fa934dbe0865369fc40e1127 Mon Sep 17 00:00:00 2001 From: "fabro-releases[bot]" Date: Tue, 5 May 2026 09:52:01 +0000 Subject: [PATCH 10/16] Bump version to 0.224.0-nightly.0 --- Cargo.lock | 86 +++++++++++++++++++++++++++--------------------------- Cargo.toml | 2 +- 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e4e138ffa..f88b44544 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1536,7 +1536,7 @@ dependencies = [ [[package]] name = "fabro-agent" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "async-trait", @@ -1575,7 +1575,7 @@ dependencies = [ [[package]] name = "fabro-api" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "chrono", "fabro-config", @@ -1596,7 +1596,7 @@ dependencies = [ [[package]] name = "fabro-auth" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "async-trait", @@ -1620,7 +1620,7 @@ dependencies = [ [[package]] name = "fabro-checkpoint" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "chrono", "fabro-config", @@ -1636,7 +1636,7 @@ dependencies = [ [[package]] name = "fabro-cli" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "assert_cmd", @@ -1732,7 +1732,7 @@ dependencies = [ [[package]] name = "fabro-client" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "bytes", @@ -1761,7 +1761,7 @@ dependencies = [ [[package]] name = "fabro-config" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "chrono", @@ -1788,7 +1788,7 @@ dependencies = [ [[package]] name = "fabro-core" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "async-trait", "fabro-types", @@ -1803,7 +1803,7 @@ dependencies = [ [[package]] name = "fabro-dev" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "assert_cmd", @@ -1823,7 +1823,7 @@ dependencies = [ [[package]] name = "fabro-devcontainer" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "fabro-http", "fabro-static", @@ -1840,7 +1840,7 @@ dependencies = [ [[package]] name = "fabro-dump" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "bytes", @@ -1854,7 +1854,7 @@ dependencies = [ [[package]] name = "fabro-github" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "base64", @@ -1876,7 +1876,7 @@ dependencies = [ [[package]] name = "fabro-graphviz" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "fabro-types", @@ -1890,7 +1890,7 @@ dependencies = [ [[package]] name = "fabro-hooks" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "async-trait", "fabro-agent", @@ -1914,7 +1914,7 @@ dependencies = [ [[package]] name = "fabro-http" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "fabro-static", "http", @@ -1924,7 +1924,7 @@ dependencies = [ [[package]] name = "fabro-install" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "base64", @@ -1939,7 +1939,7 @@ dependencies = [ [[package]] name = "fabro-interview" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "async-trait", "dialoguer", @@ -1954,7 +1954,7 @@ dependencies = [ [[package]] name = "fabro-llm" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "async-trait", @@ -1986,7 +1986,7 @@ dependencies = [ [[package]] name = "fabro-macros" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "clap", "fabro-options-metadata", @@ -1997,7 +1997,7 @@ dependencies = [ [[package]] name = "fabro-mcp" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "fabro-config", @@ -2013,7 +2013,7 @@ dependencies = [ [[package]] name = "fabro-model" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "fabro-static", "insta", @@ -2024,7 +2024,7 @@ dependencies = [ [[package]] name = "fabro-oauth" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "axum", @@ -2046,7 +2046,7 @@ dependencies = [ [[package]] name = "fabro-options-metadata" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "serde", "serde_json", @@ -2054,7 +2054,7 @@ dependencies = [ [[package]] name = "fabro-proc" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "cc", "libc", @@ -2063,7 +2063,7 @@ dependencies = [ [[package]] name = "fabro-redact" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "aho-corasick", "ref-cast", @@ -2079,7 +2079,7 @@ dependencies = [ [[package]] name = "fabro-retro" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "chrono", @@ -2098,7 +2098,7 @@ dependencies = [ [[package]] name = "fabro-sandbox" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "async-trait", @@ -2138,7 +2138,7 @@ dependencies = [ [[package]] name = "fabro-server" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "async-trait", @@ -2219,7 +2219,7 @@ dependencies = [ [[package]] name = "fabro-slack" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "fabro-http", "fabro-interview", @@ -2240,18 +2240,18 @@ dependencies = [ [[package]] name = "fabro-spa" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "rust-embed", ] [[package]] name = "fabro-static" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" [[package]] name = "fabro-store" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "async-trait", "bytes", @@ -2278,7 +2278,7 @@ dependencies = [ [[package]] name = "fabro-telemetry" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "base64", @@ -2304,7 +2304,7 @@ dependencies = [ [[package]] name = "fabro-template" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "fabro-util", @@ -2316,7 +2316,7 @@ dependencies = [ [[package]] name = "fabro-test" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "assert_cmd", "axum", @@ -2339,7 +2339,7 @@ dependencies = [ [[package]] name = "fabro-tracker" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "async-trait", @@ -2353,7 +2353,7 @@ dependencies = [ [[package]] name = "fabro-types" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "chrono", "clap", @@ -2374,7 +2374,7 @@ dependencies = [ [[package]] name = "fabro-util" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "console 0.15.11", @@ -2394,7 +2394,7 @@ dependencies = [ [[package]] name = "fabro-validate" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "fabro-graphviz", "fabro-model", @@ -2404,7 +2404,7 @@ dependencies = [ [[package]] name = "fabro-vault" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "chrono", "fabro-types", @@ -2416,7 +2416,7 @@ dependencies = [ [[package]] name = "fabro-workflow" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "assert_cmd", @@ -7120,7 +7120,7 @@ dependencies = [ [[package]] name = "twin-github" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "axum", "base64", @@ -7139,7 +7139,7 @@ dependencies = [ [[package]] name = "twin-openai" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "async-stream", diff --git a/Cargo.toml b/Cargo.toml index f7897daf1..52d5940be 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ resolver = "2" [workspace.package] edition = "2021" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" license = "MIT" [workspace.dependencies] From d9c8030e7448cbed2ef0b26dc5763a14279f78b7 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Tue, 5 May 2026 08:25:17 -0400 Subject: [PATCH 11/16] refactor(workflow): harden PR content generation Return named PR content from the builder and keep title/body fallback logic inside the builder. Move the PR body prompt to markdown and scale prompt truncation from model context windows. Keep PR creation resilient when generated bodies are empty by emitting a reviewer-visible skeleton body. --- lib/crates/fabro-workflow/src/pipeline/mod.rs | 3 +- .../src/pipeline/prompts/pr_body.md | 54 +++ .../src/pipeline/pull_request.rs | 447 ++++++++++-------- lib/crates/fabro-workflow/src/pull_request.rs | 2 +- .../fabro-workflow/tests/it/integration.rs | 7 +- 5 files changed, 303 insertions(+), 210 deletions(-) create mode 100644 lib/crates/fabro-workflow/src/pipeline/prompts/pr_body.md diff --git a/lib/crates/fabro-workflow/src/pipeline/mod.rs b/lib/crates/fabro-workflow/src/pipeline/mod.rs index 71aefbef9..bb35d298f 100644 --- a/lib/crates/fabro-workflow/src/pipeline/mod.rs +++ b/lib/crates/fabro-workflow/src/pipeline/mod.rs @@ -19,7 +19,8 @@ pub use initialize::initialize; pub use parse::parse; pub(crate) use persist::persist; pub use pull_request::{ - AutoMergeOptions, OpenPullRequestRequest, build_pr_body, maybe_open_pull_request, pull_request, + AutoMergeOptions, OpenPullRequestRequest, PrContent, build_pr_content, maybe_open_pull_request, + pull_request, }; pub use retro::{retro, run_retro}; pub use transform::transform; diff --git a/lib/crates/fabro-workflow/src/pipeline/prompts/pr_body.md b/lib/crates/fabro-workflow/src/pipeline/prompts/pr_body.md new file mode 100644 index 000000000..9175e62db --- /dev/null +++ b/lib/crates/fabro-workflow/src/pipeline/prompts/pr_body.md @@ -0,0 +1,54 @@ +You are writing a pull request title and description for a code change produced by an AI workflow. + +OUTPUT FORMAT +Return a JSON object with exactly two fields: +- "title": a one-line title, max 72 characters, no trailing period. +- "body": the markdown body as described below. + +DO NOT INCLUDE in the body +- A `#` or `##` title heading at the top -- the title goes in the `title` field. +- A "Retro" section, "Fabro Details" section, cost/duration table, or "Generated with" footer -- those are appended programmatically after your output. +- The full plan text -- the full plan is appended programmatically as a
block. +- Bare `#1`, `#2` list prefixes -- GitHub auto-links those as issue references. Use plain `1.`, `2.` instead. +- A test plan unless the testing approach is non-obvious. + +SIZE THE BODY TO THE CHANGE +First classify along two axes from the diff: +- Size: how many files changed, how large the diff is. +- Complexity: trivial (rename / typo / dep bump / config) vs. design decisions / new patterns / cross-cutting concerns. + +Then write at the matching depth: + +| Profile | Body shape | +|---|---| +| Small + simple (typo, config, dep bump) | 1-2 sentences, no headers, total under ~300 characters | +| Small + non-trivial (targeted bugfix, behavioral change) | Short "Problem / Fix" narrative, 3-5 sentences. No headers unless two distinct concerns. | +| Medium feature or refactor | Summary paragraph, then a section explaining what changed and why. Call out design decisions. | +| Large or architecturally significant | Full narrative: problem context, approach chosen (and why), key decisions, migration/rollback notes if relevant. | +| Performance improvement | Include before/after measurements if available. A markdown table works well here. | + +Brevity matters for small changes. A 3-line bugfix with a 20-line description signals miscalibration. When in doubt, shorter is better -- reviewers can read the diff. + +WRITING PRINCIPLES +- Lead with value: the first sentence tells the reviewer *why this PR exists*, not *what files changed*. +- Describe the net result, not the journey: skip intermediate failures, debugging steps, and refactors done during development. +- Trust the final diff: if the goal or plan disagree with the diff, the diff is authoritative. +- Explain the non-obvious: spend description space on what the diff doesn't show -- why this approach, what was rejected, what to look at first. +- Use structure when it earns its keep: no empty sections, no template headers without content. +- If the body uses any `##` heading, the opening summary must also be under a heading (e.g. `## Summary`); otherwise a bare paragraph is fine. + +PLAN SUMMARY +The full plan is attached separately as a
block, so do not restate it. Include a brief `### Plan Summary` with bullet points only when the change is medium or larger in the sizing matrix above. Skip it for small changes. + +VISUAL AIDS +Include a visual aid only when a reviewer would struggle to reconstruct the mental model from prose alone -- based on what changes structurally, not on PR size. Skip for trivial / mechanical changes, or when prose already communicates clearly. + +| PR changes... | Visual aid | +|---|---| +| 3+ interacting components or services | Mermaid component / interaction diagram | +| Multi-step workflow or pipeline with non-obvious sequencing | Mermaid flow diagram | +| 3+ behavioral modes or variants | Markdown comparison table | +| Before/after data or trade-offs | Markdown table | +| Data model changes with 3+ related entities | Mermaid ERD | + +Mermaid: prefer `TB` direction, <=10 nodes typical. Place inline at the point of relevance, not in a separate "Diagrams" section. diff --git a/lib/crates/fabro-workflow/src/pipeline/pull_request.rs b/lib/crates/fabro-workflow/src/pipeline/pull_request.rs index 8825f8eef..35f27aa92 100644 --- a/lib/crates/fabro-workflow/src/pipeline/pull_request.rs +++ b/lib/crates/fabro-workflow/src/pipeline/pull_request.rs @@ -11,7 +11,7 @@ use fabro_store::RunProjection; use fabro_types::PullRequestRecord; use fabro_types::settings::run::MergeStrategy; use fabro_util::text::strip_goal_decoration; -use tracing::{debug, info}; +use tracing::{debug, info, warn}; use super::types::{Concluded, Finalized, PullRequestOptions}; use crate::event::{Event, RunNoticeLevel}; @@ -19,130 +19,75 @@ use crate::outcome::{StageOutcome, format_cost as outcome_format_cost}; use crate::records::{Conclusion, RunSpec}; use crate::runtime_store::RunStoreHandle; -/// Maximum length of a PR title (Unicode scalar values). Single source of -/// truth — referenced by the structured-output schema, the system prompt, -/// and [`enforce_title_cap`]. +/// Maximum length of a PR title (Unicode scalar values). const PR_TITLE_MAX_CHARS: usize = 72; /// Structured output schema for the LLM-generated PR title and body. -/// -/// `title` is required but allows empty strings (the only signal that -/// triggers the deterministic title fallback in -/// [`maybe_open_pull_request`]). `body` requires `minLength: 1` because -/// there is no body fallback — an empty body is fatal. static PR_CONTENT_SCHEMA: LazyLock = LazyLock::new(|| { serde_json::json!({ "type": "object", "properties": { - "title": { "type": "string", "maxLength": PR_TITLE_MAX_CHARS }, - "body": { "type": "string", "minLength": 1 } + "title": { "type": "string" }, + "body": { "type": "string" } }, "required": ["title", "body"], "additionalProperties": false }) }); +/// Complete pull request content generated for a workflow run. #[derive(Debug, serde::Deserialize)] -struct GeneratedPrContent { - title: String, - body: String, +pub struct PrContent { + pub title: String, + pub body: String, } /// System prompt that instructs the LLM how to write a Fabro PR title and /// body. The trailing programmatic sections (Plan `
`, Retro, /// Fabro Details, footer) are appended after the LLM body — the prompt /// explicitly forbids the LLM from duplicating them. -// -// The "max 72 characters" instruction must stay in sync with -// `PR_TITLE_MAX_CHARS` and the schema above; the prompt is advisory and -// `enforce_title_cap` is the actual enforcement. -const PR_BODY_SYSTEM_PROMPT: &str = "You are writing a pull request title and description for a code change produced by an AI workflow. +const PR_BODY_SYSTEM_PROMPT: &str = include_str!("prompts/pr_body.md"); -OUTPUT FORMAT -Return a JSON object with exactly two fields: -- \"title\": a one-line title, max 72 characters, no trailing period. -- \"body\": the markdown body as described below. +const DEFAULT_PR_TITLE: &str = "Update workflow output"; +const EMPTY_BODY_NOTICE: &str = "> _The LLM did not produce a description for this change. The diff and the appended details are the source of truth for review._"; -DO NOT INCLUDE in the body -- A `#` or `##` title heading at the top — the title goes in the `title` field. -- A \"Retro\" section, \"Fabro Details\" section, cost/duration table, or \"Generated with\" footer — those are appended programmatically after your output. -- The full plan text — the full plan is appended programmatically as a
block. -- Bare `#1`, `#2` list prefixes — GitHub auto-links those as issue references. Use plain `1.`, `2.` instead. -- A test plan unless the testing approach is non-obvious. - -SIZE THE BODY TO THE CHANGE -First classify along two axes from the diff: -- Size: how many files changed, how large the diff is. -- Complexity: trivial (rename / typo / dep bump / config) vs. design decisions / new patterns / cross-cutting concerns. - -Then write at the matching depth: - -| Profile | Body shape | -|---|---| -| Small + simple (typo, config, dep bump) | 1–2 sentences, no headers, total under ~300 characters | -| Small + non-trivial (targeted bugfix, behavioral change) | Short \"Problem / Fix\" narrative, 3–5 sentences. No headers unless two distinct concerns. | -| Medium feature or refactor | Summary paragraph, then a section explaining what changed and why. Call out design decisions. | -| Large or architecturally significant | Full narrative: problem context, approach chosen (and why), key decisions, migration/rollback notes if relevant. | -| Performance improvement | Include before/after measurements if available. A markdown table works well here. | - -Brevity matters for small changes. A 3-line bugfix with a 20-line description signals miscalibration. When in doubt, shorter is better — reviewers can read the diff. - -WRITING PRINCIPLES -- Lead with value: the first sentence tells the reviewer *why this PR exists*, not *what files changed*. -- Describe the net result, not the journey: skip intermediate failures, debugging steps, and refactors done during development. -- Trust the final diff: if the goal or plan disagree with the diff, the diff is authoritative. -- Explain the non-obvious: spend description space on what the diff doesn't show — why this approach, what was rejected, what to look at first. -- Use structure when it earns its keep: no empty sections, no template headers without content. -- If the body uses any `##` heading, the opening summary must also be under a heading (e.g. `## Summary`); otherwise a bare paragraph is fine. - -PLAN SUMMARY -The full plan is attached separately as a
block, so do not restate it. Include a brief `### Plan Summary` with bullet points only when the change is medium or larger in the sizing matrix above. Skip it for small changes. - -VISUAL AIDS -Include a visual aid only when a reviewer would struggle to reconstruct the mental model from prose alone — based on what changes structurally, not on PR size. Skip for trivial / mechanical changes, or when prose already communicates clearly. - -| PR changes... | Visual aid | -|---|---| -| 3+ interacting components or services | Mermaid component / interaction diagram | -| Multi-step workflow or pipeline with non-obvious sequencing | Mermaid flow diagram | -| 3+ behavioral modes or variants | Markdown comparison table | -| Before/after data or trade-offs | Markdown table | -| Data model changes with 3+ related entities | Mermaid ERD | - -Mermaid: prefer `TB` direction, ≤10 nodes typical. Place inline at the point of relevance, not in a separate \"Diagrams\" section."; - -/// Truncation budget for the LLM prompt's goal / plan / diff sections. +/// Truncation budget for the LLM prompt's plan / diff sections. +#[derive(Debug, PartialEq, Eq)] struct TruncationCaps { - goal: usize, plan: usize, diff: usize, } -/// Generous tier for models with ≥200k context windows. -const TRUNCATION_LARGE: TruncationCaps = TruncationCaps { - goal: 75_000, - plan: 75_000, - diff: 250_000, -}; - -/// Conservative tier (matches the pre-refactor values). Used for smaller -/// or unknown models. -const TRUNCATION_SMALL: TruncationCaps = TruncationCaps { - goal: 20_000, - plan: 20_000, - diff: 50_000, -}; +const DIFF_HARD_CAP: usize = 500_000; +const PLAN_HARD_CAP: usize = 100_000; +const DIFF_FRACTION_NUM: usize = 4; +const PLAN_FRACTION_NUM: usize = 1; +const FRACTION_DEN: usize = 10; +const UNKNOWN_MODEL_CTX: usize = 200_000; /// Resolve truncation caps based on the model's context window. Unknown -/// models fall through to the conservative tier. -fn truncation_caps(model: &str) -> &'static TruncationCaps { - let large_enough = Catalog::builtin() +/// models use the baseline 200k context-window assumption. +fn truncation_caps(model: &str) -> TruncationCaps { + let ctx = Catalog::builtin() .get(model) - .is_some_and(|m| m.context_window() >= 200_000); - if large_enough { - &TRUNCATION_LARGE - } else { - &TRUNCATION_SMALL + .and_then(|m| usize::try_from(m.context_window()).ok()) + .unwrap_or(UNKNOWN_MODEL_CTX); + + truncation_caps_for_context_window(ctx) +} + +fn truncation_caps_for_context_window(ctx: usize) -> TruncationCaps { + TruncationCaps { + diff: ctx + .saturating_mul(DIFF_FRACTION_NUM) + .checked_div(FRACTION_DEN) + .unwrap_or(DIFF_HARD_CAP) + .min(DIFF_HARD_CAP), + plan: ctx + .saturating_mul(PLAN_FRACTION_NUM) + .checked_div(FRACTION_DEN) + .unwrap_or(PLAN_HARD_CAP) + .min(PLAN_HARD_CAP), } } @@ -172,12 +117,18 @@ fn enforce_title_cap(title: &str) -> String { /// Derive a PR title from the workflow goal. /// -/// Uses the first line, truncated to 120 characters for readability. The -/// caller is expected to apply [`enforce_title_cap`] afterwards if a -/// stricter cap is required (the wider cap here is the legacy behaviour -/// for the deterministic fallback path). +/// Uses the first line, truncated to the same cap as LLM-generated titles. fn pr_title_from_goal(goal: &str) -> String { - truncate_with_ellipsis(strip_goal_decoration(goal), 120) + truncate_with_ellipsis(strip_goal_decoration(goal), PR_TITLE_MAX_CHARS) +} + +fn fallback_pr_title(goal: &str) -> String { + let title = pr_title_from_goal(goal); + if title.trim().is_empty() { + DEFAULT_PR_TITLE.to_string() + } else { + title + } } /// Truncate a PR body to fit GitHub's 65,536 character limit. @@ -435,75 +386,43 @@ async fn load_pull_request_diff(run_store: &RunStoreHandle) -> String { .unwrap_or_default() } -/// Build a complete PR title and body by combining LLM-generated narrative -/// with programmatic sections (plan, retro, fabro details). -/// -/// Returns `(title, body)`. The title may be the empty string when the LLM -/// returned a usable body but no usable title — callers fall back to -/// [`pr_title_from_goal`] in that case. Every other generation failure is -/// surfaced as `Err`. -pub async fn build_pr_body( +/// Build complete PR content by combining LLM-generated narrative with +/// deterministic fallbacks and programmatic sections. +pub async fn build_pr_content( diff: &str, goal: &str, model: &str, run_store: &RunStoreHandle, llm_source: &dyn CredentialSource, conclusion: Option<&Conclusion>, -) -> Result<(String, String), String> { + run_state: Option<&RunProjection>, +) -> Result { let client = Client::from_source(llm_source) .await .map_err(|e| format!("Failed to create LLM client: {e}"))?; - build_pr_body_with_client(diff, goal, model, run_store, conclusion, Arc::new(client)).await -} - -async fn build_pr_body_with_client( - diff: &str, - goal: &str, - model: &str, - run_store: &RunStoreHandle, - conclusion: Option<&Conclusion>, - client: Arc, -) -> Result<(String, String), String> { - build_pr_body_with_client_and_state(diff, goal, model, run_store, conclusion, client, None) - .await -} - -async fn build_pr_body_with_source_and_state( - diff: &str, - goal: &str, - model: &str, - run_store: &RunStoreHandle, - llm_source: &dyn CredentialSource, - conclusion: Option<&Conclusion>, - run_state: Option<&fabro_store::RunProjection>, -) -> Result<(String, String), String> { - let client = Client::from_source(llm_source) - .await - .map_err(|e| format!("Failed to create LLM client: {e}"))?; - - build_pr_body_with_client_and_state( + build_pr_content_with_client( diff, goal, model, run_store, conclusion, - Arc::new(client), run_state, + Arc::new(client), ) .await } -async fn build_pr_body_with_client_and_state( +async fn build_pr_content_with_client( diff: &str, goal: &str, model: &str, run_store: &RunStoreHandle, conclusion: Option<&Conclusion>, + run_state: Option<&RunProjection>, client: Arc, - run_state: Option<&fabro_store::RunProjection>, -) -> Result<(String, String), String> { - info!("Building PR body"); +) -> Result { + info!("Building PR content"); let loaded_run_state = if run_state.is_none() { run_store @@ -524,16 +443,15 @@ async fn build_pr_body_with_client_and_state( let dot_source = run_state.and_then(|state| state.graph_source.clone()); let caps = truncation_caps(model); - let truncated_goal = truncate_chars(goal, caps.goal); let truncated_diff = truncate_chars(diff, caps.diff); let prompt = if let Some(ref plan) = plan_text { let truncated_plan = truncate_chars(plan, caps.plan); format!( - "Goal: {truncated_goal}\n\nPlan:\n```\n{truncated_plan}\n```\n\nDiff:\n```\n{truncated_diff}\n```" + "Goal: {goal}\n\nPlan:\n```\n{truncated_plan}\n```\n\nDiff:\n```\n{truncated_diff}\n```" ) } else { - format!("Goal: {truncated_goal}\n\nDiff:\n```\n{truncated_diff}\n```") + format!("Goal: {goal}\n\nDiff:\n```\n{truncated_diff}\n```") }; let params = GenerateParams::new(model, client) @@ -547,15 +465,22 @@ async fn build_pr_body_with_client_and_state( let output = result .output .ok_or_else(|| "LLM generation returned no structured output".to_string())?; - let generated: GeneratedPrContent = serde_json::from_value(output) + let generated: PrContent = serde_json::from_value(output) .map_err(|e| format!("Failed to deserialize PR content: {e}"))?; - if generated.body.trim().is_empty() { - return Err("LLM generated an empty PR body".to_string()); - } + let title = if generated.title.trim().is_empty() { + fallback_pr_title(goal) + } else { + generated.title.trim().to_string() + }; + let title = enforce_title_cap(&title); - let title = enforce_title_cap(generated.title.trim()); - let llm_body = generated.body; + let llm_body = if generated.body.trim().is_empty() { + warn!(model = %model, "LLM generated empty PR body; using skeleton PR body"); + EMPTY_BODY_NOTICE.to_string() + } else { + generated.body + }; let retro_section = retro.as_ref().map(format_retro_section).unwrap_or_default(); let arc_details_section = conclusion @@ -570,9 +495,9 @@ async fn build_pr_body_with_client_and_state( &arc_details_section, ); - info!("PR body generated"); + info!("PR content generated"); - Ok((title, body)) + Ok(PrContent { title, body }) } /// Auto-merge configuration for a pull request. @@ -594,7 +519,7 @@ pub struct OpenPullRequestRequest<'a> { pub run_store: &'a RunStoreHandle, pub llm_source: &'a dyn CredentialSource, pub conclusion: Option<&'a Conclusion>, - pub run_state: Option<&'a fabro_store::RunProjection>, + pub run_state: Option<&'a RunProjection>, } /// Optionally open a pull request after a successful workflow run. @@ -613,7 +538,7 @@ pub async fn maybe_open_pull_request( let (owner, repo) = github_app::parse_github_owner_repo(&https_url).map_err(|err| format!("{err:#}"))?; - let (llm_title, body) = build_pr_body_with_source_and_state( + let content = build_pr_content( req.diff, req.goal, req.model, @@ -624,14 +549,8 @@ pub async fn maybe_open_pull_request( ) .await .map_err(|err| format!("{err:#}"))?; - let body = truncate_pr_body(&body); - - let title = if llm_title.is_empty() { - pr_title_from_goal(req.goal) - } else { - llm_title - }; - let title = enforce_title_cap(&title); + let body = truncate_pr_body(&content.body); + let title = content.title; let created = github_app::create_pull_request( &req.github, @@ -1286,15 +1205,16 @@ mod tests { } #[tokio::test] - async fn build_pr_body_uses_in_memory_conclusion() { + async fn build_pr_content_uses_in_memory_conclusion() { let store = test_store(); let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); - let (title, body) = build_pr_body_with_client( + let PrContent { title, body } = build_pr_content_with_client( "diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n", "Implement feature", "mock-model", &run_store.clone().into(), Some(&make_test_conclusion()), + None, explicit_client( "mock", &pr_content_json("Mock title", "Narrative from mock."), @@ -1311,7 +1231,7 @@ mod tests { } #[tokio::test] - async fn build_pr_body_uses_store_records_without_legacy_files() { + async fn build_pr_content_uses_store_records_without_legacy_files() { let store = test_store(); let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); @@ -1363,19 +1283,21 @@ mod tests { .await .unwrap(); - let (_, body) = build_pr_body_with_client( + let body = build_pr_content_with_client( "diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n", "Implement feature", "mock-model", &run_store.clone().into(), Some(&make_test_conclusion()), + None, explicit_client( "mock", &pr_content_json("Mock title", "Narrative from mock."), ), ) .await - .unwrap(); + .unwrap() + .body; assert!(body.contains("Narrative from mock.")); assert!(body.contains("### Retro")); @@ -1384,7 +1306,7 @@ mod tests { } #[tokio::test] - async fn build_pr_body_uses_plan_text_from_store_without_response_md() { + async fn build_pr_content_uses_plan_text_from_store_without_response_md() { let store = test_store(); let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); @@ -1453,48 +1375,52 @@ mod tests { .await .unwrap(); - let (_, body) = build_pr_body_with_client( + let body = build_pr_content_with_client( "diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n", "Implement feature", "mock-model", &run_store.clone().into(), Some(&make_test_conclusion()), + None, explicit_client( "mock", &pr_content_json("Mock title", "Narrative from mock."), ), ) .await - .unwrap(); + .unwrap() + .body; assert!(body.contains("Full plan")); assert!(body.contains("Plan from store")); } #[tokio::test] - async fn build_pr_body_uses_explicit_llm_client() { + async fn build_pr_content_uses_explicit_llm_client() { let store = test_store(); let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); - let (_, body) = build_pr_body_with_client( + let body = build_pr_content_with_client( "diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n", "Implement feature", "gpt-5.4", &run_store.clone().into(), Some(&make_test_conclusion()), + None, explicit_client( "openai", &pr_content_json("Explicit title", "Narrative from explicit client."), ), ) .await - .unwrap(); + .unwrap() + .body; assert!(body.contains("Narrative from explicit client.")); assert!(!body.contains("Narrative from mock.")); } #[tokio::test] - async fn build_pr_body_uses_vault_only_openai_codex_source() { + async fn build_pr_content_uses_vault_only_openai_codex_source() { let server = MockServer::start_async().await; let response_mock = server .mock_async(|when, then| { @@ -1534,13 +1460,14 @@ mod tests { let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); let run_store_handle: RunStoreHandle = run_store.into(); - let (title, body) = build_pr_body( + let PrContent { title, body } = build_pr_content( "diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n", "Implement feature", "gpt-5.4", &run_store_handle, llm_source.as_ref(), Some(&make_test_conclusion()), + None, ) .await .unwrap(); @@ -1642,7 +1569,7 @@ mod tests { fn pr_title_truncates_long_line() { let long = "x".repeat(300); let title = pr_title_from_goal(&long); - assert_eq!(title.chars().count(), 120); + assert_eq!(title.chars().count(), 72); assert!(title.ends_with('…')); } @@ -1665,6 +1592,42 @@ mod tests { assert_eq!(pr_title_from_goal("Fix bug"), "Fix bug"); } + #[test] + fn truncation_caps_scale_with_context_window_and_clamp() { + assert_eq!( + truncation_caps_for_context_window(100_000), + TruncationCaps { + diff: 40_000, + plan: 10_000, + } + ); + assert_eq!( + truncation_caps_for_context_window(200_000), + TruncationCaps { + diff: 80_000, + plan: 20_000, + } + ); + assert_eq!( + truncation_caps_for_context_window(1_000_000), + TruncationCaps { + diff: 400_000, + plan: 100_000, + } + ); + assert_eq!( + truncation_caps_for_context_window(10_000_000), + TruncationCaps { + diff: 500_000, + plan: 100_000, + } + ); + assert_eq!(truncation_caps("unknown-model"), TruncationCaps { + diff: 80_000, + plan: 20_000, + }); + } + #[tokio::test] async fn empty_diff_returns_none() { let store = test_store(); @@ -1760,68 +1723,144 @@ mod tests { /// MockProvider returns an over-long title; builder must cap it at 72 /// chars and end with `…`. Exercises [`enforce_title_cap`] inside - /// [`build_pr_body_with_client_and_state`]. + /// [`build_pr_content_with_client`]. #[tokio::test] - async fn build_pr_body_truncates_long_title() { + async fn build_pr_content_truncates_long_title() { let store = test_store(); let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); let long_title = "x".repeat(200); let payload = pr_content_json(&long_title, "Body content."); - let (title, _) = build_pr_body_with_client( + let title = build_pr_content_with_client( "diff --git a/src/lib.rs b/src/lib.rs\n+fn x() {}\n", "Implement feature", "mock-model", &run_store.clone().into(), Some(&make_test_conclusion()), + None, explicit_client("mock", &payload), ) .await - .unwrap(); + .unwrap() + .title; assert_eq!(title.chars().count(), 72); assert!(title.ends_with('\u{2026}')); } - /// Empty bodies are fatal. Real providers may reject this via the - /// schema's `minLength`; the Rust-side trim check also catches it for - /// local/mock providers. #[tokio::test] - async fn build_pr_body_returns_err_when_body_empty() { + async fn build_pr_content_uses_default_title_when_generated_and_goal_titles_empty() { let store = test_store(); let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); - let payload = pr_content_json("Mock", ""); - let result = build_pr_body_with_client( + let payload = pr_content_json("", "Body content."); + let title = build_pr_content_with_client( "diff --git a/src/lib.rs b/src/lib.rs\n+fn x() {}\n", - "Implement feature", + "## Plan:", "mock-model", &run_store.clone().into(), Some(&make_test_conclusion()), + None, explicit_client("mock", &payload), ) - .await; + .await + .unwrap() + .title; - assert!(result.is_err(), "expected Err, got {result:?}"); + assert_eq!(title, DEFAULT_PR_TITLE); } - /// Whitespace-only bodies pass schema validation but fail the - /// `body.trim().is_empty()` check inside the builder. + /// Empty or whitespace-only bodies use the skeleton fallback instead of + /// aborting PR creation. #[tokio::test] - async fn build_pr_body_returns_err_when_body_whitespace() { + async fn build_pr_content_uses_skeleton_when_body_empty() { let store = test_store(); let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); + + let run_spec = RunSpec { + run_id: fixtures::RUN_1, + settings: fabro_types::WorkflowSettings::default(), + graph: Graph::new("test"), + workflow_slug: Some("test".to_string()), + source_directory: Some("/tmp/project".to_string()), + git: None, + labels: HashMap::new(), + provenance: None, + manifest_blob: None, + definition_blob: None, + fork_source_ref: None, + in_place: false, + }; + append_event(&run_store, &fixtures::RUN_1, &Event::RunCreated { + run_id: fixtures::RUN_1, + settings: serde_json::to_value(&run_spec.settings).unwrap(), + graph: serde_json::to_value(&run_spec.graph).unwrap(), + workflow_source: Some("digraph test { plan -> code }".to_string()), + workflow_config: None, + labels: run_spec.labels.clone().into_iter().collect(), + run_dir: "/tmp/project".to_string(), + source_directory: run_spec.source_directory.clone(), + workflow_slug: run_spec.workflow_slug.clone(), + db_prefix: None, + provenance: None, + manifest_blob: None, + git: None, + fork_source_ref: None, + in_place: false, + web_url: None, + }) + .await + .unwrap(); + append_event(&run_store, &fixtures::RUN_1, &Event::StageCompleted { + node_id: "plan".to_string(), + name: "plan".to_string(), + index: 0, + duration_ms: 1, + status: "succeeded".to_string(), + preferred_label: None, + suggested_next_ids: vec![], + billing: None, + failure: None, + notes: None, + files_touched: vec![], + context_updates: None, + jump_to_node: None, + context_values: None, + node_visits: None, + loop_failure_signatures: None, + restart_failure_signatures: None, + response: Some("Plan from store".to_string()), + attempt: 1, + max_attempts: 1, + }) + .await + .unwrap(); + append_event(&run_store, &fixtures::RUN_1, &Event::RetroCompleted { + duration_ms: 1, + response: Some(String::new()), + retro: Some(serde_json::to_value(make_test_retro()).unwrap()), + }) + .await + .unwrap(); + let payload = pr_content_json("Mock", " \n"); - let result = build_pr_body_with_client( + let body = build_pr_content_with_client( "diff --git a/src/lib.rs b/src/lib.rs\n+fn x() {}\n", "Implement feature", "mock-model", &run_store.clone().into(), Some(&make_test_conclusion()), + None, explicit_client("mock", &payload), ) - .await; + .await + .unwrap() + .body; - let err = result.expect_err("expected Err for whitespace-only body"); - assert!(err.contains("empty PR body"), "unexpected error: {err}"); + assert!(body.contains("The LLM did not produce a description")); + assert!(body.contains("Full plan")); + assert!(body.contains("Plan from store")); + assert!(body.contains("### Retro")); + assert!(body.contains("### Fabro Details")); + assert!(body.contains("Generated with [Fabro](https://fabro.sh)")); } // ── maybe_open_pull_request fallback tests ────────────────────────── @@ -1978,9 +2017,9 @@ mod tests { } } - /// LLM returns a usable body but an empty title; `maybe_open_pull_request` - /// must fall back to `pr_title_from_goal` (first line, decoration - /// stripped) and the PR creation must succeed with that title. + /// LLM returns a usable body but an empty title; the content builder + /// falls back to `pr_title_from_goal` (first line, decoration stripped) + /// and PR creation succeeds with that title. #[tokio::test] async fn maybe_open_pull_request_falls_back_to_goal_title_when_llm_returns_empty_title() { let payload = pr_content_json("", "Narrative."); @@ -2012,10 +2051,8 @@ mod tests { harness.assert_mocks_called_once().await; } - /// LLM returns an empty title; the fallback path produces a long title - /// (close to `pr_title_from_goal`'s 120-char cap), and the unconditional - /// `enforce_title_cap` in `maybe_open_pull_request` must still bring it - /// down to 72 chars ending with `…`. + /// LLM returns an empty title; the content builder fallback still caps + /// the deterministic goal title at 72 chars ending with `…`. #[tokio::test] async fn maybe_open_pull_request_caps_fallback_title_at_72_chars() { let payload = pr_content_json("", "Narrative."); diff --git a/lib/crates/fabro-workflow/src/pull_request.rs b/lib/crates/fabro-workflow/src/pull_request.rs index 3989bf63c..3b65dfcbc 100644 --- a/lib/crates/fabro-workflow/src/pull_request.rs +++ b/lib/crates/fabro-workflow/src/pull_request.rs @@ -1,4 +1,4 @@ pub use crate::pipeline::{ - AutoMergeOptions, OpenPullRequestRequest, PullRequestRecord, build_pr_body, + AutoMergeOptions, OpenPullRequestRequest, PrContent, PullRequestRecord, build_pr_content, maybe_open_pull_request, }; diff --git a/lib/crates/fabro-workflow/tests/it/integration.rs b/lib/crates/fabro-workflow/tests/it/integration.rs index 742218c14..f6ece331b 100644 --- a/lib/crates/fabro-workflow/tests/it/integration.rs +++ b/lib/crates/fabro-workflow/tests/it/integration.rs @@ -6896,7 +6896,7 @@ async fn workflow_run_with_vault_only_openai_codex_builds_pr_body() { let run_store = store.open_run_reader(&run_options.run_id).await.unwrap(); let run_store_handle: fabro_workflow::runtime_store::RunStoreHandle = run_store.into(); - let (title, body) = fabro_workflow::pull_request::build_pr_body( + let content = fabro_workflow::pull_request::build_pr_content( "diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n", "Implement feature", "gpt-5.4", @@ -6912,12 +6912,13 @@ async fn workflow_run_with_vault_only_openai_codex_builds_pr_body() { billing: None, total_retries: 0, }), + None, ) .await .expect("PR body should build from vault-only credentials"); - assert_eq!(title, "Vault title"); - assert!(body.contains("Narrative from vault source.")); + assert_eq!(content.title, "Vault title"); + assert!(content.body.contains("Narrative from vault source.")); response_mock.assert_async().await; } From 333b603f5bc6f177309df542304e904dfff203af Mon Sep 17 00:00:00 2001 From: "fabro-sh-0530[bot]" <281434857+fabro-sh-0530[bot]@users.noreply.github.com> Date: Tue, 5 May 2026 08:27:11 -0400 Subject: [PATCH 12/16] Encode stage visits in run stage URLs (#206) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Summary Stages that re-enter the same workflow node now get distinct `node@visit` identities end to end, so looped stages like `verify@1` and `verify@2` no longer collapse to the same sidebar link, event stream, graph selection, or turns view. ### What changed - `RunStage.id` now uses the full `StageId` string (`node_id@visit`), with required `node_id` and `visit` fields in the OpenAPI schema and generated clients. This intentionally replaces the old `dot_id` field. - The server builds `/runs/{id}/stages` from `RunProjection::iter_stages()` instead of checkpoint `completed_nodes`, preserving visit information and including in-flight stages from projection data. - Stage status is derived from the latest lifecycle event for each exact `stage_id`, so retrying stages do not appear failed while a retry is underway. - The frontend maps and displays visits with `(N)` suffixes, filters fallback turns by `stage_id`, invalidates suffixed stage-turn query keys from SSE, and aggregates graph nodes by `node_id` with latest-visit click targets. ### Plan Summary - Preserve per-visit stage identity across API, server projection, generated clients, and UI routing. - Keep graph nodes keyed by workflow node while routing clicks to the latest visit. - Add coverage for multi-visit stages, retrying status derivation, suffixed SSE invalidation, sidebar labels, and stage event filtering. ### Reviewer notes This is a breaking API shape change for `RunStage`: consumers should use `node_id` for graph/node identity and `id` for per-visit stage identity. The old `dot_id` field is removed rather than kept as a compatibility alias. ### Fabro Details
Ran 9 stages in 54m 55s for $41.40 | Stage | Duration | Cost | Retries | |---|---|---|---| | start | 0s | – | 0 | | toolchain | 1s | – | 0 | | preflight_compile | 2m 8s | – | 0 | | preflight_lint | 2m 14s | – | 0 | | implement | 31m 38s | $17.65 | 0 | | simplify_opus | 10m 2s | $2.40 | 0 | | simplify_gpt | 6m 9s | $21.35 | 0 | | verify | 2m 3s | – | 0 | | fmt | 2s | – | 0 | | **Total** | **54m 55s** | **$41.40** | **0** |
Ran ImplementPlan.fabro (12 nodes and 15 edges) ```dot digraph ImplementPlan { graph [ goal="Implement and simplify", model_stylesheet=" * { model: claude-opus-4-7; } " ] rankdir=LR start [shape=Mdiamond, label="Start"] exit [shape=Msquare, label="Exit"] toolchain [label="Toolchain", shape=parallelogram, script="command -v cargo >/dev/null || { curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && sudo ln -sf $HOME/.cargo/bin/* /usr/local/bin/; }; cargo --version 2>&1", max_retries=0] preflight_compile [label="Preflight Compile", shape=parallelogram, script="cargo check -q --workspace 2>&1", max_retries=0] preflight_lint [label="Preflight Lint", shape=parallelogram, script="cargo +nightly-2026-04-14 clippy -q --workspace --all-targets -- -D warnings 2>&1", max_retries=0] fix_lints [label="Fix Lints", prompt="The preflight lint step failed. Read the build output from context and fix all clippy lint warnings.", max_visits=3] implement [label="Implement", prompt="Read the plan file referenced in the goal and implement every step. Make all the code changes described in the plan. Use red/green TDD."] simplify_opus [label="Simplify (Opus)", prompt="@prompts/simplify.md"] simplify_gpt [label="Simplify (GPT-55)", prompt="@prompts/simplify.md", model="gpt-55"] verify [label="Verify", shape=parallelogram, script="cargo +nightly-2026-04-14 clippy -q --workspace --all-targets -- -D warnings 2>&1 && cargo nextest run --cargo-quiet --workspace --status-level fail 2>&1 && cargo dev docs refresh 2>&1 && cargo dev docs check 2>&1", goal_gate=true, retry_target="fixup"] fixup [label="Fixup", prompt="The verify step failed. Read the build output from context and fix all clippy lint warnings, test failures, and generated docs errors.", max_visits=3] fmt [label="Format", shape=parallelogram, script="cargo +nightly-2026-04-14 fmt --all 2>&1", max_retries=0] start -> toolchain toolchain -> preflight_compile [condition="outcome=succeeded"] toolchain -> exit preflight_compile -> preflight_lint [condition="outcome=succeeded"] preflight_compile -> exit preflight_lint -> implement [condition="outcome=succeeded"] preflight_lint -> fix_lints fix_lints -> preflight_lint implement -> simplify_opus -> simplify_gpt -> verify verify -> fmt [condition="outcome=succeeded"] verify -> fixup fixup -> verify fmt -> exit } ```
⚒️ Generated with [Fabro](https://fabro.sh) --------- Co-authored-by: Fabro Co-authored-by: Bryan Helmkamp Co-authored-by: Claude Opus 4.7 (1M context) --- .../app/components/stage-sidebar.tsx | 7 +- apps/fabro-web/app/lib/run-events.test.tsx | 65 ++ apps/fabro-web/app/lib/run-events.ts | 9 +- apps/fabro-web/app/lib/stage-sidebar.test.ts | 172 ++++ apps/fabro-web/app/lib/stage-sidebar.ts | 51 +- apps/fabro-web/app/routes/run-overview.tsx | 34 +- apps/fabro-web/app/routes/run-stages.test.ts | 96 ++- apps/fabro-web/app/routes/run-stages.tsx | 20 +- docs/public/api-reference/fabro-api.yaml | 30 +- lib/crates/fabro-dump/src/lib.rs | 7 +- lib/crates/fabro-server/src/demo/mod.rs | 61 +- lib/crates/fabro-server/src/server.rs | 127 +-- .../src/server/handler/billing.rs | 277 +++--- lib/crates/fabro-server/src/server/tests.rs | 791 ++++++++++++++++-- lib/crates/fabro-store/src/artifact_store.rs | 7 +- lib/crates/fabro-store/src/run_state.rs | 327 +++++++- .../tests/serializable_projection.rs | 30 +- lib/crates/fabro-types/src/lib.rs | 2 +- lib/crates/fabro-types/src/outcome.rs | 25 +- lib/crates/fabro-types/src/run_event/stage.rs | 2 + lib/crates/fabro-types/src/run_projection.rs | 123 ++- lib/crates/fabro-types/src/stage_id.rs | 39 +- .../fabro-workflow/src/billing_rollup.rs | 291 +++++++ lib/crates/fabro-workflow/src/error.rs | 1 + .../fabro-workflow/src/event/convert.rs | 29 +- lib/crates/fabro-workflow/src/event/events.rs | 1 + lib/crates/fabro-workflow/src/lib.rs | 189 ++++- .../fabro-workflow/src/lifecycle/event.rs | 2 + .../fabro-workflow/src/pipeline/finalize.rs | 137 ++- lib/crates/fabro-workflow/src/pipeline/mod.rs | 2 +- .../fabro-workflow/src/pipeline/retro.rs | 2 +- lib/crates/fabro-workflow/src/test_support.rs | 7 +- .../src/models/billing-by-model.ts | 2 +- .../src/models/billing-stage-ref.ts | 2 +- .../src/models/run-billing-stage.ts | 4 +- .../src/models/run-billing.ts | 2 +- .../fabro-api-client/src/models/run-stage.ts | 10 +- 37 files changed, 2508 insertions(+), 475 deletions(-) create mode 100644 apps/fabro-web/app/lib/stage-sidebar.test.ts create mode 100644 lib/crates/fabro-workflow/src/billing_rollup.rs diff --git a/apps/fabro-web/app/components/stage-sidebar.tsx b/apps/fabro-web/app/components/stage-sidebar.tsx index b72ee77bd..2d3122602 100644 --- a/apps/fabro-web/app/components/stage-sidebar.tsx +++ b/apps/fabro-web/app/components/stage-sidebar.tsx @@ -11,14 +11,15 @@ import { } from "@heroicons/react/24/solid"; import { Bars3BottomLeftIcon, DocumentTextIcon, MapIcon } from "@heroicons/react/24/outline"; import { formatDurationSecs } from "../lib/format"; -import { ACTIVE_STAGE_STATES } from "../lib/stage-sidebar"; +import { ACTIVE_STAGE_STATES, formatStageLabel } from "../lib/stage-sidebar"; export interface Stage { id: string; name: string; status: StageState; duration: string; - dotId?: string; + nodeId: string; + visit: number; } export const statusConfig: Record; color: string }> = { @@ -100,7 +101,7 @@ export function StageSidebar({ stages, runId, selectedStageId, activeLink }: Sta }`} > - {stage.name} + {formatStageLabel(stage)} {stageDuration(stage)} diff --git a/apps/fabro-web/app/lib/run-events.test.tsx b/apps/fabro-web/app/lib/run-events.test.tsx index e9faa93fc..c38ea1dd2 100644 --- a/apps/fabro-web/app/lib/run-events.test.tsx +++ b/apps/fabro-web/app/lib/run-events.test.tsx @@ -49,6 +49,14 @@ describe("queryKeysForRunEvent", () => { queryKeys.runs.graph("run-1", "TB"), ]); }); + + test("stage.retrying invalidates the same keys as other stage events", () => { + const keys = queryKeysForRunEvent("run-1", "stage.retrying", "verify@2"); + expect(keys).toContain(queryKeys.runs.stages("run-1")); + expect(keys).toContain(queryKeys.runs.events("run-1", 1000)); + expect(keys).toContain(queryKeys.runs.detail("run-1")); + expect(keys).toContain(queryKeys.runs.stageTurns("run-1", "verify@2")); + }); }); describe("subscribeToRunEvents", () => { @@ -172,6 +180,63 @@ describe("subscribeToRunEvents", () => { coordinator.close(); }); + test("envelope with suffixed stage_id invalidates stageTurns(runId, stageId)", async () => { + const source = new FakeEventSource(); + const keys: string[] = []; + const coordinator = createCoordinator(() => source); + const cleanup = subscribeToRunEvents( + "run-stage", + (key) => { + keys.push(key); + return Promise.resolve(); + }, + () => source, + { debounceMs: 0, coordinator }, + ); + + await waitFor(() => source.onmessage !== null); + source.emit({ + event: "stage.retrying", + run_id: "run-stage", + stage_id: "verify@2", + node_id: "verify", + }); + + expect(keys).toContain(queryKeys.runs.stageTurns("run-stage", "verify@2")); + expect(keys).toContain(queryKeys.runs.stages("run-stage")); + expect(keys).toContain(queryKeys.runs.events("run-stage", 1000)); + expect(keys).toContain(queryKeys.runs.graph("run-stage", "LR")); + expect(keys).toContain(queryKeys.runs.detail("run-stage")); + expect(keys).not.toContain(queryKeys.runs.stageTurns("run-stage", "verify")); + + cleanup(); + coordinator.close(); + }); + + test("falls back to node_id when an event has no stage_id", async () => { + const source = new FakeEventSource(); + const keys: string[] = []; + const coordinator = createCoordinator(() => source); + const cleanup = subscribeToRunEvents( + "run-stage-node", + (key) => { + keys.push(key); + return Promise.resolve(); + }, + () => source, + { debounceMs: 0, coordinator }, + ); + + await waitFor(() => source.onmessage !== null); + source.emit({ event: "stage.started", run_id: "run-stage-node", node_id: "verify" }); + + expect(keys).toContain(queryKeys.runs.stageTurns("run-stage-node", "verify")); + expect(keys).toContain(queryKeys.runs.stages("run-stage-node")); + + cleanup(); + coordinator.close(); + }); + test("fallback malformed events are ignored and StrictMode-style cleanup does not underflow", () => { const firstSource = new FakeEventSource(); const secondSource = new FakeEventSource(); diff --git a/apps/fabro-web/app/lib/run-events.ts b/apps/fabro-web/app/lib/run-events.ts index 1413d38d4..a81a11c2b 100644 --- a/apps/fabro-web/app/lib/run-events.ts +++ b/apps/fabro-web/app/lib/run-events.ts @@ -19,6 +19,7 @@ interface RunEventPayload extends EventPayload { event?: string; run_id?: string; node_id?: string; + stage_id?: string; properties?: Record; } @@ -42,7 +43,12 @@ const RUN_SUMMARY_EVENTS = new Set([ "run.archived", "run.unarchived", ]); -const STAGE_EVENTS = new Set(["stage.started", "stage.completed", "stage.failed"]); +const STAGE_EVENTS = new Set([ + "stage.started", + "stage.completed", + "stage.failed", + "stage.retrying", +]); const COMMAND_EVENTS = new Set(["command.started", "command.completed"]); const INTERVIEW_EVENTS = new Set([ "interview.started", @@ -166,6 +172,7 @@ function resyncKeysForRun(runId: string) { } function stageIdFromPayload(payload: RunEventPayload): string | undefined { + if (typeof payload.stage_id === "string") return payload.stage_id; if (typeof payload.node_id === "string") return payload.node_id; const nodeId = payload.properties?.node_id; return typeof nodeId === "string" ? nodeId : undefined; diff --git a/apps/fabro-web/app/lib/stage-sidebar.test.ts b/apps/fabro-web/app/lib/stage-sidebar.test.ts new file mode 100644 index 000000000..9e8891645 --- /dev/null +++ b/apps/fabro-web/app/lib/stage-sidebar.test.ts @@ -0,0 +1,172 @@ +import { describe, expect, test } from "bun:test"; +import type { PaginatedRunStageList, StageState } from "@qltysh/fabro-api-client"; + +import type { Stage } from "../components/stage-sidebar"; +import { aggregateGraphNodeStatus, formatStageLabel, mapRunStagesToSidebarStages } from "./stage-sidebar"; + +function makeStage(nodeId: string, visit: number, status: StageState): Stage { + return { + id: `${nodeId}@${visit}`, + name: nodeId, + nodeId, + visit, + status, + duration: "--", + }; +} + +describe("mapRunStagesToSidebarStages", () => { + test("maps two visits of the same node to distinct sidebar entries", () => { + const stages: PaginatedRunStageList = { + data: [ + { + id: "apply-changes@1", + name: "Apply Changes", + status: "succeeded", + duration_secs: 12.5, + node_id: "apply", + visit: 1, + }, + { + id: "apply-changes@2", + name: "Apply Changes", + status: "running", + node_id: "apply", + visit: 2, + }, + ], + meta: { has_more: false }, + }; + + const result = mapRunStagesToSidebarStages(stages); + expect(result).toHaveLength(2); + + expect(result[0].id).toBe("apply-changes@1"); + expect(result[0].nodeId).toBe("apply"); + expect(result[0].visit).toBe(1); + expect(formatStageLabel(result[0])).toBe("Apply Changes"); + + expect(result[1].id).toBe("apply-changes@2"); + expect(result[1].nodeId).toBe("apply"); + expect(result[1].visit).toBe(2); + expect(formatStageLabel(result[1])).toBe("Apply Changes (2)"); + }); + + test("filters by node_id (suffixed start@1 / exit@1 are still hidden)", () => { + const stages: PaginatedRunStageList = { + data: [ + { + id: "start@1", + name: "start", + status: "succeeded", + node_id: "start", + visit: 1, + }, + { + id: "verify@1", + name: "verify", + status: "succeeded", + node_id: "verify", + visit: 1, + }, + { + id: "exit@1", + name: "exit", + status: "succeeded", + node_id: "exit", + visit: 1, + }, + ], + meta: { has_more: false }, + }; + + const result = mapRunStagesToSidebarStages(stages); + expect(result.map((s) => s.id)).toEqual(["verify@1"]); + }); + + test("missing duration renders as '--'", () => { + const stages: PaginatedRunStageList = { + data: [ + { + id: "verify@1", + name: "verify", + status: "running", + node_id: "verify", + visit: 1, + }, + ], + meta: { has_more: false }, + }; + + expect(mapRunStagesToSidebarStages(stages)[0].duration).toBe("--"); + }); +}); + +describe("aggregateGraphNodeStatus", () => { + test("(failed, running) renders as running and clicks open the latest visit", () => { + const result = aggregateGraphNodeStatus([ + makeStage("verify", 1, "failed"), + makeStage("verify", 2, "running"), + ]); + expect(result.get("verify")).toEqual({ + displayStatus: "running", + latestStageId: "verify@2", + }); + }); + + test("(failed, succeeded) renders as succeeded — failure-then-fix shows healed", () => { + const result = aggregateGraphNodeStatus([ + makeStage("verify", 1, "failed"), + makeStage("verify", 2, "succeeded"), + ]); + expect(result.get("verify")).toEqual({ + displayStatus: "succeeded", + latestStageId: "verify@2", + }); + }); + + test("(succeeded, failed) renders as failed and clicks open the latest visit", () => { + const result = aggregateGraphNodeStatus([ + makeStage("verify", 1, "succeeded"), + makeStage("verify", 2, "failed"), + ]); + expect(result.get("verify")).toEqual({ + displayStatus: "failed", + latestStageId: "verify@2", + }); + }); + + test("(running, retrying) — latest active wins", () => { + const result = aggregateGraphNodeStatus([ + makeStage("verify", 1, "running"), + makeStage("verify", 2, "retrying"), + ]); + expect(result.get("verify")).toEqual({ + displayStatus: "retrying", + latestStageId: "verify@2", + }); + }); + + test("orders by visit even when input is shuffled", () => { + const result = aggregateGraphNodeStatus([ + makeStage("verify", 2, "running"), + makeStage("verify", 1, "failed"), + ]); + expect(result.get("verify")?.latestStageId).toBe("verify@2"); + }); + + test("single visit per node is unaffected", () => { + const result = aggregateGraphNodeStatus([ + makeStage("plan", 1, "succeeded"), + makeStage("apply", 1, "running"), + ]); + expect(result.get("plan")).toEqual({ + displayStatus: "succeeded", + latestStageId: "plan@1", + }); + expect(result.get("apply")).toEqual({ + displayStatus: "running", + latestStageId: "apply@1", + }); + }); +}); diff --git a/apps/fabro-web/app/lib/stage-sidebar.ts b/apps/fabro-web/app/lib/stage-sidebar.ts index 747c6a70e..e44e5f4a5 100644 --- a/apps/fabro-web/app/lib/stage-sidebar.ts +++ b/apps/fabro-web/app/lib/stage-sidebar.ts @@ -10,18 +10,65 @@ export const SUCCEEDED_STAGE_STATES: ReadonlySet = new Set([ "partially_succeeded", ]); +/** + * Display label for a stage. Suffixes `(N)` for visits > 1 so a looped node + * (e.g. `verify`) renders as `verify`, `verify (2)`, `verify (3)` in the + * sidebar and stage header. + */ +export function formatStageLabel(stage: { name: string; visit: number }): string { + return stage.visit > 1 ? `${stage.name} (${stage.visit})` : stage.name; +} + export function mapRunStagesToSidebarStages( stagesResult: PaginatedRunStageList | null | undefined, ): Stage[] { return (stagesResult?.data ?? []) - .filter((stage) => isVisibleStage(stage.id)) + .filter((stage) => isVisibleStage(stage.node_id)) .map((stage) => ({ id: stage.id, name: stage.name, - dotId: stage.dot_id ?? stage.id, + nodeId: stage.node_id, + visit: stage.visit, status: stage.status, duration: stage.duration_secs != null ? formatDurationSecs(stage.duration_secs) : "--", })); } + +/** + * Aggregate per-node display state for the workflow graph. + * + * Status policy: if any visit is active (running/retrying), the node renders + * that active state (latest active visit wins). Otherwise the node renders + * the latest visit's terminal state. The click target is always the latest + * visit's stageId. + */ +export function aggregateGraphNodeStatus(stages: readonly Stage[]): Map< + string, + { displayStatus: StageState; latestStageId: string } +> { + // Single pass per nodeId: track the visit with the highest `visit` overall + // (drives click target + terminal status) and the highest-visit *active* + // stage (drives display when any visit is in flight). + const latest = new Map(); + const latestActive = new Map(); + for (const stage of stages) { + const prevLatest = latest.get(stage.nodeId); + if (!prevLatest || stage.visit > prevLatest.visit) { + latest.set(stage.nodeId, stage); + } + if (ACTIVE_STAGE_STATES.has(stage.status)) { + const prevActive = latestActive.get(stage.nodeId); + if (!prevActive || stage.visit > prevActive.visit) { + latestActive.set(stage.nodeId, stage); + } + } + } + const result = new Map(); + for (const [nodeId, latestStage] of latest) { + const display = latestActive.get(nodeId) ?? latestStage; + result.set(nodeId, { displayStatus: display.status, latestStageId: latestStage.id }); + } + return result; +} diff --git a/apps/fabro-web/app/routes/run-overview.tsx b/apps/fabro-web/app/routes/run-overview.tsx index bf0c0dd93..48066eae0 100644 --- a/apps/fabro-web/app/routes/run-overview.tsx +++ b/apps/fabro-web/app/routes/run-overview.tsx @@ -3,7 +3,6 @@ import { useNavigate, useParams } from "react-router"; import { graphTheme } from "../lib/graph-theme"; import { useRun, useRunGraph, useRunStages } from "../lib/queries"; import { StageSidebar } from "../components/stage-sidebar"; -import type { Stage } from "../components/stage-sidebar"; import { GRAPH_DEFAULT_ZOOM_INDEX, GRAPH_ZOOM_STEPS, @@ -13,6 +12,7 @@ import { EmptyState } from "../components/state"; import { ACTIVE_STAGE_STATES, SUCCEEDED_STAGE_STATES, + aggregateGraphNodeStatus, mapRunStagesToSidebarStages, } from "../lib/stage-sidebar"; @@ -54,27 +54,27 @@ export default function RunOverview() { const inner = innerRef.current; if (!inner || !graphSvg) return; - let cancelled = false; - (async () => { - if (cancelled) return; inner.innerHTML = graphSvg; const svg = inner.querySelector("svg"); if (!svg) return; svgRef.current = svg; const gt = graphTheme; - const runningDotIds = new Set( - stages.filter((s: Stage) => ACTIVE_STAGE_STATES.has(s.status)).map((s: Stage) => s.dotId ?? s.id), - ); - const failedDotIds = new Set( - stages.filter((s: Stage) => s.status === "failed").map((s: Stage) => s.dotId ?? s.id), - ); - const completedDotIds = new Set( - stages.filter((s: Stage) => SUCCEEDED_STAGE_STATES.has(s.status)).map((s: Stage) => s.dotId ?? s.id), - ); - const dotIdToStageId = new Map( - stages.map((s: Stage) => [s.dotId ?? s.id, s.id]), - ); + const aggregated = aggregateGraphNodeStatus(stages); + const runningDotIds = new Set(); + const failedDotIds = new Set(); + const completedDotIds = new Set(); + const dotIdToStageId = new Map(); + for (const [nodeId, { displayStatus, latestStageId }] of aggregated) { + dotIdToStageId.set(nodeId, latestStageId); + if (ACTIVE_STAGE_STATES.has(displayStatus)) { + runningDotIds.add(nodeId); + } else if (displayStatus === "failed") { + failedDotIds.add(nodeId); + } else if (SUCCEEDED_STAGE_STATES.has(displayStatus)) { + completedDotIds.add(nodeId); + } + } const ns = "http://www.w3.org/2000/svg"; for (const group of svg.querySelectorAll(".node")) { @@ -148,8 +148,6 @@ export default function RunOverview() { } } } - })(); - return () => { cancelled = true; }; }, [stages, graphSvg, id, navigate, terminalOutcome]); const onPointerDown = useCallback((e: React.PointerEvent) => { diff --git a/apps/fabro-web/app/routes/run-stages.test.ts b/apps/fabro-web/app/routes/run-stages.test.ts index d10b5b2d2..704c5cac8 100644 --- a/apps/fabro-web/app/routes/run-stages.test.ts +++ b/apps/fabro-web/app/routes/run-stages.test.ts @@ -1,6 +1,7 @@ import { describe, expect, test } from "bun:test"; +import type { EventEnvelope } from "@qltysh/fabro-api-client"; -import { isSafeMarkdownHref } from "./run-stages"; +import { isSafeMarkdownHref, turnsFromEvents } from "./run-stages"; describe("isSafeMarkdownHref", () => { test("rejects protocol-relative URLs", () => { @@ -15,3 +16,96 @@ describe("isSafeMarkdownHref", () => { expect(isSafeMarkdownHref("mailto:test@example.com")).toBe(true); }); }); + +function makeEnvelope(overrides: Partial): EventEnvelope { + return { + seq: 1, + id: "evt", + ts: "2026-01-01T00:00:00Z", + run_id: "run-1", + event: "stage.prompt", + ...overrides, + } as EventEnvelope; +} + +describe("turnsFromEvents", () => { + test("filters events by stage_id (verify@1 vs verify@2 do not cross-contaminate)", () => { + const events: EventEnvelope[] = [ + makeEnvelope({ + seq: 1, + event: "stage.prompt", + stage_id: "verify@1", + node_id: "verify", + properties: { text: "first visit prompt" }, + }), + makeEnvelope({ + seq: 2, + event: "stage.prompt", + stage_id: "verify@2", + node_id: "verify", + properties: { text: "second visit prompt" }, + }), + makeEnvelope({ + seq: 3, + event: "agent.message", + stage_id: "verify@1", + node_id: "verify", + properties: { text: "first visit reply" }, + }), + makeEnvelope({ + seq: 4, + event: "agent.message", + stage_id: "verify@2", + node_id: "verify", + properties: { text: "second visit reply" }, + }), + ]; + + const firstVisit = turnsFromEvents(events, "verify@1"); + expect(firstVisit).toEqual([ + { kind: "system", content: "first visit prompt" }, + { kind: "assistant", content: "first visit reply" }, + ]); + + const secondVisit = turnsFromEvents(events, "verify@2"); + expect(secondVisit).toEqual([ + { kind: "system", content: "second visit prompt" }, + { kind: "assistant", content: "second visit reply" }, + ]); + }); + + test("command turn carries the requested stage_id, no @1 fallback", () => { + const events: EventEnvelope[] = [ + makeEnvelope({ + seq: 1, + event: "command.started", + stage_id: "verify@2", + node_id: "verify", + properties: { script: "echo hi", language: "shell" }, + }), + makeEnvelope({ + seq: 2, + event: "command.completed", + stage_id: "verify@2", + node_id: "verify", + properties: { + stdout: "hi", + stderr: "", + exit_code: 0, + duration_ms: 5, + termination: "exited", + }, + }), + ]; + + const turns = turnsFromEvents(events, "verify@2"); + expect(turns).toHaveLength(1); + const turn = turns[0]; + expect(turn.kind).toBe("command"); + if (turn.kind === "command") { + expect(turn.stageId).toBe("verify@2"); + expect(turn.script).toBe("echo hi"); + expect(turn.running).toBe(false); + } + }); +}); diff --git a/apps/fabro-web/app/routes/run-stages.tsx b/apps/fabro-web/app/routes/run-stages.tsx index 6d179d701..1176cb71a 100644 --- a/apps/fabro-web/app/routes/run-stages.tsx +++ b/apps/fabro-web/app/routes/run-stages.tsx @@ -41,7 +41,7 @@ import { EmptyState } from "../components/state"; import { CopyButton } from "../components/ui"; import { formatDurationSecs } from "../lib/format"; import { fetchRunCommandLog, useRunEventsList, useRunStageTurns, useRunStages } from "../lib/queries"; -import { mapRunStagesToSidebarStages } from "../lib/stage-sidebar"; +import { ACTIVE_STAGE_STATES, formatStageLabel, mapRunStagesToSidebarStages } from "../lib/stage-sidebar"; import { getNumber, getString, type UnknownRecord } from "../lib/unknown"; import { CommandOutputStream, @@ -68,8 +68,8 @@ function readTermination(props: UnknownRecord): CommandTermination { return CommandTermination.EXITED; } -function turnsFromEvents(events: EventEnvelope[], stageId: string): TurnType[] { - const stageEvents = events.filter((e) => e.node_id === stageId); +export function turnsFromEvents(events: EventEnvelope[], stageId: string): TurnType[] { + const stageEvents = events.filter((e) => e.stage_id === stageId); const turns: TurnType[] = []; // Collect tool pairs: started → completed const pendingTools = new Map(); @@ -114,7 +114,7 @@ function turnsFromEvents(events: EventEnvelope[], stageId: string): TurnType[] { } case "command.started": { pendingCommand = { - stageId: e.stage_id ?? `${stageId}@1`, + stageId, script: getString(props, "script") ?? "", language: getString(props, "language") ?? "shell", }; @@ -123,7 +123,7 @@ function turnsFromEvents(events: EventEnvelope[], stageId: string): TurnType[] { case "command.completed": { turns.push({ kind: "command", - stageId: pendingCommand?.stageId ?? e.stage_id ?? `${stageId}@1`, + stageId: pendingCommand?.stageId ?? stageId, script: pendingCommand?.script ?? "", language: pendingCommand?.language ?? "shell", stdout: getString(props, "stdout") ?? "", @@ -614,7 +614,7 @@ export default function RunStages() { () => mapTurns(turnsQuery.data, eventsQuery.data, selectedStage?.id), [eventsQuery.data, selectedStage?.id, turnsQuery.data], ); - const isRunning = selectedStage?.status === "running"; + const isActive = selectedStage ? ACTIVE_STAGE_STATES.has(selectedStage.status) : false; if (!id || !stages.length) { return ( @@ -636,11 +636,13 @@ export default function RunStages() {
- -

{selectedStage.name}

+ +

+ {formatStageLabel(selectedStage)} +

diff --git a/docs/public/api-reference/fabro-api.yaml b/docs/public/api-reference/fabro-api.yaml index 4137dfaad..a767a7262 100644 --- a/docs/public/api-reference/fabro-api.yaml +++ b/docs/public/api-reference/fabro-api.yaml @@ -6160,7 +6160,7 @@ components: example: 3501.0 BillingStageRef: - description: Reference to a billing stage. + description: Reference to a workflow node in a billing stage row. type: object required: - id @@ -6320,11 +6320,13 @@ components: - id - name - status + - node_id + - visit properties: id: type: string - description: Unique stage identifier within the run. - example: propose-changes + description: StageId in "node_id@visit" form, e.g. verify@2. + example: verify@2 name: type: string description: Human-readable stage name. @@ -6335,10 +6337,16 @@ components: type: number description: Time spent in this stage, in seconds. example: 154.0 - dot_id: + node_id: type: string - description: Node identifier in the Graphviz graph source. - example: propose + description: Node id in the workflow graph; multiple stages with different visits share the same node_id. + example: verify + visit: + type: integer + format: uint32 + minimum: 1 + description: 1-based visit count; bumped each time the workflow re-enters this node. + example: 2 ToolUse: description: A single tool invocation with its input, result, and execution metadata. @@ -6608,7 +6616,7 @@ components: # ── Billing Schemas ────────────────────────────────────────────────── RunBillingStage: - description: Token counts and billed totals for a single stage within a run. + description: Token counts and billed totals for one workflow node within a run. Rows are grouped by node; billing and runtime sum every visit of that node. type: object required: - stage @@ -6619,7 +6627,7 @@ components: stage: $ref: "#/components/schemas/BillingStageRef" model: - description: Model used for this stage; null for non-LLM stages. + description: Latest usage-bearing visit model for this node; null when no visit used an LLM model. oneOf: - $ref: "#/components/schemas/ModelReference" - type: "null" @@ -6627,7 +6635,7 @@ components: $ref: "#/components/schemas/BilledTokenCounts" runtime_secs: type: number - description: Wall-clock runtime in seconds. + description: Wall-clock runtime in seconds, summed across every visit of this node. example: 154.0 RunBillingTotals: @@ -6688,7 +6696,7 @@ components: $ref: "#/components/schemas/ModelReference" stages: type: integer - description: Number of stages that used this model. + description: Number of usage-bearing stage visits that used this model. example: 2 billing: $ref: "#/components/schemas/BilledTokenCounts" @@ -6703,7 +6711,7 @@ components: properties: stages: type: array - description: Per-stage billing breakdown. + description: Per-node billing breakdown. Each row sums billing and runtime across all visits of that node. items: $ref: "#/components/schemas/RunBillingStage" totals: diff --git a/lib/crates/fabro-dump/src/lib.rs b/lib/crates/fabro-dump/src/lib.rs index 69a670565..5b2da9403 100644 --- a/lib/crates/fabro-dump/src/lib.rs +++ b/lib/crates/fabro-dump/src/lib.rs @@ -66,18 +66,13 @@ impl RunDump { entries.push(RunDumpEntry::text("graph.fabro", graph_source.clone())); } - let mut stages: Vec<_> = state.iter_stages().collect(); + let stages: Vec<_> = state.iter_stages().collect(); if stages.len() > MAX_STAGES_IN_DUMP { bail!( "run dump supports at most {MAX_STAGES_IN_DUMP} stages with the current path prefix width (got {})", stages.len() ); } - stages.sort_by(|(left_id, left), (right_id, right)| { - left.first_event_seq - .cmp(&right.first_event_seq) - .then_with(|| left_id.cmp(right_id)) - }); let mut stage_ranks = HashMap::new(); for (index, (stage_id, _)) in stages.iter().enumerate() { diff --git a/lib/crates/fabro-server/src/demo/mod.rs b/lib/crates/fabro-server/src/demo/mod.rs index 08423cd55..bc527007d 100644 --- a/lib/crates/fabro-server/src/demo/mod.rs +++ b/lib/crates/fabro-server/src/demo/mod.rs @@ -784,9 +784,10 @@ mod runs { RunNamespace, RunPrepareSettings, RunSandboxSettings, }; use fabro_types::settings::{InterpString, ProjectNamespace, WorkflowNamespace}; - use fabro_types::{RunId, WorkflowSettings}; + use fabro_types::{RunId, StageId, WorkflowSettings}; use super::ts; + use crate::server::run_stage_from_stage_id; fn labels(entries: &[(&str, &str)]) -> HashMap { entries @@ -1181,34 +1182,36 @@ mod runs { pub(super) fn stages() -> Vec { vec![ - RunStage { - id: "detect-drift".into(), - name: "Detect Drift".into(), - status: StageState::Succeeded, - duration_secs: Some(72.0), - dot_id: Some("detect".into()), - }, - RunStage { - id: "propose-changes".into(), - name: "Propose Changes".into(), - status: StageState::Succeeded, - duration_secs: Some(154.0), - dot_id: Some("propose".into()), - }, - RunStage { - id: "review-changes".into(), - name: "Review Changes".into(), - status: StageState::Succeeded, - duration_secs: Some(45.0), - dot_id: Some("review".into()), - }, - RunStage { - id: "apply-changes".into(), - name: "Apply Changes".into(), - status: StageState::Running, - duration_secs: Some(118.0), - dot_id: Some("apply".into()), - }, + run_stage_from_stage_id( + &StageId::new("detect-drift", 1), + "Detect Drift", + StageState::Succeeded, + Some(72.0), + ), + run_stage_from_stage_id( + &StageId::new("propose-changes", 1), + "Propose Changes", + StageState::Succeeded, + Some(154.0), + ), + run_stage_from_stage_id( + &StageId::new("review-changes", 1), + "Review Changes", + StageState::Succeeded, + Some(45.0), + ), + run_stage_from_stage_id( + &StageId::new("apply-changes", 1), + "Apply Changes", + StageState::Succeeded, + Some(118.0), + ), + run_stage_from_stage_id( + &StageId::new("apply-changes", 2), + "Apply Changes", + StageState::Running, + None, + ), ] } diff --git a/lib/crates/fabro-server/src/server.rs b/lib/crates/fabro-server/src/server.rs index eb35ef8b0..26fbdd8d6 100644 --- a/lib/crates/fabro-server/src/server.rs +++ b/lib/crates/fabro-server/src/server.rs @@ -54,7 +54,7 @@ use fabro_llm::types::{ ContentPart, FinishReason, Message as LlmMessage, Request as LlmRequest, Role, ToolChoice, ToolDefinition, }; -use fabro_model::{BilledModelUsage, BilledTokenCounts, Catalog, ModelTestMode, Provider}; +use fabro_model::{BilledTokenCounts, Catalog, ModelTestMode, Provider}; use fabro_redact::redact_jsonl_line; use fabro_sandbox::daytona::{self, DaytonaSandbox}; use fabro_sandbox::reconnect::reconnect; @@ -536,17 +536,48 @@ pub(crate) struct ResolvedAppStateSettings { pub(crate) manifest_run_settings: std::result::Result, } -fn accumulate_model_billing(entry: &mut ModelBillingTotals, usage: &BilledModelUsage) { - let tokens = usage.tokens(); - entry.stages += 1; - entry.billing.input_tokens += tokens.input_tokens; - entry.billing.output_tokens += tokens.output_tokens; - entry.billing.reasoning_tokens += tokens.reasoning_tokens; - entry.billing.cache_read_tokens += tokens.cache_read_tokens; - entry.billing.cache_write_tokens += tokens.cache_write_tokens; - entry.billing.total_tokens += tokens.total_tokens(); - if let Some(value) = usage.total_usd_micros { - *entry.billing.total_usd_micros.get_or_insert(0) += value; +fn accumulate_billed_token_counts(target: &mut BilledTokenCounts, source: &BilledTokenCounts) { + target.input_tokens += source.input_tokens; + target.output_tokens += source.output_tokens; + target.reasoning_tokens += source.reasoning_tokens; + target.cache_read_tokens += source.cache_read_tokens; + target.cache_write_tokens += source.cache_write_tokens; + target.total_tokens += source.total_tokens; + if let Some(value) = source.total_usd_micros { + *target.total_usd_micros.get_or_insert(0) += value; + } +} + +fn accumulate_billing_rollup( + accumulator: &mut BillingAccumulator, + rollup: &fabro_workflow::ProjectionBillingRollup, +) { + accumulator.total_runs += 1; + accumulator.total_runtime_secs += rollup.runtime_ms as f64 / 1000.0; + for model in &rollup.by_model { + let entry = accumulator + .by_model + .entry(model.model_id.clone()) + .or_default(); + entry.stages += model.stages; + accumulate_billed_token_counts(&mut entry.billing, &model.billing); + } +} + +pub(crate) fn run_stage_from_stage_id( + stage_id: &StageId, + name: impl Into, + status: StageState, + duration_secs: Option, +) -> RunStage { + RunStage { + id: stage_id.to_string(), + name: name.into(), + status, + duration_secs, + node_id: stage_id.node_id().to_string(), + visit: std::num::NonZeroU32::new(stage_id.visit()) + .expect("StageId stores a non-zero visit"), } } @@ -2776,9 +2807,9 @@ async fn execute_run_in_process(state: Arc, run_id: RunId) { } } - // Save final checkpoint - let checkpoint = match run_store.state().await { - Ok(state) => state.checkpoint, + // Save final projection + let final_projection = match run_store.state().await { + Ok(state) => Some(state), Err(err) => { tracing::warn!(run_id = %run_id, error = %err, "Failed to load run state from store"); None @@ -2786,32 +2817,17 @@ async fn execute_run_in_process(state: Arc, run_id: RunId) { }; // Accumulate aggregate usage after execution completes. - if let Some(ref cp) = checkpoint { - let stage_durations = match run_store.list_events().await { - Ok(events) => fabro_workflow::extract_stage_durations_from_events(&events), - Err(err) => { - tracing::warn!(run_id = %run_id, error = %err, "Failed to load run events from store"); - HashMap::default() - } - }; - let mut agg = state - .aggregate_billing - .lock() - .expect("aggregate_billing lock poisoned"); - agg.total_runs += 1; - let mut run_runtime: f64 = 0.0; - for (node_id, outcome) in &cp.node_outcomes { - if let Some(usage) = &outcome.usage { - let entry = agg - .by_model - .entry(usage.model_id().to_string()) - .or_default(); - accumulate_model_billing(entry, usage); - } - let duration_ms = stage_durations.get(node_id).copied().unwrap_or(0); - run_runtime += duration_ms as f64 / 1000.0; + if let Some(ref projection) = final_projection { + if projection.checkpoint.is_some() { + let mut agg = state + .aggregate_billing + .lock() + .expect("aggregate_billing lock poisoned"); + accumulate_billing_rollup( + &mut agg, + &fabro_workflow::billing_rollup_from_projection(projection), + ); } - agg.total_runtime_secs += run_runtime; } let mut runs = state.runs.lock().expect("runs lock poisoned"); @@ -2860,7 +2876,9 @@ async fn execute_run_in_process(state: Arc, run_id: RunId) { }; } } - managed_run.checkpoint = checkpoint; + managed_run.checkpoint = final_projection + .as_ref() + .and_then(|projection| projection.checkpoint.clone()); managed_run.run_dir = Some(run_dir); clear_live_run_state(managed_run); } @@ -3103,32 +3121,15 @@ async fn execute_run_subprocess(state: Arc, run_id: RunId) { } }; - if let Some(ref checkpoint) = final_state.checkpoint { - let stage_durations = match run_store.list_events().await { - Ok(events) => fabro_workflow::extract_stage_durations_from_events(&events), - Err(err) => { - tracing::warn!(run_id = %run_id, error = %err, "Failed to load run events from store"); - HashMap::default() - } - }; + if final_state.checkpoint.is_some() { let mut agg = state .aggregate_billing .lock() .expect("aggregate_billing lock poisoned"); - agg.total_runs += 1; - let mut run_runtime: f64 = 0.0; - for (node_id, outcome) in &checkpoint.node_outcomes { - if let Some(usage) = &outcome.usage { - let entry = agg - .by_model - .entry(usage.model_id().to_string()) - .or_default(); - accumulate_model_billing(entry, usage); - } - let duration_ms = stage_durations.get(node_id).copied().unwrap_or(0); - run_runtime += duration_ms as f64 / 1000.0; - } - agg.total_runtime_secs += run_runtime; + accumulate_billing_rollup( + &mut agg, + &fabro_workflow::billing_rollup_from_projection(&final_state), + ); } let mut runs = state.runs.lock().expect("runs lock poisoned"); diff --git a/lib/crates/fabro-server/src/server/handler/billing.rs b/lib/crates/fabro-server/src/server/handler/billing.rs index dfe7d5176..193ebd78d 100644 --- a/lib/crates/fabro-server/src/server/handler/billing.rs +++ b/lib/crates/fabro-server/src/server/handler/billing.rs @@ -1,13 +1,13 @@ use std::sync::Arc; -use fabro_types::EventBody; +use fabro_store::RunProjectionReducer; +use fabro_types::{EventBody, RunProjection, StageId}; use super::super::{ - ApiError, AppState, BilledTokenCounts, BillingByModel, BillingStageRef, EventEnvelope, HashMap, - IntoResponse, Json, ListResponse, ModelBillingTotals, ModelReference, PaginationParams, Path, - Query, RequiredUser, Response, Router, RunBilling, RunBillingStage, RunBillingTotals, RunId, - RunStage, RunStatus, StageState, State, StatusCode, accumulate_model_billing, get, - parse_run_id_path, + ApiError, AppState, BillingByModel, BillingStageRef, EventEnvelope, HashMap, IntoResponse, + Json, ListResponse, ModelReference, PaginationParams, Path, Query, RequiredUser, Response, + Router, RunBilling, RunBillingStage, RunBillingTotals, RunId, StageState, State, StatusCode, + get, parse_run_id_path, run_stage_from_stage_id, }; pub(super) fn routes() -> Router> { @@ -16,25 +16,41 @@ pub(super) fn routes() -> Router> { .route("/runs/{id}/billing", get(get_run_billing)) } -fn active_stage_state_from_events(events: &[EventEnvelope], node_id: &str) -> StageState { - let latest = events.iter().rev().find(|envelope| { - envelope.event.node_id.as_deref() == Some(node_id) - && matches!( - &envelope.event.body, - EventBody::StageRetrying(_) - | EventBody::StageStarted(_) - | EventBody::StageCompleted(_) - | EventBody::StageFailed(_) - ) - }); - - if latest.is_some_and(|e| matches!(&e.event.body, EventBody::StageRetrying(_))) { - StageState::Retrying - } else { - StageState::Running +/// Map a `stage.*` lifecycle event body to the [`StageState`] it implies. +/// Returns `None` for any other variant. +fn stage_state_from_lifecycle(body: &EventBody) -> Option { + match body { + EventBody::StageStarted(_) => Some(StageState::Running), + EventBody::StageRetrying(_) => Some(StageState::Retrying), + EventBody::StageFailed(props) => Some(if props.will_retry { + StageState::Retrying + } else { + StageState::Failed + }), + EventBody::StageCompleted(props) => Some(StageState::from(props.status)), + _ => None, } } +/// Single-pass scan over `events` building the latest [`StageState`] for each +/// [`StageId`] from lifecycle events (started/retrying/completed/failed). Each +/// later lifecycle event overwrites earlier ones, leaving the latest as the +/// stored value — equivalent to "scan in reverse, take first match" but in O(E) +/// for the whole list rather than O(stages × events). +fn latest_stage_states(events: &[EventEnvelope]) -> HashMap { + let mut states = HashMap::new(); + for envelope in events { + let Some(stage_id) = envelope.event.stage_id.as_ref() else { + continue; + }; + let Some(state) = stage_state_from_lifecycle(&envelope.event.body) else { + continue; + }; + states.insert(stage_id.clone(), state); + } + states +} + async fn list_run_stages( _auth: RequiredUser, State(state): State>, @@ -46,80 +62,41 @@ async fn list_run_stages( Err(response) => return response, }; - // Try live run first. - let (checkpoint, run_is_active) = { - let runs = state.runs.lock().expect("runs lock poisoned"); - match runs.get(&id) { - Some(managed_run) => { - let active = !matches!( - managed_run.status, - RunStatus::Succeeded { .. } | RunStatus::Failed { .. } | RunStatus::Dead - ); - (managed_run.checkpoint.clone(), active) - } - None => (None, false), - } - }; - - // Fall back to stored run. - let (checkpoint, run_is_active) = if checkpoint.is_some() { - (checkpoint, run_is_active) - } else { - match state.store.open_run_reader(&id).await { - Ok(run_store) => match run_store.state().await { - Ok(run_state) => { - let active = run_state.status.is_some_and(|status| !status.is_terminal()); - (run_state.checkpoint, active) - } - Err(_) => (None, false), - }, - Err(_) => return ApiError::not_found("Run not found.").into_response(), - } - }; - - let Some(checkpoint) = checkpoint else { - return ( - StatusCode::OK, - Json(ListResponse::new(Vec::::new())), - ) - .into_response(); - }; - let events = match state.store.open_run_reader(&id).await { Ok(run_store) => run_store.list_events().await.unwrap_or_default(), - Err(_) => Vec::new(), + Err(_) => return ApiError::not_found("Run not found.").into_response(), }; - let stage_durations = fabro_workflow::extract_stage_durations_from_events(&events); + + let projection = match RunProjection::apply_events(&events) { + Ok(projection) => projection, + Err(err) => { + tracing::warn!( + run_id = %id, + error = %err, + "Failed to build run projection; returning empty stages list", + ); + RunProjection::default() + } + }; + let stage_durations = fabro_workflow::extract_stage_durations_by_stage_id(&events); + let lifecycle_states = latest_stage_states(&events); let mut stages = Vec::new(); - for node_id in &checkpoint.completed_nodes { - let duration_ms = stage_durations.get(node_id).copied().unwrap_or(0); - let status = match checkpoint.node_outcomes.get(node_id) { - Some(outcome) => StageState::from(outcome.status), - None => StageState::Succeeded, - }; - stages.push(RunStage { - id: node_id.clone(), - name: node_id.clone(), - status, - duration_secs: Some(duration_ms as f64 / 1000.0), - dot_id: Some(node_id.clone()), + for (stage_id, stage_projection) in projection.iter_stages() { + // Prefer the latest lifecycle event; fall back to the projection's + // stored completion (e.g. for runs recovered from snapshot only). + let status = lifecycle_states.get(stage_id).copied().unwrap_or_else(|| { + stage_projection + .completion + .as_ref() + .map_or(StageState::Pending, |c| StageState::from(c.outcome)) }); - } - - // Add next node as running if the run is still active. - // The checkpoint's current_node is the last *completed* stage; next_node_id - // is the stage that is currently executing. - if let Some(next_id) = &checkpoint.next_node_id { - if run_is_active && next_id != "exit" && !checkpoint.completed_nodes.contains(next_id) { - stages.push(RunStage { - id: next_id.clone(), - name: next_id.clone(), - status: active_stage_state_from_events(&events, next_id), - duration_secs: None, - dot_id: Some(next_id.clone()), - }); - } + stages.push(run_stage_from_stage_id( + stage_id, + stage_id.node_id().to_string(), + status, + stage_durations.get(stage_id).map(|ms| *ms as f64 / 1000.0), + )); } (StatusCode::OK, Json(ListResponse::new(stages))).into_response() @@ -137,91 +114,39 @@ async fn get_run_billing( } }; - let checkpoint = match run_store.state().await { - Ok(state) => state.checkpoint, + let projection = match run_store.state().await { + Ok(state) => state, Err(err) => { return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) .into_response(); } }; - - let Some(checkpoint) = checkpoint else { - let empty = RunBilling { - by_model: Vec::new(), - stages: Vec::new(), - totals: RunBillingTotals { - cache_read_tokens: 0, - cache_write_tokens: 0, - input_tokens: 0, - output_tokens: 0, - reasoning_tokens: 0, - runtime_secs: 0.0, - total_tokens: 0, - total_usd_micros: None, + let rollup = fabro_workflow::billing_rollup_from_projection(&projection); + let by_model = rollup + .by_model + .iter() + .map(|model| BillingByModel { + billing: model.billing.clone(), + model: ModelReference { + id: model.model_id.clone(), }, - }; - return (StatusCode::OK, Json(empty)).into_response(); - }; - - let stage_durations = match run_store.list_events().await { - Ok(events) => fabro_workflow::extract_stage_durations_from_events(&events), - Err(err) => { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) - .into_response(); - } - }; - - let mut by_model_totals = HashMap::::new(); - let mut billed_usages = Vec::new(); - let mut runtime_secs = 0.0_f64; - let mut stages = Vec::new(); - - for node_id in &checkpoint.completed_nodes { - let duration_ms = stage_durations.get(node_id).copied().unwrap_or(0); - runtime_secs += duration_ms as f64 / 1000.0; - - let usage = checkpoint - .node_outcomes - .get(node_id) - .and_then(|outcome| outcome.usage.as_ref()); - - let (billing, model) = if let Some(usage) = usage { - billed_usages.push(usage.clone()); - let tokens = usage.tokens(); - let billing = BilledTokenCounts { - cache_read_tokens: tokens.cache_read_tokens, - cache_write_tokens: tokens.cache_write_tokens, - input_tokens: tokens.input_tokens, - output_tokens: tokens.output_tokens, - reasoning_tokens: tokens.reasoning_tokens, - total_tokens: tokens.total_tokens(), - total_usd_micros: usage.total_usd_micros, - }; - let model_id = usage.model_id().to_string(); - accumulate_model_billing(by_model_totals.entry(model_id.clone()).or_default(), usage); - (billing, Some(ModelReference { id: model_id })) - } else { - (BilledTokenCounts::default(), None) - }; - - stages.push(RunBillingStage { - billing, - model, - runtime_secs: duration_ms as f64 / 1000.0, - stage: BillingStageRef { - id: node_id.clone(), - name: node_id.clone(), + stages: model.stages, + }) + .collect::>(); + let stages = rollup + .stages + .iter() + .map(|stage| RunBillingStage { + billing: stage.billing.clone(), + model: stage + .model_id + .as_ref() + .map(|id| ModelReference { id: id.clone() }), + runtime_secs: stage.duration_ms as f64 / 1000.0, + stage: BillingStageRef { + id: stage.node_id.clone(), + name: stage.node_id.clone(), }, - }); - } - - let totals = BilledTokenCounts::from_billed_usage(&billed_usages); - let by_model = by_model_totals - .into_iter() - .map(|(model, totals)| BillingByModel { - billing: totals.billing, - model: ModelReference { id: model }, - stages: totals.stages, }) .collect::>(); @@ -229,14 +154,14 @@ async fn get_run_billing( by_model, stages, totals: RunBillingTotals { - cache_read_tokens: totals.cache_read_tokens, - cache_write_tokens: totals.cache_write_tokens, - input_tokens: totals.input_tokens, - output_tokens: totals.output_tokens, - reasoning_tokens: totals.reasoning_tokens, - runtime_secs, - total_tokens: totals.total_tokens, - total_usd_micros: totals.total_usd_micros, + cache_read_tokens: rollup.totals.cache_read_tokens, + cache_write_tokens: rollup.totals.cache_write_tokens, + input_tokens: rollup.totals.input_tokens, + output_tokens: rollup.totals.output_tokens, + reasoning_tokens: rollup.totals.reasoning_tokens, + runtime_secs: rollup.runtime_ms as f64 / 1000.0, + total_tokens: rollup.totals.total_tokens, + total_usd_micros: rollup.totals.total_usd_micros, }, }; diff --git a/lib/crates/fabro-server/src/server/tests.rs b/lib/crates/fabro-server/src/server/tests.rs index 5587e1952..08196dbdd 100644 --- a/lib/crates/fabro-server/src/server/tests.rs +++ b/lib/crates/fabro-server/src/server/tests.rs @@ -18,8 +18,8 @@ use fabro_model::Provider; use fabro_types::settings::ServerAuthMethod; use fabro_types::{ AttrValue, AuthMethod, CommandTermination, FailureCategory, FailureDetail, Graph, - InterviewQuestionRecord, Outcome, QuestionType, RunBlobId, RunId, RunSpec, StageOutcome, - SystemActorKind, fixtures, + InterviewQuestionRecord, Outcome, QuestionType, RunBlobId, RunId, RunSpec, SystemActorKind, + fixtures, }; use fabro_util::check_report::CheckStatus; use httpmock::Method::{GET, POST}; @@ -2112,6 +2112,30 @@ async fn create_durable_run_with_events( } } +/// Append a stage lifecycle event with an explicit `StageScope`, so the +/// stored envelope carries the full `stage_id` (`node_id@visit`). The bare +/// [`workflow_event::append_event`] helper only writes `node_id` because +/// stage lifecycle variants don't carry visit in their payload — production +/// always emits via `Emitter::emit_scoped`. +async fn append_scoped_stage_event( + state: &Arc, + run_id: RunId, + node_id: &str, + visit: u32, + event: &workflow_event::Event, +) { + let scope = fabro_workflow::event::StageScope { + node_id: node_id.to_string(), + visit, + parallel_group_id: None, + parallel_branch_id: None, + }; + let stored = fabro_workflow::event::to_run_event_at(&run_id, event, Utc::now(), Some(&scope)); + let payload = fabro_workflow::event::build_redacted_event_payload(&stored, &run_id).unwrap(); + let run_store = state.store.open_run(&run_id).await.unwrap(); + run_store.append_event(&payload).await.unwrap(); +} + fn stage_status<'a>(body: &'a serde_json::Value, id: &str) -> &'a str { body["data"] .as_array() @@ -2134,7 +2158,58 @@ async fn list_run_stages_projects_retrying_until_completion() { }, workflow_event::Event::RunStarting, workflow_event::Event::RunRunning, - workflow_event::Event::StageStarted { + ]) + .await; + append_scoped_stage_event( + &state, + run_id, + "setup", + 1, + &workflow_event::Event::StageStarted { + node_id: "setup".to_string(), + name: "Setup".to_string(), + index: 0, + handler_type: "command".to_string(), + attempt: 1, + max_attempts: 1, + }, + ) + .await; + append_scoped_stage_event( + &state, + run_id, + "setup", + 1, + &workflow_event::Event::StageCompleted { + node_id: "setup".to_string(), + name: "Setup".to_string(), + index: 0, + duration_ms: 5, + status: "succeeded".to_string(), + preferred_label: None, + suggested_next_ids: Vec::new(), + billing: None, + failure: None, + notes: None, + files_touched: Vec::new(), + context_updates: None, + jump_to_node: None, + context_values: None, + node_visits: None, + loop_failure_signatures: None, + restart_failure_signatures: None, + response: None, + attempt: 1, + max_attempts: 1, + }, + ) + .await; + append_scoped_stage_event( + &state, + run_id, + "work", + 1, + &workflow_event::Event::StageStarted { node_id: "work".to_string(), name: "Work".to_string(), index: 1, @@ -2142,16 +2217,31 @@ async fn list_run_stages_projects_retrying_until_completion() { attempt: 1, max_attempts: 3, }, - workflow_event::Event::StageFailed { + ) + .await; + append_scoped_stage_event( + &state, + run_id, + "work", + 1, + &workflow_event::Event::StageFailed { node_id: "work".to_string(), name: "Work".to_string(), index: 1, failure: FailureDetail::new("try again", FailureCategory::TransientInfra), will_retry: true, duration_ms: 10, + billing: None, actor: None, }, - workflow_event::Event::StageRetrying { + ) + .await; + append_scoped_stage_event( + &state, + run_id, + "work", + 1, + &workflow_event::Event::StageRetrying { node_id: "work".to_string(), name: "Work".to_string(), index: 1, @@ -2159,41 +2249,9 @@ async fn list_run_stages_projects_retrying_until_completion() { max_attempts: 3, delay_ms: 100, }, - ]) + ) .await; - let mut node_outcomes = HashMap::new(); - node_outcomes.insert("setup".to_string(), Outcome::success()); - let mut checkpoint = Checkpoint { - timestamp: Utc::now(), - current_node: "setup".to_string(), - completed_nodes: vec!["setup".to_string()], - node_retries: HashMap::new(), - context_values: HashMap::new(), - node_outcomes, - next_node_id: Some("work".to_string()), - git_commit_sha: None, - loop_failure_signatures: HashMap::new(), - restart_failure_signatures: HashMap::new(), - node_visits: HashMap::new(), - }; - - let run_dir = std::env::temp_dir().join(format!("fabro-server-test-{run_id}")); - std::fs::create_dir_all(&run_dir).unwrap(); - let mut managed = managed_run( - MINIMAL_DOT.to_string(), - RunStatus::Running, - Utc::now(), - run_dir, - RunExecutionMode::Start, - ); - managed.checkpoint = Some(checkpoint.clone()); - state - .runs - .lock() - .expect("runs lock poisoned") - .insert(run_id, managed); - let response = app .clone() .oneshot( @@ -2206,29 +2264,14 @@ async fn list_run_stages_projects_retrying_until_completion() { .await .unwrap(); let body = response_json!(response, StatusCode::OK).await; - assert_eq!(stage_status(&body, "setup"), "succeeded"); - assert_eq!(stage_status(&body, "work"), "retrying"); + assert_eq!(stage_status(&body, "setup@1"), "succeeded"); + assert_eq!(stage_status(&body, "work@1"), "retrying"); - let mut work_outcome = Outcome::success(); - work_outcome.status = StageOutcome::PartiallySucceeded; - checkpoint.completed_nodes.push("work".to_string()); - checkpoint - .node_outcomes - .insert("work".to_string(), work_outcome); - checkpoint.current_node = "work".to_string(); - checkpoint.next_node_id = Some("exit".to_string()); - state - .runs - .lock() - .expect("runs lock poisoned") - .get_mut(&run_id) - .unwrap() - .checkpoint = Some(checkpoint); - - let run_store = state.store.open_run(&run_id).await.unwrap(); - workflow_event::append_event( - &run_store, - &run_id, + append_scoped_stage_event( + &state, + run_id, + "work", + 1, &workflow_event::Event::StageCompleted { node_id: "work".to_string(), name: "Work".to_string(), @@ -2252,8 +2295,7 @@ async fn list_run_stages_projects_retrying_until_completion() { max_attempts: 3, }, ) - .await - .unwrap(); + .await; let response = app .oneshot( @@ -2266,7 +2308,574 @@ async fn list_run_stages_projects_retrying_until_completion() { .await .unwrap(); let body = response_json!(response, StatusCode::OK).await; - assert_eq!(stage_status(&body, "work"), "partially_succeeded"); + assert_eq!(stage_status(&body, "work@1"), "partially_succeeded"); +} + +fn stage_entry<'a>(body: &'a serde_json::Value, id: &str) -> &'a serde_json::Value { + body["data"] + .as_array() + .unwrap() + .iter() + .find(|stage| stage["id"] == id) + .unwrap_or_else(|| panic!("stage {id} not found in {body:#?}")) +} + +fn test_billed_usage( + model_id: &str, + input_tokens: i64, + output_tokens: i64, +) -> fabro_model::BilledModelUsage { + serde_json::from_value(json!({ + "input": { + "usage": { + "model": { + "provider": "openai", + "model_id": model_id + }, + "tokens": { + "input_tokens": input_tokens, + "output_tokens": output_tokens + } + }, + "facts": { + "provider": "open_ai" + } + }, + "total_usd_micros": input_tokens + output_tokens + })) + .unwrap() +} + +#[tokio::test] +async fn list_run_stages_distinguishes_visits() { + let state = test_app_state_with_isolated_storage(); + let app = crate::test_support::build_test_router(Arc::clone(&state)); + let run_id = RunId::new(); + + create_durable_run_with_events(&state, run_id, &[ + workflow_event::Event::RunSubmitted { + definition_blob: None, + }, + workflow_event::Event::RunStarting, + workflow_event::Event::RunRunning, + ]) + .await; + + // First visit of `verify` — failed. + append_scoped_stage_event( + &state, + run_id, + "verify", + 1, + &workflow_event::Event::StageStarted { + node_id: "verify".to_string(), + name: "Verify".to_string(), + index: 1, + handler_type: "command".to_string(), + attempt: 1, + max_attempts: 1, + }, + ) + .await; + append_scoped_stage_event( + &state, + run_id, + "verify", + 1, + &workflow_event::Event::StageCompleted { + node_id: "verify".to_string(), + name: "Verify".to_string(), + index: 1, + duration_ms: 1500, + status: "failed".to_string(), + preferred_label: None, + suggested_next_ids: Vec::new(), + billing: None, + failure: None, + notes: None, + files_touched: Vec::new(), + context_updates: None, + jump_to_node: None, + context_values: None, + node_visits: None, + loop_failure_signatures: None, + restart_failure_signatures: None, + response: None, + attempt: 1, + max_attempts: 1, + }, + ) + .await; + + // Second visit of `verify` — running. + append_scoped_stage_event( + &state, + run_id, + "verify", + 2, + &workflow_event::Event::StageStarted { + node_id: "verify".to_string(), + name: "Verify".to_string(), + index: 1, + handler_type: "command".to_string(), + attempt: 1, + max_attempts: 1, + }, + ) + .await; + + let response = app + .clone() + .oneshot( + Request::builder() + .method("GET") + .uri(api(&format!("/runs/{run_id}/stages"))) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let body = response_json!(response, StatusCode::OK).await; + + let data = body["data"].as_array().unwrap(); + let verify_entries: Vec<_> = data.iter().filter(|s| s["node_id"] == "verify").collect(); + assert_eq!(verify_entries.len(), 2, "expected two verify visits"); + + let first = stage_entry(&body, "verify@1"); + assert_eq!(first["node_id"], "verify"); + assert_eq!(first["visit"], 1); + assert_eq!(first["status"], "failed"); + assert_eq!(first["duration_secs"], 1.5); + + let second = stage_entry(&body, "verify@2"); + assert_eq!(second["node_id"], "verify"); + assert_eq!(second["visit"], 2); + assert_eq!(second["status"], "running"); + + // Old `dot_id` field must be gone. + assert!(first.get("dot_id").is_none(), "dot_id should be removed"); +} + +/// `checkpoint.completed_nodes` records every visit, so a looped node appears +/// once per re-entry. Billing must dedup so a retried node renders as one row +/// and `runtime_secs` is summed across all visits exactly once. +#[tokio::test] +async fn run_billing_dedups_retried_nodes_and_sums_their_durations() { + let state = test_app_state_with_isolated_storage(); + let app = crate::test_support::build_test_router(Arc::clone(&state)); + let run_id = RunId::new(); + + create_durable_run_with_events(&state, run_id, &[ + workflow_event::Event::RunSubmitted { + definition_blob: None, + }, + workflow_event::Event::RunStarting, + workflow_event::Event::RunRunning, + ]) + .await; + + // Visit 1 of `verify` — completed in 1.5s. + append_scoped_stage_event( + &state, + run_id, + "verify", + 1, + &workflow_event::Event::StageCompleted { + node_id: "verify".to_string(), + name: "Verify".to_string(), + index: 1, + duration_ms: 1500, + status: "failed".to_string(), + preferred_label: None, + suggested_next_ids: Vec::new(), + billing: None, + failure: None, + notes: None, + files_touched: Vec::new(), + context_updates: None, + jump_to_node: None, + context_values: None, + node_visits: None, + loop_failure_signatures: None, + restart_failure_signatures: None, + response: None, + attempt: 1, + max_attempts: 1, + }, + ) + .await; + + // Visit 2 of `verify` — completed in 0.8s. + append_scoped_stage_event( + &state, + run_id, + "verify", + 2, + &workflow_event::Event::StageCompleted { + node_id: "verify".to_string(), + name: "Verify".to_string(), + index: 1, + duration_ms: 800, + status: "succeeded".to_string(), + preferred_label: None, + suggested_next_ids: Vec::new(), + billing: None, + failure: None, + notes: None, + files_touched: Vec::new(), + context_updates: None, + jump_to_node: None, + context_values: None, + node_visits: None, + loop_failure_signatures: None, + restart_failure_signatures: None, + response: None, + attempt: 1, + max_attempts: 1, + }, + ) + .await; + + // Checkpoint records `verify` twice (once per visit) — this is what makes + // the dedup necessary. + let run_store = state.store.open_run(&run_id).await.unwrap(); + workflow_event::append_event( + &run_store, + &run_id, + &workflow_event::Event::CheckpointCompleted { + node_id: "verify".to_string(), + status: "running".to_string(), + current_node: "verify".to_string(), + completed_nodes: vec!["verify".to_string(), "verify".to_string()], + node_retries: std::collections::BTreeMap::new(), + context_values: std::collections::BTreeMap::new(), + node_outcomes: std::collections::BTreeMap::from([( + "verify".to_string(), + Outcome::default(), + )]), + next_node_id: Some("done".to_string()), + git_commit_sha: None, + loop_failure_signatures: std::collections::BTreeMap::new(), + restart_failure_signatures: std::collections::BTreeMap::new(), + node_visits: std::collections::BTreeMap::from([("verify".to_string(), 2usize)]), + diff: None, + }, + ) + .await + .unwrap(); + + let response = app + .clone() + .oneshot( + Request::builder() + .method("GET") + .uri(api(&format!("/runs/{run_id}/billing"))) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let body = response_json!(response, StatusCode::OK).await; + + let stages = body["stages"].as_array().unwrap(); + assert_eq!( + stages.len(), + 1, + "expected one row for the retried verify node" + ); + assert_eq!(stages[0]["stage"]["id"], "verify"); + // Duration on the row is the sum across visits (1.5s + 0.8s = 2.3s). + assert!( + (stages[0]["runtime_secs"].as_f64().unwrap() - 2.3).abs() < f64::EPSILON, + "row runtime_secs should sum visits, got {}", + stages[0]["runtime_secs"] + ); + + // Totals must not double-count: a single 2.3s, not 4.6s. + assert!( + (body["totals"]["runtime_secs"].as_f64().unwrap() - 2.3).abs() < f64::EPSILON, + "totals.runtime_secs should sum visits exactly once, got {}", + body["totals"]["runtime_secs"] + ); +} + +#[tokio::test] +async fn run_billing_sums_usage_across_retry_visits_and_uses_latest_model() { + let state = test_app_state_with_isolated_storage(); + let app = crate::test_support::build_test_router(Arc::clone(&state)); + let run_id = RunId::new(); + let failed_usage = test_billed_usage("gpt-old", 100, 10); + let success_usage = test_billed_usage("gpt-new", 200, 20); + + create_durable_run_with_events(&state, run_id, &[ + workflow_event::Event::RunSubmitted { + definition_blob: None, + }, + workflow_event::Event::RunStarting, + workflow_event::Event::RunRunning, + ]) + .await; + + append_scoped_stage_event( + &state, + run_id, + "verify", + 1, + &workflow_event::Event::StageFailed { + node_id: "verify".to_string(), + name: "Verify".to_string(), + index: 1, + failure: FailureDetail::new("try again", FailureCategory::TransientInfra), + will_retry: true, + duration_ms: 1200, + billing: Some(failed_usage), + actor: None, + }, + ) + .await; + append_scoped_stage_event( + &state, + run_id, + "verify", + 2, + &workflow_event::Event::StageCompleted { + node_id: "verify".to_string(), + name: "Verify".to_string(), + index: 1, + duration_ms: 800, + status: "succeeded".to_string(), + preferred_label: None, + suggested_next_ids: Vec::new(), + billing: Some(success_usage.clone()), + failure: None, + notes: None, + files_touched: Vec::new(), + context_updates: None, + jump_to_node: None, + context_values: None, + node_visits: None, + loop_failure_signatures: None, + restart_failure_signatures: None, + response: None, + attempt: 2, + max_attempts: 2, + }, + ) + .await; + + let mut latest_outcome: Outcome> = Outcome::success(); + latest_outcome.usage = Some(success_usage); + latest_outcome.duration_ms = Some(800); + let run_store = state.store.open_run(&run_id).await.unwrap(); + workflow_event::append_event( + &run_store, + &run_id, + &workflow_event::Event::CheckpointCompleted { + node_id: "verify".to_string(), + status: "running".to_string(), + current_node: "verify".to_string(), + completed_nodes: vec!["verify".to_string(), "verify".to_string()], + node_retries: std::collections::BTreeMap::from([("verify".to_string(), 2)]), + context_values: std::collections::BTreeMap::new(), + node_outcomes: std::collections::BTreeMap::from([( + "verify".to_string(), + latest_outcome, + )]), + next_node_id: None, + git_commit_sha: None, + loop_failure_signatures: std::collections::BTreeMap::new(), + restart_failure_signatures: std::collections::BTreeMap::new(), + node_visits: std::collections::BTreeMap::from([("verify".to_string(), 2usize)]), + diff: None, + }, + ) + .await + .unwrap(); + + let response = app + .clone() + .oneshot( + Request::builder() + .method("GET") + .uri(api(&format!("/runs/{run_id}/billing"))) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let body = response_json!(response, StatusCode::OK).await; + + let stages = body["stages"].as_array().unwrap(); + assert_eq!(stages.len(), 1); + assert_eq!(stages[0]["stage"]["id"], "verify"); + assert_eq!(stages[0]["model"]["id"], "gpt-new"); + assert_eq!(stages[0]["billing"]["input_tokens"], 300); + assert_eq!(stages[0]["billing"]["output_tokens"], 30); + assert_eq!(stages[0]["billing"]["total_usd_micros"], 330); + assert!((stages[0]["runtime_secs"].as_f64().unwrap() - 2.0).abs() < f64::EPSILON); + + assert_eq!(body["totals"]["input_tokens"], 300); + assert_eq!(body["totals"]["output_tokens"], 30); + assert_eq!(body["totals"]["total_usd_micros"], 330); + assert!((body["totals"]["runtime_secs"].as_f64().unwrap() - 2.0).abs() < f64::EPSILON); + + let by_model = body["by_model"].as_array().unwrap(); + assert_eq!(by_model.len(), 2); + let old_model = by_model + .iter() + .find(|entry| entry["model"]["id"] == "gpt-old") + .unwrap(); + let new_model = by_model + .iter() + .find(|entry| entry["model"]["id"] == "gpt-new") + .unwrap(); + assert_eq!(old_model["stages"], 1); + assert_eq!(old_model["billing"]["input_tokens"], 100); + assert_eq!(new_model["stages"], 1); + assert_eq!(new_model["billing"]["input_tokens"], 200); +} + +#[tokio::test] +async fn list_run_stages_shows_retrying_after_failed_event() { + let state = test_app_state_with_isolated_storage(); + let app = crate::test_support::build_test_router(Arc::clone(&state)); + let run_id = RunId::new(); + + create_durable_run_with_events(&state, run_id, &[ + workflow_event::Event::RunSubmitted { + definition_blob: None, + }, + workflow_event::Event::RunStarting, + workflow_event::Event::RunRunning, + ]) + .await; + + append_scoped_stage_event( + &state, + run_id, + "work", + 1, + &workflow_event::Event::StageStarted { + node_id: "work".to_string(), + name: "Work".to_string(), + index: 0, + handler_type: "command".to_string(), + attempt: 1, + max_attempts: 3, + }, + ) + .await; + append_scoped_stage_event( + &state, + run_id, + "work", + 1, + &workflow_event::Event::StageFailed { + node_id: "work".to_string(), + name: "Work".to_string(), + index: 0, + failure: FailureDetail::new("flake", FailureCategory::TransientInfra), + will_retry: true, + duration_ms: 5, + billing: None, + actor: None, + }, + ) + .await; + append_scoped_stage_event( + &state, + run_id, + "work", + 1, + &workflow_event::Event::StageRetrying { + node_id: "work".to_string(), + name: "Work".to_string(), + index: 0, + attempt: 2, + max_attempts: 3, + delay_ms: 50, + }, + ) + .await; + + let response = app + .clone() + .oneshot( + Request::builder() + .method("GET") + .uri(api(&format!("/runs/{run_id}/stages"))) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let body = response_json!(response, StatusCode::OK).await; + assert_eq!(stage_status(&body, "work@1"), "retrying"); +} + +#[tokio::test] +async fn list_run_stages_shows_retrying_when_failed_will_retry() { + let state = test_app_state_with_isolated_storage(); + let app = crate::test_support::build_test_router(Arc::clone(&state)); + let run_id = RunId::new(); + + create_durable_run_with_events(&state, run_id, &[ + workflow_event::Event::RunSubmitted { + definition_blob: None, + }, + workflow_event::Event::RunStarting, + workflow_event::Event::RunRunning, + ]) + .await; + + append_scoped_stage_event( + &state, + run_id, + "work", + 1, + &workflow_event::Event::StageStarted { + node_id: "work".to_string(), + name: "Work".to_string(), + index: 0, + handler_type: "command".to_string(), + attempt: 1, + max_attempts: 3, + }, + ) + .await; + // Only StageFailed, no StageRetrying yet — should still render retrying + // because props.will_retry is true. + append_scoped_stage_event( + &state, + run_id, + "work", + 1, + &workflow_event::Event::StageFailed { + node_id: "work".to_string(), + name: "Work".to_string(), + index: 0, + failure: FailureDetail::new("flake", FailureCategory::TransientInfra), + will_retry: true, + duration_ms: 5, + billing: None, + actor: None, + }, + ) + .await; + + let response = app + .clone() + .oneshot( + Request::builder() + .method("GET") + .uri(api(&format!("/runs/{run_id}/stages"))) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let body = response_json!(response, StatusCode::OK).await; + assert_eq!(stage_status(&body, "work@1"), "retrying"); } async fn append_raw_run_event( @@ -5835,6 +6444,62 @@ async fn get_aggregate_billing_returns_zeros_initially() { assert!(body["by_model"].as_array().unwrap().is_empty()); } +#[test] +fn aggregate_billing_counts_projection_rollup_usage_visits() { + let mut accumulator = BillingAccumulator::default(); + let rollup = fabro_workflow::ProjectionBillingRollup { + stages: Vec::new(), + totals: BilledTokenCounts { + input_tokens: 300, + output_tokens: 30, + total_tokens: 330, + reasoning_tokens: 0, + cache_read_tokens: 0, + cache_write_tokens: 0, + total_usd_micros: Some(330), + }, + by_model: vec![ + fabro_workflow::ProjectionBillingByModel { + model_id: "gpt-old".to_string(), + stages: 1, + billing: BilledTokenCounts { + input_tokens: 100, + output_tokens: 10, + total_tokens: 110, + reasoning_tokens: 0, + cache_read_tokens: 0, + cache_write_tokens: 0, + total_usd_micros: Some(110), + }, + }, + fabro_workflow::ProjectionBillingByModel { + model_id: "gpt-new".to_string(), + stages: 1, + billing: BilledTokenCounts { + input_tokens: 200, + output_tokens: 20, + total_tokens: 220, + reasoning_tokens: 0, + cache_read_tokens: 0, + cache_write_tokens: 0, + total_usd_micros: Some(220), + }, + }, + ], + runtime_ms: 2000, + billed_visit_count: 2, + }; + + accumulate_billing_rollup(&mut accumulator, &rollup); + + assert_eq!(accumulator.total_runs, 1); + assert_eq!(accumulator.total_runtime_secs, 2.0); + assert_eq!(accumulator.by_model["gpt-old"].stages, 1); + assert_eq!(accumulator.by_model["gpt-old"].billing.input_tokens, 100); + assert_eq!(accumulator.by_model["gpt-new"].stages, 1); + assert_eq!(accumulator.by_model["gpt-new"].billing.input_tokens, 200); +} + #[tokio::test] async fn post_runs_returns_submitted_status() { let state = test_app_state(); diff --git a/lib/crates/fabro-store/src/artifact_store.rs b/lib/crates/fabro-store/src/artifact_store.rs index 4cd4f85e1..5be00d792 100644 --- a/lib/crates/fabro-store/src/artifact_store.rs +++ b/lib/crates/fabro-store/src/artifact_store.rs @@ -297,8 +297,13 @@ fn decode_artifact_location( )) })?; let (retry, filename) = decode_retry_and_filename(location, &mut parts)?; + let stage_id = StageId::try_new(node_id, visit).map_err(|err| { + Error::Other(format!( + "artifact location {location} has an invalid stage id: {err}" + )) + })?; Ok(NodeArtifact { - node: StageId::new(node_id, visit), + node: stage_id, retry, filename, size, diff --git a/lib/crates/fabro-store/src/run_state.rs b/lib/crates/fabro-store/src/run_state.rs index 77b36796e..af467f557 100644 --- a/lib/crates/fabro-store/src/run_state.rs +++ b/lib/crates/fabro-store/src/run_state.rs @@ -9,8 +9,8 @@ use fabro_types::run_event::{ use fabro_types::{ BilledModelUsage, Checkpoint, Conclusion, EventBody, FailureSignature, InterviewQuestionRecord, Outcome, PendingInterviewRecord, PullRequestRecord, RunControlAction, RunEvent, RunId, - RunProjection, RunSpec, RunStatus, RunSummary, SandboxRecord, StageCompletion, StageOutcome, - StageProjection, StartRecord, TerminalStatus, first_event_seq, + RunProjection, RunSpec, RunStatus, RunSummary, SandboxRecord, StageCompletion, StageId, + StageOutcome, StageProjection, StartRecord, TerminalStatus, first_event_seq, }; use fabro_util::error::render_with_causes; use serde_json::Value; @@ -297,52 +297,59 @@ impl RunProjectionReducer for RunProjection { ); } EventBody::StagePrompt(props) => { - let Some(stage) = stage_at_visit(self, stored, props.visit, event.seq) else { + let Some(stage) = stage_at_stored_or_visit(self, stored, props.visit, event.seq) + else { return Ok(()); }; stage.prompt = Some(props.text.clone()); stage.provider_used = provider_used_from_prompt(props); } EventBody::PromptCompleted(props) => { - let Some(stage) = stage_at_current_visit(self, stored, event.seq) else { + let Some(stage) = stage_at_stored_or_current_visit(self, stored, event.seq) else { return Ok(()); }; stage.response = Some(props.response.clone()); } EventBody::StageCompleted(props) => { - let Some(node_id) = stored.node_id.as_deref() else { - return Ok(()); - }; - let visit = stage_visit(node_id, props.node_visits.as_ref(), self).unwrap_or(1); let response = props.response.clone(); let outcome = stage_outcome_from_props(props); let completion = stage_completion_from_outcome(&outcome, ts); - let stage = self.stage_entry(node_id, visit, first_event_seq(event.seq)); + let Some(stage) = + stage_at_completed_visit(self, stored, props.node_visits.as_ref(), event.seq) + else { + return Ok(()); + }; stage.response = response; stage.completion = Some(completion); + stage.duration_ms = Some(props.duration_ms); + stage.usage.clone_from(&props.billing); } EventBody::StageFailed(props) => { let failure_reason = props.failure.as_ref().map(|detail| detail.message.clone()); - let Some(stage) = stage_at_current_visit(self, stored, event.seq) else { + let Some(stage) = stage_at_stored_or_current_visit(self, stored, event.seq) else { return Ok(()); }; stage.completion = Some(StageCompletion { outcome: StageOutcome::Failed { - retry_requested: false, + retry_requested: props.will_retry, }, notes: None, failure_reason, timestamp: ts, }); + stage.duration_ms = Some(props.duration_ms); + stage.usage.clone_from(&props.billing); } EventBody::AgentSessionStarted(props) => { - let Some(stage) = stage_at_visit(self, stored, props.visit, event.seq) else { + let Some(stage) = stage_at_stored_or_visit(self, stored, props.visit, event.seq) + else { return Ok(()); }; stage.provider_used = Some(provider_used_from_agent_session_started(props)); } EventBody::AgentCliStarted(props) => { - let Some(stage) = stage_at_visit(self, stored, props.visit, event.seq) else { + let Some(stage) = stage_at_stored_or_visit(self, stored, props.visit, event.seq) + else { return Ok(()); }; stage.provider_used = Some(provider_used_from_agent_cli_started(props)); @@ -351,7 +358,7 @@ impl RunProjectionReducer for RunProjection { let script_invocation = serde_json::to_value(props).map_err(|err| { Error::InvalidEvent(format!("invalid command.started payload: {err}")) })?; - let Some(stage) = stage_at_current_visit(self, stored, event.seq) else { + let Some(stage) = stage_at_stored_or_current_visit(self, stored, event.seq) else { return Ok(()); }; stage.script_invocation = Some(script_invocation); @@ -360,7 +367,7 @@ impl RunProjectionReducer for RunProjection { let script_timing = serde_json::to_value(props).map_err(|err| { Error::InvalidEvent(format!("invalid command.completed payload: {err}")) })?; - let Some(stage) = stage_at_current_visit(self, stored, event.seq) else { + let Some(stage) = stage_at_stored_or_current_visit(self, stored, event.seq) else { return Ok(()); }; stage.stdout = Some(props.stdout.clone()); @@ -376,7 +383,7 @@ impl RunProjectionReducer for RunProjection { let parallel_results = serde_json::to_value(&props.results).map_err(|err| { Error::InvalidEvent(format!("invalid parallel.completed payload: {err}")) })?; - let Some(stage) = stage_at_current_visit(self, stored, event.seq) else { + let Some(stage) = stage_at_stored_or_current_visit(self, stored, event.seq) else { return Ok(()); }; stage.parallel_results = Some(parallel_results); @@ -394,6 +401,9 @@ fn stage_at_visit<'a>( visit: u32, seq: u32, ) -> Option<&'a mut StageProjection> { + if visit == 0 { + return None; + } let node_id = stored.node_id.as_deref()?; Some(state.stage_entry(node_id, visit, first_event_seq(seq))) } @@ -408,6 +418,51 @@ fn stage_at_current_visit<'a>( Some(state.stage_entry(node_id, visit, first_event_seq(seq))) } +fn stage_at_stored_stage_id<'a>( + state: &'a mut RunProjection, + stage_id: &StageId, + seq: u32, +) -> &'a mut StageProjection { + state.stage_entry(stage_id.node_id(), stage_id.visit(), first_event_seq(seq)) +} + +fn stage_at_stored_or_visit<'a>( + state: &'a mut RunProjection, + stored: &RunEvent, + visit: u32, + seq: u32, +) -> Option<&'a mut StageProjection> { + if let Some(stage_id) = stored.stage_id.as_ref() { + return Some(stage_at_stored_stage_id(state, stage_id, seq)); + } + stage_at_visit(state, stored, visit, seq) +} + +fn stage_at_stored_or_current_visit<'a>( + state: &'a mut RunProjection, + stored: &RunEvent, + seq: u32, +) -> Option<&'a mut StageProjection> { + if let Some(stage_id) = stored.stage_id.as_ref() { + return Some(stage_at_stored_stage_id(state, stage_id, seq)); + } + stage_at_current_visit(state, stored, seq) +} + +fn stage_at_completed_visit<'a>( + state: &'a mut RunProjection, + stored: &RunEvent, + node_visits: Option<&BTreeMap>, + seq: u32, +) -> Option<&'a mut StageProjection> { + if let Some(stage_id) = stored.stage_id.as_ref() { + return Some(stage_at_stored_stage_id(state, stage_id, seq)); + } + let node_id = stored.node_id.as_deref()?; + let visit = stage_visit(node_id, node_visits, state).unwrap_or(1); + Some(state.stage_entry(node_id, visit, first_event_seq(seq))) +} + pub(crate) fn build_summary(state: &RunProjection, run_id: &RunId) -> RunSummary { let workflow_name = state.spec.as_ref().map(|spec| { if spec.graph.name.is_empty() { @@ -529,6 +584,7 @@ fn stage_visit( node_visits .and_then(|visits| visits.get(node_id).copied()) .and_then(|visit| u32::try_from(visit).ok()) + .filter(|visit| *visit > 0) .or_else(|| state.current_visit_for(node_id)) } @@ -613,12 +669,13 @@ mod tests { use fabro_types::run_event::run::RunFailedProps; use fabro_types::run_event::{ CheckpointCompletedProps, InterviewCompletedProps, InterviewOption, InterviewStartedProps, - RunControlEffectProps, StagePromptProps, StageStartedProps, + RunControlEffectProps, StageCompletedProps, StageFailedProps, StagePromptProps, + StageStartedProps, }; use fabro_types::{ - BlockedReason, Checkpoint, EventBody, FailureReason, Outcome, QuestionType, RunBlobId, - RunControlAction, RunEvent, RunStatus, StageOutcome, SuccessReason, TerminalStatus, - WorkflowSettings, first_event_seq, fixtures, + BilledModelUsage, BlockedReason, Checkpoint, EventBody, FailureReason, Outcome, + QuestionType, RunBlobId, RunControlAction, RunEvent, RunStatus, StageOutcome, + SuccessReason, TerminalStatus, WorkflowSettings, first_event_seq, fixtures, }; use serde_json::json; @@ -651,6 +708,32 @@ mod tests { event } + fn test_usage(model_id: &str, input_tokens: i64, output_tokens: i64) -> BilledModelUsage { + serde_json::from_value(json!({ + "input": { + "usage": { + "model": { + "provider": "openai", + "model_id": model_id + }, + "tokens": { + "input_tokens": input_tokens, + "output_tokens": output_tokens + } + }, + "facts": { + "provider": "open_ai" + } + }, + "total_usd_micros": input_tokens + output_tokens + })) + .unwrap() + } + + fn usage_json(usage: &BilledModelUsage) -> serde_json::Value { + serde_json::to_value(usage).unwrap() + } + fn test_raw_event( seq: u32, event: &str, @@ -861,6 +944,210 @@ mod tests { assert_eq!(stage.prompt.as_deref(), Some("prompt")); } + #[test] + fn stage_completed_event_captures_duration_and_usage_per_visit() { + let mut state = RunProjection::default(); + let usage = test_usage("gpt-5.2", 123, 45); + + state + .apply_event(&test_event( + 3, + EventBody::StageCompleted(StageCompletedProps { + index: 0, + duration_ms: 789, + status: StageOutcome::Succeeded, + preferred_label: None, + suggested_next_ids: Vec::new(), + billing: Some(usage.clone()), + failure: None, + notes: None, + files_touched: Vec::new(), + context_updates: None, + jump_to_node: None, + context_values: None, + node_visits: None, + loop_failure_signatures: None, + restart_failure_signatures: None, + response: Some("done".to_string()), + attempt: 1, + max_attempts: 1, + }), + Some("build"), + )) + .unwrap(); + + let stage = state.stage(&StageId::new("build", 1)).unwrap(); + assert_eq!(stage.duration_ms, Some(789)); + assert_eq!(stage.usage.as_ref(), Some(&usage)); + } + + #[test] + fn stage_failed_event_captures_duration_and_usage_per_visit() { + let mut state = RunProjection::default(); + let stage_id = StageId::new("build", 1); + let usage = test_usage("gpt-5.2", 321, 54); + + state + .apply_event(&test_stage_event( + 2, + EventBody::StageStarted(StageStartedProps { + index: 0, + handler_type: "agent".to_string(), + attempt: 1, + max_attempts: 1, + }), + stage_id.clone(), + )) + .unwrap(); + state + .apply_event(&test_raw_event( + 3, + "stage.failed", + &json!({ + "index": 0, + "failure": { + "message": "provider failed", + "failure_class": "transient_infra" + }, + "will_retry": false, + "duration_ms": 654, + "billing": usage_json(&usage) + }), + Some("build"), + )) + .unwrap(); + + let stage = state.stage(&stage_id).unwrap(); + assert_eq!(stage.duration_ms, Some(654)); + assert_eq!(stage.usage.as_ref(), Some(&usage)); + } + + #[test] + fn two_visits_of_one_node_retain_distinct_usage() { + let mut state = RunProjection::default(); + let first_usage = test_usage("gpt-5.2", 100, 10); + let second_usage = test_usage("gpt-5.2", 200, 20); + + for (seq, visit, duration_ms, usage) in [ + (3, 1usize, 111, first_usage.clone()), + (4, 2usize, 222, second_usage.clone()), + ] { + state + .apply_event(&test_event( + seq, + EventBody::StageCompleted(StageCompletedProps { + index: 0, + duration_ms, + status: StageOutcome::Succeeded, + preferred_label: None, + suggested_next_ids: Vec::new(), + billing: Some(usage), + failure: None, + notes: None, + files_touched: Vec::new(), + context_updates: None, + jump_to_node: None, + context_values: None, + node_visits: Some(BTreeMap::from([("build".to_string(), visit)])), + loop_failure_signatures: None, + restart_failure_signatures: None, + response: None, + attempt: 1, + max_attempts: 1, + }), + Some("build"), + )) + .unwrap(); + } + + let first_stage = state.stage(&StageId::new("build", 1)).unwrap(); + let second_stage = state.stage(&StageId::new("build", 2)).unwrap(); + assert_eq!(first_stage.duration_ms, Some(111)); + assert_eq!(first_stage.usage.as_ref(), Some(&first_usage)); + assert_eq!(second_stage.duration_ms, Some(222)); + assert_eq!(second_stage.usage.as_ref(), Some(&second_usage)); + } + + #[test] + fn stage_completed_prefers_stored_stage_id_over_legacy_node_visits() { + let mut state = RunProjection::default(); + let usage = test_usage("gpt-5.2", 300, 30); + let scoped_stage_id = StageId::new("build", 2); + + state + .apply_event(&test_stage_event( + 3, + EventBody::StageCompleted(StageCompletedProps { + index: 0, + duration_ms: 333, + status: StageOutcome::Succeeded, + preferred_label: None, + suggested_next_ids: Vec::new(), + billing: Some(usage.clone()), + failure: None, + notes: None, + files_touched: Vec::new(), + context_updates: None, + jump_to_node: None, + context_values: None, + node_visits: Some(BTreeMap::from([("build".to_string(), 1usize)])), + loop_failure_signatures: None, + restart_failure_signatures: None, + response: Some("done".to_string()), + attempt: 1, + max_attempts: 1, + }), + scoped_stage_id.clone(), + )) + .unwrap(); + + assert!( + state.stage(&StageId::new("build", 1)).is_none(), + "legacy node_visits must not override stored stage_id" + ); + let stage = state.stage(&scoped_stage_id).unwrap(); + assert_eq!(stage.duration_ms, Some(333)); + assert_eq!(stage.usage.as_ref(), Some(&usage)); + assert_eq!(stage.response.as_deref(), Some("done")); + } + + #[test] + fn stage_failed_prefers_stored_stage_id_and_preserves_retry_request() { + let mut state = RunProjection::default(); + let usage = test_usage("gpt-5.2", 400, 40); + let scoped_stage_id = StageId::new("build", 2); + + state + .apply_event(&test_stage_event( + 3, + EventBody::StageFailed(StageFailedProps { + index: 0, + failure: Some(fabro_types::FailureDetail::new( + "try again", + fabro_types::FailureCategory::TransientInfra, + )), + will_retry: true, + duration_ms: 444, + billing: Some(usage.clone()), + }), + scoped_stage_id.clone(), + )) + .unwrap(); + + assert!( + state.stage(&StageId::new("build", 1)).is_none(), + "current-visit fallback must not override stored stage_id" + ); + let stage = state.stage(&scoped_stage_id).unwrap(); + assert_eq!(stage.duration_ms, Some(444)); + assert_eq!(stage.usage.as_ref(), Some(&usage)); + let completion = stage.completion.as_ref().unwrap(); + assert_eq!(completion.outcome, StageOutcome::Failed { + retry_requested: true, + }); + assert_eq!(completion.failure_reason.as_deref(), Some("try again")); + } + #[test] fn checkpoint_completed_creates_projection_entry_for_skipped_stage() { let mut state = RunProjection::default(); diff --git a/lib/crates/fabro-store/tests/serializable_projection.rs b/lib/crates/fabro-store/tests/serializable_projection.rs index 6d51065ba..2b24cd54e 100644 --- a/lib/crates/fabro-store/tests/serializable_projection.rs +++ b/lib/crates/fabro-store/tests/serializable_projection.rs @@ -5,8 +5,8 @@ use fabro_store::{RunProjection, SerializableProjection, StageId}; use fabro_types::graph::Graph; use fabro_types::run::RunSpec; use fabro_types::{ - Checkpoint, RunStatus, SandboxRecord, StageCompletion, StageOutcome, StartRecord, - TerminalStatus, WorkflowSettings, first_event_seq, fixtures, + BilledModelUsage, Checkpoint, RunStatus, SandboxRecord, StageCompletion, StageOutcome, + StartRecord, TerminalStatus, WorkflowSettings, first_event_seq, fixtures, }; use serde_json::json; @@ -52,6 +52,28 @@ fn sample_checkpoint() -> Checkpoint { } } +fn sample_usage() -> BilledModelUsage { + serde_json::from_value(json!({ + "input": { + "usage": { + "model": { + "provider": "openai", + "model_id": "gpt-5.2" + }, + "tokens": { + "input_tokens": 123, + "output_tokens": 45 + } + }, + "facts": { + "provider": "open_ai" + } + }, + "total_usd_micros": 168 + })) + .expect("sample usage should deserialize") +} + #[test] fn serializable_projection_round_trips_and_trims_bulky_node_fields() { let stage_id = StageId::new("build", 2); @@ -94,6 +116,8 @@ fn serializable_projection_round_trips_and_trims_bulky_node_fields() { stage.script_invocation = Some(json!({ "command": "cargo test" })); stage.script_timing = Some(json!({ "duration_ms": 10 })); stage.parallel_results = Some(json!([{ "stage": "fanout@1" }])); + stage.duration_ms = Some(1234); + stage.usage = Some(sample_usage()); stage.stdout = Some("stdout".to_string()); stage.stderr = Some("stderr".to_string()); @@ -138,6 +162,8 @@ fn serializable_projection_round_trips_and_trims_bulky_node_fields() { node.parallel_results, Some(json!([{ "stage": "fanout@1" }])) ); + assert_eq!(node.duration_ms, Some(1234)); + assert_eq!(node.usage, Some(sample_usage())); } #[test] diff --git a/lib/crates/fabro-types/src/lib.rs b/lib/crates/fabro-types/src/lib.rs index c04702c1e..22675855c 100644 --- a/lib/crates/fabro-types/src/lib.rs +++ b/lib/crates/fabro-types/src/lib.rs @@ -80,7 +80,7 @@ pub use run_summary::RunSummary; pub use sandbox_record::SandboxRecord; pub use secret::{SecretMetadata, SecretType}; pub use stage_completion::StageCompletion; -pub use stage_id::{ParallelBranchId, StageId}; +pub use stage_id::{InvalidStageVisit, ParallelBranchId, StageId}; pub use start::StartRecord; pub use status::{ BlockedReason, FailureReason, InvalidTransition, ParseFailureReasonError, diff --git a/lib/crates/fabro-types/src/outcome.rs b/lib/crates/fabro-types/src/outcome.rs index af34cd10b..8ef0c67dc 100644 --- a/lib/crates/fabro-types/src/outcome.rs +++ b/lib/crates/fabro-types/src/outcome.rs @@ -138,12 +138,32 @@ impl From for StageState { match outcome { StageOutcome::Succeeded => Self::Succeeded, StageOutcome::PartiallySucceeded => Self::PartiallySucceeded, - StageOutcome::Failed { .. } => Self::Failed, + StageOutcome::Failed { + retry_requested: true, + } => Self::Retrying, + StageOutcome::Failed { + retry_requested: false, + } => Self::Failed, StageOutcome::Skipped => Self::Skipped, } } } +#[cfg(test)] +mod stage_state_tests { + use super::{StageOutcome, StageState}; + + #[test] + fn retry_requested_failure_projects_as_retrying() { + assert_eq!( + StageState::from(StageOutcome::Failed { + retry_requested: true, + }), + StageState::Retrying + ); + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum FailureCategory { @@ -340,9 +360,10 @@ mod tests { StageState::from(StageOutcome::Failed { retry_requested: true, }), - StageState::Failed + StageState::Retrying ); assert!(StageState::Cancelled.is_terminal()); + assert!(!StageState::Retrying.is_terminal()); assert!(!StageState::Running.is_terminal()); } } diff --git a/lib/crates/fabro-types/src/run_event/stage.rs b/lib/crates/fabro-types/src/run_event/stage.rs index 9f1781d82..1b6609884 100644 --- a/lib/crates/fabro-types/src/run_event/stage.rs +++ b/lib/crates/fabro-types/src/run_event/stage.rs @@ -57,6 +57,8 @@ pub struct StageFailedProps { pub will_retry: bool, #[serde(default)] pub duration_ms: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub billing: Option, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] diff --git a/lib/crates/fabro-types/src/run_projection.rs b/lib/crates/fabro-types/src/run_projection.rs index de2e4a1a5..87321d348 100644 --- a/lib/crates/fabro-types/src/run_projection.rs +++ b/lib/crates/fabro-types/src/run_projection.rs @@ -4,9 +4,9 @@ use std::num::NonZeroU32; use chrono::{DateTime, Utc}; use crate::{ - Checkpoint, Conclusion, InterviewQuestionRecord, InvalidTransition, PullRequestRecord, Retro, - RunControlAction, RunId, RunSpec, RunStatus, SandboxRecord, StageCompletion, StageId, - StartRecord, + BilledModelUsage, Checkpoint, Conclusion, InterviewQuestionRecord, InvalidTransition, + PullRequestRecord, Retro, RunControlAction, RunId, RunSpec, RunStatus, SandboxRecord, + StageCompletion, StageId, StartRecord, }; #[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] @@ -44,6 +44,10 @@ pub struct StageProjection { pub prompt: Option, pub response: Option, pub completion: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub duration_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub usage: Option, pub provider_used: Option, pub diff: Option, pub script_invocation: Option, @@ -78,6 +82,8 @@ impl StageProjection { prompt: None, response: None, completion: None, + duration_ms: None, + usage: None, provider_used: None, diff: None, script_invocation: None, @@ -99,12 +105,32 @@ impl RunProjection { self.stages.get(stage) } + /// Iterate stages in `first_event_seq` order (the chronological order in + /// which each stage's first lifecycle event was recorded). Internal + /// storage is a `HashMap`, so iteration would otherwise be + /// non-deterministic; every caller wants chronological order, so we sort + /// here once instead of asking each caller to remember. pub fn iter_stages(&self) -> impl Iterator { - self.stages.iter() + let mut entries: Vec<(&StageId, &StageProjection)> = self.stages.iter().collect(); + entries.sort_by(|(left_id, left_stage), (right_id, right_stage)| { + left_stage + .first_event_seq + .cmp(&right_stage.first_event_seq) + .then_with(|| left_id.cmp(right_id)) + }); + entries.into_iter() } + /// Mutable counterpart of [`iter_stages`]. Same chronological ordering. pub fn iter_stages_mut(&mut self) -> impl Iterator { - self.stages.iter_mut() + let mut entries: Vec<(&StageId, &mut StageProjection)> = self.stages.iter_mut().collect(); + entries.sort_by(|(left_id, left_stage), (right_id, right_stage)| { + left_stage + .first_event_seq + .cmp(&right_stage.first_event_seq) + .then_with(|| left_id.cmp(right_id)) + }); + entries.into_iter() } pub fn is_empty(&self) -> bool { @@ -186,3 +212,90 @@ impl RunProjection { } } } + +#[cfg(test)] +mod iter_stages_tests { + use std::num::NonZeroU32; + + use super::RunProjection; + + fn seq(n: u32) -> NonZeroU32 { + NonZeroU32::new(n).unwrap() + } + + #[test] + fn iter_stages_yields_chronological_order_across_nodes() { + let mut p = RunProjection::default(); + // Insert in non-monotonic seq order to exercise the sort. + p.stage_entry("c", 1, seq(30)); + p.stage_entry("a", 1, seq(10)); + p.stage_entry("b", 1, seq(20)); + + let order: Vec<&str> = p + .iter_stages() + .map(|(stage_id, _)| stage_id.node_id()) + .collect(); + assert_eq!(order, vec!["a", "b", "c"]); + } + + #[test] + fn iter_stages_orders_visits_within_a_node() { + let mut p = RunProjection::default(); + // Visit 2 inserted first; visit 1's earlier first_event_seq must still + // win the chronological ordering. + p.stage_entry("verify", 2, seq(50)); + p.stage_entry("verify", 1, seq(20)); + + let visits: Vec = p + .iter_stages() + .map(|(stage_id, _)| stage_id.visit()) + .collect(); + assert_eq!(visits, vec![1, 2]); + } + + #[test] + fn iter_stages_mut_yields_chronological_order() { + let mut p = RunProjection::default(); + p.stage_entry("c", 1, seq(30)); + p.stage_entry("a", 1, seq(10)); + p.stage_entry("b", 1, seq(20)); + + let order: Vec = p + .iter_stages_mut() + .map(|(stage_id, _)| stage_id.node_id().to_string()) + .collect(); + assert_eq!(order, vec!["a", "b", "c"]); + } + + #[test] + fn iter_stages_tie_breaks_same_first_event_seq_by_stage_id() { + for _ in 0..128 { + let mut p = RunProjection::default(); + p.stage_entry("verify", 2, seq(10)); + p.stage_entry("build", 1, seq(10)); + p.stage_entry("verify", 1, seq(10)); + + let order: Vec = p + .iter_stages() + .map(|(stage_id, _)| stage_id.to_string()) + .collect(); + assert_eq!(order, vec!["build@1", "verify@1", "verify@2"]); + } + } + + #[test] + fn iter_stages_mut_tie_breaks_same_first_event_seq_by_stage_id() { + for _ in 0..128 { + let mut p = RunProjection::default(); + p.stage_entry("verify", 2, seq(10)); + p.stage_entry("build", 1, seq(10)); + p.stage_entry("verify", 1, seq(10)); + + let order: Vec = p + .iter_stages_mut() + .map(|(stage_id, _)| stage_id.to_string()) + .collect(); + assert_eq!(order, vec!["build@1", "verify@1", "verify@2"]); + } + } +} diff --git a/lib/crates/fabro-types/src/stage_id.rs b/lib/crates/fabro-types/src/stage_id.rs index baae58864..91711cb79 100644 --- a/lib/crates/fabro-types/src/stage_id.rs +++ b/lib/crates/fabro-types/src/stage_id.rs @@ -1,4 +1,5 @@ use std::fmt; +use std::num::NonZeroU32; use std::str::FromStr; use serde::de::Error as _; @@ -7,16 +8,21 @@ use serde::{Deserialize, Deserializer, Serialize, Serializer}; #[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct StageId { node_id: String, - visit: u32, + visit: NonZeroU32, } impl StageId { #[must_use] pub fn new(node_id: impl Into, visit: u32) -> Self { - Self { + Self::try_new(node_id, visit).expect("stage id visit must be greater than zero") + } + + pub fn try_new(node_id: impl Into, visit: u32) -> Result { + let visit = NonZeroU32::new(visit).ok_or(InvalidStageVisit)?; + Ok(Self { node_id: node_id.into(), visit, - } + }) } #[must_use] @@ -26,7 +32,7 @@ impl StageId { #[must_use] pub fn visit(&self) -> u32 { - self.visit + self.visit.get() } } @@ -47,6 +53,17 @@ impl fmt::Display for ParseStageIdError { impl std::error::Error for ParseStageIdError {} +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct InvalidStageVisit; + +impl fmt::Display for InvalidStageVisit { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("stage id visit must be greater than zero") + } +} + +impl std::error::Error for InvalidStageVisit {} + impl FromStr for StageId { type Err = ParseStageIdError; @@ -67,7 +84,7 @@ impl FromStr for StageId { let visit = visit .parse() .map_err(|err| ParseStageIdError(format!("invalid stage id visit: {err}")))?; - Ok(Self::new(node_id, visit)) + Self::try_new(node_id, visit).map_err(|err| ParseStageIdError(err.to_string())) } } @@ -224,6 +241,18 @@ mod tests { assert!(err.to_string().starts_with("invalid stage id visit:")); } + #[test] + fn parse_rejects_zero_visit() { + let err = "code@0".parse::().unwrap_err(); + assert_eq!(err.to_string(), "stage id visit must be greater than zero"); + } + + #[test] + fn try_new_rejects_zero_visit() { + let err = StageId::try_new("code", 0).unwrap_err(); + assert_eq!(err.to_string(), "stage id visit must be greater than zero"); + } + #[test] fn parse_rejects_empty_node_id() { let err = "@3".parse::().unwrap_err(); diff --git a/lib/crates/fabro-workflow/src/billing_rollup.rs b/lib/crates/fabro-workflow/src/billing_rollup.rs new file mode 100644 index 000000000..03d5207d3 --- /dev/null +++ b/lib/crates/fabro-workflow/src/billing_rollup.rs @@ -0,0 +1,291 @@ +use std::collections::{BTreeMap, HashMap}; + +use fabro_types::{BilledModelUsage, BilledTokenCounts, RunProjection}; + +#[derive(Debug, Clone, PartialEq)] +pub struct ProjectionBillingStage { + pub node_id: String, + pub billing: BilledTokenCounts, + pub duration_ms: u64, + pub model_id: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ProjectionBillingByModel { + pub model_id: String, + pub stages: i64, + pub billing: BilledTokenCounts, +} + +#[derive(Debug, Clone, Default, PartialEq)] +pub struct ProjectionBillingRollup { + pub stages: Vec, + pub totals: BilledTokenCounts, + pub by_model: Vec, + pub runtime_ms: u64, + pub billed_visit_count: usize, +} + +impl ProjectionBillingRollup { + #[must_use] + pub fn billing_if_present(&self) -> Option { + (self.billed_visit_count > 0).then(|| self.totals.clone()) + } +} + +#[must_use] +pub fn billing_rollup_from_projection(projection: &RunProjection) -> ProjectionBillingRollup { + let mut stage_indices = HashMap::::new(); + let mut stages = Vec::::new(); + let mut by_model = BTreeMap::::new(); + let mut totals = BilledTokenCounts::default(); + let mut runtime_ms = 0_u64; + let mut billed_visit_count = 0_usize; + + for (stage_id, stage) in projection.iter_stages() { + if is_exit_stage(projection, stage_id.node_id()) { + continue; + } + if stage.completion.is_none() && stage.duration_ms.is_none() && stage.usage.is_none() { + continue; + } + + let node_id = stage_id.node_id(); + let index = *stage_indices.entry(node_id.to_string()).or_insert_with(|| { + let index = stages.len(); + stages.push(ProjectionBillingStage { + node_id: node_id.to_string(), + billing: BilledTokenCounts::default(), + duration_ms: 0, + model_id: None, + }); + index + }); + let row = &mut stages[index]; + + if let Some(duration_ms) = stage.duration_ms { + row.duration_ms = row.duration_ms.saturating_add(duration_ms); + runtime_ms = runtime_ms.saturating_add(duration_ms); + } + + if let Some(usage) = stage.usage.as_ref() { + billed_visit_count += 1; + row.model_id = Some(usage.model_id().to_string()); + accumulate_usage(&mut row.billing, usage); + accumulate_usage(&mut totals, usage); + + let model_id = usage.model_id().to_string(); + let model_entry = + by_model + .entry(model_id.clone()) + .or_insert_with(|| ProjectionBillingByModel { + model_id, + stages: 0, + billing: BilledTokenCounts::default(), + }); + model_entry.stages += 1; + accumulate_usage(&mut model_entry.billing, usage); + } + } + + ProjectionBillingRollup { + stages, + totals, + by_model: by_model.into_values().collect(), + runtime_ms, + billed_visit_count, + } +} + +fn is_exit_stage(projection: &RunProjection, node_id: &str) -> bool { + projection + .spec() + .and_then(|spec| spec.graph().nodes.get(node_id)) + .is_some_and(|node| node.handler_type() == Some("exit")) +} + +fn accumulate_usage(counts: &mut BilledTokenCounts, usage: &BilledModelUsage) { + let tokens = usage.tokens(); + counts.input_tokens += tokens.input_tokens; + counts.output_tokens += tokens.output_tokens; + counts.reasoning_tokens += tokens.reasoning_tokens; + counts.cache_read_tokens += tokens.cache_read_tokens; + counts.cache_write_tokens += tokens.cache_write_tokens; + counts.total_tokens += tokens.total_tokens(); + if let Some(value) = usage.total_usd_micros { + *counts.total_usd_micros.get_or_insert(0) += value; + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use fabro_types::{ + AttrValue, BilledModelUsage, Graph, Node, RunProjection, RunSpec, StageCompletion, + StageOutcome, WorkflowSettings, first_event_seq, fixtures, + }; + use serde_json::json; + + use super::billing_rollup_from_projection; + + fn test_usage(model_id: &str, input_tokens: i64, output_tokens: i64) -> BilledModelUsage { + serde_json::from_value(json!({ + "input": { + "usage": { + "model": { + "provider": "openai", + "model_id": model_id + }, + "tokens": { + "input_tokens": input_tokens, + "output_tokens": output_tokens + } + }, + "facts": { + "provider": "open_ai" + } + }, + "total_usd_micros": input_tokens + output_tokens + })) + .unwrap() + } + + #[test] + fn rollup_groups_stage_rows_by_node_and_sums_retry_visit_usage() { + let mut projection = RunProjection::default(); + let failed_usage = test_usage("gpt-old", 100, 10); + let success_usage = test_usage("gpt-new", 200, 20); + let first = projection.stage_entry("verify", 1, first_event_seq(1)); + first.duration_ms = Some(1200); + first.usage = Some(failed_usage); + first.completion = Some(StageCompletion { + outcome: StageOutcome::Failed { + retry_requested: true, + }, + notes: None, + failure_reason: Some("try again".to_string()), + timestamp: chrono::Utc::now(), + }); + let second = projection.stage_entry("verify", 2, first_event_seq(2)); + second.duration_ms = Some(800); + second.usage = Some(success_usage); + second.completion = Some(StageCompletion { + outcome: StageOutcome::Succeeded, + notes: None, + failure_reason: None, + timestamp: chrono::Utc::now(), + }); + + let rollup = billing_rollup_from_projection(&projection); + + assert_eq!(rollup.stages.len(), 1); + assert_eq!(rollup.stages[0].node_id, "verify"); + assert_eq!(rollup.stages[0].model_id.as_deref(), Some("gpt-new")); + assert_eq!(rollup.stages[0].duration_ms, 2000); + assert_eq!(rollup.stages[0].billing.input_tokens, 300); + assert_eq!(rollup.stages[0].billing.output_tokens, 30); + assert_eq!(rollup.stages[0].billing.total_usd_micros, Some(330)); + + assert_eq!(rollup.runtime_ms, 2000); + assert_eq!(rollup.totals.input_tokens, 300); + assert_eq!(rollup.totals.output_tokens, 30); + assert_eq!(rollup.totals.total_usd_micros, Some(330)); + assert_eq!(rollup.billed_visit_count, 2); + + assert_eq!(rollup.by_model.len(), 2); + assert_eq!(rollup.by_model[0].model_id, "gpt-new"); + assert_eq!(rollup.by_model[0].stages, 1); + assert_eq!(rollup.by_model[0].billing.input_tokens, 200); + assert_eq!(rollup.by_model[1].model_id, "gpt-old"); + assert_eq!(rollup.by_model[1].stages, 1); + assert_eq!(rollup.by_model[1].billing.input_tokens, 100); + } + + #[test] + fn rollup_includes_completed_non_llm_stage_rows_with_zero_billing() { + let mut projection = RunProjection::default(); + let stage = projection.stage_entry("start", 1, first_event_seq(1)); + stage.duration_ms = Some(25); + stage.completion = Some(StageCompletion { + outcome: StageOutcome::Succeeded, + notes: None, + failure_reason: None, + timestamp: chrono::Utc::now(), + }); + + let rollup = billing_rollup_from_projection(&projection); + + assert_eq!(rollup.stages.len(), 1); + assert_eq!(rollup.stages[0].node_id, "start"); + assert_eq!(rollup.stages[0].duration_ms, 25); + assert!(rollup.stages[0].model_id.is_none()); + assert_eq!(rollup.stages[0].billing.input_tokens, 0); + assert_eq!(rollup.runtime_ms, 25); + assert!(rollup.by_model.is_empty()); + assert!(rollup.billing_if_present().is_none()); + } + + #[test] + fn rollup_excludes_terminal_exit_stage_rows() { + let mut projection = RunProjection::default(); + projection.spec = Some(run_spec_with_exit_node()); + let start = projection.stage_entry("start", 1, first_event_seq(1)); + start.duration_ms = Some(25); + start.completion = Some(StageCompletion { + outcome: StageOutcome::Succeeded, + notes: None, + failure_reason: None, + timestamp: chrono::Utc::now(), + }); + let exit = projection.stage_entry("exit", 1, first_event_seq(2)); + exit.duration_ms = Some(7); + exit.completion = Some(StageCompletion { + outcome: StageOutcome::Succeeded, + notes: None, + failure_reason: None, + timestamp: chrono::Utc::now(), + }); + + let rollup = billing_rollup_from_projection(&projection); + + assert_eq!(rollup.stages.len(), 1); + assert_eq!(rollup.stages[0].node_id, "start"); + assert_eq!(rollup.runtime_ms, 25); + } + + fn run_spec_with_exit_node() -> RunSpec { + let mut graph = Graph::new("test"); + graph.nodes.insert("start".to_string(), { + let mut node = Node::new("start"); + node.attrs.insert( + "shape".to_string(), + AttrValue::String("Mdiamond".to_string()), + ); + node + }); + graph.nodes.insert("exit".to_string(), { + let mut node = Node::new("exit"); + node.attrs.insert( + "shape".to_string(), + AttrValue::String("Msquare".to_string()), + ); + node + }); + + RunSpec { + run_id: fixtures::RUN_1, + settings: WorkflowSettings::default(), + graph, + workflow_slug: None, + source_directory: None, + labels: HashMap::new(), + provenance: None, + manifest_blob: None, + definition_blob: None, + git: None, + fork_source_ref: None, + in_place: false, + } + } +} diff --git a/lib/crates/fabro-workflow/src/error.rs b/lib/crates/fabro-workflow/src/error.rs index 1d08b2777..6a3ff0d06 100644 --- a/lib/crates/fabro-workflow/src/error.rs +++ b/lib/crates/fabro-workflow/src/error.rs @@ -1838,6 +1838,7 @@ mod tests { failure: failure.clone(), will_retry: false, duration_ms: 0, + billing: None, actor: None, }; diff --git a/lib/crates/fabro-workflow/src/event/convert.rs b/lib/crates/fabro-workflow/src/event/convert.rs index 815aaec11..4aa9df5b1 100644 --- a/lib/crates/fabro-workflow/src/event/convert.rs +++ b/lib/crates/fabro-workflow/src/event/convert.rs @@ -290,12 +290,14 @@ fn event_body_from_event(event: &Event) -> EventBody { failure, will_retry, duration_ms, + billing, .. } => EventBody::StageFailed(fabro_types::StageFailedProps { index: *index, failure: Some(failure.clone()), will_retry: *will_retry, duration_ms: *duration_ms, + billing: billing.clone(), }), Event::StageRetrying { index, @@ -1178,7 +1180,7 @@ mod tests { use crate::error::Error; use crate::event::test_support::user_principal; use crate::event::{Event, StageScope}; - use crate::outcome::FailureDetail; + use crate::outcome::{BilledModelUsage, FailureDetail}; #[derive(Debug)] struct EventTestCause; @@ -1200,6 +1202,28 @@ mod tests { } } + fn test_usage(model_id: &str, input_tokens: i64, output_tokens: i64) -> BilledModelUsage { + serde_json::from_value(serde_json::json!({ + "input": { + "usage": { + "model": { + "provider": "openai", + "model_id": model_id + }, + "tokens": { + "input_tokens": input_tokens, + "output_tokens": output_tokens + } + }, + "facts": { + "provider": "open_ai" + } + }, + "total_usd_micros": input_tokens + output_tokens + })) + .unwrap() + } + #[test] fn run_event_stage_completed_places_node_fields_in_header() { let stored = to_run_event_at( @@ -1279,6 +1303,7 @@ mod tests { #[test] fn run_event_stage_failure_keeps_failure_detail() { + let usage = test_usage("gpt-5.2", 321, 54); let stored = to_run_event(&fixtures::RUN_3, &Event::StageFailed { node_id: "code".to_string(), name: "Code".to_string(), @@ -1289,6 +1314,7 @@ mod tests { ), will_retry: true, duration_ms: 5000, + billing: Some(usage.clone()), actor: None, }); @@ -1297,6 +1323,7 @@ mod tests { assert_eq!(properties["failure"]["message"], "lint failed"); assert_eq!(properties["failure"]["failure_class"], "deterministic"); assert_eq!(properties["will_retry"], true); + assert_eq!(properties["billing"], serde_json::to_value(&usage).unwrap()); } #[test] diff --git a/lib/crates/fabro-workflow/src/event/events.rs b/lib/crates/fabro-workflow/src/event/events.rs index a3b2e2b5b..c753e20c6 100644 --- a/lib/crates/fabro-workflow/src/event/events.rs +++ b/lib/crates/fabro-workflow/src/event/events.rs @@ -206,6 +206,7 @@ pub enum Event { failure: FailureDetail, will_retry: bool, duration_ms: u64, + billing: Option, #[serde(default, skip_serializing_if = "Option::is_none")] actor: Option, }, diff --git a/lib/crates/fabro-workflow/src/lib.rs b/lib/crates/fabro-workflow/src/lib.rs index 9af074d97..5478e611d 100644 --- a/lib/crates/fabro-workflow/src/lib.rs +++ b/lib/crates/fabro-workflow/src/lib.rs @@ -20,6 +20,7 @@ use std::sync::Arc; use fabro_retro::retro::CompletedStage; use fabro_store::EventEnvelope; +use fabro_types::{EventBody, StageId}; /// Callback invoked when a workflow node starts executing. pub type OnNodeCallback = Option>; @@ -86,34 +87,190 @@ pub fn build_completed_stages(cp: &records::Checkpoint, run_failed: bool) -> Vec stages } -pub fn extract_stage_durations_from_events(events: &[EventEnvelope]) -> HashMap { +/// Extract the `duration_ms` from a `stage.completed` / `stage.failed` +/// event body, or `None` for any other variant. +fn stage_completion_duration_ms(body: &EventBody) -> Option { + match body { + EventBody::StageCompleted(props) => Some(props.duration_ms), + EventBody::StageFailed(props) => Some(props.duration_ms), + _ => None, + } +} + +/// Extract per-stage (node_id, visit) durations from `stage.completed` / +/// `stage.failed` events. Keys on the full [`StageId`] so multi-visit stages +/// (e.g. a looped `verify` node) keep distinct durations. +/// +/// This is the canonical primitive; [`total_stage_duration_by_node`] and +/// [`latest_stage_duration_by_node`] are explicit rollups built on top of it. +pub fn extract_stage_durations_by_stage_id(events: &[EventEnvelope]) -> HashMap { let mut durations = HashMap::new(); for envelope in events { - let event = &envelope.event; - let event_name = event.event_name(); - if event_name != "stage.completed" && event_name != "stage.failed" { - continue; - } - let Some(node_id) = event.node_id.as_deref() else { + let Some(duration_ms) = stage_completion_duration_ms(&envelope.event.body) else { continue; }; - let Some(duration_ms) = event - .properties() - .ok() - .and_then(|properties| properties.get("duration_ms").cloned()) - .and_then(|duration| duration.as_u64()) - else { + let Some(stage_id) = envelope.event.stage_id.as_ref() else { continue; }; - durations.insert(node_id.to_string(), duration_ms); + durations.insert(stage_id.clone(), duration_ms); } durations } +/// Total duration spent in each node, summed across every visit. Use for +/// billing/usage where a retried node should count its full time. +pub fn total_stage_duration_by_node(events: &[EventEnvelope]) -> HashMap { + let mut totals: HashMap = HashMap::new(); + for (stage_id, duration_ms) in extract_stage_durations_by_stage_id(events) { + *totals.entry(stage_id.node_id().to_string()).or_default() += duration_ms; + } + totals +} + +/// Duration of each node's most recent visit (the highest visit number). Use +/// for run summaries and retros where the table shows one row per node and +/// "the last attempt" is the right representative. +pub fn latest_stage_duration_by_node(events: &[EventEnvelope]) -> HashMap { + let mut entries: Vec<(StageId, u64)> = extract_stage_durations_by_stage_id(events) + .into_iter() + .collect(); + entries.sort_by_key(|(stage_id, _)| stage_id.visit()); + let mut latest = HashMap::new(); + for (stage_id, duration_ms) in entries { + latest.insert(stage_id.node_id().to_string(), duration_ms); + } + latest +} + +#[cfg(test)] +mod duration_tests { + use chrono::{TimeZone, Utc}; + use fabro_store::EventEnvelope; + use fabro_types::run_event::{StageCompletedProps, StageFailedProps}; + use fabro_types::{EventBody, RunEvent, StageId, StageOutcome, fixtures}; + + use super::{ + extract_stage_durations_by_stage_id, latest_stage_duration_by_node, + total_stage_duration_by_node, + }; + + fn completed_event(seq: u32, node: &str, visit: u32, duration_ms: u64) -> EventEnvelope { + let event = RunEvent { + id: format!("evt_{seq}"), + ts: Utc.with_ymd_and_hms(2026, 1, 1, 0, 0, 0).unwrap(), + run_id: fixtures::RUN_1, + node_id: Some(node.to_string()), + node_label: None, + stage_id: Some(StageId::new(node, visit)), + parallel_group_id: None, + parallel_branch_id: None, + session_id: None, + parent_session_id: None, + tool_call_id: None, + actor: None, + body: EventBody::StageCompleted(StageCompletedProps { + index: 0, + duration_ms, + status: StageOutcome::Succeeded, + preferred_label: None, + suggested_next_ids: vec![], + billing: None, + failure: None, + notes: None, + files_touched: vec![], + context_updates: None, + jump_to_node: None, + context_values: None, + node_visits: None, + loop_failure_signatures: None, + restart_failure_signatures: None, + response: None, + attempt: 1, + max_attempts: 1, + }), + }; + EventEnvelope { seq, event } + } + + fn failed_event(seq: u32, node: &str, visit: u32, duration_ms: u64) -> EventEnvelope { + let event = RunEvent { + id: format!("evt_{seq}"), + ts: Utc.with_ymd_and_hms(2026, 1, 1, 0, 0, 0).unwrap(), + run_id: fixtures::RUN_1, + node_id: Some(node.to_string()), + node_label: None, + stage_id: Some(StageId::new(node, visit)), + parallel_group_id: None, + parallel_branch_id: None, + session_id: None, + parent_session_id: None, + tool_call_id: None, + actor: None, + body: EventBody::StageFailed(StageFailedProps { + index: 0, + failure: None, + will_retry: true, + duration_ms, + billing: None, + }), + }; + EventEnvelope { seq, event } + } + + #[test] + fn extract_keys_durations_by_full_stage_id() { + let events = vec![ + completed_event(1, "verify", 1, 100), + completed_event(2, "verify", 2, 200), + ]; + let durations = extract_stage_durations_by_stage_id(&events); + assert_eq!( + durations.get(&StageId::new("verify", 1)).copied(), + Some(100) + ); + assert_eq!( + durations.get(&StageId::new("verify", 2)).copied(), + Some(200) + ); + } + + #[test] + fn total_sums_across_visits_per_node() { + let events = vec![ + completed_event(1, "verify", 1, 100), + completed_event(2, "verify", 2, 200), + completed_event(3, "build", 1, 50), + ]; + let totals = total_stage_duration_by_node(&events); + assert_eq!(totals.get("verify").copied(), Some(300)); + assert_eq!(totals.get("build").copied(), Some(50)); + } + + #[test] + fn latest_picks_highest_visit_regardless_of_input_order() { + // Visit 2 appears in the events vector before visit 1; the result + // must still reflect visit 2's duration (the latest visit). + let events = vec![ + completed_event(1, "verify", 2, 999), + completed_event(2, "verify", 1, 100), + ]; + let latest = latest_stage_duration_by_node(&events); + assert_eq!(latest.get("verify").copied(), Some(999)); + } + + #[test] + fn stage_failed_durations_are_included() { + let events = vec![failed_event(1, "verify", 1, 75)]; + let durations = extract_stage_durations_by_stage_id(&events); + assert_eq!(durations.get(&StageId::new("verify", 1)).copied(), Some(75)); + } +} + #[doc(hidden)] pub mod artifact; pub mod artifact_snapshot; pub mod artifact_upload; +pub mod billing_rollup; pub mod command_log; pub(crate) mod condition; pub mod context; @@ -142,6 +299,10 @@ pub mod run_control; pub(crate) mod run_dir; pub mod run_lookup; +pub use billing_rollup::{ + ProjectionBillingByModel, ProjectionBillingRollup, ProjectionBillingStage, + billing_rollup_from_projection, +}; pub use error::{Error, FailureCategory, FailureSignature, FailureSignatureExt, Result}; pub use manifest_path::ManifestPath; pub mod run_materialization; diff --git a/lib/crates/fabro-workflow/src/lifecycle/event.rs b/lib/crates/fabro-workflow/src/lifecycle/event.rs index 482f0311a..82dbd0501 100644 --- a/lib/crates/fabro-workflow/src/lifecycle/event.rs +++ b/lib/crates/fabro-workflow/src/lifecycle/event.rs @@ -224,6 +224,7 @@ impl RunLifecycle for EventLifecycle { failure, will_retry: true, duration_ms, + billing: outcome.usage.clone(), actor, }, &scope, @@ -275,6 +276,7 @@ impl RunLifecycle for EventLifecycle { failure, will_retry: false, duration_ms, + billing: outcome.usage.clone(), actor, }, &scope, diff --git a/lib/crates/fabro-workflow/src/pipeline/finalize.rs b/lib/crates/fabro-workflow/src/pipeline/finalize.rs index 921697369..927d2d833 100644 --- a/lib/crates/fabro-workflow/src/pipeline/finalize.rs +++ b/lib/crates/fabro-workflow/src/pipeline/finalize.rs @@ -19,6 +19,7 @@ use crate::run_status::{FailureReason, RunStatus, SuccessReason}; use crate::runtime_store::RunStoreHandle; use crate::sandbox_git::git_diff_with_timeout; use crate::services::RunServices; +use crate::{ProjectionBillingRollup, billing_rollup_from_projection}; pub fn classify_engine_result( engine_result: &Result, @@ -68,22 +69,22 @@ pub(crate) async fn build_conclusion_from_store( run_duration_ms: u64, final_git_commit_sha: Option, ) -> Conclusion { - let (state_result, events_result) = tokio::join!(run_store.state(), run_store.list_events()); - let projection = state_result.ok(); + let projection = run_store.state().await.ok(); let projection_order = projection .as_ref() .map(stage_projection_order) .unwrap_or_default(); + let projection_billing = projection + .as_ref() + .map(billing_rollup_from_projection) + .unwrap_or_default(); let checkpoint = projection .as_ref() .and_then(|state| state.checkpoint.as_ref()); - let stage_durations = events_result - .map(|events| crate::extract_stage_durations_from_events(&events)) - .unwrap_or_default(); build_conclusion_from_parts( checkpoint, - &stage_durations, + &projection_billing, &projection_order, status, failure_reason, @@ -94,7 +95,7 @@ pub(crate) async fn build_conclusion_from_store( fn build_conclusion_from_parts( checkpoint: Option<&Checkpoint>, - stage_durations: &HashMap, + projection_billing: &ProjectionBillingRollup, projection_order: &HashMap, status: StageOutcome, failure_reason: Option, @@ -105,6 +106,11 @@ fn build_conclusion_from_parts( // while the other checkpoint maps are keyed by node_id. Dedupe to one row // per node so the stages table matches the deduped billing total. let (stages, total_retries) = if let Some(cp) = checkpoint { + let billing_by_node = projection_billing + .stages + .iter() + .map(|stage| (stage.node_id.as_str(), stage)) + .collect::>(); let mut stage_rows = Vec::new(); let mut seen = std::collections::HashSet::new(); let mut retries_sum: u32 = 0; @@ -130,7 +136,6 @@ fn build_conclusion_from_parts( } for (original_checkpoint_order, node_id) in stage_order { - let outcome = cp.node_outcomes.get(node_id); let retries = cp .node_retries .get(node_id) @@ -138,14 +143,13 @@ fn build_conclusion_from_parts( .unwrap_or(1) .saturating_sub(1); retries_sum += retries; + let billing = billing_by_node.get(node_id); let summary = StageSummary { stage_id: node_id.to_string(), stage_label: node_id.to_string(), - duration_ms: stage_durations.get(node_id).copied().unwrap_or(0), - billing_usd_micros: outcome - .and_then(|o| o.usage.as_ref()) - .and_then(|usage| usage.total_usd_micros), + duration_ms: billing.map_or(0, |stage| stage.duration_ms), + billing_usd_micros: billing.and_then(|stage| stage.billing.total_usd_micros), retries, }; stage_rows.push(( @@ -176,7 +180,7 @@ fn build_conclusion_from_parts( failure_reason, final_git_commit_sha, stages, - billing: checkpoint.and_then(billing_from_checkpoint), + billing: projection_billing.billing_if_present(), total_retries, } } @@ -391,15 +395,8 @@ async fn compute_final_patch( } } -/// Iterates `node_outcomes.values()` rather than `completed_nodes` to avoid -/// over-counting the last visit's usage on looping workflows. -pub(crate) fn billing_from_checkpoint(cp: &Checkpoint) -> Option { - let usage: Vec<_> = cp - .node_outcomes - .values() - .filter_map(|o| o.usage.clone()) - .collect(); - (!usage.is_empty()).then(|| BilledTokenCounts::from_billed_usage(&usage)) +pub(crate) fn billing_from_projection(projection: &RunProjection) -> Option { + billing_rollup_from_projection(projection).billing_if_present() } pub(crate) fn build_terminal_event( @@ -503,7 +500,6 @@ pub async fn finalize(retroed: Retroed, options: &FinalizeOptions) -> Result Result BilledModelUsage { + serde_json::from_value(serde_json::json!({ + "input": { + "usage": { + "model": { + "provider": "openai", + "model_id": model_id + }, + "tokens": { + "input_tokens": input_tokens, + "output_tokens": output_tokens + } + }, + "facts": { + "provider": "open_ai" + } + }, + "total_usd_micros": input_tokens + output_tokens + })) + .unwrap() + } + #[test] fn conclusion_stage_order_follows_projection_first_event_order() { let mut projection = RunProjection::default(); @@ -753,7 +776,7 @@ mod tests { let conclusion = build_conclusion_from_parts( Some(&checkpoint), - &HashMap::new(), + &ProjectionBillingRollup::default(), &projection_order, StageOutcome::Succeeded, None, @@ -788,7 +811,7 @@ mod tests { let conclusion = build_conclusion_from_parts( Some(&checkpoint), - &HashMap::new(), + &ProjectionBillingRollup::default(), &projection_order, StageOutcome::Succeeded, None, @@ -804,6 +827,66 @@ mod tests { assert_eq!(stage_ids, vec!["skipped", "finished"]); } + #[test] + fn conclusion_billing_sums_retry_visit_usage_from_projection() { + let mut projection = RunProjection::default(); + let failed_usage = test_usage("gpt-old", 100, 10); + let success_usage = test_usage("gpt-new", 200, 20); + let failed = projection.stage_entry("verify", 1, first_event_seq(1)); + failed.duration_ms = Some(1200); + failed.usage = Some(failed_usage); + failed.completion = Some(StageCompletion { + outcome: StageOutcome::Failed { + retry_requested: true, + }, + notes: None, + failure_reason: Some("try again".to_string()), + timestamp: chrono::Utc::now(), + }); + let succeeded = projection.stage_entry("verify", 2, first_event_seq(2)); + succeeded.duration_ms = Some(800); + succeeded.usage = Some(success_usage.clone()); + succeeded.completion = Some(StageCompletion { + outcome: StageOutcome::Succeeded, + notes: None, + failure_reason: None, + timestamp: chrono::Utc::now(), + }); + + let projection_order = stage_projection_order(&projection); + let projection_billing = billing_rollup_from_projection(&projection); + let mut latest_outcome = Outcome::success(); + latest_outcome.usage = Some(success_usage); + latest_outcome.duration_ms = Some(800); + let mut checkpoint = checkpoint_with( + vec!["verify", "verify"], + HashMap::from([("verify".to_string(), latest_outcome)]), + ); + checkpoint.node_retries.insert("verify".to_string(), 2); + + let conclusion = build_conclusion_from_parts( + Some(&checkpoint), + &projection_billing, + &projection_order, + StageOutcome::Succeeded, + None, + 10, + None, + ); + + assert_eq!(conclusion.billing.as_ref().unwrap().input_tokens, 300); + assert_eq!(conclusion.billing.as_ref().unwrap().output_tokens, 30); + assert_eq!( + conclusion.billing.as_ref().unwrap().total_usd_micros, + Some(330) + ); + assert_eq!(conclusion.stages.len(), 1); + assert_eq!(conclusion.stages[0].stage_id, "verify"); + assert_eq!(conclusion.stages[0].duration_ms, 2000); + assert_eq!(conclusion.stages[0].billing_usd_micros, Some(330)); + assert_eq!(conclusion.stages[0].retries, 1); + } + fn test_services( run_store: RunStoreHandle, emitter: Arc, diff --git a/lib/crates/fabro-workflow/src/pipeline/mod.rs b/lib/crates/fabro-workflow/src/pipeline/mod.rs index 71aefbef9..284c32a1f 100644 --- a/lib/crates/fabro-workflow/src/pipeline/mod.rs +++ b/lib/crates/fabro-workflow/src/pipeline/mod.rs @@ -12,7 +12,7 @@ mod validate; pub use execute::execute; pub use fabro_types::PullRequestRecord; pub(crate) use finalize::{ - billing_from_checkpoint, build_conclusion_from_store, build_terminal_event, + billing_from_projection, build_conclusion_from_store, build_terminal_event, }; pub use finalize::{classify_engine_result, finalize, write_finalize_commit}; pub use initialize::initialize; diff --git a/lib/crates/fabro-workflow/src/pipeline/retro.rs b/lib/crates/fabro-workflow/src/pipeline/retro.rs index 649435c2e..d4c492373 100644 --- a/lib/crates/fabro-workflow/src/pipeline/retro.rs +++ b/lib/crates/fabro-workflow/src/pipeline/retro.rs @@ -53,7 +53,7 @@ pub async fn run_retro(options: &RetroOptions, dry_run: bool) -> Option { return None; } }; - let stage_durations = crate::extract_stage_durations_from_events(&events); + let stage_durations = crate::latest_stage_duration_by_node(&events); let mut retro = derive_retro( options.run_id, &options.workflow_name, diff --git a/lib/crates/fabro-workflow/src/test_support.rs b/lib/crates/fabro-workflow/src/test_support.rs index 12aa40f10..0bf6377a6 100644 --- a/lib/crates/fabro-workflow/src/test_support.rs +++ b/lib/crates/fabro-workflow/src/test_support.rs @@ -17,7 +17,7 @@ use crate::handler::HandlerRegistry; use crate::outcome::Outcome; use crate::pipeline; use crate::pipeline::types::{Executed, Initialized}; -use crate::pipeline::{billing_from_checkpoint, build_terminal_event}; +use crate::pipeline::{billing_from_projection, build_terminal_event}; use crate::records::Checkpoint; use crate::run_metadata::RunMetadataRuntime; use crate::run_options::RunOptions; @@ -36,10 +36,7 @@ async fn execute_and_emit_terminal(initialized: InitializedState) -> Executed { let executed = Box::pin(pipeline::execute(initialized.initialized)).await; initialized.store_logger.flush().await; let state = executed.engine.run.run_store.state().await.ok(); - let billing = state - .as_ref() - .and_then(|s| s.checkpoint.as_ref()) - .and_then(billing_from_checkpoint); + let billing = state.as_ref().and_then(billing_from_projection); let event = build_terminal_event( &executed.outcome, executed.duration_ms, diff --git a/lib/packages/fabro-api-client/src/models/billing-by-model.ts b/lib/packages/fabro-api-client/src/models/billing-by-model.ts index d8d802575..74f074401 100644 --- a/lib/packages/fabro-api-client/src/models/billing-by-model.ts +++ b/lib/packages/fabro-api-client/src/models/billing-by-model.ts @@ -26,7 +26,7 @@ import type { ModelReference } from './model-reference'; export interface BillingByModel { 'model': ModelReference; /** - * Number of stages that used this model. + * Number of usage-bearing stage visits that used this model. */ 'stages': number; 'billing': BilledTokenCounts; diff --git a/lib/packages/fabro-api-client/src/models/billing-stage-ref.ts b/lib/packages/fabro-api-client/src/models/billing-stage-ref.ts index bcc94a656..7b87ef631 100644 --- a/lib/packages/fabro-api-client/src/models/billing-stage-ref.ts +++ b/lib/packages/fabro-api-client/src/models/billing-stage-ref.ts @@ -15,7 +15,7 @@ /** - * Reference to a billing stage. + * Reference to a workflow node in a billing stage row. */ export interface BillingStageRef { /** diff --git a/lib/packages/fabro-api-client/src/models/run-billing-stage.ts b/lib/packages/fabro-api-client/src/models/run-billing-stage.ts index b6d807d8f..5d376b7cc 100644 --- a/lib/packages/fabro-api-client/src/models/run-billing-stage.ts +++ b/lib/packages/fabro-api-client/src/models/run-billing-stage.ts @@ -24,14 +24,14 @@ import type { BillingStageRef } from './billing-stage-ref'; import type { ModelReference } from './model-reference'; /** - * Token counts and billed totals for a single stage within a run. + * Token counts and billed totals for one workflow node within a run. Rows are grouped by node; billing and runtime sum every visit of that node. */ export interface RunBillingStage { 'stage': BillingStageRef; 'model': ModelReference | null; 'billing': BilledTokenCounts; /** - * Wall-clock runtime in seconds. + * Wall-clock runtime in seconds, summed across every visit of this node. */ 'runtime_secs': number; } diff --git a/lib/packages/fabro-api-client/src/models/run-billing.ts b/lib/packages/fabro-api-client/src/models/run-billing.ts index 88544b1cc..09cfed9e3 100644 --- a/lib/packages/fabro-api-client/src/models/run-billing.ts +++ b/lib/packages/fabro-api-client/src/models/run-billing.ts @@ -28,7 +28,7 @@ import type { RunBillingTotals } from './run-billing-totals'; */ export interface RunBilling { /** - * Per-stage billing breakdown. + * Per-node billing breakdown. Each row sums billing and runtime across all visits of that node. */ 'stages': Array; 'totals': RunBillingTotals; diff --git a/lib/packages/fabro-api-client/src/models/run-stage.ts b/lib/packages/fabro-api-client/src/models/run-stage.ts index c98ec9bee..ad01b9791 100644 --- a/lib/packages/fabro-api-client/src/models/run-stage.ts +++ b/lib/packages/fabro-api-client/src/models/run-stage.ts @@ -22,7 +22,7 @@ import type { StageState } from './stage-state'; */ export interface RunStage { /** - * Unique stage identifier within the run. + * StageId in \"node_id@visit\" form, e.g. verify@2. */ 'id': string; /** @@ -35,9 +35,13 @@ export interface RunStage { */ 'duration_secs'?: number; /** - * Node identifier in the Graphviz graph source. + * Node id in the workflow graph; multiple stages with different visits share the same node_id. */ - 'dot_id'?: string; + 'node_id': string; + /** + * 1-based visit count; bumped each time the workflow re-enters this node. + */ + 'visit': number; } From 6e36d8350ed859949f8d0c39d97ef31f04f246c3 Mon Sep 17 00:00:00 2001 From: "fabro-sh-0530[bot]" <281434857+fabro-sh-0530[bot]@users.noreply.github.com> Date: Tue, 5 May 2026 09:16:52 -0400 Subject: [PATCH 13/16] Render stage activity from scoped events endpoint (#212) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Stage detail now loads activity from a canonical stage-scoped events endpoint instead of falling back to the first 1000 run-wide events. This fixes empty panes for late stages in long runs and removes the presentation-shaped `StageTurn` API from the wire. ### Plan Summary - Add `GET /runs/{id}/stages/{stageId}/events` with cursor pagination and server-side `node_id` filtering. - Replace frontend stage-turn/fallback loading with paginated stage-events loading and local event-to-activity projection. - Broaden SSE/SWR invalidation so every activity event consumed by the reducer refreshes the per-stage cache. - Remove `StageTurn` schemas/client models and update demo fixtures plus pagination/handler/reducer tests. ## What changed and why The store now scans the run event prefix and filters by `node_id` before applying the `limit + 1` cutoff. That preserves sparse late-stage matches that would otherwise be dropped if we reused the run-wide limited scan and filtered afterward. The real-mode handler returns an empty page for an unknown stage id in an existing run, while preserving 404 for missing runs. On the frontend, `run-stages` fetches all pages for the selected stage and feeds them through `eventsToActivity`, keeping `TurnType` as a local presentation model. Invalidation now targets `runs.stageEvents(runId, stageId)` for lifecycle and reducer-consumed activity events (`stage.prompt`, agent messages/tools, and command events), so active panes refresh from the existing run event subscription. The OpenAPI document and generated TS client now expose `listStageEvents` and drop stale `StageTurn` models. Demo mode serves a `detect-drift` stage-events fixture using the same cursor semantics as the real endpoint. ## API notes `/runs/{id}/stages/{stageId}/turns` is removed; clients should use `/runs/{id}/stages/{stageId}/events?since_seq=&limit=` and project events locally. The `stageId` path segment for this endpoint is the workflow node id, not the visit-qualified `node_id@visit` form used by command logs/artifacts. ⚒️ Generated with [Fabro](https://fabro.sh) --------- Co-authored-by: Fabro Co-authored-by: Bryan Helmkamp --- apps/fabro-web/app/lib/api-client.ts | 65 +++- apps/fabro-web/app/lib/queries.ts | 25 +- apps/fabro-web/app/lib/query-keys.test.ts | 26 +- apps/fabro-web/app/lib/query-keys.ts | 9 +- apps/fabro-web/app/lib/run-events.test.tsx | 10 +- apps/fabro-web/app/lib/run-events.ts | 33 +- apps/fabro-web/app/routes/run-stages.test.ts | 139 ++++++-- apps/fabro-web/app/routes/run-stages.tsx | 88 +++-- docs/public/api-reference/fabro-api.yaml | 129 +------- lib/crates/fabro-server/src/demo/mod.rs | 159 ++++++++- .../fabro-server/src/principal_middleware.rs | 18 + lib/crates/fabro-server/src/server.rs | 3 +- .../fabro-server/src/server/handler/events.rs | 311 +++++++++++++++++- .../fabro-server/src/server/handler/mod.rs | 5 +- .../fabro-server/tests/it/event_pagination.rs | 75 +++++ lib/crates/fabro-server/tests/it/main.rs | 1 + .../fabro-server/tests/it/pagination.rs | 4 - lib/crates/fabro-store/src/slate/run_store.rs | 285 +++++++++++++++- .../src/.openapi-generator/FILES | 6 - .../src/api/run-internals-api.ts | 64 ++-- .../src/models/assistant-stage-turn.ts | 34 -- .../src/models/board-column-definition.ts | 3 + .../fabro-api-client/src/models/index.ts | 6 - .../src/models/manifest-git.ts | 41 --- .../models/manifest-pre-run-push-outcome.ts | 36 -- .../src/models/paginated-stage-turn-list.ts | 30 -- .../src/models/pre-run-git-context.ts | 30 -- .../src/models/set-secret-request.ts | 25 -- .../fabro-api-client/src/models/stage-turn.ts | 35 -- .../src/models/system-stage-turn.ts | 34 -- .../src/models/tool-stage-turn.ts | 41 --- .../fabro-api-client/src/models/tool-use.ts | 46 --- 32 files changed, 1153 insertions(+), 663 deletions(-) create mode 100644 lib/crates/fabro-server/tests/it/event_pagination.rs delete mode 100644 lib/packages/fabro-api-client/src/models/assistant-stage-turn.ts delete mode 100644 lib/packages/fabro-api-client/src/models/manifest-git.ts delete mode 100644 lib/packages/fabro-api-client/src/models/manifest-pre-run-push-outcome.ts delete mode 100644 lib/packages/fabro-api-client/src/models/paginated-stage-turn-list.ts delete mode 100644 lib/packages/fabro-api-client/src/models/pre-run-git-context.ts delete mode 100644 lib/packages/fabro-api-client/src/models/set-secret-request.ts delete mode 100644 lib/packages/fabro-api-client/src/models/stage-turn.ts delete mode 100644 lib/packages/fabro-api-client/src/models/system-stage-turn.ts delete mode 100644 lib/packages/fabro-api-client/src/models/tool-stage-turn.ts delete mode 100644 lib/packages/fabro-api-client/src/models/tool-use.ts diff --git a/apps/fabro-web/app/lib/api-client.ts b/apps/fabro-web/app/lib/api-client.ts index bd990361b..87792ed69 100644 --- a/apps/fabro-web/app/lib/api-client.ts +++ b/apps/fabro-web/app/lib/api-client.ts @@ -217,6 +217,69 @@ export async function apiPaginatedFetcher( } } +function stageEventsPagePath(key: string, sinceSeq: number, limit: number): string { + const url = new URL(apiPath(key), "http://fabro.local"); + url.searchParams.set("since_seq", String(sinceSeq)); + url.searchParams.set("limit", String(limit)); + return `${url.pathname}${url.search}`; +} + +/** + * Cursor-paginated fetcher for `/runs/{id}/stages/{stageId}/events`. + * + * Loops from `since_seq=1` with a 1000-event page size, advancing the cursor + * to `highestSeq + 1` until the server reports `meta.has_more === false`. + * The empty-page guard mirrors `apiPaginatedFetcher`: if the server claims + * `has_more` but returns no rows we exit and `console.warn` to surface the + * server invariant violation without spinning the UI. + */ +export async function fetchAllStageEvents( + key: string, +): Promise { + const PAGE_LIMIT = 1000; + const MAX_PAGES = 50; + const data: TItem[] = []; + let sinceSeq = 1; + let pagesLoaded = 0; + + while (true) { + const response = await apiRequest(stageEventsPagePath(key, sinceSeq, PAGE_LIMIT)); + if (!response.ok) { + throw await apiErrorFromResponse(response); + } + const page = (await response.json()) as PaginatedEnvelope; + pagesLoaded += 1; + + if (page.data.length === 0) { + if (page.meta.has_more) { + console.warn( + `Stage events fetch for ${key} returned an empty page with has_more=true; stopping at ${data.length} items to avoid spinning.`, + ); + } + return data; + } + + data.push(...page.data); + if (!page.meta.has_more) return data; + + if (pagesLoaded >= MAX_PAGES) { + console.warn( + `Stopped stage events fetch for ${key} after ${pagesLoaded} pages and ${data.length} items because the safety cap was reached.`, + ); + return data; + } + + const highestSeq = page.data.reduce((max, event) => Math.max(max, event.seq), sinceSeq - 1); + if (highestSeq < sinceSeq) { + console.warn( + `Stage events fetch for ${key} returned a non-advancing page at since_seq=${sinceSeq}; stopping at ${data.length} items to avoid spinning.`, + ); + return data; + } + sinceSeq = highestSeq + 1; + } +} + export async function apiJsonMutation( key: string, { arg }: { arg: TArg }, @@ -233,4 +296,4 @@ export async function apiJsonMutation( } if (response.status === 204) return undefined as TResponse; return response.json() as Promise; -} +} \ No newline at end of file diff --git a/apps/fabro-web/app/lib/queries.ts b/apps/fabro-web/app/lib/queries.ts index 0da17785e..880cf97b6 100644 --- a/apps/fabro-web/app/lib/queries.ts +++ b/apps/fabro-web/app/lib/queries.ts @@ -1,12 +1,11 @@ import useSWR, { type SWRConfiguration } from "swr"; import type { ApiQuestion, + EventEnvelope, PaginatedBoardRunList, - PaginatedEventList, PaginatedRunFileList, PaginatedRunList, PaginatedRunStageList, - PaginatedStageTurnList, CommandLogResponse, CommandOutputStream, RunBilling, @@ -24,6 +23,7 @@ import { apiNullableTextFetcher, apiPaginatedFetcher, apiTextFetcher, + fetchAllStageEvents, type PaginatedEnvelope, } from "./api-client"; import { queryKeys } from "./query-keys"; @@ -140,21 +140,10 @@ export function useRunQuestions(id: string | undefined, enabled: boolean) { ); } -export function useRunStageTurns( - id: string | undefined, - stageId: string | undefined, - enabled = true, -) { - return useSWR( - id && stageId && enabled ? queryKeys.runs.stageTurns(id, stageId) : null, - apiNullableFetcher, - ); -} - -export function useRunEventsList(id: string | undefined, enabled = true) { - return useSWR( - id && enabled ? queryKeys.runs.events(id, 1000) : null, - apiNullableFetcher, +export function useRunStageEvents(id: string | undefined, stageId: string | undefined) { + return useSWR( + id && stageId ? queryKeys.runs.stageEvents(id, stageId) : null, + fetchAllStageEvents, ); } @@ -205,4 +194,4 @@ export function useServerSettings() { return useSWR(queryKeys.settings.server(), apiFetcher, immutableOptions); } -export { apiTextFetcher }; +export { apiTextFetcher }; \ No newline at end of file diff --git a/apps/fabro-web/app/lib/query-keys.test.ts b/apps/fabro-web/app/lib/query-keys.test.ts index 8f036ac3e..5d7b3a53a 100644 --- a/apps/fabro-web/app/lib/query-keys.test.ts +++ b/apps/fabro-web/app/lib/query-keys.test.ts @@ -11,6 +11,9 @@ describe("queryKeys", () => { expect(queryKeys.runs.stageLog("run 1", "build step@2", "stderr", 12, 34)).toBe( "/api/v1/runs/run%201/stages/build%20step%402/logs/stderr?offset=12&limit=34", ); + expect(queryKeys.runs.stageEvents("run 1", "build step", 7, 25)).toBe( + "/api/v1/runs/run%201/stages/build%20step/events?since_seq=7&limit=25", + ); }); test("event-mapped keys match query hook resources", () => { @@ -23,7 +26,26 @@ describe("queryKeys", () => { queryKeys.runs.graph("run-1", "LR"), queryKeys.runs.graph("run-1", "TB"), queryKeys.runs.detail("run-1"), - queryKeys.runs.stageTurns("run-1", "stage-1"), + queryKeys.runs.stageEvents("run-1", "stage-1"), ]); }); -}); + + test("agent activity events invalidate the per-stage events key", () => { + for (const event of [ + "stage.prompt", + "agent.message", + "agent.tool.started", + "agent.tool.completed", + "command.started", + "command.completed", + ]) { + expect(queryKeysForRunEvent("run-1", event, "stage-1")).toEqual([ + queryKeys.runs.stageEvents("run-1", "stage-1"), + ]); + } + }); + + test("agent activity events without a node_id invalidate nothing", () => { + expect(queryKeysForRunEvent("run-1", "agent.message")).toEqual([]); + }); +}); \ No newline at end of file diff --git a/apps/fabro-web/app/lib/query-keys.ts b/apps/fabro-web/app/lib/query-keys.ts index 7d3e847e7..402739897 100644 --- a/apps/fabro-web/app/lib/query-keys.ts +++ b/apps/fabro-web/app/lib/query-keys.ts @@ -44,8 +44,11 @@ export const queryKeys = { }), events: (id: string, limit = 1000) => withQuery(`/api/v1/runs/${pathSegment(id)}/events`, { limit }), - stageTurns: (id: string, stageId: string) => - `/api/v1/runs/${pathSegment(id)}/stages/${pathSegment(stageId)}/turns`, + stageEvents: (id: string, stageId: string, sinceSeq?: number, limit?: number) => + withQuery(`/api/v1/runs/${pathSegment(id)}/stages/${pathSegment(stageId)}/events`, { + since_seq: sinceSeq, + limit, + }), stageLog: ( id: string, stageId: string, @@ -75,4 +78,4 @@ export const queryKeys = { settings: { server: () => "/api/v1/settings", }, -}; +}; \ No newline at end of file diff --git a/apps/fabro-web/app/lib/run-events.test.tsx b/apps/fabro-web/app/lib/run-events.test.tsx index c38ea1dd2..14db0e3f2 100644 --- a/apps/fabro-web/app/lib/run-events.test.tsx +++ b/apps/fabro-web/app/lib/run-events.test.tsx @@ -55,7 +55,7 @@ describe("queryKeysForRunEvent", () => { expect(keys).toContain(queryKeys.runs.stages("run-1")); expect(keys).toContain(queryKeys.runs.events("run-1", 1000)); expect(keys).toContain(queryKeys.runs.detail("run-1")); - expect(keys).toContain(queryKeys.runs.stageTurns("run-1", "verify@2")); + expect(keys).toContain(queryKeys.runs.stageEvents("run-1", "verify@2")); }); }); @@ -180,7 +180,7 @@ describe("subscribeToRunEvents", () => { coordinator.close(); }); - test("envelope with suffixed stage_id invalidates stageTurns(runId, stageId)", async () => { + test("envelope with suffixed stage_id invalidates stageEvents(runId, stageId)", async () => { const source = new FakeEventSource(); const keys: string[] = []; const coordinator = createCoordinator(() => source); @@ -202,12 +202,12 @@ describe("subscribeToRunEvents", () => { node_id: "verify", }); - expect(keys).toContain(queryKeys.runs.stageTurns("run-stage", "verify@2")); + expect(keys).toContain(queryKeys.runs.stageEvents("run-stage", "verify@2")); expect(keys).toContain(queryKeys.runs.stages("run-stage")); expect(keys).toContain(queryKeys.runs.events("run-stage", 1000)); expect(keys).toContain(queryKeys.runs.graph("run-stage", "LR")); expect(keys).toContain(queryKeys.runs.detail("run-stage")); - expect(keys).not.toContain(queryKeys.runs.stageTurns("run-stage", "verify")); + expect(keys).not.toContain(queryKeys.runs.stageEvents("run-stage", "verify")); cleanup(); coordinator.close(); @@ -230,7 +230,7 @@ describe("subscribeToRunEvents", () => { await waitFor(() => source.onmessage !== null); source.emit({ event: "stage.started", run_id: "run-stage-node", node_id: "verify" }); - expect(keys).toContain(queryKeys.runs.stageTurns("run-stage-node", "verify")); + expect(keys).toContain(queryKeys.runs.stageEvents("run-stage-node", "verify")); expect(keys).toContain(queryKeys.runs.stages("run-stage-node")); cleanup(); diff --git a/apps/fabro-web/app/lib/run-events.ts b/apps/fabro-web/app/lib/run-events.ts index a81a11c2b..1546d1578 100644 --- a/apps/fabro-web/app/lib/run-events.ts +++ b/apps/fabro-web/app/lib/run-events.ts @@ -49,7 +49,25 @@ const STAGE_EVENTS = new Set([ "stage.failed", "stage.retrying", ]); -const COMMAND_EVENTS = new Set(["command.started", "command.completed"]); +// Single source of truth: every event type the `eventsToActivity` reducer in +// `routes/run-stages.tsx` consumes. When any of these arrive for a stage we +// currently view, the stage-events SWR key for that stage must be invalidated +// so the panel refetches. The reducer imports this list so the switch stays +// in sync with the invalidation set; if the reducer grows a new case, this +// list is the single edit point. +// +// The lifecycle `STAGE_EVENTS` set is kept separate because it also fans out +// to run-scoped invalidations (stages list, graph, detail). +export const STAGE_ACTIVITY_EVENT_TYPES = [ + "stage.prompt", + "agent.message", + "agent.tool.started", + "agent.tool.completed", + "command.started", + "command.completed", +] as const; +export type StageActivityEventType = (typeof STAGE_ACTIVITY_EVENT_TYPES)[number]; +const STAGE_ACTIVITY_EVENTS = new Set(STAGE_ACTIVITY_EVENT_TYPES); const INTERVIEW_EVENTS = new Set([ "interview.started", "interview.completed", @@ -97,20 +115,13 @@ export function queryKeysForRunEvent( queryKeys.runs.detail(runId), ]; if (stageId) { - keys.push(queryKeys.runs.stageTurns(runId, stageId)); + keys.push(queryKeys.runs.stageEvents(runId, stageId)); } return keys; } - if (COMMAND_EVENTS.has(event)) { - const keys = [ - queryKeys.runs.stages(runId), - queryKeys.runs.events(runId, 1000), - ]; - if (stageId) { - keys.push(queryKeys.runs.stageTurns(runId, stageId)); - } - return keys; + if (STAGE_ACTIVITY_EVENTS.has(event)) { + return stageId ? [queryKeys.runs.stageEvents(runId, stageId)] : []; } return []; diff --git a/apps/fabro-web/app/routes/run-stages.test.ts b/apps/fabro-web/app/routes/run-stages.test.ts index 704c5cac8..52f10ed05 100644 --- a/apps/fabro-web/app/routes/run-stages.test.ts +++ b/apps/fabro-web/app/routes/run-stages.test.ts @@ -1,7 +1,7 @@ import { describe, expect, test } from "bun:test"; import type { EventEnvelope } from "@qltysh/fabro-api-client"; -import { isSafeMarkdownHref, turnsFromEvents } from "./run-stages"; +import { eventsToActivity, isSafeMarkdownHref } from "./run-stages"; describe("isSafeMarkdownHref", () => { test("rejects protocol-relative URLs", () => { @@ -17,43 +17,39 @@ describe("isSafeMarkdownHref", () => { }); }); -function makeEnvelope(overrides: Partial): EventEnvelope { +function envelope(seq: number, partial: Partial): EventEnvelope { return { - seq: 1, - id: "evt", - ts: "2026-01-01T00:00:00Z", + seq, + id: `evt-${seq}`, + ts: "2026-04-09T12:00:00Z", run_id: "run-1", event: "stage.prompt", - ...overrides, + ...partial, } as EventEnvelope; } -describe("turnsFromEvents", () => { +describe("eventsToActivity", () => { test("filters events by stage_id (verify@1 vs verify@2 do not cross-contaminate)", () => { const events: EventEnvelope[] = [ - makeEnvelope({ - seq: 1, + envelope(1, { event: "stage.prompt", stage_id: "verify@1", node_id: "verify", properties: { text: "first visit prompt" }, }), - makeEnvelope({ - seq: 2, + envelope(2, { event: "stage.prompt", stage_id: "verify@2", node_id: "verify", properties: { text: "second visit prompt" }, }), - makeEnvelope({ - seq: 3, + envelope(3, { event: "agent.message", stage_id: "verify@1", node_id: "verify", properties: { text: "first visit reply" }, }), - makeEnvelope({ - seq: 4, + envelope(4, { event: "agent.message", stage_id: "verify@2", node_id: "verify", @@ -61,30 +57,61 @@ describe("turnsFromEvents", () => { }), ]; - const firstVisit = turnsFromEvents(events, "verify@1"); + const firstVisit = eventsToActivity(events, "verify@1"); expect(firstVisit).toEqual([ { kind: "system", content: "first visit prompt" }, { kind: "assistant", content: "first visit reply" }, ]); - const secondVisit = turnsFromEvents(events, "verify@2"); + const secondVisit = eventsToActivity(events, "verify@2"); expect(secondVisit).toEqual([ { kind: "system", content: "second visit prompt" }, { kind: "assistant", content: "second visit reply" }, ]); }); + test("pairs command.started + command.completed into a single command turn", () => { + const events: EventEnvelope[] = [ + envelope(1, { + event: "command.started", + node_id: "fmt", + properties: { script: "cargo fmt", language: "shell" }, + }), + envelope(2, { + event: "command.completed", + node_id: "fmt", + properties: { + stdout: "ok", + stderr: "", + exit_code: 0, + duration_ms: 12, + termination: "exited", + }, + }), + ]; + + const turns = eventsToActivity(events, "fmt"); + expect(turns).toHaveLength(1); + expect(turns[0]).toMatchObject({ + kind: "command", + stageId: "fmt", + script: "cargo fmt", + language: "shell", + stdout: "ok", + exitCode: 0, + running: false, + }); + }); + test("command turn carries the requested stage_id, no @1 fallback", () => { const events: EventEnvelope[] = [ - makeEnvelope({ - seq: 1, + envelope(1, { event: "command.started", stage_id: "verify@2", node_id: "verify", properties: { script: "echo hi", language: "shell" }, }), - makeEnvelope({ - seq: 2, + envelope(2, { event: "command.completed", stage_id: "verify@2", node_id: "verify", @@ -98,7 +125,7 @@ describe("turnsFromEvents", () => { }), ]; - const turns = turnsFromEvents(events, "verify@2"); + const turns = eventsToActivity(events, "verify@2"); expect(turns).toHaveLength(1); const turn = turns[0]; expect(turn.kind).toBe("command"); @@ -108,4 +135,72 @@ describe("turnsFromEvents", () => { expect(turn.running).toBe(false); } }); + + test("pairs agent.tool.started + agent.tool.completed into a single tool turn", () => { + const events: EventEnvelope[] = [ + envelope(1, { + event: "agent.tool.started", + node_id: "detect-drift", + properties: { + tool_call_id: "call-1", + tool_name: "read_file", + arguments: { path: "config.toml" }, + }, + }), + envelope(2, { + event: "agent.tool.completed", + node_id: "detect-drift", + properties: { + tool_call_id: "call-1", + tool_name: "read_file", + output: "[redis]", + is_error: false, + }, + }), + ]; + + const turns = eventsToActivity(events, "detect-drift"); + expect(turns).toHaveLength(1); + expect(turns[0].kind).toBe("tool"); + if (turns[0].kind === "tool") { + expect(turns[0].tools).toHaveLength(1); + expect(turns[0].tools[0]).toMatchObject({ + id: "call-1", + toolName: "read_file", + result: "[redis]", + isError: false, + }); + } + }); + + test("ignores unknown event types and events for other stages", () => { + const events: EventEnvelope[] = [ + envelope(1, { + event: "stage.started", + node_id: "detect-drift", + properties: {}, + }), + envelope(2, { + event: "agent.message", + node_id: "detect-drift", + properties: { text: "signal" }, + }), + envelope(3, { + event: "run.running", + node_id: "detect-drift", + properties: {}, + }), + envelope(4, { + event: "agent.message", + node_id: "other-stage", + properties: { text: "wrong stage" }, + }), + ]; + + const turns = eventsToActivity(events, "detect-drift"); + expect(turns).toHaveLength(1); + if (turns[0].kind === "assistant") { + expect(turns[0].content).toBe("signal"); + } + }); }); diff --git a/apps/fabro-web/app/routes/run-stages.tsx b/apps/fabro-web/app/routes/run-stages.tsx index 1176cb71a..a8fc69168 100644 --- a/apps/fabro-web/app/routes/run-stages.tsx +++ b/apps/fabro-web/app/routes/run-stages.tsx @@ -40,16 +40,14 @@ import type { Stage } from "../components/stage-sidebar"; import { EmptyState } from "../components/state"; import { CopyButton } from "../components/ui"; import { formatDurationSecs } from "../lib/format"; -import { fetchRunCommandLog, useRunEventsList, useRunStageTurns, useRunStages } from "../lib/queries"; +import { fetchRunCommandLog, useRunStageEvents, useRunStages } from "../lib/queries"; +import { STAGE_ACTIVITY_EVENT_TYPES, type StageActivityEventType } from "../lib/run-events"; import { ACTIVE_STAGE_STATES, formatStageLabel, mapRunStagesToSidebarStages } from "../lib/stage-sidebar"; import { getNumber, getString, type UnknownRecord } from "../lib/unknown"; import { CommandOutputStream, CommandTermination, type EventEnvelope, - type StageTurn as ApiStageTurn, - type PaginatedStageTurnList, - type PaginatedEventList, } from "@qltysh/fabro-api-client"; export const handle = { wide: true }; @@ -68,17 +66,40 @@ function readTermination(props: UnknownRecord): CommandTermination { return CommandTermination.EXITED; } -export function turnsFromEvents(events: EventEnvelope[], stageId: string): TurnType[] { - const stageEvents = events.filter((e) => e.stage_id === stageId); +const STAGE_ACTIVITY_EVENT_SET = new Set(STAGE_ACTIVITY_EVENT_TYPES); + +function assertNever(value: never): never { + throw new Error(`Unhandled stage activity event type: ${value}`); +} + +function activityEventStageId(event: EventEnvelope): string | undefined { + if (typeof event.stage_id === "string") return event.stage_id; + if (typeof event.node_id === "string") return event.node_id; + return getString(event.properties ?? {}, "node_id"); +} + +export function eventsToActivity(events: EventEnvelope[], stageId: string): TurnType[] { const turns: TurnType[] = []; // Collect tool pairs: started → completed const pendingTools = new Map(); // Track pending command for pairing started → completed let pendingCommand: { stageId: string; script: string; language: string } | undefined; - for (const e of stageEvents) { + for (const e of events) { + const eventName = e.event; + if ( + activityEventStageId(e) !== stageId || + !eventName || + !STAGE_ACTIVITY_EVENT_SET.has(eventName) + ) { + continue; + } + // Exhaustive switch over StageActivityEventType: adding a new variant to + // STAGE_ACTIVITY_EVENT_TYPES forces a TS error here until the case is + // handled, keeping the SWR invalidation set and the reducer in sync. + const eventType = eventName as StageActivityEventType; const props = e.properties ?? {}; - switch (e.event) { + switch (eventType) { case "stage.prompt": turns.push({ kind: "system", content: getString(props, "text") ?? e.text ?? "" }); break; @@ -136,6 +157,8 @@ export function turnsFromEvents(events: EventEnvelope[], stageId: string): TurnT pendingCommand = undefined; break; } + default: + assertNever(eventType); } } @@ -153,41 +176,6 @@ export function turnsFromEvents(events: EventEnvelope[], stageId: string): TurnT return turns; } -function mapApiStageTurn(t: ApiStageTurn): TurnType { - switch (t.kind) { - case "tool": - return { - kind: "tool", - tools: (t.tools ?? []).map((tu) => ({ - id: tu.id, - toolName: tu.tool_name, - input: tu.input, - result: tu.result, - isError: tu.is_error, - durationMs: tu.duration_ms, - })), - }; - case "system": - case "assistant": - return { kind: t.kind, content: t.content ?? "" }; - } -} - -function mapTurns( - turnsResult: PaginatedStageTurnList | null | undefined, - eventsResult: PaginatedEventList | null | undefined, - selectedStageId: string | undefined, -): TurnType[] { - if (!selectedStageId) return []; - if (turnsResult?.data?.length) { - return turnsResult.data.map(mapApiStageTurn); - } - if (eventsResult?.data) { - return turnsFromEvents(eventsResult.data, selectedStageId); - } - return []; -} - function Markdown({ content }: { content: string }) { const html = useMemo(() => markedSafe.parse(content, { async: false }) as string, [content]); return ( @@ -605,14 +593,14 @@ export default function RunStages() { ); const selectedStage = stages.find((s: Stage) => s.id === stageId) ?? stages[0]; - const turnsQuery = useRunStageTurns(id, selectedStage?.id); - const hasStageTurns = (turnsQuery.data?.data.length ?? 0) > 0; - const shouldLoadEventFallback = - !!selectedStage?.id && !turnsQuery.isLoading && !turnsQuery.error && !hasStageTurns; - const eventsQuery = useRunEventsList(id, shouldLoadEventFallback); + const selectedStageId = selectedStage?.id; + const stageEventsQuery = useRunStageEvents(id, selectedStageId); const turns = useMemo( - () => mapTurns(turnsQuery.data, eventsQuery.data, selectedStage?.id), - [eventsQuery.data, selectedStage?.id, turnsQuery.data], + () => + selectedStageId + ? eventsToActivity(stageEventsQuery.data ?? [], selectedStageId) + : [], + [stageEventsQuery.data, selectedStageId], ); const isActive = selectedStage ? ACTIVE_STAGE_STATES.has(selectedStage.status) : false; diff --git a/docs/public/api-reference/fabro-api.yaml b/docs/public/api-reference/fabro-api.yaml index a767a7262..4319f6774 100644 --- a/docs/public/api-reference/fabro-api.yaml +++ b/docs/public/api-reference/fabro-api.yaml @@ -1906,26 +1906,26 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" - /api/v1/runs/{id}/stages/{stageId}/turns: + /api/v1/runs/{id}/stages/{stageId}/events: get: - operationId: listStageTurns + operationId: listStageEvents tags: [Run Internals] - summary: List Stage Turns - description: Returns a paginated list of conversation turns within a specific stage, including system prompts, assistant responses, and tool invocations. + summary: List Stage Events + description: Returns a paginated JSON list of stored run events scoped to a single stage visit. parameters: - $ref: "#/components/parameters/RunId" - $ref: "#/components/parameters/StageId" - - $ref: "#/components/parameters/PageLimit" - - $ref: "#/components/parameters/PageOffset" + - $ref: "#/components/parameters/SinceSeq" + - $ref: "#/components/parameters/EventLimit" responses: "200": - description: Paginated list of conversation turns + description: Paginated list of stage events content: application/json: schema: - $ref: "#/components/schemas/PaginatedStageTurnList" + $ref: "#/components/schemas/PaginatedEventList" "404": - description: Run or stage not found + description: Run not found. headers: x-request-id: $ref: "#/components/headers/XRequestId" @@ -3856,20 +3856,6 @@ components: meta: $ref: "#/components/schemas/PaginationMeta" - PaginatedStageTurnList: - description: Paginated list of stage turns. - type: object - required: - - data - - meta - properties: - data: - type: array - items: - $ref: "#/components/schemas/StageTurn" - meta: - $ref: "#/components/schemas/PaginationMeta" - PaginatedApiQuestionList: description: Paginated list of pending questions. type: object @@ -6348,103 +6334,6 @@ components: description: 1-based visit count; bumped each time the workflow re-enters this node. example: 2 - ToolUse: - description: A single tool invocation with its input, result, and execution metadata. - type: object - required: - - id - - tool_name - - input - - result - - is_error - properties: - id: - type: string - description: Unique identifier for this tool invocation. Enables correlation in parallel tool use. - example: toolu_01A09q90qw90lq917835lq9 - tool_name: - type: string - description: Name of the tool that was invoked. - example: read_file - input: - type: string - description: JSON-encoded input passed to the tool. - example: '{ "path": "src/routes/auth.ts" }' - result: - type: string - description: Output returned by the tool. Contains the error message when is_error is true. - example: 'import { Router } from "express";' - is_error: - type: boolean - description: Whether the tool invocation failed. When true, the result field contains the error message. - example: false - duration_ms: - type: integer - description: Wall-clock execution time of the tool invocation in milliseconds. - example: 142 - - StageTurn: - description: A single turn in a stage conversation — a system prompt, assistant response, or tool invocation block. - discriminator: - propertyName: kind - mapping: - system: "#/components/schemas/SystemStageTurn" - assistant: "#/components/schemas/AssistantStageTurn" - tool: "#/components/schemas/ToolStageTurn" - oneOf: - - $ref: "#/components/schemas/SystemStageTurn" - - $ref: "#/components/schemas/AssistantStageTurn" - - $ref: "#/components/schemas/ToolStageTurn" - - SystemStageTurn: - description: A system prompt turn that sets the stage's instructions. - type: object - required: - - kind - - content - properties: - kind: - type: string - enum: [system] - content: - type: string - description: System prompt text. - example: You are a drift detection agent. Compare the production and staging environments. - - AssistantStageTurn: - description: An assistant response turn within a stage. - type: object - required: - - kind - - content - properties: - kind: - type: string - enum: [assistant] - content: - type: string - description: Assistant response text. - example: I'll start by loading the environment configurations for both production and staging. - - ToolStageTurn: - description: A tool invocation turn containing one or more tool calls. - type: object - required: - - kind - - tools - properties: - kind: - type: string - enum: [tool] - content: - type: string - description: Text accompanying the tool invocations, or null when the turn contains only tool calls. - tools: - type: array - description: Tool invocations executed in this turn. - items: - $ref: "#/components/schemas/ToolUse" - # ── File Diff Schemas ────────────────────────────────────────────── FileCheckpoint: diff --git a/lib/crates/fabro-server/src/demo/mod.rs b/lib/crates/fabro-server/src/demo/mod.rs index bc527007d..c900c8b91 100644 --- a/lib/crates/fabro-server/src/demo/mod.rs +++ b/lib/crates/fabro-server/src/demo/mod.rs @@ -15,15 +15,16 @@ use axum::http::StatusCode; use axum::response::sse::{Event, Sse}; use axum::response::{IntoResponse, Response}; use fabro_api::types::{ - CreateSecretRequest, DeleteSecretRequest, DiffFile, DiffStats, FileDiff, FileDiffChangeKind, - PaginatedRunFileList, RunArtifactListResponse, RunFilesMeta, + CreateSecretRequest, DeleteSecretRequest, DiffFile, DiffStats, EventEnvelope, FileDiff, + FileDiffChangeKind, PaginatedEventList, PaginatedRunFileList, PaginationMeta, + RunArtifactListResponse, RunFilesMeta, }; use serde_json::json; use crate::error::ApiError; use crate::principal_middleware::RequiredUser; use crate::run_selector::{ResolveRunError, resolve_run_by_selector}; -use crate::server::{AppState, PaginationParams}; +use crate::server::{AppState, EventListParams, PaginationParams, parse_stage_id_path}; fn paginated_response( items: Vec, @@ -133,13 +134,39 @@ pub(crate) async fn get_run_stages( paginated_response(runs::stages(), &pagination) } -pub(crate) async fn get_stage_turns( +pub(crate) async fn get_stage_events( _auth: RequiredUser, State(_state): State>, - Path((_id, _stage_id)): Path<(String, String)>, - Query(pagination): Query, + Path((_id, stage_id)): Path<(String, String)>, + Query(params): Query, ) -> Response { - paginated_response(runs::turns(), &pagination) + let stage_id = match parse_stage_id_path(&stage_id) { + Ok(stage_id) => stage_id, + Err(response) => return response, + }; + let since_seq = params.since_seq(); + let limit = params.limit(); + let mut matches: Vec = runs::stage_events() + .into_iter() + .filter(|envelope| { + envelope.seq >= since_seq + && (envelope.event.stage_id.as_ref() == Some(&stage_id) + || (envelope.event.stage_id.is_none() + && stage_id.visit() == 1 + && envelope.event.node_id.as_deref() == Some(stage_id.node_id()))) + }) + .take(limit + 1) + .collect(); + let has_more = matches.len() > limit; + matches.truncate(limit); + ( + StatusCode::OK, + Json(PaginatedEventList { + data: matches, + meta: PaginationMeta { has_more }, + }), + ) + .into_response() } pub(crate) async fn list_run_artifacts_stub( @@ -1215,18 +1242,114 @@ mod runs { ] } - pub(super) fn turns() -> Vec { + pub(super) fn stage_events() -> Vec { + use fabro_model::BilledTokenCounts; + use fabro_types::run_event::agent::{ + AgentMessageProps, AgentToolCompletedProps, AgentToolStartedProps, + }; + use fabro_types::run_event::stage::StagePromptProps; + use fabro_types::{EventBody, EventEnvelope, RunEvent}; + + let run_id = demo_run_id(1); + let node_id = "detect-drift"; + let stage_id = fabro_types::StageId::new(node_id, 1); + let ts = ts("2026-03-06T14:30:00Z"); + + let make_envelope = |seq: u32, id: &str, body: EventBody| EventEnvelope { + seq, + event: RunEvent { + id: id.into(), + ts, + run_id, + node_id: Some(node_id.into()), + node_label: Some("Detect Drift".into()), + stage_id: Some(stage_id.clone()), + parallel_group_id: None, + parallel_branch_id: None, + session_id: None, + parent_session_id: None, + tool_call_id: None, + actor: None, + body, + }, + }; + vec![ - StageTurn::SystemStageTurn(SystemStageTurn { kind: SystemStageTurnKind::System, content: "You are a drift detection agent. Compare the production and staging environments and identify any configuration or code drift.".into() }), - StageTurn::AssistantStageTurn(AssistantStageTurn { kind: AssistantStageTurnKind::Assistant, content: "I'll start by loading the environment configurations for both production and staging to compare them.".into() }), - StageTurn::ToolStageTurn(ToolStageTurn { - kind: ToolStageTurnKind::Tool, content: None, - tools: vec![ - ToolUse { id: "toolu_01".into(), tool_name: "read_file".into(), input: r#"{ "path": "environments/production/config.toml" }"#.into(), result: "[redis]\nhost = \"redis-prod.internal\"\nport = 6379".into(), is_error: false, duration_ms: Some(45) }, - ToolUse { id: "toolu_02".into(), tool_name: "read_file".into(), input: r#"{ "path": "environments/staging/config.toml" }"#.into(), result: "[redis]\nhost = \"redis-staging.internal\"\nport = 6379".into(), is_error: false, duration_ms: Some(38) }, - ], - }), - StageTurn::AssistantStageTurn(AssistantStageTurn { kind: AssistantStageTurnKind::Assistant, content: "I've detected drift in 3 resources between production and staging:\n\n1. **redis.max_connections** — production has 200, staging has 100\n2. **redis.tls** — enabled in production, disabled in staging\n3. **iam.session_duration** — production uses 3600s, staging uses 1800s".into() }), + make_envelope( + 1, + "evt-detect-drift-1", + EventBody::StagePrompt(StagePromptProps { + visit: 1, + text: "You are a drift detection agent. Compare the production and staging environments and identify any configuration or code drift.".into(), + mode: None, + provider: None, + model: None, + }), + ), + make_envelope( + 2, + "evt-detect-drift-2", + EventBody::AgentMessage(AgentMessageProps { + text: "I'll start by loading the environment configurations for both production and staging to compare them.".into(), + model: "Opus 4.6".into(), + billing: BilledTokenCounts::default(), + tool_call_count: 0, + visit: 1, + }), + ), + make_envelope( + 3, + "evt-detect-drift-3", + EventBody::AgentToolStarted(AgentToolStartedProps { + tool_name: "read_file".into(), + tool_call_id: "toolu_01".into(), + arguments: serde_json::json!({ "path": "environments/production/config.toml" }), + visit: 1, + }), + ), + make_envelope( + 4, + "evt-detect-drift-4", + EventBody::AgentToolCompleted(AgentToolCompletedProps { + tool_name: "read_file".into(), + tool_call_id: "toolu_01".into(), + output: serde_json::json!("[redis]\nhost = \"redis-prod.internal\"\nport = 6379"), + is_error: false, + visit: 1, + }), + ), + make_envelope( + 5, + "evt-detect-drift-5", + EventBody::AgentToolStarted(AgentToolStartedProps { + tool_name: "read_file".into(), + tool_call_id: "toolu_02".into(), + arguments: serde_json::json!({ "path": "environments/staging/config.toml" }), + visit: 1, + }), + ), + make_envelope( + 6, + "evt-detect-drift-6", + EventBody::AgentToolCompleted(AgentToolCompletedProps { + tool_name: "read_file".into(), + tool_call_id: "toolu_02".into(), + output: serde_json::json!("[redis]\nhost = \"redis-staging.internal\"\nport = 6379"), + is_error: false, + visit: 1, + }), + ), + make_envelope( + 7, + "evt-detect-drift-7", + EventBody::AgentMessage(AgentMessageProps { + text: "I've detected drift in 3 resources between production and staging:\n\n1. **redis.max_connections** — production has 200, staging has 100\n2. **redis.tls** — enabled in production, disabled in staging\n3. **iam.session_duration** — production uses 3600s, staging uses 1800s".into(), + model: "Opus 4.6".into(), + billing: BilledTokenCounts::default(), + tool_call_count: 0, + visit: 1, + }), + ), ] } diff --git a/lib/crates/fabro-server/src/principal_middleware.rs b/lib/crates/fabro-server/src/principal_middleware.rs index e8971831c..822d87dc6 100644 --- a/lib/crates/fabro-server/src/principal_middleware.rs +++ b/lib/crates/fabro-server/src/principal_middleware.rs @@ -52,6 +52,7 @@ pub(crate) struct RequestAuth(pub(crate) AuthContextSlot); pub(crate) struct RequiredUser(pub(crate) UserPrincipal); pub(crate) struct RequireRunScoped(pub(crate) RunId); pub(crate) struct RequireRunBlob(pub(crate) RunId, pub(crate) RunBlobId); +pub(crate) struct RequireRunStageScoped(pub(crate) RunId, pub(crate) String); pub(crate) struct RequireStageArtifact(pub(crate) RunId, pub(crate) StageId); pub(crate) struct RequireCommandLog( pub(crate) RunId, @@ -203,6 +204,23 @@ impl FromRequestParts> for RequireRunBlob { } } +impl FromRequestParts> for RequireRunStageScoped { + type Rejection = Response; + + async fn from_request_parts( + parts: &mut Parts, + state: &Arc, + ) -> Result { + let Path((id, stage_id)): Path<(String, String)> = Path::from_request_parts(parts, state) + .await + .map_err(IntoResponse::into_response)?; + let run_id = parse_run_id_path(&id)?; + require_worker_or_user_for_run(&auth_slot_from_parts(parts), &run_id) + .map_err(IntoResponse::into_response)?; + Ok(Self(run_id, stage_id)) + } +} + impl FromRequestParts> for RequireStageArtifact { type Rejection = Response; diff --git a/lib/crates/fabro-server/src/server.rs b/lib/crates/fabro-server/src/server.rs index 26fbdd8d6..965989c8c 100644 --- a/lib/crates/fabro-server/src/server.rs +++ b/lib/crates/fabro-server/src/server.rs @@ -125,7 +125,7 @@ use crate::ip_allowlist::{IpAllowlistConfig, ip_allowlist_middleware}; use crate::jwt_auth::{self, AuthMode}; use crate::principal_middleware::{ AuthContextSlot, RequestAuth, RequestAuthContext, RequireRunBlob, RequireRunScoped, - RequireStageArtifact, RequiredUser, principal_middleware, + RequireRunStageScoped, RequireStageArtifact, RequiredUser, principal_middleware, }; use crate::request_id::{self, RequestId}; use crate::run_files::{FilesInFlight, new_files_in_flight}; @@ -138,6 +138,7 @@ use crate::{ mod handler; +pub(crate) use handler::events::EventListParams; #[cfg(test)] pub(in crate::server) use handler::events::filtered_global_events; pub(crate) use handler::graph::render_graph_bytes; diff --git a/lib/crates/fabro-server/src/server/handler/events.rs b/lib/crates/fabro-server/src/server/handler/events.rs index ff3889cd5..fce4a4d11 100644 --- a/lib/crates/fabro-server/src/server/handler/events.rs +++ b/lib/crates/fabro-server/src/server/handler/events.rs @@ -3,9 +3,10 @@ use std::sync::Arc; use super::super::{ ApiError, AppState, AppendEventResponse, BroadcastStream, Event, EventBody, EventEnvelope, EventPayload, HashSet, IntoResponse, Json, KeepAlive, PaginatedEventList, PaginationMeta, Path, - Query, RequireRunScoped, RequiredUser, Response, Router, RunEvent, RunId, RunStatus, Sse, - State, StatusCode, StreamExt, UnboundedReceiverStream, broadcast, get, mpsc, parse_run_id_path, - redact_jsonl_line, reject_if_archived, update_live_run_from_event, + Query, RequireRunScoped, RequireRunStageScoped, RequiredUser, Response, Router, RunEvent, + RunId, RunStatus, Sse, State, StatusCode, StreamExt, UnboundedReceiverStream, broadcast, get, + mpsc, parse_run_id_path, parse_stage_id_path, redact_jsonl_line, reject_if_archived, + update_live_run_from_event, }; pub(super) fn routes() -> Router> { @@ -15,11 +16,15 @@ pub(super) fn routes() -> Router> { "/runs/{id}/events", get(list_run_events).post(append_run_event), ) + .route( + "/runs/{id}/stages/{stageId}/events", + get(list_run_stage_events), + ) .route("/runs/{id}/attach", get(attach_run_events)) } #[derive(serde::Deserialize)] -struct EventListParams { +pub(crate) struct EventListParams { #[serde(default)] since_seq: Option, #[serde(default)] @@ -27,11 +32,11 @@ struct EventListParams { } impl EventListParams { - fn since_seq(&self) -> u32 { + pub(crate) fn since_seq(&self) -> u32 { self.since_seq.unwrap_or(1).max(1) } - fn limit(&self) -> usize { + pub(crate) fn limit(&self) -> usize { self.limit.unwrap_or(100).clamp(1, 1000) } } @@ -200,6 +205,39 @@ async fn list_run_events( } } +async fn list_run_stage_events( + RequireRunStageScoped(id, stage_id): RequireRunStageScoped, + State(state): State>, + Query(params): Query, +) -> Response { + let stage_id = match parse_stage_id_path(&stage_id) { + Ok(stage_id) => stage_id, + Err(response) => return response, + }; + let since_seq = params.since_seq(); + let limit = params.limit(); + match state.store.open_run_reader(&id).await { + Ok(run_store) => match run_store + .list_events_for_stage_from_with_limit(&stage_id, since_seq, limit) + .await + { + Ok(mut events) => { + let has_more = events.len() > limit; + events.truncate(limit); + Json(PaginatedEventList { + data: events, + meta: PaginationMeta { has_more }, + }) + .into_response() + } + Err(err) => { + ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() + } + }, + Err(_) => ApiError::not_found("Run not found.").into_response(), + } +} + async fn attach_run_events( _auth: RequiredUser, State(state): State>, @@ -342,3 +380,264 @@ fn denied_lifecycle_event_name(body: &EventBody) -> Option<&'static str> { _ => None, } } + +#[cfg(test)] +mod stage_events_tests { + use axum::body::{Body, to_bytes}; + use axum::http::{Request, StatusCode, header}; + use fabro_store::EventPayload; + use fabro_types::RunId; + use serde_json::json; + use tower::ServiceExt; + + use crate::test_support::{build_test_router, test_app_state}; + + fn req_get(uri: &str) -> Request { + Request::builder() + .method("GET") + .uri(uri) + .body(Body::empty()) + .expect("stage events GET request should build") + } + + fn make_event(run_id: &RunId, idx: u32, node_id: Option<&str>) -> EventPayload { + make_event_with_stage_id(run_id, idx, node_id, None) + } + + fn make_event_with_stage_id( + run_id: &RunId, + idx: u32, + node_id: Option<&str>, + stage_id: Option<&str>, + ) -> EventPayload { + let mut value = json!({ + "id": format!("evt-{idx}"), + "ts": "2026-04-09T12:00:00Z", + "run_id": run_id.to_string(), + "event": "stage.prompt", + "properties": { + "visit": 1, + "text": format!("prompt {idx}"), + }, + }); + if let Some(node) = node_id { + value + .as_object_mut() + .unwrap() + .insert("node_id".into(), json!(node)); + } + if let Some(stage_id) = stage_id { + value + .as_object_mut() + .unwrap() + .insert("stage_id".into(), json!(stage_id)); + } + EventPayload::new(value, run_id).expect("event payload should validate") + } + + async fn body_json(response: axum::response::Response) -> serde_json::Value { + let bytes = to_bytes(response.into_body(), usize::MAX) + .await + .expect("response body should fit in memory"); + serde_json::from_slice(&bytes).expect("response body should be valid JSON") + } + + async fn seed_run_with_mixed_events() -> (RunId, axum::Router) { + let state = test_app_state(); + let app = build_test_router(state.clone()); + let run_id = RunId::new(); + let run_store = state + .store_ref() + .create_run(&run_id) + .await + .expect("test run should be creatable"); + + // Seed 200 unrelated 'beta' events first so any node-blind + // truncation would lose the sparse 'alpha' tail. Then 3 'alpha' + // events past seq 100, plus a couple with no node_id at all. + for idx in 1..=200_u32 { + run_store + .append_event(&make_event(&run_id, idx, Some("beta"))) + .await + .expect("append should succeed"); + } + run_store + .append_event(&make_event(&run_id, 201, None)) + .await + .expect("append should succeed"); + for idx in 202..=204_u32 { + run_store + .append_event(&make_event(&run_id, idx, Some("alpha"))) + .await + .expect("append should succeed"); + } + + (run_id, app) + } + + #[tokio::test] + async fn returns_only_matching_node_events_in_seq_order() { + let (run_id, app) = seed_run_with_mixed_events().await; + let response = app + .oneshot(req_get(&format!( + "/api/v1/runs/{run_id}/stages/alpha@1/events" + ))) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let body = body_json(response).await; + let data = body["data"].as_array().expect("data is array"); + let seqs: Vec = data.iter().map(|e| e["seq"].as_u64().unwrap()).collect(); + assert_eq!(seqs, vec![202, 203, 204]); + assert_eq!(body["meta"]["has_more"], false); + } + + #[tokio::test] + async fn since_seq_filters_to_events_with_seq_at_least_k() { + let (run_id, app) = seed_run_with_mixed_events().await; + let response = app + .oneshot(req_get(&format!( + "/api/v1/runs/{run_id}/stages/alpha@1/events?since_seq=203" + ))) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let body = body_json(response).await; + let seqs: Vec = body["data"] + .as_array() + .unwrap() + .iter() + .map(|e| e["seq"].as_u64().unwrap()) + .collect(); + assert_eq!(seqs, vec![203, 204]); + } + + #[tokio::test] + async fn limit_one_returns_first_envelope_with_has_more_true() { + let (run_id, app) = seed_run_with_mixed_events().await; + let response = app + .oneshot(req_get(&format!( + "/api/v1/runs/{run_id}/stages/alpha@1/events?limit=1" + ))) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let body = body_json(response).await; + let data = body["data"].as_array().unwrap(); + assert_eq!(data.len(), 1); + assert_eq!(data[0]["seq"].as_u64().unwrap(), 202); + assert_eq!(body["meta"]["has_more"], true); + } + + #[tokio::test] + async fn unknown_stage_in_existing_run_returns_empty_list_with_no_more() { + let (run_id, app) = seed_run_with_mixed_events().await; + let response = app + .oneshot(req_get(&format!( + "/api/v1/runs/{run_id}/stages/unknown-stage@1/events" + ))) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let body = body_json(response).await; + assert_eq!(body["data"].as_array().unwrap().len(), 0); + assert_eq!(body["meta"]["has_more"], false); + } + + #[tokio::test] + async fn missing_run_returns_404_with_run_not_found() { + let app = build_test_router(test_app_state()); + // A syntactically valid RunId that the store has never seen, so + // `parse_run_id_path` succeeds but `open_run_reader` fails — that + // exercises the handler's not-found branch rather than the path + // parser's 400 branch. + let absent = RunId::new(); + let response = app + .oneshot(req_get(&format!( + "/api/v1/runs/{absent}/stages/alpha@1/events" + ))) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::NOT_FOUND); + + let body = body_json(response).await; + let detail = body["errors"][0]["detail"] + .as_str() + .expect("error detail string"); + assert!( + detail.contains("Run not found."), + "unexpected error body: {body}" + ); + } + + #[tokio::test] + async fn unauthenticated_request_is_rejected() { + let state = test_app_state(); + // Bypass `build_test_router`'s auto-injected bearer token by + // building the raw router directly. The principal middleware sees + // a missing Authorization header and the extractor enforces auth. + let app = crate::server::build_router(state, crate::test_support::test_auth_mode()); + let run_id = RunId::new(); + + let request = Request::builder() + .method("GET") + .uri(format!("/api/v1/runs/{run_id}/stages/alpha@1/events")) + .header(header::ACCEPT, "application/json") + .body(Body::empty()) + .unwrap(); + let response = app.oneshot(request).await.unwrap(); + + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + } + + #[tokio::test] + async fn returns_only_requested_visit_when_stage_id_is_present() { + let state = test_app_state(); + let app = build_test_router(state.clone()); + let run_id = RunId::new(); + let run_store = state + .store_ref() + .create_run(&run_id) + .await + .expect("test run should be creatable"); + run_store + .append_event(&make_event_with_stage_id( + &run_id, + 1, + Some("verify"), + Some("verify@1"), + )) + .await + .expect("append should succeed"); + run_store + .append_event(&make_event_with_stage_id( + &run_id, + 2, + Some("verify"), + Some("verify@2"), + )) + .await + .expect("append should succeed"); + + let response = app + .oneshot(req_get(&format!( + "/api/v1/runs/{run_id}/stages/verify@2/events" + ))) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let body = body_json(response).await; + let seqs: Vec = body["data"] + .as_array() + .unwrap() + .iter() + .map(|e| e["seq"].as_u64().unwrap()) + .collect(); + assert_eq!(seqs, vec![2]); + } +} diff --git a/lib/crates/fabro-server/src/server/handler/mod.rs b/lib/crates/fabro-server/src/server/handler/mod.rs index d2625395b..79a3d8246 100644 --- a/lib/crates/fabro-server/src/server/handler/mod.rs +++ b/lib/crates/fabro-server/src/server/handler/mod.rs @@ -57,8 +57,8 @@ pub(super) fn demo_routes() -> Router> { .route("/runs/{id}/artifacts", get(demo::list_run_artifacts_stub)) .route("/runs/{id}/files", get(demo::list_run_files_stub)) .route( - "/runs/{id}/stages/{stageId}/turns", - get(demo::get_stage_turns), + "/runs/{id}/stages/{stageId}/events", + get(demo::get_stage_events), ) .route( "/runs/{id}/stages/{stageId}/artifacts", @@ -113,7 +113,6 @@ pub(super) fn demo_routes() -> Router> { pub(super) fn real_routes() -> Router> { Router::new() - .route("/runs/{id}/stages/{stageId}/turns", get(not_implemented)) .route("/runs/{id}/steer", post(not_implemented)) .route("/workflows", get(not_implemented)) .route("/workflows/{name}", get(not_implemented)) diff --git a/lib/crates/fabro-server/tests/it/event_pagination.rs b/lib/crates/fabro-server/tests/it/event_pagination.rs new file mode 100644 index 000000000..cd6ceec94 --- /dev/null +++ b/lib/crates/fabro-server/tests/it/event_pagination.rs @@ -0,0 +1,75 @@ +//! Cursor-pagination tests for the per-stage events endpoint (demo mode). +//! +//! The stage-events route uses `since_seq=` + `limit=` (cursor-based) instead +//! of the offset-based `page[limit]/page[offset]` pagination used by other +//! list endpoints, so it gets its own test rather than living in the generic +//! offset-shape matrix. + +#![allow( + clippy::absolute_paths, + reason = "This test module prefers explicit type paths over extra imports." +)] + +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use tower::ServiceExt; + +use super::helpers::{response_json, test_app_state}; + +async fn get_json(app: &axum::Router, uri: &str) -> serde_json::Value { + let req = Request::builder() + .method("GET") + .uri(uri) + .header("x-fabro-demo", "1") + .body(Body::empty()) + .expect("event pagination request should build"); + let response = app.clone().oneshot(req).await.unwrap(); + response_json(response, StatusCode::OK, format!("GET {uri}")).await +} + +#[tokio::test] +async fn demo_stage_events_default_returns_all_fixture_events_with_no_more() { + let app = fabro_server::test_support::build_test_router(test_app_state()); + + let body = get_json(&app, "/api/v1/runs/run-1/stages/detect-drift@1/events").await; + let data = body["data"].as_array().expect("data is an array"); + + assert_eq!(data.len(), 7, "all seven fixture events should be returned"); + assert_eq!(body["meta"]["has_more"], false); +} + +#[tokio::test] +async fn demo_stage_events_limit_one_signals_has_more() { + let app = fabro_server::test_support::build_test_router(test_app_state()); + + let body = get_json( + &app, + "/api/v1/runs/run-1/stages/detect-drift@1/events?limit=1", + ) + .await; + let data = body["data"].as_array().expect("data is an array"); + + assert_eq!(data.len(), 1); + assert_eq!(body["meta"]["has_more"], true); +} + +#[tokio::test] +async fn demo_stage_events_since_seq_filters_out_earlier_events() { + let app = fabro_server::test_support::build_test_router(test_app_state()); + + // The fixture seqs are 1..=7. since_seq=4 should skip the first three. + let body = get_json( + &app, + "/api/v1/runs/run-1/stages/detect-drift@1/events?since_seq=4", + ) + .await; + let data = body["data"].as_array().expect("data is an array"); + + assert_eq!(data.len(), 4); + let seqs: Vec = data + .iter() + .map(|envelope| envelope["seq"].as_u64().expect("seq is a number")) + .collect(); + assert_eq!(seqs, vec![4, 5, 6, 7]); + assert_eq!(body["meta"]["has_more"], false); +} diff --git a/lib/crates/fabro-server/tests/it/main.rs b/lib/crates/fabro-server/tests/it/main.rs index 9e5ddab9f..865bb8132 100644 --- a/lib/crates/fabro-server/tests/it/main.rs +++ b/lib/crates/fabro-server/tests/it/main.rs @@ -4,6 +4,7 @@ )] mod api; +mod event_pagination; mod helpers; mod openapi_conformance; mod pagination; diff --git a/lib/crates/fabro-server/tests/it/pagination.rs b/lib/crates/fabro-server/tests/it/pagination.rs index 64174fd78..f773ac207 100644 --- a/lib/crates/fabro-server/tests/it/pagination.rs +++ b/lib/crates/fabro-server/tests/it/pagination.rs @@ -56,10 +56,6 @@ const ENDPOINTS: &[PaginatedEndpoint] = &[ path: "/api/v1/models", name: "listModels", }, - PaginatedEndpoint { - path: "/api/v1/runs/run-1/stages/detect-drift/turns", - name: "listStageTurns", - }, PaginatedEndpoint { path: "/api/v1/runs/run-1/questions", name: "listRunQuestions", diff --git a/lib/crates/fabro-store/src/slate/run_store.rs b/lib/crates/fabro-store/src/slate/run_store.rs index a334281b9..f28e52341 100644 --- a/lib/crates/fabro-store/src/slate/run_store.rs +++ b/lib/crates/fabro-store/src/slate/run_store.rs @@ -12,7 +12,7 @@ use tokio_stream::wrappers::UnboundedReceiverStream; use super::blob_store::BlobStore; use crate::run_state::{EventProjectionCache, RunProjectionReducer, build_summary}; -use crate::{Error, EventEnvelope, EventPayload, Result, RunProjection, keys}; +use crate::{Error, EventEnvelope, EventPayload, Result, RunProjection, StageId, keys}; const DEFAULT_EVENT_TAIL_LIMIT: usize = 1024; #[derive(Clone)] @@ -213,6 +213,30 @@ impl RunDatabase { list_events_from_with_limit(&self.inner.db, &self.inner.run_id, start_seq, limit).await } + /// Returns up to `limit + 1` events for the given stage visit, + /// starting at `start_seq`. The `+1` lets callers compute `has_more`. + /// + /// Implementation note: scans the unbounded run-event prefix and + /// filters by stage identity *before* applying `limit`, so a stage with + /// matches sparsely scattered late in the event log still returns its + /// full slice (no premature truncation from a generic `limit`-bounded + /// scan). + pub async fn list_events_for_stage_from_with_limit( + &self, + stage_id: &StageId, + start_seq: u32, + limit: usize, + ) -> Result> { + list_events_for_stage_from_with_limit( + &self.inner.db, + &self.inner.run_id, + stage_id, + start_seq, + limit, + ) + .await + } + pub fn watch_events_from( &self, seq: u32, @@ -348,6 +372,74 @@ where Ok(events) } +async fn list_events_for_stage_from_with_limit( + db: &R, + run_id: &RunId, + stage_id: &StageId, + start_seq: u32, + limit: usize, +) -> Result> +where + R: DbRead + Sync, +{ + // Unbounded scan first: filtering by stage identity with a generic + // limit-bounded scan would silently drop matches whenever the stage's + // events are sparse late in the event log. + // + // We probe just the stage identity fields with a small partial deserialize and + // only run the full `RunEvent` parse on matches. Most events in a run + // belong to other nodes, so this avoids deserializing large payloads + // (`agent.tool.completed.output`, `agent.message.text`, …) we'd discard. + #[derive(serde::Deserialize)] + struct StageIdProbe<'a> { + #[serde(default, borrow)] + stage_id: Option<&'a str>, + #[serde(default, borrow)] + node_id: Option<&'a str>, + } + + let stage_id_string = stage_id.to_string(); + let max_events = limit.saturating_add(1); + let mut iter = db.scan_prefix(keys::run_events_prefix(run_id)).await?; + let mut events: Vec = Vec::new(); + while let Some(entry) = iter.next().await? { + let key = key_to_string(&entry.key)?; + let Some(seq) = keys::parse_event_seq(&key) else { + continue; + }; + if seq < start_seq { + continue; + } + let probe: StageIdProbe = serde_json::from_slice(&entry.value)?; + let matches_stage_id = probe.stage_id == Some(stage_id_string.as_str()); + let matches_legacy_node_id = probe.stage_id.is_none() + && stage_id.visit() == 1 + && probe.node_id == Some(stage_id.node_id()); + if !matches_stage_id && !matches_legacy_node_id { + continue; + } + let event: RunEvent = serde_json::from_slice(&entry.value)?; + let envelope = EventEnvelope { seq, event }; + if events.len() < max_events { + events.push(envelope); + continue; + } + + if let Some((max_index, max_seq)) = events + .iter() + .enumerate() + .max_by_key(|(_, existing)| existing.seq) + .map(|(index, existing)| (index, existing.seq)) + { + if seq < max_seq { + events[max_index] = envelope; + } + } + } + events.sort_by_key(|event| event.seq); + Ok(events) +} + async fn list_blobs(db: &R) -> Result> where R: DbRead + Sync, @@ -375,9 +467,12 @@ mod tests { use std::sync::Arc; use std::time::Duration; + use fabro_types::{RunId, StageId}; use object_store::memory::InMemory; + use serde_json::json; + + use crate::{Database, EventPayload}; - use crate::Database; #[tokio::test] async fn list_blobs_reads_global_cas_namespace() { let object_store = Arc::new(InMemory::new()); @@ -394,4 +489,190 @@ mod tests { assert_eq!(blob_ids, vec![first_id, second_id]); } + + fn stage_prompt_payload(run_id: &RunId, idx: u32, node_id: Option<&str>) -> EventPayload { + stage_prompt_payload_for_stage(run_id, idx, node_id, None) + } + + fn stage_prompt_payload_for_stage( + run_id: &RunId, + idx: u32, + node_id: Option<&str>, + stage_id: Option<&StageId>, + ) -> EventPayload { + let mut value = json!({ + "id": format!("evt-{idx}"), + "ts": "2026-04-09T12:00:00Z", + "run_id": run_id.to_string(), + "event": "stage.prompt", + "properties": { + "visit": 1, + "text": format!("prompt {idx}"), + }, + }); + if let Some(node_id) = node_id { + value + .as_object_mut() + .unwrap() + .insert("node_id".into(), json!(node_id)); + } + if let Some(stage_id) = stage_id { + value + .as_object_mut() + .unwrap() + .insert("stage_id".into(), json!(stage_id.to_string())); + } + EventPayload::new(value, run_id).unwrap() + } + + async fn fresh_run() -> super::RunDatabase { + let object_store = Arc::new(InMemory::new()); + let store = Database::new(object_store, "", Duration::from_millis(1), None); + let run_id: RunId = "01JT56VE4Z5NZ814GZN2JZD65A".parse().unwrap(); + store.create_run(&run_id).await.unwrap() + } + + #[tokio::test] + async fn list_events_for_stage_returns_only_matching_events_in_seq_order() { + let run = fresh_run().await; + let run_id = run.run_id(); + run.append_event(&stage_prompt_payload(&run_id, 1, Some("alpha"))) + .await + .unwrap(); + run.append_event(&stage_prompt_payload(&run_id, 2, Some("beta"))) + .await + .unwrap(); + run.append_event(&stage_prompt_payload(&run_id, 3, Some("alpha"))) + .await + .unwrap(); + + let events = run + .list_events_for_stage_from_with_limit(&StageId::new("alpha", 1), 1, 100) + .await + .unwrap(); + + let seqs: Vec = events.iter().map(|e| e.seq).collect(); + assert_eq!(seqs, vec![1, 3]); + } + + #[tokio::test] + async fn list_events_for_stage_skips_events_with_no_stage_identity() { + let run = fresh_run().await; + let run_id = run.run_id(); + run.append_event(&stage_prompt_payload(&run_id, 1, None)) + .await + .unwrap(); + run.append_event(&stage_prompt_payload(&run_id, 2, Some("alpha"))) + .await + .unwrap(); + + let events = run + .list_events_for_stage_from_with_limit(&StageId::new("alpha", 1), 1, 100) + .await + .unwrap(); + + let seqs: Vec = events.iter().map(|e| e.seq).collect(); + assert_eq!(seqs, vec![2]); + } + + #[tokio::test] + async fn list_events_for_stage_paginates_via_start_seq_on_filtered_slice() { + let run = fresh_run().await; + let run_id = run.run_id(); + for idx in 1..=5 { + let node = if idx % 2 == 0 { "beta" } else { "alpha" }; + run.append_event(&stage_prompt_payload(&run_id, idx, Some(node))) + .await + .unwrap(); + } + + // alpha events live at seqs 1, 3, 5. Start at seq=2 should skip seq=1. + let events = run + .list_events_for_stage_from_with_limit(&StageId::new("alpha", 1), 2, 100) + .await + .unwrap(); + + let seqs: Vec = events.iter().map(|e| e.seq).collect(); + assert_eq!(seqs, vec![3, 5]); + } + + #[tokio::test] + async fn list_events_for_stage_walks_past_unrelated_events_for_sparse_matches() { + let run = fresh_run().await; + let run_id = run.run_id(); + // 200 unrelated events first. + for idx in 1..=200 { + run.append_event(&stage_prompt_payload(&run_id, idx, Some("noise"))) + .await + .unwrap(); + } + // Then 3 sparse "alpha" events at the tail. + for idx in 201..=203 { + run.append_event(&stage_prompt_payload(&run_id, idx, Some("alpha"))) + .await + .unwrap(); + } + + // limit smaller than the number of unrelated events would have + // truncated the upstream scan if we had post-filtered. + let events = run + .list_events_for_stage_from_with_limit(&StageId::new("alpha", 1), 1, 5) + .await + .unwrap(); + + let seqs: Vec = events.iter().map(|e| e.seq).collect(); + assert_eq!(seqs, vec![201, 202, 203]); + } + + #[tokio::test] + async fn list_events_for_stage_returns_limit_plus_one_for_has_more_signal() { + let run = fresh_run().await; + let run_id = run.run_id(); + for idx in 1..=5 { + run.append_event(&stage_prompt_payload(&run_id, idx, Some("alpha"))) + .await + .unwrap(); + } + + let events = run + .list_events_for_stage_from_with_limit(&StageId::new("alpha", 1), 1, 2) + .await + .unwrap(); + + // With limit=2, we expect up to limit+1 = 3 envelopes so the + // caller can compute has_more. + assert_eq!(events.len(), 3); + } + + #[tokio::test] + async fn list_events_for_stage_prefers_stage_id_over_node_id() { + let run = fresh_run().await; + let run_id = run.run_id(); + let first_visit = StageId::new("verify", 1); + let second_visit = StageId::new("verify", 2); + run.append_event(&stage_prompt_payload_for_stage( + &run_id, + 1, + Some("verify"), + Some(&first_visit), + )) + .await + .unwrap(); + run.append_event(&stage_prompt_payload_for_stage( + &run_id, + 2, + Some("verify"), + Some(&second_visit), + )) + .await + .unwrap(); + + let events = run + .list_events_for_stage_from_with_limit(&second_visit, 1, 100) + .await + .unwrap(); + + let seqs: Vec = events.iter().map(|e| e.seq).collect(); + assert_eq!(seqs, vec![2]); + } } diff --git a/lib/packages/fabro-api-client/src/.openapi-generator/FILES b/lib/packages/fabro-api-client/src/.openapi-generator/FILES index 3113a0298..2dffdba54 100644 --- a/lib/packages/fabro-api-client/src/.openapi-generator/FILES +++ b/lib/packages/fabro-api-client/src/.openapi-generator/FILES @@ -30,7 +30,6 @@ models/artifact-batch-upload-manifest.ts models/artifact-entry.ts models/artifact-list-response.ts models/artifacts-settings.ts -models/assistant-stage-turn.ts models/auth-method.ts models/billed-token-counts.ts models/billing-by-model.ts @@ -168,7 +167,6 @@ models/paginated-run-file-list.ts models/paginated-run-list.ts models/paginated-run-stage-list.ts models/paginated-saved-query-list.ts -models/paginated-stage-turn-list.ts models/pagination-meta.ts models/pending-interview-record.ts models/pre-run-push-outcome-failed.ts @@ -300,7 +298,6 @@ models/stage-completion.ts models/stage-outcome.ts models/stage-projection.ts models/stage-state.ts -models/stage-turn.ts models/start-run-request.ts models/submit-answer-request.ts models/success-reason.ts @@ -308,13 +305,10 @@ models/system-actor-kind.ts models/system-features.ts models/system-info-response.ts models/system-run-counts.ts -models/system-stage-turn.ts models/teams-integration-settings.ts models/terminal-status.ts models/timeline-entry-response.ts models/tls-mode.ts -models/tool-stage-turn.ts -models/tool-use.ts models/user-response.ts models/validate-response.ts models/webhook-strategy.ts diff --git a/lib/packages/fabro-api-client/src/api/run-internals-api.ts b/lib/packages/fabro-api-client/src/api/run-internals-api.ts index 0852b16f6..6520b0fc6 100644 --- a/lib/packages/fabro-api-client/src/api/run-internals-api.ts +++ b/lib/packages/fabro-api-client/src/api/run-internals-api.ts @@ -36,8 +36,6 @@ import type { PaginatedEventList } from '../models'; // @ts-ignore import type { PaginatedRunStageList } from '../models'; // @ts-ignore -import type { PaginatedStageTurnList } from '../models'; -// @ts-ignore import type { RunArtifactListResponse } from '../models'; // @ts-ignore import type { RunCheckpoint } from '../models'; @@ -527,21 +525,21 @@ export const RunInternalsApiAxiosParamCreator = function (configuration?: Config }; }, /** - * Returns a paginated list of conversation turns within a specific stage, including system prompts, assistant responses, and tool invocations. - * @summary List Stage Turns + * Returns a paginated JSON list of stored run events scoped to a single stage visit. + * @summary List Stage Events * @param {string} id Unique run identifier (ULID). * @param {string} stageId Identifier of a stage within a run\'s workflow graph, serialized as `node_id@visit`. - * @param {number} [pageLimit] Maximum number of items to return per page. - * @param {number} [pageOffset] Number of items to skip before returning results. + * @param {number} [sinceSeq] First event sequence number to include. + * @param {number} [limit] Maximum number of events to return. * @param {*} [options] Override http request option. * @throws {RequiredError} */ - listStageTurns: async (id: string, stageId: string, pageLimit?: number, pageOffset?: number, options: RawAxiosRequestConfig = {}): Promise => { + listStageEvents: async (id: string, stageId: string, sinceSeq?: number, limit?: number, options: RawAxiosRequestConfig = {}): Promise => { // verify required parameter 'id' is not null or undefined - assertParamExists('listStageTurns', 'id', id) + assertParamExists('listStageEvents', 'id', id) // verify required parameter 'stageId' is not null or undefined - assertParamExists('listStageTurns', 'stageId', stageId) - const localVarPath = `/api/v1/runs/{id}/stages/{stageId}/turns` + assertParamExists('listStageEvents', 'stageId', stageId) + const localVarPath = `/api/v1/runs/{id}/stages/{stageId}/events` .replace(`{${"id"}}`, encodeURIComponent(String(id))) .replace(`{${"stageId"}}`, encodeURIComponent(String(stageId))); // use dummy base URL string because the URL constructor only accepts absolute URLs. @@ -561,12 +559,12 @@ export const RunInternalsApiAxiosParamCreator = function (configuration?: Config // http bearer authentication required await setBearerAuthToObject(localVarHeaderParameter, configuration) - if (pageLimit !== undefined) { - localVarQueryParameter['page[limit]'] = pageLimit; + if (sinceSeq !== undefined) { + localVarQueryParameter['since_seq'] = sinceSeq; } - if (pageOffset !== undefined) { - localVarQueryParameter['page[offset]'] = pageOffset; + if (limit !== undefined) { + localVarQueryParameter['limit'] = limit; } localVarHeaderParameter['Accept'] = 'application/json'; @@ -964,19 +962,19 @@ export const RunInternalsApiFp = function(configuration?: Configuration) { return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, localVarOperationServerBasePath || basePath); }, /** - * Returns a paginated list of conversation turns within a specific stage, including system prompts, assistant responses, and tool invocations. - * @summary List Stage Turns + * Returns a paginated JSON list of stored run events scoped to a single stage visit. + * @summary List Stage Events * @param {string} id Unique run identifier (ULID). * @param {string} stageId Identifier of a stage within a run\'s workflow graph, serialized as `node_id@visit`. - * @param {number} [pageLimit] Maximum number of items to return per page. - * @param {number} [pageOffset] Number of items to skip before returning results. + * @param {number} [sinceSeq] First event sequence number to include. + * @param {number} [limit] Maximum number of events to return. * @param {*} [options] Override http request option. * @throws {RequiredError} */ - async listStageTurns(id: string, stageId: string, pageLimit?: number, pageOffset?: number, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise> { - const localVarAxiosArgs = await localVarAxiosParamCreator.listStageTurns(id, stageId, pageLimit, pageOffset, options); + async listStageEvents(id: string, stageId: string, sinceSeq?: number, limit?: number, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise> { + const localVarAxiosArgs = await localVarAxiosParamCreator.listStageEvents(id, stageId, sinceSeq, limit, options); const localVarOperationServerIndex = configuration?.serverIndex ?? 0; - const localVarOperationServerBasePath = operationServerMap['RunInternalsApi.listStageTurns']?.[localVarOperationServerIndex]?.url; + const localVarOperationServerBasePath = operationServerMap['RunInternalsApi.listStageEvents']?.[localVarOperationServerIndex]?.url; return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, localVarOperationServerBasePath || basePath); }, /** @@ -1174,17 +1172,17 @@ export const RunInternalsApiFactory = function (configuration?: Configuration, b return localVarFp.listStageArtifacts(id, stageId, options).then((request) => request(axios, basePath)); }, /** - * Returns a paginated list of conversation turns within a specific stage, including system prompts, assistant responses, and tool invocations. - * @summary List Stage Turns + * Returns a paginated JSON list of stored run events scoped to a single stage visit. + * @summary List Stage Events * @param {string} id Unique run identifier (ULID). * @param {string} stageId Identifier of a stage within a run\'s workflow graph, serialized as `node_id@visit`. - * @param {number} [pageLimit] Maximum number of items to return per page. - * @param {number} [pageOffset] Number of items to skip before returning results. + * @param {number} [sinceSeq] First event sequence number to include. + * @param {number} [limit] Maximum number of events to return. * @param {*} [options] Override http request option. * @throws {RequiredError} */ - listStageTurns(id: string, stageId: string, pageLimit?: number, pageOffset?: number, options?: RawAxiosRequestConfig): AxiosPromise { - return localVarFp.listStageTurns(id, stageId, pageLimit, pageOffset, options).then((request) => request(axios, basePath)); + listStageEvents(id: string, stageId: string, sinceSeq?: number, limit?: number, options?: RawAxiosRequestConfig): AxiosPromise { + return localVarFp.listStageEvents(id, stageId, sinceSeq, limit, options).then((request) => request(axios, basePath)); }, /** * Uploads one or more artifacts for a stage. Intended for trusted internal callers. The server accepts both: - `application/octet-stream` for single-file uploads with the `filename` query parameter - strict manifest-first `multipart/form-data` uploads documented by `ArtifactBatchUploadManifest` The generated Rust client currently exposes the octet-stream variant because the OpenAPI code generator in this repo does not support multiple request media types on one operation. @@ -1374,17 +1372,17 @@ export class RunInternalsApi extends BaseAPI { } /** - * Returns a paginated list of conversation turns within a specific stage, including system prompts, assistant responses, and tool invocations. - * @summary List Stage Turns + * Returns a paginated JSON list of stored run events scoped to a single stage visit. + * @summary List Stage Events * @param {string} id Unique run identifier (ULID). * @param {string} stageId Identifier of a stage within a run\'s workflow graph, serialized as `node_id@visit`. - * @param {number} [pageLimit] Maximum number of items to return per page. - * @param {number} [pageOffset] Number of items to skip before returning results. + * @param {number} [sinceSeq] First event sequence number to include. + * @param {number} [limit] Maximum number of events to return. * @param {*} [options] Override http request option. * @throws {RequiredError} */ - public listStageTurns(id: string, stageId: string, pageLimit?: number, pageOffset?: number, options?: RawAxiosRequestConfig) { - return RunInternalsApiFp(this.configuration).listStageTurns(id, stageId, pageLimit, pageOffset, options).then((request) => request(this.axios, this.basePath)); + public listStageEvents(id: string, stageId: string, sinceSeq?: number, limit?: number, options?: RawAxiosRequestConfig) { + return RunInternalsApiFp(this.configuration).listStageEvents(id, stageId, sinceSeq, limit, options).then((request) => request(this.axios, this.basePath)); } /** diff --git a/lib/packages/fabro-api-client/src/models/assistant-stage-turn.ts b/lib/packages/fabro-api-client/src/models/assistant-stage-turn.ts deleted file mode 100644 index 8e0a8edcf..000000000 --- a/lib/packages/fabro-api-client/src/models/assistant-stage-turn.ts +++ /dev/null @@ -1,34 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ -/** - * Fabro Run API - * HTTP API for managing Fabro workflow run executions. - * - * The version of the OpenAPI document: 0.1.0 - * - * - * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). - * https://openapi-generator.tech - * Do not edit the class manually. - */ - - - -/** - * An assistant response turn within a stage. - */ -export interface AssistantStageTurn { - 'kind': AssistantStageTurnKindEnum; - /** - * Assistant response text. - */ - 'content': string; -} - -export const AssistantStageTurnKindEnum = { - ASSISTANT: 'assistant' -} as const; - -export type AssistantStageTurnKindEnum = typeof AssistantStageTurnKindEnum[keyof typeof AssistantStageTurnKindEnum]; - - diff --git a/lib/packages/fabro-api-client/src/models/board-column-definition.ts b/lib/packages/fabro-api-client/src/models/board-column-definition.ts index 20deba1f5..314cbcde1 100644 --- a/lib/packages/fabro-api-client/src/models/board-column-definition.ts +++ b/lib/packages/fabro-api-client/src/models/board-column-definition.ts @@ -21,3 +21,6 @@ export interface BoardColumnDefinition { 'id': BoardColumn; 'name': string; } + + + diff --git a/lib/packages/fabro-api-client/src/models/index.ts b/lib/packages/fabro-api-client/src/models/index.ts index 0b72c59bd..fcb9e05ef 100644 --- a/lib/packages/fabro-api-client/src/models/index.ts +++ b/lib/packages/fabro-api-client/src/models/index.ts @@ -10,7 +10,6 @@ export * from './artifact-batch-upload-manifest'; export * from './artifact-entry'; export * from './artifact-list-response'; export * from './artifacts-settings'; -export * from './assistant-stage-turn'; export * from './auth-method'; export * from './billed-token-counts'; export * from './billing-by-model'; @@ -147,7 +146,6 @@ export * from './paginated-run-file-list'; export * from './paginated-run-list'; export * from './paginated-run-stage-list'; export * from './paginated-saved-query-list'; -export * from './paginated-stage-turn-list'; export * from './pagination-meta'; export * from './pending-interview-record'; export * from './pre-run-push-outcome'; @@ -279,7 +277,6 @@ export * from './stage-completion'; export * from './stage-outcome'; export * from './stage-projection'; export * from './stage-state'; -export * from './stage-turn'; export * from './start-run-request'; export * from './submit-answer-request'; export * from './success-reason'; @@ -287,13 +284,10 @@ export * from './system-actor-kind'; export * from './system-features'; export * from './system-info-response'; export * from './system-run-counts'; -export * from './system-stage-turn'; export * from './teams-integration-settings'; export * from './terminal-status'; export * from './timeline-entry-response'; export * from './tls-mode'; -export * from './tool-stage-turn'; -export * from './tool-use'; export * from './user-response'; export * from './validate-response'; export * from './webhook-strategy'; diff --git a/lib/packages/fabro-api-client/src/models/manifest-git.ts b/lib/packages/fabro-api-client/src/models/manifest-git.ts deleted file mode 100644 index bef882689..000000000 --- a/lib/packages/fabro-api-client/src/models/manifest-git.ts +++ /dev/null @@ -1,41 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ -/** - * Fabro Run API - * HTTP API for managing Fabro workflow run executions. - * - * The version of the OpenAPI document: 0.1.0 - * - * - * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). - * https://openapi-generator.tech - * Do not edit the class manually. - */ - - -// May contain unused imports in some cases -// @ts-ignore -import type { ManifestPreRunPushOutcome } from './manifest-pre-run-push-outcome'; - -/** - * Observable git state from the CLI working directory. - */ -export interface ManifestGit { - /** - * Remote origin URL with any embedded credentials removed. - */ - 'origin_url': string; - /** - * Current branch name. - */ - 'branch': string; - /** - * Current commit SHA. - */ - 'sha': string; - /** - * Whether the working tree has uncommitted changes. - */ - 'clean': boolean; - 'push_outcome': ManifestPreRunPushOutcome; -} diff --git a/lib/packages/fabro-api-client/src/models/manifest-pre-run-push-outcome.ts b/lib/packages/fabro-api-client/src/models/manifest-pre-run-push-outcome.ts deleted file mode 100644 index 93f6be234..000000000 --- a/lib/packages/fabro-api-client/src/models/manifest-pre-run-push-outcome.ts +++ /dev/null @@ -1,36 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ -/** - * Fabro Run API - * HTTP API for managing Fabro workflow run executions. - * - * The version of the OpenAPI document: 0.1.0 - * - * - * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). - * https://openapi-generator.tech - * Do not edit the class manually. - */ - - - -/** - * Outcome of the CLI\'s best-effort pre-run push. - */ -export interface ManifestPreRunPushOutcome { - 'type': ManifestPreRunPushOutcomeTypeEnum; - 'remote'?: string | null; - 'branch'?: string | null; - 'message'?: string | null; - 'repo_origin_url'?: string | null; -} - -export const ManifestPreRunPushOutcomeTypeEnum = { - NOT_ATTEMPTED: 'not_attempted', - SUCCEEDED: 'succeeded', - FAILED: 'failed', - SKIPPED_NO_REMOTE: 'skipped_no_remote', - SKIPPED_REMOTE_MISMATCH: 'skipped_remote_mismatch' -} as const; - -export type ManifestPreRunPushOutcomeTypeEnum = typeof ManifestPreRunPushOutcomeTypeEnum[keyof typeof ManifestPreRunPushOutcomeTypeEnum]; diff --git a/lib/packages/fabro-api-client/src/models/paginated-stage-turn-list.ts b/lib/packages/fabro-api-client/src/models/paginated-stage-turn-list.ts deleted file mode 100644 index 26f8d9cf2..000000000 --- a/lib/packages/fabro-api-client/src/models/paginated-stage-turn-list.ts +++ /dev/null @@ -1,30 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ -/** - * Fabro Run API - * HTTP API for managing Fabro workflow run executions. - * - * The version of the OpenAPI document: 0.1.0 - * - * - * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). - * https://openapi-generator.tech - * Do not edit the class manually. - */ - - -// May contain unused imports in some cases -// @ts-ignore -import type { PaginationMeta } from './pagination-meta'; -// May contain unused imports in some cases -// @ts-ignore -import type { StageTurn } from './stage-turn'; - -/** - * Paginated list of stage turns. - */ -export interface PaginatedStageTurnList { - 'data': Array; - 'meta': PaginationMeta; -} - diff --git a/lib/packages/fabro-api-client/src/models/pre-run-git-context.ts b/lib/packages/fabro-api-client/src/models/pre-run-git-context.ts deleted file mode 100644 index aaddc93e7..000000000 --- a/lib/packages/fabro-api-client/src/models/pre-run-git-context.ts +++ /dev/null @@ -1,30 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ -/** - * Fabro Run API - * HTTP API for managing Fabro workflow run executions. - * - * The version of the OpenAPI document: 0.1.0 - * - * - * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). - * https://openapi-generator.tech - * Do not edit the class manually. - */ - - -// May contain unused imports in some cases -// @ts-ignore -import type { DirtyStatus } from './dirty-status'; -// May contain unused imports in some cases -// @ts-ignore -import type { PreRunPushOutcome } from './pre-run-push-outcome'; - -/** - * Submitter-side git context captured before run creation. - */ -export interface PreRunGitContext { - 'display_base_sha'?: string | null; - 'local_dirty': DirtyStatus; - 'push_outcome': PreRunPushOutcome; -} diff --git a/lib/packages/fabro-api-client/src/models/set-secret-request.ts b/lib/packages/fabro-api-client/src/models/set-secret-request.ts deleted file mode 100644 index b343832c1..000000000 --- a/lib/packages/fabro-api-client/src/models/set-secret-request.ts +++ /dev/null @@ -1,25 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ -/** - * Fabro Run API - * HTTP API for managing Fabro workflow run executions. - * - * The version of the OpenAPI document: 0.1.0 - * - * - * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). - * https://openapi-generator.tech - * Do not edit the class manually. - */ - - - -/** - * Request to store a secret value. - */ -export interface SetSecretRequest { - /** - * The secret value to store. - */ - 'value': string; -} diff --git a/lib/packages/fabro-api-client/src/models/stage-turn.ts b/lib/packages/fabro-api-client/src/models/stage-turn.ts deleted file mode 100644 index 3c5ffdf3d..000000000 --- a/lib/packages/fabro-api-client/src/models/stage-turn.ts +++ /dev/null @@ -1,35 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ -/** - * Fabro Run API - * HTTP API for managing Fabro workflow run executions. - * - * The version of the OpenAPI document: 0.1.0 - * - * - * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). - * https://openapi-generator.tech - * Do not edit the class manually. - */ - - -// May contain unused imports in some cases -// @ts-ignore -import type { AssistantStageTurn } from './assistant-stage-turn'; -// May contain unused imports in some cases -// @ts-ignore -import type { SystemStageTurn } from './system-stage-turn'; -// May contain unused imports in some cases -// @ts-ignore -import type { ToolStageTurn } from './tool-stage-turn'; -// May contain unused imports in some cases -// @ts-ignore -import type { ToolUse } from './tool-use'; - -/** - * @type StageTurn - * A single turn in a stage conversation — a system prompt, assistant response, or tool invocation block. - */ -export type StageTurn = { kind: 'assistant' } & AssistantStageTurn | { kind: 'system' } & SystemStageTurn | { kind: 'tool' } & ToolStageTurn; - - diff --git a/lib/packages/fabro-api-client/src/models/system-stage-turn.ts b/lib/packages/fabro-api-client/src/models/system-stage-turn.ts deleted file mode 100644 index 68ad6386f..000000000 --- a/lib/packages/fabro-api-client/src/models/system-stage-turn.ts +++ /dev/null @@ -1,34 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ -/** - * Fabro Run API - * HTTP API for managing Fabro workflow run executions. - * - * The version of the OpenAPI document: 0.1.0 - * - * - * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). - * https://openapi-generator.tech - * Do not edit the class manually. - */ - - - -/** - * A system prompt turn that sets the stage\'s instructions. - */ -export interface SystemStageTurn { - 'kind': SystemStageTurnKindEnum; - /** - * System prompt text. - */ - 'content': string; -} - -export const SystemStageTurnKindEnum = { - SYSTEM: 'system' -} as const; - -export type SystemStageTurnKindEnum = typeof SystemStageTurnKindEnum[keyof typeof SystemStageTurnKindEnum]; - - diff --git a/lib/packages/fabro-api-client/src/models/tool-stage-turn.ts b/lib/packages/fabro-api-client/src/models/tool-stage-turn.ts deleted file mode 100644 index e637a0520..000000000 --- a/lib/packages/fabro-api-client/src/models/tool-stage-turn.ts +++ /dev/null @@ -1,41 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ -/** - * Fabro Run API - * HTTP API for managing Fabro workflow run executions. - * - * The version of the OpenAPI document: 0.1.0 - * - * - * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). - * https://openapi-generator.tech - * Do not edit the class manually. - */ - - -// May contain unused imports in some cases -// @ts-ignore -import type { ToolUse } from './tool-use'; - -/** - * A tool invocation turn containing one or more tool calls. - */ -export interface ToolStageTurn { - 'kind': ToolStageTurnKindEnum; - /** - * Text accompanying the tool invocations, or null when the turn contains only tool calls. - */ - 'content'?: string; - /** - * Tool invocations executed in this turn. - */ - 'tools': Array; -} - -export const ToolStageTurnKindEnum = { - TOOL: 'tool' -} as const; - -export type ToolStageTurnKindEnum = typeof ToolStageTurnKindEnum[keyof typeof ToolStageTurnKindEnum]; - - diff --git a/lib/packages/fabro-api-client/src/models/tool-use.ts b/lib/packages/fabro-api-client/src/models/tool-use.ts deleted file mode 100644 index c6f5e721e..000000000 --- a/lib/packages/fabro-api-client/src/models/tool-use.ts +++ /dev/null @@ -1,46 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ -/** - * Fabro Run API - * HTTP API for managing Fabro workflow run executions. - * - * The version of the OpenAPI document: 0.1.0 - * - * - * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). - * https://openapi-generator.tech - * Do not edit the class manually. - */ - - - -/** - * A single tool invocation with its input, result, and execution metadata. - */ -export interface ToolUse { - /** - * Unique identifier for this tool invocation. Enables correlation in parallel tool use. - */ - 'id': string; - /** - * Name of the tool that was invoked. - */ - 'tool_name': string; - /** - * JSON-encoded input passed to the tool. - */ - 'input': string; - /** - * Output returned by the tool. Contains the error message when is_error is true. - */ - 'result': string; - /** - * Whether the tool invocation failed. When true, the result field contains the error message. - */ - 'is_error': boolean; - /** - * Wall-clock execution time of the tool invocation in milliseconds. - */ - 'duration_ms'?: number; -} - From e901cd3a813dc82f16738ab6bba9154b55ae1fea Mon Sep 17 00:00:00 2001 From: "fabro-sh-0530[bot]" <281434857+fabro-sh-0530[bot]@users.noreply.github.com> Date: Tue, 5 May 2026 09:18:05 -0400 Subject: [PATCH 14/16] Surface silent fallback warnings in runs and logs (#205) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Summary Silent fallback paths now emit stable warnings instead of degrading without a user-visible signal. The fallback behavior is unchanged; runs still continue, but worktree, Git, checkpoint, and LLM failover issues now show up in the run feed and logs. ### Plan Summary - Emit run notices for workflow Git/worktree fallback paths. - Reuse the existing failover event for one-shot LLM provider fallback. - Add tracing for sandbox pipe drain failures. ### What changed - Added `worktree_skipped_no_git` and gated `sandbox_git_unavailable` notices during initialization. - Added `git_push_failed` and `parallel_base_checkpoint_failed` notices, including redacted output tails where available. - Logged GitHub token mint failures with a structured `error` field before the existing notice. - Plumbed `Emitter` and `StageScope` through `CodergenBackend::one_shot` so the API backend emits the existing `agent.failover` event instead of a duplicate tracing-only warning. - Extracted sandbox pipe draining into a helper that warns on stdout/stderr read failures, with unit coverage for the error path. - Updated CLI snapshots for the new worktree warning in stderr and JSON event output. ⚒️ Generated with [Fabro](https://fabro.sh) --------- Co-authored-by: Fabro Co-authored-by: Bryan Helmkamp --- lib/crates/fabro-cli/src/commands/run/logs.rs | 51 +++++++++++++--- .../src/commands/run/run_progress/event.rs | 7 ++- .../src/commands/run/run_progress/mod.rs | 14 +++-- lib/crates/fabro-cli/tests/it/cmd/attach.rs | 17 ++++++ lib/crates/fabro-cli/tests/it/cmd/dump.rs | 6 +- lib/crates/fabro-cli/tests/it/cmd/run.rs | 1 + .../tests/it/workflow/dry_run_examples.rs | 5 ++ lib/crates/fabro-sandbox/src/local.rs | 59 ++++++++++++++----- lib/crates/fabro-types/src/lib.rs | 2 +- lib/crates/fabro-types/src/run_event/infra.rs | 53 +++++++++++++---- lib/crates/fabro-types/src/run_event/mod.rs | 4 +- lib/crates/fabro-workflow/src/event.rs | 2 +- .../fabro-workflow/src/event/convert.rs | 6 +- .../fabro-workflow/src/event/emitter.rs | 15 ++--- .../fabro-workflow/src/handler/agent.rs | 2 + .../fabro-workflow/src/handler/llm/api.rs | 20 ++++--- .../fabro-workflow/src/handler/llm/cli.rs | 6 +- .../fabro-workflow/src/handler/parallel.rs | 8 ++- .../fabro-workflow/src/handler/prompt.rs | 23 ++++++-- .../fabro-workflow/src/lifecycle/artifact.rs | 10 ++-- .../fabro-workflow/src/lifecycle/git.rs | 43 +++++++++----- .../fabro-workflow/src/pipeline/finalize.rs | 42 +++++++++---- .../fabro-workflow/src/pipeline/initialize.rs | 59 +++++++++++++++---- .../src/pipeline/pull_request.rs | 4 +- .../fabro-workflow/tests/it/integration.rs | 2 + 25 files changed, 337 insertions(+), 124 deletions(-) diff --git a/lib/crates/fabro-cli/src/commands/run/logs.rs b/lib/crates/fabro-cli/src/commands/run/logs.rs index 8f01378cc..7d7e9063b 100644 --- a/lib/crates/fabro-cli/src/commands/run/logs.rs +++ b/lib/crates/fabro-cli/src/commands/run/logs.rs @@ -14,7 +14,7 @@ use std::time::Duration; use anyhow::{Context, Result, bail}; use chrono::{DateTime, Utc}; use fabro_redact::redact_jsonl_line; -use fabro_types::run_event::is_metadata_snapshot_compat_notice_code; +use fabro_types::RunNoticeCode; use fabro_util::json::normalize_json_value; use fabro_util::terminal::Styles; use tokio::time; @@ -801,7 +801,9 @@ fn format_event_pretty_value(envelope: &serde_json::Value, styles: &Styles) -> O } fn is_metadata_snapshot_compat_notice(envelope: &serde_json::Value) -> bool { - prop_str_field(envelope, "code").is_some_and(is_metadata_snapshot_compat_notice_code) + prop_str_field(envelope, "code") + .and_then(|code| code.parse::().ok()) + .is_some_and(RunNoticeCode::is_metadata_snapshot_compat) } fn str_field<'a>(value: &'a serde_json::Value, key: &str) -> Option<&'a str> { @@ -1150,14 +1152,27 @@ mod tests { #[test] fn pretty_run_notice_warn() { let styles = no_color_styles(); - let line = r#"{"ts":"2026-01-01T14:25:00Z","event":"run.notice","properties":{"level":"warn","code":"sandbox_cleanup_failed","message":"sandbox cleanup failed: boom"}}"#; - let result = format_event_pretty(line, &styles).unwrap(); + let code = RunNoticeCode::SandboxCleanupFailed.to_string(); + let line = serde_json::json!({ + "ts": "2026-01-01T14:25:00Z", + "event": "run.notice", + "properties": { + "level": "warn", + "code": code, + "message": "sandbox cleanup failed: boom", + }, + }) + .to_string(); + let result = format_event_pretty(&line, &styles).unwrap(); assert!(result.contains("Warning:"), "got: {result}"); assert!( result.contains("sandbox cleanup failed: boom"), "got: {result}" ); - assert!(result.contains("[sandbox_cleanup_failed]"), "got: {result}"); + assert!( + result.contains(&format!("[{}]", RunNoticeCode::SandboxCleanupFailed)), + "got: {result}" + ); } #[test] @@ -1235,13 +1250,31 @@ mod tests { fn pretty_stream_suppresses_metadata_compat_notice_only() { let styles = no_color_styles(); let failed = r#"{"ts":"2026-01-01T14:25:00Z","event":"metadata.snapshot.failed","properties":{"phase":"checkpoint","branch":"fabro/meta","duration_ms":900,"failure_kind":"write","error":"write failed"}}"#; - let compat_notice = r#"{"ts":"2026-01-01T14:25:01Z","event":"run.notice","properties":{"level":"warn","code":"checkpoint_metadata_write_failed","message":"legacy metadata warning"}}"#; - let degraded_notice = r#"{"ts":"2026-01-01T14:25:02Z","event":"run.notice","properties":{"level":"warn","code":"checkpoint_metadata_degraded","message":"metadata snapshots disabled"}}"#; + let compat_notice = serde_json::json!({ + "ts": "2026-01-01T14:25:01Z", + "event": "run.notice", + "properties": { + "level": "warn", + "code": RunNoticeCode::CheckpointMetadataWriteFailed, + "message": "legacy metadata warning", + }, + }) + .to_string(); + let degraded_notice = serde_json::json!({ + "ts": "2026-01-01T14:25:02Z", + "event": "run.notice", + "properties": { + "level": "warn", + "code": RunNoticeCode::CheckpointMetadataDegraded, + "message": "metadata snapshots disabled", + }, + }) + .to_string(); let mut state = PrettyEventState::default(); assert!(format_event_pretty_streamed(failed, &styles, &mut state).is_some()); - assert!(format_event_pretty_streamed(compat_notice, &styles, &mut state).is_none()); - let degraded = format_event_pretty_streamed(degraded_notice, &styles, &mut state).unwrap(); + assert!(format_event_pretty_streamed(&compat_notice, &styles, &mut state).is_none()); + let degraded = format_event_pretty_streamed(°raded_notice, &styles, &mut state).unwrap(); assert!( degraded.contains("metadata snapshots disabled"), "got: {degraded}" diff --git a/lib/crates/fabro-cli/src/commands/run/run_progress/event.rs b/lib/crates/fabro-cli/src/commands/run/run_progress/event.rs index 9e40f0d8b..ec56dce46 100644 --- a/lib/crates/fabro-cli/src/commands/run/run_progress/event.rs +++ b/lib/crates/fabro-cli/src/commands/run/run_progress/event.rs @@ -527,7 +527,7 @@ fn display_value(value: &Value) -> Option { mod tests { use fabro_agent::AgentEvent; use fabro_types::{MetadataSnapshotFailureKind, MetadataSnapshotPhase, fixtures}; - use fabro_workflow::event::{Event, to_run_event}; + use fabro_workflow::event::{Event, RunNoticeCode, to_run_event}; use super::*; @@ -804,10 +804,11 @@ mod tests { fn round_trip_run_notice() { let event = Event::RunNotice { level: RunNoticeLevel::Warn, - code: "sandbox_cleanup_failed".into(), + code: RunNoticeCode::SandboxCleanupFailed.to_string(), message: "sandbox cleanup failed".into(), exec_output_tail: None, }; + let expected_code = RunNoticeCode::SandboxCleanupFailed.to_string(); let stored = to_run_event(&fixtures::RUN_1, &event); let parsed = from_run_event(&stored).unwrap(); @@ -817,7 +818,7 @@ mod tests { level: RunNoticeLevel::Warn, code, message, - } if code == "sandbox_cleanup_failed" && message == "sandbox cleanup failed" + } if code == expected_code && message == "sandbox cleanup failed" )); } diff --git a/lib/crates/fabro-cli/src/commands/run/run_progress/mod.rs b/lib/crates/fabro-cli/src/commands/run/run_progress/mod.rs index 2004ac0ba..7e7df7f2d 100644 --- a/lib/crates/fabro-cli/src/commands/run/run_progress/mod.rs +++ b/lib/crates/fabro-cli/src/commands/run/run_progress/mod.rs @@ -3,8 +3,7 @@ reason = "sync CLI run-progress renderer: writes to std::io::stderr directly" )] -use fabro_types::RunEvent; -use fabro_types::run_event::is_metadata_snapshot_compat_notice_code; +use fabro_types::{RunEvent, RunNoticeCode}; mod event; mod info_display; @@ -444,7 +443,10 @@ impl ProgressUI { message, } => { if self.saw_metadata_snapshot_failure - && is_metadata_snapshot_compat_notice_code(&code) + && code + .parse::() + .ok() + .is_some_and(RunNoticeCode::is_metadata_snapshot_compat) { return; } @@ -1208,7 +1210,7 @@ mod tests { emit(&mut ui, Event::RunNotice { level: RunNoticeLevel::Warn, - code: "sandbox_cleanup_failed".into(), + code: RunNoticeCode::SandboxCleanupFailed.to_string(), message: "sandbox cleanup failed".into(), exec_output_tail: None, }); @@ -1279,13 +1281,13 @@ mod tests { }); emit(&mut ui, Event::RunNotice { level: RunNoticeLevel::Warn, - code: "checkpoint_metadata_write_failed".into(), + code: RunNoticeCode::CheckpointMetadataWriteFailed.to_string(), message: "legacy metadata warning".into(), exec_output_tail: None, }); emit(&mut ui, Event::RunNotice { level: RunNoticeLevel::Warn, - code: "checkpoint_metadata_degraded".into(), + code: RunNoticeCode::CheckpointMetadataDegraded.to_string(), message: "metadata snapshots are disabled for this run".into(), exec_output_tail: None, }); diff --git a/lib/crates/fabro-cli/tests/it/cmd/attach.rs b/lib/crates/fabro-cli/tests/it/cmd/attach.rs index e703e3207..7c6728f39 100644 --- a/lib/crates/fabro-cli/tests/it/cmd/attach.rs +++ b/lib/crates/fabro-cli/tests/it/cmd/attach.rs @@ -160,6 +160,7 @@ fn attach_replays_completed_detached_run() { ----- stdout ----- ----- stderr ----- Web UI: http://localhost:3000/runs/[ULID] + Warning: Worktree mode `always` requested but no Git repository was found; running without a worktree. [worktree_skipped_no_git] Sandbox: local (ready in [TIME]) ✓ Start [TIME] ✓ Run Tests [TIME] @@ -267,6 +268,7 @@ fn attach_before_completion_streams_to_finished_state() { ----- stdout ----- ----- stderr ----- Web UI: http://localhost:3000/runs/[ULID] + Warning: Worktree mode `always` requested but no Git repository was found; running without a worktree. [worktree_skipped_no_git] Sandbox: local (ready in [TIME]) ✓ start [DURATION] ✓ wait [DURATION] @@ -699,6 +701,21 @@ fn attach_json_errors_without_prompting_for_human_input() { "run_id": "[ULID]", "ts": "[TIMESTAMP]" }, + { + "actor": { + "kind": "worker", + "run_id": "[ULID]" + }, + "event": "run.notice", + "id": "[EVENT_ID]", + "properties": { + "code": "worktree_skipped_no_git", + "level": "warn", + "message": "Worktree mode `always` requested but no Git repository was found; running without a worktree." + }, + "run_id": "[ULID]", + "ts": "[TIMESTAMP]" + }, { "actor": { "kind": "worker", diff --git a/lib/crates/fabro-cli/tests/it/cmd/dump.rs b/lib/crates/fabro-cli/tests/it/cmd/dump.rs index 12b13ee9a..c541b59eb 100644 --- a/lib/crates/fabro-cli/tests/it/cmd/dump.rs +++ b/lib/crates/fabro-cli/tests/it/cmd/dump.rs @@ -278,9 +278,9 @@ fn dump_exports_completed_run_snapshot() { "); assert_snapshot!(dump_file_summary(&output_dir), @" - checkpoints/0013.json - checkpoints/0017.json - checkpoints/0021.json + checkpoints/0014.json + checkpoints/0018.json + checkpoints/0022.json events.jsonl graph.fabro run.json diff --git a/lib/crates/fabro-cli/tests/it/cmd/run.rs b/lib/crates/fabro-cli/tests/it/cmd/run.rs index 19d876cfc..2aa614979 100644 --- a/lib/crates/fabro-cli/tests/it/cmd/run.rs +++ b/lib/crates/fabro-cli/tests/it/cmd/run.rs @@ -684,6 +684,7 @@ fn dry_run_simple() { Run: [ULID] Web UI: http://localhost:3000/runs/[ULID] + Warning: Worktree mode `always` requested but no Git repository was found; running without a worktree. [worktree_skipped_no_git] Sandbox: local (ready in [TIME]) ✓ Start [TIME] ✓ Run Tests [TIME] diff --git a/lib/crates/fabro-cli/tests/it/workflow/dry_run_examples.rs b/lib/crates/fabro-cli/tests/it/workflow/dry_run_examples.rs index bb8a50c85..c3f8d836c 100644 --- a/lib/crates/fabro-cli/tests/it/workflow/dry_run_examples.rs +++ b/lib/crates/fabro-cli/tests/it/workflow/dry_run_examples.rs @@ -21,6 +21,7 @@ fn dry_run_branching() { warning [node: implement]: Node 'implement' has goal_gate=true but no retry_target or fallback_retry_target (goal_gate_has_retry) Run: [ULID] Web UI: http://localhost:3000/runs/[ULID] + Warning: Worktree mode `always` requested but no Git repository was found; running without a worktree. [worktree_skipped_no_git] Sandbox: local (ready in [TIME]) ✓ Start [TIME] ✓ Plan [TIME] @@ -57,6 +58,7 @@ fn dry_run_conditions() { Run: [ULID] Web UI: http://localhost:3000/runs/[ULID] + Warning: Worktree mode `always` requested but no Git repository was found; running without a worktree. [worktree_skipped_no_git] Sandbox: local (ready in [TIME]) ✓ start [TIME] ✓ Decide [TIME] @@ -91,6 +93,7 @@ fn dry_run_parallel() { Run: [ULID] Web UI: http://localhost:3000/runs/[ULID] + Warning: Worktree mode `always` requested but no Git repository was found; running without a worktree. [worktree_skipped_no_git] Sandbox: local (ready in [TIME]) ✓ start [TIME] ✓ Fork Work [TIME] @@ -126,6 +129,7 @@ fn dry_run_styled() { Run: [ULID] Web UI: http://localhost:3000/runs/[ULID] + Warning: Worktree mode `always` requested but no Git repository was found; running without a worktree. [worktree_skipped_no_git] Sandbox: local (ready in [TIME]) ✓ start [TIME] ✓ Plan [TIME] @@ -161,6 +165,7 @@ fn dry_run_legacy_tool() { Run: [ULID] Web UI: http://localhost:3000/runs/[ULID] + Warning: Worktree mode `always` requested but no Git repository was found; running without a worktree. [worktree_skipped_no_git] Sandbox: local (ready in [TIME]) ✓ Start [TIME] ✓ Echo [TIME] diff --git a/lib/crates/fabro-sandbox/src/local.rs b/lib/crates/fabro-sandbox/src/local.rs index 8b69b7377..66d084b5f 100644 --- a/lib/crates/fabro-sandbox/src/local.rs +++ b/lib/crates/fabro-sandbox/src/local.rs @@ -126,6 +126,19 @@ fn process_env_vars() -> Vec<(String, String)> { std::env::vars().collect() } +async fn drain_pipe(mut pipe: Option, stream: CommandOutputStream) -> String +where + R: AsyncRead + Unpin, +{ + let mut buf = String::new(); + if let Some(ref mut reader) = pipe { + if let Err(err) = reader.read_to_string(&mut buf).await { + tracing::warn!(error = %err, ?stream, "Failed to drain child output"); + } + } + buf +} + #[async_trait] impl Sandbox for LocalSandbox { async fn read_file( @@ -277,22 +290,12 @@ impl Sandbox for LocalSandbox { // it writes more than the OS pipe buffer (~64 KB) the write() syscall // blocks until the parent drains the pipe, but the parent is blocked // on child.wait(). - let mut stdout_pipe = child.stdout.take(); - let mut stderr_pipe = child.stderr.take(); - let stdout_task = tokio::spawn(async move { - let mut buf = String::new(); - if let Some(ref mut r) = stdout_pipe { - let _ = r.read_to_string(&mut buf).await; - } - buf - }); - let stderr_task = tokio::spawn(async move { - let mut buf = String::new(); - if let Some(ref mut r) = stderr_pipe { - let _ = r.read_to_string(&mut buf).await; - } - buf - }); + let stdout_pipe = child.stdout.take(); + let stderr_pipe = child.stderr.take(); + let stdout_task = + tokio::spawn(async move { drain_pipe(stdout_pipe, CommandOutputStream::Stdout).await }); + let stderr_task = + tokio::spawn(async move { drain_pipe(stderr_pipe, CommandOutputStream::Stderr).await }); let (termination, exit_code) = tokio::select! { status_result = child.wait() => { @@ -712,7 +715,12 @@ where )] mod tests { use std::collections::HashMap; + use std::io; use std::path::PathBuf; + use std::pin::Pin; + use std::task::{Context as TaskContext, Poll}; + + use tokio::io::ReadBuf; use super::*; @@ -722,6 +730,25 @@ mod tests { dir } + #[tokio::test] + async fn drain_pipe_returns_empty_buffer_after_read_failure() { + struct FailingReader; + + impl AsyncRead for FailingReader { + fn poll_read( + self: Pin<&mut Self>, + _cx: &mut TaskContext<'_>, + _buf: &mut ReadBuf<'_>, + ) -> Poll> { + Poll::Ready(Err(io::Error::other("simulated read failure"))) + } + } + + let output = drain_pipe(Some(FailingReader), CommandOutputStream::Stdout).await; + + assert!(output.is_empty()); + } + #[tokio::test] async fn read_file_with_line_numbers() { let dir = temp_dir(); diff --git a/lib/crates/fabro-types/src/lib.rs b/lib/crates/fabro-types/src/lib.rs index 22675855c..e5d3b65a5 100644 --- a/lib/crates/fabro-types/src/lib.rs +++ b/lib/crates/fabro-types/src/lib.rs @@ -72,7 +72,7 @@ pub use run::{ pub use run_blob_id::RunBlobId; pub use run_event::{ EventBody, ExecOutputTail, InterviewOption, MetadataSnapshotFailureKind, MetadataSnapshotPhase, - RunEvent, RunNoticeLevel, + RunEvent, RunNoticeCode, RunNoticeLevel, }; pub use run_id::{RunId, fixtures}; pub use run_projection::{PendingInterviewRecord, RunProjection, StageProjection, first_event_seq}; diff --git a/lib/crates/fabro-types/src/run_event/infra.rs b/lib/crates/fabro-types/src/run_event/infra.rs index 8ee1bc32c..77689087b 100644 --- a/lib/crates/fabro-types/src/run_event/infra.rs +++ b/lib/crates/fabro-types/src/run_event/infra.rs @@ -1,17 +1,48 @@ use serde::{Deserialize, Serialize}; -/// Legacy `run.notice` codes paired with the new `metadata.snapshot.failed` -/// event for backward compatibility. Display layers suppress these so the -/// typed event renders without a duplicate raw warning. -pub const NOTICE_CODE_CHECKPOINT_METADATA_WRITE_FAILED: &str = "checkpoint_metadata_write_failed"; -pub const NOTICE_CODE_CHECKPOINT_METADATA_PUSH_FAILED: &str = "checkpoint_metadata_push_failed"; +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + Hash, + Serialize, + Deserialize, + strum::Display, + strum::EnumString, + strum::IntoStaticStr, +)] +#[serde(rename_all = "snake_case")] +#[strum(serialize_all = "snake_case")] +pub enum RunNoticeCode { + ArtifactCollectionFailed, + ArtifactOffloadFailed, + ArtifactSyncFailed, + ArtifactUploadFailed, + CheckpointMetadataDegraded, + CheckpointMetadataPushFailed, + CheckpointMetadataWriteFailed, + DirtyWorktree, + GitDiffFailed, + GitPushFailed, + GithubTokenFailed, + ParallelBaseCheckpointFailed, + PullRequestFailed, + SandboxCleanupFailed, + SandboxGitUnavailable, + SandboxPreserved, + WorktreeSkippedNoGit, +} -#[must_use] -pub fn is_metadata_snapshot_compat_notice_code(code: &str) -> bool { - matches!( - code, - NOTICE_CODE_CHECKPOINT_METADATA_WRITE_FAILED | NOTICE_CODE_CHECKPOINT_METADATA_PUSH_FAILED - ) +impl RunNoticeCode { + #[must_use] + pub fn is_metadata_snapshot_compat(self) -> bool { + matches!( + self, + Self::CheckpointMetadataWriteFailed | Self::CheckpointMetadataPushFailed + ) + } } #[derive( diff --git a/lib/crates/fabro-types/src/run_event/mod.rs b/lib/crates/fabro-types/src/run_event/mod.rs index 9d5fbf11d..647efc381 100644 --- a/lib/crates/fabro-types/src/run_event/mod.rs +++ b/lib/crates/fabro-types/src/run_event/mod.rs @@ -1366,7 +1366,7 @@ mod tests { for body in [ EventBody::RunNotice(RunNoticeProps { level: RunNoticeLevel::Warn, - code: "git_diff_failed".to_string(), + code: RunNoticeCode::GitDiffFailed.to_string(), message: "git diff failed".to_string(), exec_output_tail: Some(tail.clone()), }), @@ -1402,7 +1402,7 @@ mod tests { for body in [ EventBody::RunNotice(RunNoticeProps { level: RunNoticeLevel::Warn, - code: "git_diff_failed".to_string(), + code: RunNoticeCode::GitDiffFailed.to_string(), message: "git diff failed".to_string(), exec_output_tail: None, }), diff --git a/lib/crates/fabro-workflow/src/event.rs b/lib/crates/fabro-workflow/src/event.rs index f5120e62e..e64b2ae30 100644 --- a/lib/crates/fabro-workflow/src/event.rs +++ b/lib/crates/fabro-workflow/src/event.rs @@ -8,7 +8,7 @@ mod stored_fields; #[cfg(test)] mod test_support; -pub use fabro_types::{EventBody, RunNoticeLevel}; +pub use fabro_types::{EventBody, RunNoticeCode, RunNoticeLevel}; pub use self::convert::{to_run_event, to_run_event_at}; pub use self::emitter::Emitter; diff --git a/lib/crates/fabro-workflow/src/event/convert.rs b/lib/crates/fabro-workflow/src/event/convert.rs index 4aa9df5b1..255d235ea 100644 --- a/lib/crates/fabro-workflow/src/event/convert.rs +++ b/lib/crates/fabro-workflow/src/event/convert.rs @@ -1169,8 +1169,8 @@ mod tests { use std::collections::BTreeMap; use ::fabro_types::{ - EventBody, FailureReason, ParallelBranchId, Principal, RunNoticeLevel, RunProvenance, - StageId, SystemActorKind, fixtures, run_event as fabro_types, + EventBody, FailureReason, ParallelBranchId, Principal, RunNoticeCode, RunNoticeLevel, + RunProvenance, StageId, SystemActorKind, fixtures, run_event as fabro_types, }; use chrono::Utc; use fabro_agent::{AgentEvent, SandboxEvent}; @@ -1643,7 +1643,7 @@ mod tests { fn run_notice_maps_exec_output_tail_to_props() { let stored = to_run_event(&fixtures::RUN_1, &Event::RunNotice { level: RunNoticeLevel::Warn, - code: "git_diff_failed".to_string(), + code: RunNoticeCode::GitDiffFailed.to_string(), message: "git diff failed".to_string(), exec_output_tail: Some(exec_tail()), }); diff --git a/lib/crates/fabro-workflow/src/event/emitter.rs b/lib/crates/fabro-workflow/src/event/emitter.rs index a1bba727a..dc50a56dd 100644 --- a/lib/crates/fabro-workflow/src/event/emitter.rs +++ b/lib/crates/fabro-workflow/src/event/emitter.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use std::sync::atomic::{AtomicI64, Ordering}; -use ::fabro_types::{ExecOutputTail, RunEvent, RunId, RunNoticeLevel}; +use ::fabro_types::{ExecOutputTail, RunEvent, RunId, RunNoticeCode, RunNoticeLevel}; use chrono::Utc; use fabro_agent::{WorktreeEvent, WorktreeEventCallback}; @@ -76,15 +76,10 @@ impl Emitter { self.emit_with_scope(event, Some(scope)); } - pub fn notice( - &self, - level: RunNoticeLevel, - code: impl Into, - message: impl Into, - ) { + pub fn notice(&self, level: RunNoticeLevel, code: RunNoticeCode, message: impl Into) { self.emit(&Event::RunNotice { level, - code: code.into(), + code: code.to_string(), message: message.into(), exec_output_tail: None, }); @@ -93,13 +88,13 @@ impl Emitter { pub fn notice_with_tail( &self, level: RunNoticeLevel, - code: impl Into, + code: RunNoticeCode, message: impl Into, exec_output_tail: Option, ) { self.emit(&Event::RunNotice { level, - code: code.into(), + code: code.to_string(), message: message.into(), exec_output_tail, }); diff --git a/lib/crates/fabro-workflow/src/handler/agent.rs b/lib/crates/fabro-workflow/src/handler/agent.rs index 676e2f142..7e3a4be0c 100644 --- a/lib/crates/fabro-workflow/src/handler/agent.rs +++ b/lib/crates/fabro-workflow/src/handler/agent.rs @@ -52,6 +52,8 @@ pub trait CodergenBackend: Send + Sync { _node: &Node, _prompt: &str, _system_prompt: Option<&str>, + _emitter: &Arc, + _stage_scope: &StageScope, ) -> Result { Err(Error::Validation( "one_shot mode not supported by this backend".into(), diff --git a/lib/crates/fabro-workflow/src/handler/llm/api.rs b/lib/crates/fabro-workflow/src/handler/llm/api.rs index 5d21c7d32..9247c19ec 100644 --- a/lib/crates/fabro-workflow/src/handler/llm/api.rs +++ b/lib/crates/fabro-workflow/src/handler/llm/api.rs @@ -285,6 +285,8 @@ impl CodergenBackend for AgentApiBackend { node: &Node, prompt: &str, system_prompt: Option<&str>, + emitter: &Arc, + stage_scope: &StageScope, ) -> Result { let client = Client::from_source(self.source.as_ref()) .await @@ -358,14 +360,16 @@ impl CodergenBackend for AgentApiBackend { let mut found = None; for target in fallback_chain { - tracing::warn!( - stage = node.id.as_str(), - from_provider = from_provider.as_str(), - from_model = from_model.as_str(), - to_provider = target.provider.as_str(), - to_model = target.model.as_str(), - error = error_msg.as_str(), - "LLM provider failover (prompt)" + emitter.emit_scoped( + &Event::Failover { + stage: node.id.clone(), + from_provider: from_provider.clone(), + from_model: from_model.clone(), + to_provider: target.provider.clone(), + to_model: target.model.clone(), + error: error_msg.clone(), + }, + stage_scope, ); let max_tokens = node.max_tokens().or_else(|| { diff --git a/lib/crates/fabro-workflow/src/handler/llm/cli.rs b/lib/crates/fabro-workflow/src/handler/llm/cli.rs index 5036eec97..7fcf60ed8 100644 --- a/lib/crates/fabro-workflow/src/handler/llm/cli.rs +++ b/lib/crates/fabro-workflow/src/handler/llm/cli.rs @@ -810,9 +810,13 @@ impl CodergenBackend for BackendRouter { node: &Node, prompt: &str, system_prompt: Option<&str>, + emitter: &Arc, + stage_scope: &StageScope, ) -> Result { // CLI backend doesn't support one_shot, always route to API - self.api_backend.one_shot(node, prompt, system_prompt).await + self.api_backend + .one_shot(node, prompt, system_prompt, emitter, stage_scope) + .await } } diff --git a/lib/crates/fabro-workflow/src/handler/parallel.rs b/lib/crates/fabro-workflow/src/handler/parallel.rs index 39862586c..c58104f83 100644 --- a/lib/crates/fabro-workflow/src/handler/parallel.rs +++ b/lib/crates/fabro-workflow/src/handler/parallel.rs @@ -12,7 +12,7 @@ use tokio::sync::Semaphore; use super::{EngineServices, Handler}; use crate::context::{Context, WorkflowContext, keys}; use crate::error::Error; -use crate::event::{Event, StageScope}; +use crate::event::{Event, RunNoticeCode, RunNoticeLevel, StageScope}; use crate::git::sanitize_ref_component; use crate::hook_context::set_hook_node; use crate::millis_u64; @@ -207,6 +207,12 @@ impl Handler for ParallelHandler { error = %fabro_sandbox::display_for_log(&e), "parallel base checkpoint failed" ); + services.run.emitter.notice_with_tail( + RunNoticeLevel::Warn, + RunNoticeCode::ParallelBaseCheckpointFailed, + format!("Could not checkpoint base state before parallel branches: {e}"), + fabro_sandbox::default_redacted_output_tail(&e), + ); None } } diff --git a/lib/crates/fabro-workflow/src/handler/prompt.rs b/lib/crates/fabro-workflow/src/handler/prompt.rs index a76099b80..caad1e058 100644 --- a/lib/crates/fabro-workflow/src/handler/prompt.rs +++ b/lib/crates/fabro-workflow/src/handler/prompt.rs @@ -105,7 +105,13 @@ impl Handler for PromptHandler { let (response_text, stage_usage, backend_files_touched) = if let Some(backend) = &self.backend { let result = backend - .one_shot(node, &prompt, system_prompt.as_deref()) + .one_shot( + node, + &prompt, + system_prompt.as_deref(), + &services.run.emitter, + &stage_scope, + ) .await; match result { Ok(CodergenResult::Full(outcome)) => return Ok(outcome), @@ -187,6 +193,7 @@ mod tests { use tempfile::TempDir; use super::*; + use crate::event::Emitter; fn make_services() -> EngineServices { EngineServices::test_default() @@ -211,7 +218,7 @@ mod tests { let mut services = EngineServices::test_default(); services.run = services .run - .with_emitter(Arc::new(crate::event::Emitter::new(fixtures::RUN_1))) + .with_emitter(Arc::new(Emitter::new(fixtures::RUN_1))) .with_run_store(run_store.clone().into()); let logger = crate::event::StoreProgressLogger::new(run_store.clone()); logger.register(services.run.emitter.as_ref()); @@ -267,7 +274,7 @@ mod tests { _prompt: &str, _context: &Context, _thread_id: Option<&str>, - _emitter: &Arc, + _emitter: &Arc, _sandbox: &Arc, _tool_hooks: Option>, ) -> Result { @@ -279,6 +286,8 @@ mod tests { _node: &Node, _prompt: &str, _system_prompt: Option<&str>, + _emitter: &Arc, + _stage_scope: &StageScope, ) -> Result { Ok(CodergenResult::Text { text: "one-shot response".to_string(), @@ -327,7 +336,7 @@ mod tests { _prompt: &str, _context: &Context, _thread_id: Option<&str>, - _emitter: &Arc, + _emitter: &Arc, _sandbox: &Arc, _tool_hooks: Option>, ) -> Result { @@ -339,6 +348,8 @@ mod tests { _node: &Node, _prompt: &str, _system_prompt: Option<&str>, + _emitter: &Arc, + _stage_scope: &StageScope, ) -> Result { Ok(CodergenResult::Text { text: "one-shot response".to_string(), @@ -384,7 +395,7 @@ mod tests { _prompt: &str, _context: &Context, _thread_id: Option<&str>, - _emitter: &Arc, + _emitter: &Arc, _sandbox: &Arc, _tool_hooks: Option>, ) -> Result { @@ -396,6 +407,8 @@ mod tests { _node: &Node, prompt: &str, system_prompt: Option<&str>, + _emitter: &Arc, + _stage_scope: &StageScope, ) -> Result { *self.captured_prompt.lock().unwrap() = Some(prompt.to_string()); *self.captured_system_prompt.lock().unwrap() = Some(system_prompt.map(String::from)); diff --git a/lib/crates/fabro-workflow/src/lifecycle/artifact.rs b/lib/crates/fabro-workflow/src/lifecycle/artifact.rs index f65fcee86..50d6ccded 100644 --- a/lib/crates/fabro-workflow/src/lifecycle/artifact.rs +++ b/lib/crates/fabro-workflow/src/lifecycle/artifact.rs @@ -16,7 +16,7 @@ use tokio::time::sleep; use crate::artifact::{normalize_durable_updates, offload_large_values, sync_artifacts_to_env}; use crate::artifact_snapshot::collect_artifacts; use crate::artifact_upload::ArtifactSink; -use crate::event::{Emitter, Event, RunNoticeLevel}; +use crate::event::{Emitter, Event, RunNoticeCode, RunNoticeLevel}; use crate::graph::{WorkflowGraph, WorkflowNode}; use crate::lifecycle::event::{stage_scope_for, stage_visit}; use crate::outcome::BilledModelUsage; @@ -125,7 +125,7 @@ impl RunLifecycle for ArtifactLifecycle { { self.emitter.notice( RunNoticeLevel::Warn, - "artifact_upload_failed", + RunNoticeCode::ArtifactUploadFailed, format!("[node: {node_id}] artifact upload failed: {err}"), ); return Ok(()); @@ -151,7 +151,7 @@ impl RunLifecycle for ArtifactLifecycle { Err(e) => { self.emitter.notice( RunNoticeLevel::Warn, - "artifact_collection_failed", + RunNoticeCode::ArtifactCollectionFailed, format!("[node: {node_id}] artifact collection failed: {e}"), ); } @@ -174,7 +174,7 @@ impl RunLifecycle for ArtifactLifecycle { { self.emitter.notice( RunNoticeLevel::Warn, - "artifact_offload_failed", + RunNoticeCode::ArtifactOffloadFailed, format!("[node: {node_id}] artifact offload failed: {e}"), ); } @@ -187,7 +187,7 @@ impl RunLifecycle for ArtifactLifecycle { { self.emitter.notice( RunNoticeLevel::Warn, - "artifact_sync_failed", + RunNoticeCode::ArtifactSyncFailed, format!("[node: {node_id}] artifact sync failed: {e}"), ); } diff --git a/lib/crates/fabro-workflow/src/lifecycle/git.rs b/lib/crates/fabro-workflow/src/lifecycle/git.rs index 0e0e9f195..2f9d5bfdb 100644 --- a/lib/crates/fabro-workflow/src/lifecycle/git.rs +++ b/lib/crates/fabro-workflow/src/lifecycle/git.rs @@ -14,7 +14,7 @@ use fabro_util::error::collect_causes; use fabro_util::time::elapsed_ms; use crate::artifact; -use crate::event::{Emitter, Event, RunNoticeLevel, StageScope}; +use crate::event::{Emitter, Event, RunNoticeCode, RunNoticeLevel, StageScope}; use crate::graph::{WorkflowGraph, WorkflowNode}; use crate::lifecycle::event::stage_scope_for; use crate::outcome::BilledModelUsage; @@ -128,7 +128,10 @@ impl RunLifecycle for GitLifecycle { None, None, ); - self.emit_metadata_warning("checkpoint_metadata_write_failed", message); + self.emit_metadata_warning( + RunNoticeCode::CheckpointMetadataWriteFailed, + message, + ); } }, Err(err) => { @@ -145,7 +148,10 @@ impl RunLifecycle for GitLifecycle { None, None, ); - self.emit_metadata_warning("checkpoint_metadata_write_failed", message); + self.emit_metadata_warning( + RunNoticeCode::CheckpointMetadataWriteFailed, + message, + ); } } } @@ -217,7 +223,7 @@ impl RunLifecycle for GitLifecycle { Some(&scope), ); self.emit_metadata_warning( - "checkpoint_metadata_write_failed", + RunNoticeCode::CheckpointMetadataWriteFailed, message, ); None @@ -239,7 +245,10 @@ impl RunLifecycle for GitLifecycle { None, Some(&scope), ); - self.emit_metadata_warning("checkpoint_metadata_write_failed", message); + self.emit_metadata_warning( + RunNoticeCode::CheckpointMetadataWriteFailed, + message, + ); None } } @@ -292,6 +301,12 @@ impl RunLifecycle for GitLifecycle { error = %fabro_sandbox::display_for_log(&err), "git push from run lifecycle failed" ); + self.emitter.notice_with_tail( + RunNoticeLevel::Warn, + RunNoticeCode::GitPushFailed, + format!("Failed to push run branch {branch}: {err}"), + exec_output_tail.clone(), + ); (false, exec_output_tail) } }; @@ -321,7 +336,7 @@ impl RunLifecycle for GitLifecycle { fabro_sandbox::default_redacted_output_tail(&err); self.emitter.notice_with_tail( RunNoticeLevel::Warn, - "git_diff_failed", + RunNoticeCode::GitDiffFailed, format!("[node: {node_id}] git diff failed: {err}"), exec_output_tail, ); @@ -395,7 +410,10 @@ impl GitLifecycle { Some(snapshot.bytes), scope, ); - self.emit_metadata_warning("checkpoint_metadata_push_failed", message); + self.emit_metadata_warning( + RunNoticeCode::CheckpointMetadataPushFailed, + message, + ); } else { self.emit_metadata_snapshot_completed( phase, @@ -421,7 +439,7 @@ impl GitLifecycle { None, scope, ); - self.emit_metadata_warning("checkpoint_metadata_write_failed", message); + self.emit_metadata_warning(RunNoticeCode::CheckpointMetadataWriteFailed, message); None } } @@ -506,14 +524,9 @@ impl GitLifecycle { } } - fn emit_metadata_warning(&self, code: &str, message: String) { + fn emit_metadata_warning(&self, code: RunNoticeCode, message: String) { if self.metadata_runtime.mark_metadata_degraded() { - self.emitter.emit(&Event::RunNotice { - level: RunNoticeLevel::Warn, - code: code.to_string(), - message, - exec_output_tail: None, - }); + self.emitter.notice(RunNoticeLevel::Warn, code, message); } } } diff --git a/lib/crates/fabro-workflow/src/pipeline/finalize.rs b/lib/crates/fabro-workflow/src/pipeline/finalize.rs index 927d2d833..cb58bd345 100644 --- a/lib/crates/fabro-workflow/src/pipeline/finalize.rs +++ b/lib/crates/fabro-workflow/src/pipeline/finalize.rs @@ -10,7 +10,7 @@ use fabro_util::time::elapsed_ms; use super::types::{Concluded, FinalizeOptions, Retroed}; use crate::error::Error; -use crate::event::{Event, RunNoticeLevel}; +use crate::event::{Event, RunNoticeCode, RunNoticeLevel}; use crate::outcome::{Outcome, OutcomeExt, StageOutcome}; use crate::records::{Checkpoint, Conclusion, StageSummary}; use crate::run_metadata::MetadataSnapshot; @@ -239,7 +239,11 @@ pub async fn write_finalize_commit( None, None, ); - emit_metadata_warning(services, "checkpoint_metadata_write_failed", message); + emit_metadata_warning( + services, + RunNoticeCode::CheckpointMetadataWriteFailed, + message, + ); return; } }; @@ -260,7 +264,11 @@ pub async fn write_finalize_commit( None, None, ); - emit_metadata_warning(services, "checkpoint_metadata_write_failed", message); + emit_metadata_warning( + services, + RunNoticeCode::CheckpointMetadataWriteFailed, + message, + ); return; } }; @@ -281,7 +289,11 @@ pub async fn write_finalize_commit( Some(snapshot.entry_count), Some(snapshot.bytes), ); - emit_metadata_warning(services, "checkpoint_metadata_push_failed", message); + emit_metadata_warning( + services, + RunNoticeCode::CheckpointMetadataPushFailed, + message, + ); } else { emit_metadata_snapshot_completed(services, phase, meta_branch, started, &snapshot); } @@ -300,7 +312,11 @@ pub async fn write_finalize_commit( None, None, ); - emit_metadata_warning(services, "checkpoint_metadata_write_failed", message); + emit_metadata_warning( + services, + RunNoticeCode::CheckpointMetadataWriteFailed, + message, + ); } } } @@ -363,7 +379,7 @@ fn emit_metadata_snapshot_failed( }); } -fn emit_metadata_warning(services: &RunServices, code: &str, message: String) { +fn emit_metadata_warning(services: &RunServices, code: RunNoticeCode, message: String) { if services.metadata_runtime.mark_metadata_degraded() { services.emitter.notice(RunNoticeLevel::Warn, code, message); } @@ -387,7 +403,7 @@ async fn compute_final_patch( Err(err) => { services.emitter.notice( RunNoticeLevel::Warn, - "git_diff_failed", + RunNoticeCode::GitDiffFailed, format!("final diff failed: {err}"), ); None @@ -534,7 +550,7 @@ pub async fn finalize(retroed: Retroed, options: &FinalizeOptions) -> Result Result Result Option { if let Some(env_name) = env_name { options.emitter.notice( RunNoticeLevel::Warn, - "dirty_worktree", + RunNoticeCode::DirtyWorktree, format!("Uncommitted changes will not be included in the {env_name}."), ); } @@ -200,6 +200,14 @@ fn resolve_worktree_plan(options: &mut InitOptions) -> Option { }) } +fn worktree_skipped_notice(mode: Option) -> Option<(RunNoticeCode, &'static str)> { + matches!(mode, Some(WorktreeMode::Always)).then_some(( + RunNoticeCode::WorktreeSkippedNoGit, + "Worktree mode `always` requested but no Git repository was found; running without a \ + worktree.", + )) +} + fn git_setup_intent(run_options: &RunOptions) -> GitSetupIntent { if let Some(source) = run_options.fork_source_ref.as_ref() { GitSetupIntent::ForkFromCheckpoint { @@ -239,11 +247,14 @@ async fn build_sandbox_env( Ok(token) => { env.insert("GITHUB_TOKEN".to_string(), token); } - Err(e) => emitter.notice( - RunNoticeLevel::Warn, - "github_token_failed", - format!("Failed to mint GitHub token: {e}"), - ), + Err(e) => { + tracing::warn!(error = %e, "Failed to mint GitHub token"); + emitter.notice( + RunNoticeLevel::Warn, + RunNoticeCode::GithubTokenFailed, + format!("Failed to mint GitHub token: {e}"), + ); + } } } } @@ -516,6 +527,13 @@ pub async fn initialize( )) }; if worktree_plan.is_some() && !worktree_created { + if let Some((code, message)) = worktree_skipped_notice(options.worktree_mode) { + tracing::warn!( + worktree_mode = ?options.worktree_mode, + "worktree skipped: cwd is not a git repository" + ); + options.emitter.notice(RunNoticeLevel::Warn, code, message); + } options.run_options.git = None; } let cleanup_guard = scopeguard::guard(Arc::clone(&sandbox), |sandbox| { @@ -593,7 +611,8 @@ pub async fn initialize( .is_some(); if !has_run_branch { let intent = git_setup_intent(&options.run_options); - if sandbox.origin_url().is_some() { + let sandbox_has_origin = sandbox.origin_url().is_some(); + if sandbox_has_origin { sandbox_git .ensure_git_available(&*sandbox) .await @@ -619,7 +638,16 @@ pub async fn initialize( options.run_options.base_branch = info.base_branch; } } - Ok(None) => {} + Ok(None) => { + if sandbox_has_origin { + options.emitter.notice( + RunNoticeLevel::Warn, + RunNoticeCode::SandboxGitUnavailable, + "Sandbox could not set up Git despite a configured origin; running \ + without checkpointing or PR support.", + ); + } + } Err(e) => { return Err(Error::engine_with_source("Sandbox git setup failed", &e)); } @@ -702,7 +730,7 @@ pub async fn initialize( if metadata_runtime.mark_metadata_degraded() { options.emitter.notice( RunNoticeLevel::Warn, - "checkpoint_metadata_write_failed", + RunNoticeCode::CheckpointMetadataWriteFailed, message, ); } @@ -1011,6 +1039,17 @@ mod tests { assert!(options.run_options.git.is_none()); } + #[test] + fn worktree_skipped_notice_only_warns_for_always() { + assert!(worktree_skipped_notice(None).is_none()); + assert!(worktree_skipped_notice(Some(WorktreeMode::Clean)).is_none()); + assert!(worktree_skipped_notice(Some(WorktreeMode::Dirty)).is_none()); + assert!(worktree_skipped_notice(Some(WorktreeMode::Never)).is_none()); + + let (code, _) = worktree_skipped_notice(Some(WorktreeMode::Always)).unwrap(); + assert_eq!(code, RunNoticeCode::WorktreeSkippedNoGit); + } + #[tokio::test] async fn initialize_prepares_sandbox_and_uses_persisted_run_dir() { let temp = tempfile::tempdir().unwrap(); diff --git a/lib/crates/fabro-workflow/src/pipeline/pull_request.rs b/lib/crates/fabro-workflow/src/pipeline/pull_request.rs index 35f27aa92..37aec323e 100644 --- a/lib/crates/fabro-workflow/src/pipeline/pull_request.rs +++ b/lib/crates/fabro-workflow/src/pipeline/pull_request.rs @@ -14,7 +14,7 @@ use fabro_util::text::strip_goal_decoration; use tracing::{debug, info, warn}; use super::types::{Concluded, Finalized, PullRequestOptions}; -use crate::event::{Event, RunNoticeLevel}; +use crate::event::{Event, RunNoticeCode, RunNoticeLevel}; use crate::outcome::{StageOutcome, format_cost as outcome_format_cost}; use crate::records::{Conclusion, RunSpec}; use crate::runtime_store::RunStoreHandle; @@ -675,7 +675,7 @@ pub async fn pull_request(concluded: Concluded, options: &PullRequestOptions) -> .emit(&Event::PullRequestFailed { error: e.clone() }); services.emitter.notice( RunNoticeLevel::Warn, - "pull_request_failed", + RunNoticeCode::PullRequestFailed, format!("PR creation failed: {e}"), ); } diff --git a/lib/crates/fabro-workflow/tests/it/integration.rs b/lib/crates/fabro-workflow/tests/it/integration.rs index f6ece331b..b89fc7f5a 100644 --- a/lib/crates/fabro-workflow/tests/it/integration.rs +++ b/lib/crates/fabro-workflow/tests/it/integration.rs @@ -6221,6 +6221,8 @@ mod real_llm { _node: &Node, prompt: &str, _system_prompt: Option<&str>, + _emitter: &Arc, + _stage_scope: &fabro_workflow::event::StageScope, ) -> Result { self.complete(prompt).await } From 786a6f67e15dc16eb5da10cf29f7b2438879485d Mon Sep 17 00:00:00 2001 From: "fabro-sh-0530[bot]" <281434857+fabro-sh-0530[bot]@users.noreply.github.com> Date: Tue, 5 May 2026 09:32:33 -0400 Subject: [PATCH 15/16] Read billing and stages from RunProjection with live runtimes (#213) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Summary Billing and stage lists now use the event-sourced `RunProjection` as their source of truth, so running and retrying stages appear immediately and runtimes keep advancing in the UI. This removes the checkpoint completed-node bypass that hid in-flight work and froze totals until the next server response. ### Plan Summary - Store stage `started_at`, terminal `duration_ms`, server-internal `usage`, and lifecycle `state` on `StageProjection`. - Populate those fields from stage lifecycle events, including retry transitions and per-attempt reset on new starts. - Render `/runs/{id}/stages` and `/runs/{id}/billing` from `RunProjection.iter_stages()`. - Expose the new API/client fields and tick in-flight billing runtimes on the web UI. ```mermaid flowchart TB Events["Stage lifecycle events"] --> Projection["RunProjection StageProjection"] Projection --> StagesAPI["GET /runs/{id}/stages"] Projection --> BillingAPI["GET /runs/{id}/billing"] StagesAPI --> StageUI["Stage sidebar/stages view"] BillingAPI --> BillingUI["Billing tab live totals"] ``` ### Key decisions Retry and revisit handling stays one row per node id: latest visit data wins, while first-seen event sequence keeps ordering stable with finalize output. `state` is stored rather than derived so `Retrying` is representable, and old serialized projections still work through the `effective_state()` fallback. Billing `usage` remains server-internal and is skipped on the wire; public schemas only expose the fields needed by `/stages`, `/billing`, and the frontend live timer. Added focused reducer, server retry/revisit, API round-trip, billing UI, and event invalidation coverage. ⚒️ Generated with [Fabro](https://fabro.sh) --------- Co-authored-by: Fabro Co-authored-by: Bryan Helmkamp --- .../app/components/stage-sidebar.tsx | 13 +- apps/fabro-web/app/lib/query-keys.test.ts | 3 +- apps/fabro-web/app/lib/run-events.test.tsx | 16 +- apps/fabro-web/app/lib/run-events.ts | 1 + apps/fabro-web/app/lib/stage-sidebar.ts | 17 +- apps/fabro-web/app/lib/time.ts | 19 +- .../fabro-web/app/routes/run-billing.test.tsx | 62 ++++- apps/fabro-web/app/routes/run-billing.tsx | 161 ++++++----- apps/fabro-web/app/routes/run-stages.tsx | 10 +- docs/public/api-reference/fabro-api.yaml | 29 ++ .../tests/run_billing_stage_round_trip.rs | 57 ++++ .../tests/stage_projection_round_trip.rs | 5 +- lib/crates/fabro-server/src/demo/mod.rs | 13 + lib/crates/fabro-server/src/server.rs | 2 + .../src/server/handler/billing.rs | 214 ++++++++------- lib/crates/fabro-server/src/server/tests.rs | 192 ++++++++++++++ lib/crates/fabro-store/src/run_state.rs | 249 +++++++++++++++++- .../tests/serializable_projection.rs | 6 +- lib/crates/fabro-types/src/run_projection.rs | 64 ++++- .../src/models/board-column-definition.ts | 3 - .../src/models/run-billing-stage.ts | 9 +- .../fabro-api-client/src/models/run-stage.ts | 7 +- .../src/models/stage-projection.ts | 15 +- 23 files changed, 954 insertions(+), 213 deletions(-) diff --git a/apps/fabro-web/app/components/stage-sidebar.tsx b/apps/fabro-web/app/components/stage-sidebar.tsx index 2d3122602..e2cfe88f8 100644 --- a/apps/fabro-web/app/components/stage-sidebar.tsx +++ b/apps/fabro-web/app/components/stage-sidebar.tsx @@ -1,4 +1,4 @@ -import { useState, useEffect, useRef, type ComponentType } from "react"; +import { useEffect, useRef, type ComponentType } from "react"; import { Link } from "react-router"; import type { StageState } from "@qltysh/fabro-api-client"; import { @@ -12,6 +12,7 @@ import { import { Bars3BottomLeftIcon, DocumentTextIcon, MapIcon } from "@heroicons/react/24/outline"; import { formatDurationSecs } from "../lib/format"; import { ACTIVE_STAGE_STATES, formatStageLabel } from "../lib/stage-sidebar"; +import { useTickingNow } from "../lib/time"; export interface Stage { id: string; @@ -43,7 +44,6 @@ interface StageSidebarProps { export function StageSidebar({ stages, runId, selectedStageId, activeLink }: StageSidebarProps) { // Track when we first observed each running stage (for ticking timer) const runningStartRef = useRef>(new Map()); - const [, setTick] = useState(0); // Track start times for running stages useEffect(() => { @@ -63,16 +63,13 @@ export function StageSidebar({ stages, runId, selectedStageId, activeLink }: Sta }, [stages]); // Tick every second while any stage is running - useEffect(() => { - if (!stages.some((s) => ACTIVE_STAGE_STATES.has(s.status))) return; - const interval = setInterval(() => setTick((t) => t + 1), 1000); - return () => clearInterval(interval); - }, [stages]); + const hasActive = stages.some((s) => ACTIVE_STAGE_STATES.has(s.status)); + const now = useTickingNow(hasActive); function stageDuration(stage: Stage): string { if (ACTIVE_STAGE_STATES.has(stage.status)) { const start = runningStartRef.current.get(stage.id); - if (start) return formatDurationSecs(Math.floor((Date.now() - start) / 1000)); + if (start) return formatDurationSecs(Math.floor((now - start) / 1000)); return "0s"; } return stage.duration; diff --git a/apps/fabro-web/app/lib/query-keys.test.ts b/apps/fabro-web/app/lib/query-keys.test.ts index 5d7b3a53a..a06acaedb 100644 --- a/apps/fabro-web/app/lib/query-keys.test.ts +++ b/apps/fabro-web/app/lib/query-keys.test.ts @@ -22,6 +22,7 @@ describe("queryKeys", () => { ]); expect(queryKeysForRunEvent("run-1", "stage.completed", "stage-1")).toEqual([ queryKeys.runs.stages("run-1"), + queryKeys.runs.billing("run-1"), queryKeys.runs.events("run-1", 1000), queryKeys.runs.graph("run-1", "LR"), queryKeys.runs.graph("run-1", "TB"), @@ -48,4 +49,4 @@ describe("queryKeys", () => { test("agent activity events without a node_id invalidate nothing", () => { expect(queryKeysForRunEvent("run-1", "agent.message")).toEqual([]); }); -}); \ No newline at end of file +}); diff --git a/apps/fabro-web/app/lib/run-events.test.tsx b/apps/fabro-web/app/lib/run-events.test.tsx index 14db0e3f2..432791352 100644 --- a/apps/fabro-web/app/lib/run-events.test.tsx +++ b/apps/fabro-web/app/lib/run-events.test.tsx @@ -50,12 +50,16 @@ describe("queryKeysForRunEvent", () => { ]); }); - test("stage.retrying invalidates the same keys as other stage events", () => { - const keys = queryKeysForRunEvent("run-1", "stage.retrying", "verify@2"); - expect(keys).toContain(queryKeys.runs.stages("run-1")); - expect(keys).toContain(queryKeys.runs.events("run-1", 1000)); - expect(keys).toContain(queryKeys.runs.detail("run-1")); - expect(keys).toContain(queryKeys.runs.stageEvents("run-1", "verify@2")); + test("stage.retrying invalidates stages, billing, events, graph, detail, and stage events", () => { + expect(queryKeysForRunEvent("run-1", "stage.retrying", "verify@2")).toEqual([ + queryKeys.runs.stages("run-1"), + queryKeys.runs.billing("run-1"), + queryKeys.runs.events("run-1", 1000), + queryKeys.runs.graph("run-1", "LR"), + queryKeys.runs.graph("run-1", "TB"), + queryKeys.runs.detail("run-1"), + queryKeys.runs.stageEvents("run-1", "verify@2"), + ]); }); }); diff --git a/apps/fabro-web/app/lib/run-events.ts b/apps/fabro-web/app/lib/run-events.ts index 1546d1578..63855296e 100644 --- a/apps/fabro-web/app/lib/run-events.ts +++ b/apps/fabro-web/app/lib/run-events.ts @@ -109,6 +109,7 @@ export function queryKeysForRunEvent( if (STAGE_EVENTS.has(event)) { const keys = [ queryKeys.runs.stages(runId), + queryKeys.runs.billing(runId), queryKeys.runs.events(runId, 1000), queryKeys.runs.graph(runId, "LR"), queryKeys.runs.graph(runId, "TB"), diff --git a/apps/fabro-web/app/lib/stage-sidebar.ts b/apps/fabro-web/app/lib/stage-sidebar.ts index e44e5f4a5..84a5de2de 100644 --- a/apps/fabro-web/app/lib/stage-sidebar.ts +++ b/apps/fabro-web/app/lib/stage-sidebar.ts @@ -1,13 +1,22 @@ -import type { PaginatedRunStageList, StageState } from "@qltysh/fabro-api-client"; +import { StageState } from "@qltysh/fabro-api-client"; +import type { PaginatedRunStageList } from "@qltysh/fabro-api-client"; import type { Stage } from "../components/stage-sidebar"; import { isVisibleStage } from "../data/runs"; import { formatDurationSecs } from "./format"; -export const ACTIVE_STAGE_STATES: ReadonlySet = new Set(["running", "retrying"]); +export const ACTIVE_STAGE_STATES: ReadonlySet = new Set([ + StageState.RUNNING, + StageState.RETRYING, +]); +export const IN_FLIGHT_STAGE_STATES: ReadonlySet = new Set([ + StageState.PENDING, + StageState.RUNNING, + StageState.RETRYING, +]); export const SUCCEEDED_STAGE_STATES: ReadonlySet = new Set([ - "succeeded", - "partially_succeeded", + StageState.SUCCEEDED, + StageState.PARTIALLY_SUCCEEDED, ]); /** diff --git a/apps/fabro-web/app/lib/time.ts b/apps/fabro-web/app/lib/time.ts index 1d8a818d4..fb6f9b87f 100644 --- a/apps/fabro-web/app/lib/time.ts +++ b/apps/fabro-web/app/lib/time.ts @@ -1,3 +1,21 @@ +import { useEffect, useState } from "react"; + +/** + * Re-renders the calling component every `intervalMs` milliseconds while + * `active` is true, returning the current `Date.now()` value at each tick. + * Returns the captured value when paused, so renders are stable. + */ +export function useTickingNow(active: boolean, intervalMs = 1000): number { + const [now, setNow] = useState(() => Date.now()); + useEffect(() => { + if (!active) return; + setNow(Date.now()); + const interval = setInterval(() => setNow(Date.now()), intervalMs); + return () => clearInterval(interval); + }, [active, intervalMs]); + return now; +} + function relativeTime(seconds: number, past: boolean): string { if (seconds < 60) return past ? "just now" : "in <1m"; const minutes = Math.floor(seconds / 60); @@ -21,4 +39,3 @@ export function timeAgo(iso: string): string { export function timeUntil(iso: string): string { return relativeTime(Math.floor((new Date(iso).getTime() - Date.now()) / 1000), false); } - diff --git a/apps/fabro-web/app/routes/run-billing.test.tsx b/apps/fabro-web/app/routes/run-billing.test.tsx index ed9e3040c..c35ab52cb 100644 --- a/apps/fabro-web/app/routes/run-billing.test.tsx +++ b/apps/fabro-web/app/routes/run-billing.test.tsx @@ -75,12 +75,14 @@ describe("RunBilling", () => { model: null, billing: zeroBilling(), runtime_secs: 0, + state: "succeeded", }, { stage: { id: "command", name: "command" }, model: null, billing: zeroBilling(), runtime_secs: 61, + state: "succeeded", }, ], totals: { @@ -96,7 +98,7 @@ describe("RunBilling", () => { expect(text).toMatch(/—\s*\/\s*—/); expect(text).toContain("1m 1s"); expect(text).not.toContain("By model"); - expect(text).not.toContain("No completed stages yet"); + expect(text).not.toContain("No stages yet"); }); test("renders mixed LLM and non-LLM rows while counting only LLM rows by model", () => { @@ -108,6 +110,7 @@ describe("RunBilling", () => { model: null, billing: zeroBilling(), runtime_secs: 0, + state: "succeeded", }, { stage: { id: "agent", name: "agent" }, @@ -119,6 +122,7 @@ describe("RunBilling", () => { total_usd_micros: 240000, }), runtime_secs: 42, + state: "succeeded", }, ], totals: { @@ -155,11 +159,61 @@ describe("RunBilling", () => { expect(textFromInstance(byModelFooterCells[1])).toBe("1"); }); - test("keeps the empty state for runs with no completed stages", () => { + test("keeps the empty state for runs with no stages", () => { const renderer = renderBilling(billing()); const text = textFromNode(renderer.toJSON()); - expect(text).toContain("No completed stages yet"); - expect(text).toContain("Stages will appear once the run produces completed nodes."); + expect(text).toContain("No stages yet"); + expect(text).toContain("Stages will appear as soon as the run starts executing."); + }); + + test("renders an in-flight row with live runtime and includes its elapsed time in the footer", () => { + const originalNow = Date.now; + // Pin "now" to 30s after the in-flight row started. + const startedAt = "2026-04-29T12:00:00.000Z"; + const fakeNow = new Date("2026-04-29T12:00:30.000Z").getTime(); + Date.now = () => fakeNow; + + try { + const renderer = renderBilling( + billing({ + stages: [ + { + stage: { id: "in-flight", name: "in-flight" }, + model: null, + // Server reports 0 runtime / no billing; the row is still being executed. + billing: zeroBilling(), + runtime_secs: 0, + started_at: startedAt, + state: "running", + }, + ], + // Server total is 0 because the in-flight row hasn't been finalized. + totals: { + runtime_secs: 0, + ...zeroBilling(), + }, + }), + ); + + const text = textFromNode(renderer.toJSON()); + // Empty-state must NOT show — the table should appear as soon as the + // first stage starts. + expect(text).not.toContain("No stages yet"); + expect(text).toContain("in-flight"); + + // Both the row's runtime cell and the footer total should reflect + // ~30s elapsed since started_at. + expect(text).toContain("30s"); + + const footers = renderer.root.findAll((node) => node.type === "tfoot"); + const footerCells = footers[0].findAll((node) => node.type === "td"); + // The Run time column in the footer is index 3 (Total / [empty Model] / + // Tokens / Run time / Billing). + const footerRuntime = textFromInstance(footerCells[3]); + expect(footerRuntime).toContain("30s"); + } finally { + Date.now = originalNow; + } }); }); diff --git a/apps/fabro-web/app/routes/run-billing.tsx b/apps/fabro-web/app/routes/run-billing.tsx index 6bb6ab763..be0d52386 100644 --- a/apps/fabro-web/app/routes/run-billing.tsx +++ b/apps/fabro-web/app/routes/run-billing.tsx @@ -1,7 +1,11 @@ +import { useMemo } from "react"; + import { EmptyState } from "../components/state"; import { formatDurationSecs } from "../lib/format"; import { useRunBilling } from "../lib/queries"; -import type { RunBilling } from "@qltysh/fabro-api-client"; +import { IN_FLIGHT_STAGE_STATES } from "../lib/stage-sidebar"; +import { useTickingNow } from "../lib/time"; +import type { RunBilling, RunBillingStage } from "@qltysh/fabro-api-client"; const EMPTY_VALUE = "—"; @@ -14,78 +18,103 @@ function formatUsdMicros(usdMicros?: number | null) { return usdMicros == null ? EMPTY_VALUE : `$${(usdMicros / 1_000_000).toFixed(2)}`; } -function mapBilling(billing: RunBilling | undefined) { - if (!billing) { - return { - stages: [], - totalRuntime: formatDurationSecs(0), - totalUsdMicros: undefined, - totalInput: null, - totalOutput: null, - modelBreakdown: [], - modelStageCount: 0, - }; - } +function isInFlight(stage: RunBillingStage): boolean { + return stage.state != null && IN_FLIGHT_STAGE_STATES.has(stage.state); +} - const stages = billing.stages.map((stage) => { - const hasModel = stage.model != null; - return { - stage: stage.stage.name, - model: stage.model?.id ?? null, - inputTokens: hasModel ? stage.billing.input_tokens : null, - outputTokens: hasModel - ? stage.billing.output_tokens + stage.billing.reasoning_tokens - : null, - runtime: formatDurationSecs(stage.runtime_secs), - totalUsdMicros: stage.billing.total_usd_micros, - }; - }); - const totalRuntime = formatDurationSecs(billing.totals.runtime_secs); - const hasLlmStages = billing.by_model.length > 0; - const totalInput = hasLlmStages ? billing.totals.input_tokens : null; - const totalOutput = hasLlmStages - ? billing.totals.output_tokens + billing.totals.reasoning_tokens - : null; - const totalUsdMicros = billing.totals.total_usd_micros; - const modelBreakdown = billing.by_model - .map((entry) => ({ - model: entry.model.id, - stages: entry.stages, - inputTokens: entry.billing.input_tokens, - outputTokens: entry.billing.output_tokens + entry.billing.reasoning_tokens, - totalUsdMicros: entry.billing.total_usd_micros, - })) - .sort((a, b) => (b.totalUsdMicros ?? -1) - (a.totalUsdMicros ?? -1)); - const modelStageCount = modelBreakdown.reduce((sum, row) => sum + row.stages, 0); +interface MappedStageRow { + stage: string; + model: string | null; + inputTokens: number | null; + outputTokens: number | null; + runtimeSecs: number; + totalUsdMicros: number | null | undefined; +} + +function liveRuntimeSecs(stage: RunBillingStage, now: number): number { + if (stage.started_at) { + const startedMs = new Date(stage.started_at).getTime(); + if (Number.isFinite(startedMs)) { + return Math.max(0, (now - startedMs) / 1000); + } + } + return stage.runtime_secs; +} + +function mapStageRow(stage: RunBillingStage, runtimeSecs: number): MappedStageRow { + const hasModel = stage.model != null; return { - stages, - totalRuntime, - totalUsdMicros, - totalInput, - totalOutput, - modelBreakdown, - modelStageCount, + stage: stage.stage.name, + model: stage.model?.id ?? null, + inputTokens: hasModel ? stage.billing.input_tokens : null, + outputTokens: hasModel + ? stage.billing.output_tokens + stage.billing.reasoning_tokens + : null, + runtimeSecs, + totalUsdMicros: stage.billing.total_usd_micros, }; } export default function RunBilling({ params }: { params: { id: string } }) { const billingQuery = useRunBilling(params.id); - const { - stages, - totalRuntime, - totalUsdMicros, - totalInput, - totalOutput, - modelBreakdown, - modelStageCount, - } = mapBilling(billingQuery.data); + const billing = billingQuery.data; + const hasInFlight = billing?.stages.some(isInFlight) ?? false; - if (!stages.length) { + // Tick once per second only while a stage is in-flight. + const now = useTickingNow(hasInFlight); + + // Completed rows don't depend on `now`; memoize them by `billing` so we + // don't reallocate them every tick. + const completedRows = useMemo(() => { + if (!billing) return []; + return billing.stages.map((stage) => mapStageRow(stage, stage.runtime_secs)); + }, [billing]); + + // The model breakdown is server-derived and stable across ticks too. + const modelBreakdown = useMemo(() => { + if (!billing) return []; + return billing.by_model + .map((entry) => ({ + model: entry.model.id, + stages: entry.stages, + inputTokens: entry.billing.input_tokens, + outputTokens: entry.billing.output_tokens + entry.billing.reasoning_tokens, + totalUsdMicros: entry.billing.total_usd_micros, + })) + .sort((a, b) => (b.totalUsdMicros ?? -1) - (a.totalUsdMicros ?? -1)); + }, [billing]); + + // Re-derive only the in-flight rows on each tick; everything else stays put. + const rows = useMemo(() => { + if (!billing) return []; + if (!hasInFlight) return completedRows; + return billing.stages.map((stage, idx) => + isInFlight(stage) + ? mapStageRow(stage, liveRuntimeSecs(stage, now)) + : completedRows[idx], + ); + }, [billing, completedRows, hasInFlight, now]); + + // While ticking, sum the displayed row runtimes so the footer updates in + // lock-step. Otherwise trust the server's authoritative total. + const totalRuntimeSecs = hasInFlight + ? rows.reduce((sum, row) => sum + row.runtimeSecs, 0) + : (billing?.totals.runtime_secs ?? 0); + + const hasLlmStages = (billing?.by_model.length ?? 0) > 0; + const totalInput = hasLlmStages ? (billing?.totals.input_tokens ?? null) : null; + const totalOutput = hasLlmStages && billing + ? billing.totals.output_tokens + billing.totals.reasoning_tokens + : null; + const totalUsdMicros = billing?.totals.total_usd_micros; + const modelStageCount = modelBreakdown.reduce((sum, row) => sum + row.stages, 0); + + if (!rows.length) { return (
); @@ -105,7 +134,7 @@ export default function RunBilling({ params }: { params: { id: string } }) { - {stages.map((row) => ( + {rows.map((row) => ( {row.stage} @@ -115,7 +144,9 @@ export default function RunBilling({ params }: { params: { id: string } }) { {formatTokens(row.inputTokens)} /{" "} {formatTokens(row.outputTokens)} - {row.runtime} + + {formatDurationSecs(row.runtimeSecs)} + {formatUsdMicros(row.totalUsdMicros)} @@ -131,7 +162,7 @@ export default function RunBilling({ params }: { params: { id: string } }) { {formatTokens(totalOutput)} - {totalRuntime} + {formatDurationSecs(totalRuntimeSecs)} {formatUsdMicros(totalUsdMicros)} diff --git a/apps/fabro-web/app/routes/run-stages.tsx b/apps/fabro-web/app/routes/run-stages.tsx index a8fc69168..2b01fc5f3 100644 --- a/apps/fabro-web/app/routes/run-stages.tsx +++ b/apps/fabro-web/app/routes/run-stages.tsx @@ -40,6 +40,7 @@ import type { Stage } from "../components/stage-sidebar"; import { EmptyState } from "../components/state"; import { CopyButton } from "../components/ui"; import { formatDurationSecs } from "../lib/format"; +import { useTickingNow } from "../lib/time"; import { fetchRunCommandLog, useRunStageEvents, useRunStages } from "../lib/queries"; import { STAGE_ACTIVITY_EVENT_TYPES, type StageActivityEventType } from "../lib/run-events"; import { ACTIVE_STAGE_STATES, formatStageLabel, mapRunStagesToSidebarStages } from "../lib/stage-sidebar"; @@ -563,7 +564,6 @@ function RunningStageDuration({ const [startedAt, setStartedAt] = useState(() => isRunning ? Date.now() : null, ); - const [, setTick] = useState(0); useEffect(() => { setStartedAt((current) => { @@ -572,14 +572,10 @@ function RunningStageDuration({ }); }, [isRunning]); - useEffect(() => { - if (!isRunning) return; - const interval = setInterval(() => setTick((tick) => tick + 1), 1000); - return () => clearInterval(interval); - }, [isRunning]); + const now = useTickingNow(isRunning); if (isRunning && startedAt) { - return formatDurationSecs(Math.floor((Date.now() - startedAt) / 1000)); + return formatDurationSecs(Math.floor((now - startedAt) / 1000)); } return duration; } diff --git a/docs/public/api-reference/fabro-api.yaml b/docs/public/api-reference/fabro-api.yaml index 4319f6774..0b499e7fa 100644 --- a/docs/public/api-reference/fabro-api.yaml +++ b/docs/public/api-reference/fabro-api.yaml @@ -5314,6 +5314,20 @@ components: oneOf: - $ref: "#/components/schemas/CommandTermination" - type: "null" + started_at: + type: ["string", "null"] + format: date-time + description: Wall-clock time the latest attempt of this stage started, if known. + duration_ms: + type: ["integer", "null"] + format: uint64 + minimum: 0 + description: Wall-clock duration of the stage's latest terminal attempt, if known. + state: + oneOf: + - $ref: "#/components/schemas/StageState" + - type: "null" + description: Lifecycle state of the stage projection. InterviewOption: description: Option stored with an interview question in the event log. @@ -6333,6 +6347,11 @@ components: minimum: 1 description: 1-based visit count; bumped each time the workflow re-enters this node. example: 2 + started_at: + type: ["string", "null"] + format: date-time + description: Wall-clock time the latest attempt of this stage started, if known. + example: "2026-04-29T12:34:56Z" # ── File Diff Schemas ────────────────────────────────────────────── @@ -6526,6 +6545,16 @@ components: type: number description: Wall-clock runtime in seconds, summed across every visit of this node. example: 154.0 + started_at: + type: ["string", "null"] + format: date-time + description: Wall-clock time the latest attempt of this stage started, if known. + example: "2026-04-29T12:34:56Z" + state: + oneOf: + - $ref: "#/components/schemas/StageState" + - type: "null" + description: Lifecycle state of the stage. Use to detect in-flight rows for client-side runtime ticking. RunBillingTotals: description: Aggregate billing totals across all stages of a run. diff --git a/lib/crates/fabro-api/tests/run_billing_stage_round_trip.rs b/lib/crates/fabro-api/tests/run_billing_stage_round_trip.rs index c160a840a..52fb4ece1 100644 --- a/lib/crates/fabro-api/tests/run_billing_stage_round_trip.rs +++ b/lib/crates/fabro-api/tests/run_billing_stage_round_trip.rs @@ -1,4 +1,5 @@ use fabro_api::types::RunBillingStage; +use fabro_types::StageState; use serde_json::json; #[test] @@ -28,3 +29,59 @@ fn run_billing_stage_model_accepts_required_null() { assert!(encoded.get("model").is_some()); assert!(encoded["model"].is_null()); } + +#[test] +fn run_billing_stage_round_trips_terminal_row_with_started_at_and_state() { + let value = json!({ + "stage": { + "id": "build", + "name": "build" + }, + "model": { "id": "claude-sonnet-4-5" }, + "billing": { + "input_tokens": 12, + "output_tokens": 34, + "total_tokens": 46, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "runtime_secs": 5.5, + "started_at": "2026-04-29T12:34:56Z", + "state": "succeeded" + }); + + let stage: RunBillingStage = + serde_json::from_value(value.clone()).expect("terminal stage row should deserialize"); + assert!(stage.started_at.is_some()); + assert_eq!(stage.state, Some(StageState::Succeeded)); + assert_eq!(serde_json::to_value(stage).unwrap(), value); +} + +#[test] +fn run_billing_stage_round_trips_in_flight_row() { + let value = json!({ + "stage": { + "id": "build", + "name": "build" + }, + "model": null, + "billing": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "runtime_secs": 1.25, + "started_at": "2026-04-29T12:34:56Z", + "state": "running" + }); + + let stage: RunBillingStage = + serde_json::from_value(value.clone()).expect("in-flight stage row should deserialize"); + assert!(stage.model.is_none()); + assert_eq!(stage.state, Some(StageState::Running)); + assert_eq!(serde_json::to_value(stage).unwrap(), value); +} diff --git a/lib/crates/fabro-api/tests/stage_projection_round_trip.rs b/lib/crates/fabro-api/tests/stage_projection_round_trip.rs index 0523197cf..ad4dcb579 100644 --- a/lib/crates/fabro-api/tests/stage_projection_round_trip.rs +++ b/lib/crates/fabro-api/tests/stage_projection_round_trip.rs @@ -28,7 +28,10 @@ fn stage_projection_round_trips_representative_json() { "parallel_results": [{ "branch": 0, "status": "succeeded" }], "stdout": "ok", "stderr": "", - "termination": "exited" + "termination": "exited", + "started_at": "2026-04-29T12:34:00Z", + "duration_ms": 56000, + "state": "succeeded" }); let state: StageProjection = serde_json::from_value(value.clone()).unwrap(); diff --git a/lib/crates/fabro-server/src/demo/mod.rs b/lib/crates/fabro-server/src/demo/mod.rs index c900c8b91..539731c82 100644 --- a/lib/crates/fabro-server/src/demo/mod.rs +++ b/lib/crates/fabro-server/src/demo/mod.rs @@ -1214,30 +1214,35 @@ mod runs { "Detect Drift", StageState::Succeeded, Some(72.0), + None, ), run_stage_from_stage_id( &StageId::new("propose-changes", 1), "Propose Changes", StageState::Succeeded, Some(154.0), + None, ), run_stage_from_stage_id( &StageId::new("review-changes", 1), "Review Changes", StageState::Succeeded, Some(45.0), + None, ), run_stage_from_stage_id( &StageId::new("apply-changes", 1), "Apply Changes", StageState::Succeeded, Some(118.0), + None, ), run_stage_from_stage_id( &StageId::new("apply-changes", 2), "Apply Changes", StageState::Running, None, + None, ), ] } @@ -1374,6 +1379,8 @@ mod runs { total_usd_micros: Some(480_000), }, runtime_secs: 72.0, + started_at: None, + state: Some(StageState::Succeeded), }, RunBillingStage { stage: BillingStageRef { @@ -1393,6 +1400,8 @@ mod runs { total_usd_micros: Some(720_000), }, runtime_secs: 154.0, + started_at: None, + state: Some(StageState::Succeeded), }, RunBillingStage { stage: BillingStageRef { @@ -1412,6 +1421,8 @@ mod runs { total_usd_micros: Some(190_000), }, runtime_secs: 45.0, + started_at: None, + state: Some(StageState::Succeeded), }, RunBillingStage { stage: BillingStageRef { @@ -1431,6 +1442,8 @@ mod runs { total_usd_micros: Some(870_000), }, runtime_secs: 118.0, + started_at: None, + state: Some(StageState::Running), }, ], totals: RunBillingTotals { diff --git a/lib/crates/fabro-server/src/server.rs b/lib/crates/fabro-server/src/server.rs index 965989c8c..276374d2f 100644 --- a/lib/crates/fabro-server/src/server.rs +++ b/lib/crates/fabro-server/src/server.rs @@ -570,6 +570,7 @@ pub(crate) fn run_stage_from_stage_id( name: impl Into, status: StageState, duration_secs: Option, + started_at: Option>, ) -> RunStage { RunStage { id: stage_id.to_string(), @@ -579,6 +580,7 @@ pub(crate) fn run_stage_from_stage_id( node_id: stage_id.node_id().to_string(), visit: std::num::NonZeroU32::new(stage_id.visit()) .expect("StageId stores a non-zero visit"), + started_at, } } diff --git a/lib/crates/fabro-server/src/server/handler/billing.rs b/lib/crates/fabro-server/src/server/handler/billing.rs index 193ebd78d..5c000869c 100644 --- a/lib/crates/fabro-server/src/server/handler/billing.rs +++ b/lib/crates/fabro-server/src/server/handler/billing.rs @@ -1,13 +1,14 @@ +use std::collections::HashMap; use std::sync::Arc; -use fabro_store::RunProjectionReducer; -use fabro_types::{EventBody, RunProjection, StageId}; +use chrono::{DateTime, Utc}; +use fabro_types::{RunProjection, StageProjection, StageState}; use super::super::{ - ApiError, AppState, BillingByModel, BillingStageRef, EventEnvelope, HashMap, IntoResponse, - Json, ListResponse, ModelReference, PaginationParams, Path, Query, RequiredUser, Response, - Router, RunBilling, RunBillingStage, RunBillingTotals, RunId, StageState, State, StatusCode, - get, parse_run_id_path, run_stage_from_stage_id, + ApiError, AppState, BillingByModel, BillingStageRef, IntoResponse, Json, ListResponse, + ModelReference, PaginationParams, Path, Query, RequiredUser, Response, Router, RunBilling, + RunBillingStage, RunBillingTotals, RunId, State, StatusCode, get, parse_run_id_path, + run_stage_from_stage_id, }; pub(super) fn routes() -> Router> { @@ -16,41 +17,6 @@ pub(super) fn routes() -> Router> { .route("/runs/{id}/billing", get(get_run_billing)) } -/// Map a `stage.*` lifecycle event body to the [`StageState`] it implies. -/// Returns `None` for any other variant. -fn stage_state_from_lifecycle(body: &EventBody) -> Option { - match body { - EventBody::StageStarted(_) => Some(StageState::Running), - EventBody::StageRetrying(_) => Some(StageState::Retrying), - EventBody::StageFailed(props) => Some(if props.will_retry { - StageState::Retrying - } else { - StageState::Failed - }), - EventBody::StageCompleted(props) => Some(StageState::from(props.status)), - _ => None, - } -} - -/// Single-pass scan over `events` building the latest [`StageState`] for each -/// [`StageId`] from lifecycle events (started/retrying/completed/failed). Each -/// later lifecycle event overwrites earlier ones, leaving the latest as the -/// stored value — equivalent to "scan in reverse, take first match" but in O(E) -/// for the whole list rather than O(stages × events). -fn latest_stage_states(events: &[EventEnvelope]) -> HashMap { - let mut states = HashMap::new(); - for envelope in events { - let Some(stage_id) = envelope.event.stage_id.as_ref() else { - continue; - }; - let Some(state) = stage_state_from_lifecycle(&envelope.event.body) else { - continue; - }; - states.insert(stage_id.clone(), state); - } - states -} - async fn list_run_stages( _auth: RequiredUser, State(state): State>, @@ -62,42 +28,30 @@ async fn list_run_stages( Err(response) => return response, }; - let events = match state.store.open_run_reader(&id).await { - Ok(run_store) => run_store.list_events().await.unwrap_or_default(), - Err(_) => return ApiError::not_found("Run not found.").into_response(), + let Ok(run_store) = state.store.open_run_reader(&id).await else { + return ApiError::not_found("Run not found.").into_response(); }; - - let projection = match RunProjection::apply_events(&events) { - Ok(projection) => projection, + let projection = match run_store.state().await { + Ok(state) => state, Err(err) => { - tracing::warn!( - run_id = %id, - error = %err, - "Failed to build run projection; returning empty stages list", - ); - RunProjection::default() + return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) + .into_response(); } }; - let stage_durations = fabro_workflow::extract_stage_durations_by_stage_id(&events); - let lifecycle_states = latest_stage_states(&events); - let mut stages = Vec::new(); - for (stage_id, stage_projection) in projection.iter_stages() { - // Prefer the latest lifecycle event; fall back to the projection's - // stored completion (e.g. for runs recovered from snapshot only). - let status = lifecycle_states.get(stage_id).copied().unwrap_or_else(|| { - stage_projection - .completion - .as_ref() - .map_or(StageState::Pending, |c| StageState::from(c.outcome)) - }); - stages.push(run_stage_from_stage_id( - stage_id, - stage_id.node_id().to_string(), - status, - stage_durations.get(stage_id).map(|ms| *ms as f64 / 1000.0), - )); - } + let now = Utc::now(); + let stages = projection + .iter_stages() + .map(|(stage_id, stage)| { + run_stage_from_stage_id( + stage_id, + stage_id.node_id().to_string(), + stage.effective_state(), + stage.runtime_secs(now), + stage.started_at, + ) + }) + .collect::>(); (StatusCode::OK, Json(ListResponse::new(stages))).into_response() } @@ -121,6 +75,7 @@ async fn get_run_billing( .into_response(); } }; + let rollup = fabro_workflow::billing_rollup_from_projection(&projection); let by_model = rollup .by_model @@ -133,20 +88,33 @@ async fn get_run_billing( stages: model.stages, }) .collect::>(); - let stages = rollup + + let rollup_by_node = rollup .stages .iter() - .map(|stage| RunBillingStage { - billing: stage.billing.clone(), - model: stage - .model_id - .as_ref() - .map(|id| ModelReference { id: id.clone() }), - runtime_secs: stage.duration_ms as f64 / 1000.0, - stage: BillingStageRef { - id: stage.node_id.clone(), - name: stage.node_id.clone(), - }, + .map(|stage| (stage.node_id.as_str(), stage)) + .collect::>(); + let live_rows = live_billing_rows(&projection, Utc::now()); + let runtime_secs = live_rows.iter().map(|row| row.runtime_secs).sum::(); + let stages = live_rows + .into_iter() + .map(|row| { + let rollup_stage = rollup_by_node.get(row.node_id.as_str()); + RunBillingStage { + billing: rollup_stage + .map(|stage| stage.billing.clone()) + .unwrap_or_default(), + model: rollup_stage + .and_then(|stage| stage.model_id.as_ref()) + .map(|id| ModelReference { id: id.clone() }), + runtime_secs: row.runtime_secs, + stage: BillingStageRef { + id: row.node_id.clone(), + name: row.node_id, + }, + started_at: row.started_at, + state: row.state, + } }) .collect::>(); @@ -154,16 +122,80 @@ async fn get_run_billing( by_model, stages, totals: RunBillingTotals { - cache_read_tokens: rollup.totals.cache_read_tokens, + cache_read_tokens: rollup.totals.cache_read_tokens, cache_write_tokens: rollup.totals.cache_write_tokens, - input_tokens: rollup.totals.input_tokens, - output_tokens: rollup.totals.output_tokens, - reasoning_tokens: rollup.totals.reasoning_tokens, - runtime_secs: rollup.runtime_ms as f64 / 1000.0, - total_tokens: rollup.totals.total_tokens, - total_usd_micros: rollup.totals.total_usd_micros, + input_tokens: rollup.totals.input_tokens, + output_tokens: rollup.totals.output_tokens, + reasoning_tokens: rollup.totals.reasoning_tokens, + runtime_secs, + total_tokens: rollup.totals.total_tokens, + total_usd_micros: rollup.totals.total_usd_micros, }, }; (StatusCode::OK, Json(response)).into_response() } + +struct LiveBillingRow { + node_id: String, + runtime_secs: f64, + started_at: Option>, + state: Option, + latest_visit: u32, +} + +fn live_billing_rows(projection: &RunProjection, now: DateTime) -> Vec { + let mut row_indices = HashMap::::new(); + let mut rows = Vec::::new(); + + for (stage_id, stage) in projection.iter_stages() { + let node_id = stage_id.node_id(); + if is_exit_stage(projection, node_id) || !stage_has_billing_row(stage) { + continue; + } + + let index = *row_indices.entry(node_id.to_string()).or_insert_with(|| { + let index = rows.len(); + rows.push(LiveBillingRow { + node_id: node_id.to_string(), + runtime_secs: 0.0, + started_at: None, + state: None, + latest_visit: 0, + }); + index + }); + let row = &mut rows[index]; + row.runtime_secs += billing_runtime_secs(stage, now).unwrap_or(0.0); + + if stage_id.visit() >= row.latest_visit { + row.latest_visit = stage_id.visit(); + row.started_at = stage.started_at; + row.state = Some(stage.effective_state()); + } + } + + rows +} + +fn billing_runtime_secs(stage: &StageProjection, now: DateTime) -> Option { + stage + .duration_ms + .map(|ms| ms as f64 / 1000.0) + .or_else(|| stage.runtime_secs(now)) +} + +fn stage_has_billing_row(stage: &StageProjection) -> bool { + stage.completion.is_some() + || stage.duration_ms.is_some() + || stage.usage.is_some() + || stage.started_at.is_some() + || stage.state.is_some() +} + +fn is_exit_stage(projection: &RunProjection, node_id: &str) -> bool { + projection + .spec() + .and_then(|spec| spec.graph().nodes.get(node_id)) + .is_some_and(|node| node.handler_type() == Some("exit")) +} diff --git a/lib/crates/fabro-server/src/server/tests.rs b/lib/crates/fabro-server/src/server/tests.rs index 08196dbdd..aadc30c8e 100644 --- a/lib/crates/fabro-server/src/server/tests.rs +++ b/lib/crates/fabro-server/src/server/tests.rs @@ -2878,6 +2878,198 @@ async fn list_run_stages_shows_retrying_when_failed_will_retry() { assert_eq!(stage_status(&body, "work@1"), "retrying"); } +#[tokio::test] +async fn run_billing_retried_node_then_succeeded_emits_one_row_with_final_attempt_duration() { + let state = test_app_state_with_isolated_storage(); + let app = crate::test_support::build_test_router(Arc::clone(&state)); + let run_id = RunId::new(); + + create_durable_run_with_events(&state, run_id, &[ + workflow_event::Event::RunSubmitted { + definition_blob: None, + }, + workflow_event::Event::RunStarting, + workflow_event::Event::RunRunning, + workflow_event::Event::StageStarted { + node_id: "work".to_string(), + name: "Work".to_string(), + index: 0, + handler_type: "command".to_string(), + attempt: 1, + max_attempts: 3, + }, + workflow_event::Event::StageFailed { + node_id: "work".to_string(), + name: "Work".to_string(), + index: 0, + failure: FailureDetail::new("transient", FailureCategory::TransientInfra), + will_retry: true, + duration_ms: 10, + billing: None, + actor: None, + }, + workflow_event::Event::StageRetrying { + node_id: "work".to_string(), + name: "Work".to_string(), + index: 0, + attempt: 2, + max_attempts: 3, + delay_ms: 0, + }, + workflow_event::Event::StageStarted { + node_id: "work".to_string(), + name: "Work".to_string(), + index: 0, + handler_type: "command".to_string(), + attempt: 2, + max_attempts: 3, + }, + workflow_event::Event::StageCompleted { + node_id: "work".to_string(), + name: "Work".to_string(), + index: 0, + duration_ms: 25, + status: "succeeded".to_string(), + preferred_label: None, + suggested_next_ids: Vec::new(), + billing: None, + failure: None, + notes: None, + files_touched: Vec::new(), + context_updates: None, + jump_to_node: None, + context_values: None, + node_visits: None, + loop_failure_signatures: None, + restart_failure_signatures: None, + response: None, + attempt: 2, + max_attempts: 3, + }, + ]) + .await; + + let response = app + .oneshot( + Request::builder() + .method("GET") + .uri(api(&format!("/runs/{run_id}/billing"))) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let body = response_json!(response, StatusCode::OK).await; + let stages = body["stages"].as_array().unwrap(); + assert_eq!(stages.len(), 1, "retry collapses to one row per node_id"); + let row = &stages[0]; + assert_eq!(row["stage"]["id"], "work"); + assert_eq!( + row["state"], "succeeded", + "final state mirrors the latest StageCompleted" + ); + let runtime = row["runtime_secs"].as_f64().unwrap(); + assert!( + (runtime - 0.025).abs() < f64::EPSILON, + "runtime should equal final attempt's 25ms, got {runtime}" + ); +} + +fn revisit_test_started(node_id: &str) -> workflow_event::Event { + workflow_event::Event::StageStarted { + node_id: node_id.to_string(), + name: node_id.to_string(), + index: 0, + handler_type: "command".to_string(), + attempt: 1, + max_attempts: 1, + } +} + +fn revisit_test_completed_with_visit( + node_id: &str, + duration_ms: u64, + visit: usize, +) -> workflow_event::Event { + let mut node_visits = std::collections::BTreeMap::new(); + node_visits.insert(node_id.to_string(), visit); + workflow_event::Event::StageCompleted { + node_id: node_id.to_string(), + name: node_id.to_string(), + index: 0, + duration_ms, + status: "succeeded".to_string(), + preferred_label: None, + suggested_next_ids: Vec::new(), + billing: None, + failure: None, + notes: None, + files_touched: Vec::new(), + context_updates: None, + jump_to_node: None, + context_values: None, + node_visits: Some(node_visits), + loop_failure_signatures: None, + restart_failure_signatures: None, + response: None, + attempt: 1, + max_attempts: 1, + } +} + +#[tokio::test] +async fn run_billing_revisited_node_collapses_to_two_rows_with_summed_visit_duration() { + let state = test_app_state_with_isolated_storage(); + let app = crate::test_support::build_test_router(Arc::clone(&state)); + let run_id = RunId::new(); + + create_durable_run_with_events(&state, run_id, &[ + workflow_event::Event::RunSubmitted { + definition_blob: None, + }, + workflow_event::Event::RunStarting, + workflow_event::Event::RunRunning, + // A → B → A loop. Per-visit `node_visits` payload steers the reducer + // to attribute each StageCompleted to the right visit. + revisit_test_started("a"), + revisit_test_completed_with_visit("a", 1, 1), + revisit_test_started("b"), + revisit_test_completed_with_visit("b", 2, 1), + revisit_test_started("a"), + revisit_test_completed_with_visit("a", 99, 2), + ]) + .await; + + let response = app + .oneshot( + Request::builder() + .method("GET") + .uri(api(&format!("/runs/{run_id}/billing"))) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let body = response_json!(response, StatusCode::OK).await; + let stages = body["stages"].as_array().unwrap(); + assert_eq!(stages.len(), 2, "two distinct node_ids → two rows"); + assert_eq!( + stages[0]["stage"]["id"], "a", + "A appeared first → A's row first" + ); + assert_eq!(stages[1]["stage"]["id"], "b"); + let a_runtime = stages[0]["runtime_secs"].as_f64().unwrap(); + assert!( + (a_runtime - 0.1).abs() < f64::EPSILON, + "A should sum both visit durations (1ms + 99ms), got {a_runtime}" + ); + let b_runtime = stages[1]["runtime_secs"].as_f64().unwrap(); + assert!( + (b_runtime - 0.002).abs() < f64::EPSILON, + "B should carry its single visit's duration (2ms), got {b_runtime}" + ); +} + async fn append_raw_run_event( state: &Arc, run_id: RunId, diff --git a/lib/crates/fabro-store/src/run_state.rs b/lib/crates/fabro-store/src/run_state.rs index af467f557..ae23ff9fd 100644 --- a/lib/crates/fabro-store/src/run_state.rs +++ b/lib/crates/fabro-store/src/run_state.rs @@ -10,7 +10,7 @@ use fabro_types::{ BilledModelUsage, Checkpoint, Conclusion, EventBody, FailureSignature, InterviewQuestionRecord, Outcome, PendingInterviewRecord, PullRequestRecord, RunControlAction, RunEvent, RunId, RunProjection, RunSpec, RunStatus, RunSummary, SandboxRecord, StageCompletion, StageId, - StageOutcome, StageProjection, StartRecord, TerminalStatus, first_event_seq, + StageOutcome, StageProjection, StageState, StartRecord, TerminalStatus, first_event_seq, }; use fabro_util::error::render_with_causes; use serde_json::Value; @@ -290,11 +290,18 @@ impl RunProjectionReducer for RunProjection { let Some(stage_id) = stored.stage_id.as_ref() else { return Ok(()); }; - self.stage_entry( + let stage = self.stage_entry( stage_id.node_id(), stage_id.visit(), first_event_seq(event.seq), ); + stage.begin_attempt(ts); + } + EventBody::StageRetrying(_) => { + let Some(stage) = stage_at_stored_or_current_visit(self, stored, event.seq) else { + return Ok(()); + }; + stage.state = Some(StageState::Retrying); } EventBody::StagePrompt(props) => { let Some(stage) = stage_at_stored_or_visit(self, stored, props.visit, event.seq) @@ -323,22 +330,25 @@ impl RunProjectionReducer for RunProjection { stage.completion = Some(completion); stage.duration_ms = Some(props.duration_ms); stage.usage.clone_from(&props.billing); + stage.state = Some(StageState::from(outcome.status)); } EventBody::StageFailed(props) => { let failure_reason = props.failure.as_ref().map(|detail| detail.message.clone()); let Some(stage) = stage_at_stored_or_current_visit(self, stored, event.seq) else { return Ok(()); }; + let outcome = StageOutcome::Failed { + retry_requested: props.will_retry, + }; stage.completion = Some(StageCompletion { - outcome: StageOutcome::Failed { - retry_requested: props.will_retry, - }, + outcome, notes: None, failure_reason, timestamp: ts, }); stage.duration_ms = Some(props.duration_ms); stage.usage.clone_from(&props.billing); + stage.state = Some(StageState::from(outcome)); } EventBody::AgentSessionStarted(props) => { let Some(stage) = stage_at_stored_or_visit(self, stored, props.visit, event.seq) @@ -670,12 +680,13 @@ mod tests { use fabro_types::run_event::{ CheckpointCompletedProps, InterviewCompletedProps, InterviewOption, InterviewStartedProps, RunControlEffectProps, StageCompletedProps, StageFailedProps, StagePromptProps, - StageStartedProps, + StageRetryingProps, StageStartedProps, }; use fabro_types::{ - BilledModelUsage, BlockedReason, Checkpoint, EventBody, FailureReason, Outcome, - QuestionType, RunBlobId, RunControlAction, RunEvent, RunStatus, StageOutcome, - SuccessReason, TerminalStatus, WorkflowSettings, first_event_seq, fixtures, + BilledModelUsage, BlockedReason, Checkpoint, EventBody, FailureCategory, FailureDetail, + FailureReason, Outcome, QuestionType, RunBlobId, RunControlAction, RunEvent, RunStatus, + StageOutcome, StageState, SuccessReason, TerminalStatus, WorkflowSettings, first_event_seq, + fixtures, }; use serde_json::json; @@ -1775,4 +1786,224 @@ mod tests { ); assert_eq!(state.status_updated_at, updated_at); } + + fn started_props() -> StageStartedProps { + StageStartedProps { + index: 0, + handler_type: "agent".to_string(), + attempt: 1, + max_attempts: 3, + } + } + + fn failed_props(duration_ms: u64, will_retry: bool) -> StageFailedProps { + StageFailedProps { + index: 0, + failure: Some(FailureDetail::new("boom", FailureCategory::TransientInfra)), + will_retry, + duration_ms, + billing: None, + } + } + + fn retrying_props() -> StageRetryingProps { + StageRetryingProps { + index: 0, + attempt: 2, + max_attempts: 3, + delay_ms: 0, + } + } + + fn completed_props(duration_ms: u64, status: StageOutcome) -> StageCompletedProps { + StageCompletedProps { + index: 0, + duration_ms, + status, + preferred_label: None, + suggested_next_ids: Vec::new(), + billing: None, + failure: None, + notes: None, + files_touched: Vec::new(), + context_updates: None, + jump_to_node: None, + context_values: None, + node_visits: None, + loop_failure_signatures: None, + restart_failure_signatures: None, + response: None, + attempt: 1, + max_attempts: 3, + } + } + + fn billed_usage() -> BilledModelUsage { + serde_json::from_value(json!({ + "input": { + "usage": { + "model": { + "provider": "openai", + "model_id": "gpt-test" + }, + "tokens": { + "input_tokens": 10, + "output_tokens": 5, + "reasoning_tokens": 2, + "cache_read_tokens": 3, + "cache_write_tokens": 4 + } + }, + "facts": { "provider": "open_ai" } + }, + "total_usd_micros": 123 + })) + .expect("billing fixture should deserialize") + } + + #[test] + fn stage_started_records_started_at_and_running_state() { + let mut state = RunProjection::default(); + let stage_id = StageId::new("build", 1); + + state + .apply_event(&test_stage_event( + 3, + EventBody::StageStarted(started_props()), + stage_id.clone(), + )) + .unwrap(); + + let stage = state.stage(&stage_id).unwrap(); + assert_eq!(stage.state, Some(StageState::Running)); + assert!(stage.started_at.is_some()); + assert_eq!(stage.effective_state(), StageState::Running); + } + + #[test] + fn stage_completed_records_duration_usage_and_terminal_state() { + let mut state = RunProjection::default(); + let stage_id = StageId::new("build", 1); + let usage = billed_usage(); + + state + .apply_event(&test_stage_event( + 1, + EventBody::StageStarted(started_props()), + stage_id.clone(), + )) + .unwrap(); + let mut props = completed_props(42, StageOutcome::Succeeded); + props.billing = Some(usage.clone()); + state + .apply_event(&test_event( + 2, + EventBody::StageCompleted(props), + Some("build"), + )) + .unwrap(); + + let stage = state.stage(&stage_id).unwrap(); + assert_eq!(stage.duration_ms, Some(42)); + assert_eq!(stage.usage.as_ref(), Some(&usage)); + assert_eq!(stage.state, Some(StageState::Succeeded)); + assert_eq!(stage.effective_state(), StageState::Succeeded); + } + + #[test] + fn stage_failed_records_duration_and_failed_state() { + let mut state = RunProjection::default(); + let stage_id = StageId::new("build", 1); + + state + .apply_event(&test_stage_event( + 1, + EventBody::StageStarted(started_props()), + stage_id.clone(), + )) + .unwrap(); + state + .apply_event(&test_event( + 2, + EventBody::StageFailed(failed_props(10, false)), + Some("build"), + )) + .unwrap(); + + let stage = state.stage(&stage_id).unwrap(); + assert_eq!(stage.duration_ms, Some(10)); + assert_eq!(stage.state, Some(StageState::Failed)); + } + + #[test] + fn stage_retrying_sets_retrying_state() { + let mut state = RunProjection::default(); + let stage_id = StageId::new("build", 1); + + state + .apply_event(&test_stage_event( + 1, + EventBody::StageStarted(started_props()), + stage_id.clone(), + )) + .unwrap(); + state + .apply_event(&test_event( + 2, + EventBody::StageFailed(failed_props(10, true)), + Some("build"), + )) + .unwrap(); + state + .apply_event(&test_event( + 3, + EventBody::StageRetrying(retrying_props()), + Some("build"), + )) + .unwrap(); + + let stage = state.stage(&stage_id).unwrap(); + assert_eq!(stage.state, Some(StageState::Retrying)); + } + + #[test] + fn stage_started_after_retrying_returns_to_running_and_resets_attempt_data() { + let mut state = RunProjection::default(); + let stage_id = StageId::new("build", 1); + + state + .apply_event(&test_stage_event( + 1, + EventBody::StageStarted(started_props()), + stage_id.clone(), + )) + .unwrap(); + state + .apply_event(&test_event( + 2, + EventBody::StageFailed(failed_props(10, true)), + Some("build"), + )) + .unwrap(); + state + .apply_event(&test_event( + 3, + EventBody::StageRetrying(retrying_props()), + Some("build"), + )) + .unwrap(); + state + .apply_event(&test_stage_event( + 4, + EventBody::StageStarted(started_props()), + stage_id.clone(), + )) + .unwrap(); + + let stage = state.stage(&stage_id).unwrap(); + assert_eq!(stage.state, Some(StageState::Running)); + // Prior attempt's terminal data must not leak into the new attempt. + assert!(stage.completion.is_none()); + assert_eq!(stage.duration_ms, None); + } } diff --git a/lib/crates/fabro-store/tests/serializable_projection.rs b/lib/crates/fabro-store/tests/serializable_projection.rs index 2b24cd54e..8809acda2 100644 --- a/lib/crates/fabro-store/tests/serializable_projection.rs +++ b/lib/crates/fabro-store/tests/serializable_projection.rs @@ -123,6 +123,10 @@ fn serializable_projection_round_trips_and_trims_bulky_node_fields() { let serialized = serde_json::to_value(SerializableProjection(&projection)) .expect("projection should serialize"); + assert!( + serialized["stages"]["build@2"].get("usage").is_none(), + "stage usage is server-internal and should not be serialized" + ); let round_tripped: RunProjection = serde_json::from_value(serialized).expect("serialized projection should deserialize"); let node = round_tripped.stage(&stage_id).expect("node should remain"); @@ -163,7 +167,7 @@ fn serializable_projection_round_trips_and_trims_bulky_node_fields() { Some(json!([{ "stage": "fanout@1" }])) ); assert_eq!(node.duration_ms, Some(1234)); - assert_eq!(node.usage, Some(sample_usage())); + assert_eq!(node.usage, None); } #[test] diff --git a/lib/crates/fabro-types/src/run_projection.rs b/lib/crates/fabro-types/src/run_projection.rs index 87321d348..90da9b59b 100644 --- a/lib/crates/fabro-types/src/run_projection.rs +++ b/lib/crates/fabro-types/src/run_projection.rs @@ -6,7 +6,7 @@ use chrono::{DateTime, Utc}; use crate::{ BilledModelUsage, Checkpoint, Conclusion, InterviewQuestionRecord, InvalidTransition, PullRequestRecord, Retro, RunControlAction, RunId, RunSpec, RunStatus, SandboxRecord, - StageCompletion, StageId, StartRecord, + StageCompletion, StageId, StageState, StartRecord, }; #[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] @@ -44,10 +44,6 @@ pub struct StageProjection { pub prompt: Option, pub response: Option, pub completion: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub duration_ms: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub usage: Option, pub provider_used: Option, pub diff: Option, pub script_invocation: Option, @@ -65,6 +61,17 @@ pub struct StageProjection { pub live_streaming: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub termination: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub started_at: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub duration_ms: Option, + /// Server-internal billing usage for the latest attempt; not part of the + /// wire contract because `BilledModelUsage` is not modeled in OpenAPI. + /// Read only in-process by the billing handler. + #[serde(skip)] + pub usage: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub state: Option, } /// Convert a 1-based event sequence number into the `NonZeroU32` form used for @@ -96,8 +103,55 @@ impl StageProjection { streams_separated: None, live_streaming: None, termination: None, + started_at: None, + state: None, } } + + /// Effective lifecycle state derived from stored event data. + /// + /// Falls back to deriving from `completion` for projections that predate + /// the stored `state` field, so old serialized projections still work + /// without a backfill. + #[must_use] + pub fn effective_state(&self) -> StageState { + self.state.unwrap_or_else(|| match &self.completion { + Some(completion) => StageState::from(completion.outcome), + None => StageState::Running, + }) + } + + /// Live wall-clock runtime in seconds. + /// + /// While the stage is non-terminal (`Pending`, `Running`, or `Retrying`), + /// this returns the elapsed time since `started_at` so the UI can tick + /// client-side. Once terminal, the stored `duration_ms` is returned. This + /// also handles retries safely: a new `StageStarted` resets the state + /// back to `Running` and keeps the live computation correct even if a + /// previous attempt left a stale `duration_ms`. + #[must_use] + pub fn runtime_secs(&self, now: DateTime) -> Option { + let state = self.effective_state(); + if matches!( + state, + StageState::Running | StageState::Retrying | StageState::Pending + ) { + return self.started_at.map(|started| { + now.signed_duration_since(started).num_milliseconds().max(0) as f64 / 1000.0 + }); + } + self.duration_ms.map(|ms| ms as f64 / 1000.0) + } + + /// Begin a new attempt (or visit) for this stage: clear every + /// per-attempt field so prior-attempt data does not leak, then record + /// `started_at` and `state = Running`. Preserves `first_event_seq` + /// (identity / sort key). + pub fn begin_attempt(&mut self, started_at: DateTime) { + *self = Self::new(self.first_event_seq); + self.started_at = Some(started_at); + self.state = Some(StageState::Running); + } } impl RunProjection { diff --git a/lib/packages/fabro-api-client/src/models/board-column-definition.ts b/lib/packages/fabro-api-client/src/models/board-column-definition.ts index 314cbcde1..20deba1f5 100644 --- a/lib/packages/fabro-api-client/src/models/board-column-definition.ts +++ b/lib/packages/fabro-api-client/src/models/board-column-definition.ts @@ -21,6 +21,3 @@ export interface BoardColumnDefinition { 'id': BoardColumn; 'name': string; } - - - diff --git a/lib/packages/fabro-api-client/src/models/run-billing-stage.ts b/lib/packages/fabro-api-client/src/models/run-billing-stage.ts index 5d376b7cc..acdf57c97 100644 --- a/lib/packages/fabro-api-client/src/models/run-billing-stage.ts +++ b/lib/packages/fabro-api-client/src/models/run-billing-stage.ts @@ -22,6 +22,9 @@ import type { BillingStageRef } from './billing-stage-ref'; // May contain unused imports in some cases // @ts-ignore import type { ModelReference } from './model-reference'; +// May contain unused imports in some cases +// @ts-ignore +import type { StageState } from './stage-state'; /** * Token counts and billed totals for one workflow node within a run. Rows are grouped by node; billing and runtime sum every visit of that node. @@ -34,5 +37,9 @@ export interface RunBillingStage { * Wall-clock runtime in seconds, summed across every visit of this node. */ 'runtime_secs': number; + /** + * Wall-clock time the latest attempt of this stage started, if known. + */ + 'started_at'?: string | null; + 'state'?: StageState | null; } - diff --git a/lib/packages/fabro-api-client/src/models/run-stage.ts b/lib/packages/fabro-api-client/src/models/run-stage.ts index ad01b9791..060a6166e 100644 --- a/lib/packages/fabro-api-client/src/models/run-stage.ts +++ b/lib/packages/fabro-api-client/src/models/run-stage.ts @@ -42,7 +42,8 @@ export interface RunStage { * 1-based visit count; bumped each time the workflow re-enters this node. */ 'visit': number; + /** + * Wall-clock time the latest attempt of this stage started, if known. + */ + 'started_at'?: string | null; } - - - diff --git a/lib/packages/fabro-api-client/src/models/stage-projection.ts b/lib/packages/fabro-api-client/src/models/stage-projection.ts index 41321ec78..139fdd437 100644 --- a/lib/packages/fabro-api-client/src/models/stage-projection.ts +++ b/lib/packages/fabro-api-client/src/models/stage-projection.ts @@ -19,6 +19,9 @@ import type { CommandTermination } from './command-termination'; // May contain unused imports in some cases // @ts-ignore import type { StageCompletion } from './stage-completion'; +// May contain unused imports in some cases +// @ts-ignore +import type { StageState } from './stage-state'; /** * Observable projection data for one workflow stage execution. @@ -52,7 +55,13 @@ export interface StageProjection { 'streams_separated'?: boolean | null; 'live_streaming'?: boolean | null; 'termination'?: CommandTermination | null; + /** + * Wall-clock time the latest attempt of this stage started, if known. + */ + 'started_at'?: string | null; + /** + * Wall-clock duration of the stage\'s latest terminal attempt, if known. + */ + 'duration_ms'?: number | null; + 'state'?: StageState | null; } - - - From 5e4035981fbdd05201382a1a6445ce8c38154eb6 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Mon, 4 May 2026 17:08:44 -0400 Subject: [PATCH 16/16] refactor(cli): format auth status timestamps to seconds precision Use to_rfc3339_opts with SecondsFormat::Secs so auth status output shows clean second-precision timestamps instead of nanoseconds. Co-Authored-By: Claude Opus 4.7 (1M context) --- lib/crates/fabro-cli/src/commands/auth/status.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/crates/fabro-cli/src/commands/auth/status.rs b/lib/crates/fabro-cli/src/commands/auth/status.rs index db30eb115..2002fa1cd 100644 --- a/lib/crates/fabro-cli/src/commands/auth/status.rs +++ b/lib/crates/fabro-cli/src/commands/auth/status.rs @@ -1,5 +1,5 @@ use anyhow::Result; -use chrono::{DateTime, Utc}; +use chrono::{DateTime, SecondsFormat, Utc}; use fabro_client::{AuthEntry, AuthStore, OAuthEntry}; use serde::Serialize; @@ -108,12 +108,12 @@ pub(super) fn status_command(args: &AuthStatusArgs, ctx: &CommandContext) -> Res fabro_util::printerr!( printer, " Access expires: {}", - access_token_expires_at.to_rfc3339() + access_token_expires_at.to_rfc3339_opts(SecondsFormat::Secs, true) ); fabro_util::printerr!( printer, " Refresh expires: {}", - refresh_token_expires_at.to_rfc3339() + refresh_token_expires_at.to_rfc3339_opts(SecondsFormat::Secs, true) ); } StatusRow::DevToken { @@ -122,7 +122,11 @@ pub(super) fn status_command(args: &AuthStatusArgs, ctx: &CommandContext) -> Res } => { fabro_util::printerr!(printer, "{server}"); fabro_util::printerr!(printer, " Auth: dev-token"); - fabro_util::printerr!(printer, " Logged in: {}", logged_in_at.to_rfc3339()); + fabro_util::printerr!( + printer, + " Logged in: {}", + logged_in_at.to_rfc3339_opts(SecondsFormat::Secs, true) + ); } } }