diff --git a/Cargo.lock b/Cargo.lock index e4e138ffa..f88b44544 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1536,7 +1536,7 @@ dependencies = [ [[package]] name = "fabro-agent" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "async-trait", @@ -1575,7 +1575,7 @@ dependencies = [ [[package]] name = "fabro-api" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "chrono", "fabro-config", @@ -1596,7 +1596,7 @@ dependencies = [ [[package]] name = "fabro-auth" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "async-trait", @@ -1620,7 +1620,7 @@ dependencies = [ [[package]] name = "fabro-checkpoint" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "chrono", "fabro-config", @@ -1636,7 +1636,7 @@ dependencies = [ [[package]] name = "fabro-cli" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "assert_cmd", @@ -1732,7 +1732,7 @@ dependencies = [ [[package]] name = "fabro-client" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "bytes", @@ -1761,7 +1761,7 @@ dependencies = [ [[package]] name = "fabro-config" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "chrono", @@ -1788,7 +1788,7 @@ dependencies = [ [[package]] name = "fabro-core" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "async-trait", "fabro-types", @@ -1803,7 +1803,7 @@ dependencies = [ [[package]] name = "fabro-dev" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "assert_cmd", @@ -1823,7 +1823,7 @@ dependencies = [ [[package]] name = "fabro-devcontainer" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "fabro-http", "fabro-static", @@ -1840,7 +1840,7 @@ dependencies = [ [[package]] name = "fabro-dump" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "bytes", @@ -1854,7 +1854,7 @@ dependencies = [ [[package]] name = "fabro-github" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "base64", @@ -1876,7 +1876,7 @@ dependencies = [ [[package]] name = "fabro-graphviz" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "fabro-types", @@ -1890,7 +1890,7 @@ dependencies = [ [[package]] name = "fabro-hooks" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "async-trait", "fabro-agent", @@ -1914,7 +1914,7 @@ dependencies = [ [[package]] name = "fabro-http" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "fabro-static", "http", @@ -1924,7 +1924,7 @@ dependencies = [ [[package]] name = "fabro-install" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "base64", @@ -1939,7 +1939,7 @@ dependencies = [ [[package]] name = "fabro-interview" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "async-trait", "dialoguer", @@ -1954,7 +1954,7 @@ dependencies = [ [[package]] name = "fabro-llm" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "async-trait", @@ -1986,7 +1986,7 @@ dependencies = [ [[package]] name = "fabro-macros" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "clap", "fabro-options-metadata", @@ -1997,7 +1997,7 @@ dependencies = [ [[package]] name = "fabro-mcp" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "fabro-config", @@ -2013,7 +2013,7 @@ dependencies = [ [[package]] name = "fabro-model" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "fabro-static", "insta", @@ -2024,7 +2024,7 @@ dependencies = [ [[package]] name = "fabro-oauth" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "axum", @@ -2046,7 +2046,7 @@ dependencies = [ [[package]] name = "fabro-options-metadata" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "serde", "serde_json", @@ -2054,7 +2054,7 @@ dependencies = [ [[package]] name = "fabro-proc" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "cc", "libc", @@ -2063,7 +2063,7 @@ dependencies = [ [[package]] name = "fabro-redact" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "aho-corasick", "ref-cast", @@ -2079,7 +2079,7 @@ dependencies = [ [[package]] name = "fabro-retro" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "chrono", @@ -2098,7 +2098,7 @@ dependencies = [ [[package]] name = "fabro-sandbox" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "async-trait", @@ -2138,7 +2138,7 @@ dependencies = [ [[package]] name = "fabro-server" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "async-trait", @@ -2219,7 +2219,7 @@ dependencies = [ [[package]] name = "fabro-slack" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "fabro-http", "fabro-interview", @@ -2240,18 +2240,18 @@ dependencies = [ [[package]] name = "fabro-spa" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "rust-embed", ] [[package]] name = "fabro-static" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" [[package]] name = "fabro-store" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "async-trait", "bytes", @@ -2278,7 +2278,7 @@ dependencies = [ [[package]] name = "fabro-telemetry" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "base64", @@ -2304,7 +2304,7 @@ dependencies = [ [[package]] name = "fabro-template" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "fabro-util", @@ -2316,7 +2316,7 @@ dependencies = [ [[package]] name = "fabro-test" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "assert_cmd", "axum", @@ -2339,7 +2339,7 @@ dependencies = [ [[package]] name = "fabro-tracker" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "async-trait", @@ -2353,7 +2353,7 @@ dependencies = [ [[package]] name = "fabro-types" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "chrono", "clap", @@ -2374,7 +2374,7 @@ dependencies = [ [[package]] name = "fabro-util" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "console 0.15.11", @@ -2394,7 +2394,7 @@ dependencies = [ [[package]] name = "fabro-validate" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "fabro-graphviz", "fabro-model", @@ -2404,7 +2404,7 @@ dependencies = [ [[package]] name = "fabro-vault" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "chrono", "fabro-types", @@ -2416,7 +2416,7 @@ dependencies = [ [[package]] name = "fabro-workflow" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "assert_cmd", @@ -7120,7 +7120,7 @@ dependencies = [ [[package]] name = "twin-github" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "axum", "base64", @@ -7139,7 +7139,7 @@ dependencies = [ [[package]] name = "twin-openai" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" dependencies = [ "anyhow", "async-stream", diff --git a/Cargo.toml b/Cargo.toml index f7897daf1..52d5940be 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ resolver = "2" [workspace.package] edition = "2021" -version = "0.223.0-nightly.0" +version = "0.224.0-nightly.0" license = "MIT" [workspace.dependencies] diff --git a/apps/fabro-web/app/components/stage-sidebar.tsx b/apps/fabro-web/app/components/stage-sidebar.tsx index b72ee77bd..e2cfe88f8 100644 --- a/apps/fabro-web/app/components/stage-sidebar.tsx +++ b/apps/fabro-web/app/components/stage-sidebar.tsx @@ -1,4 +1,4 @@ -import { useState, useEffect, useRef, type ComponentType } from "react"; +import { useEffect, useRef, type ComponentType } from "react"; import { Link } from "react-router"; import type { StageState } from "@qltysh/fabro-api-client"; import { @@ -11,14 +11,16 @@ import { } from "@heroicons/react/24/solid"; import { Bars3BottomLeftIcon, DocumentTextIcon, MapIcon } from "@heroicons/react/24/outline"; import { formatDurationSecs } from "../lib/format"; -import { ACTIVE_STAGE_STATES } from "../lib/stage-sidebar"; +import { ACTIVE_STAGE_STATES, formatStageLabel } from "../lib/stage-sidebar"; +import { useTickingNow } from "../lib/time"; export interface Stage { id: string; name: string; status: StageState; duration: string; - dotId?: string; + nodeId: string; + visit: number; } export const statusConfig: Record; color: string }> = { @@ -42,7 +44,6 @@ interface StageSidebarProps { export function StageSidebar({ stages, runId, selectedStageId, activeLink }: StageSidebarProps) { // Track when we first observed each running stage (for ticking timer) const runningStartRef = useRef>(new Map()); - const [, setTick] = useState(0); // Track start times for running stages useEffect(() => { @@ -62,16 +63,13 @@ export function StageSidebar({ stages, runId, selectedStageId, activeLink }: Sta }, [stages]); // Tick every second while any stage is running - useEffect(() => { - if (!stages.some((s) => ACTIVE_STAGE_STATES.has(s.status))) return; - const interval = setInterval(() => setTick((t) => t + 1), 1000); - return () => clearInterval(interval); - }, [stages]); + const hasActive = stages.some((s) => ACTIVE_STAGE_STATES.has(s.status)); + const now = useTickingNow(hasActive); function stageDuration(stage: Stage): string { if (ACTIVE_STAGE_STATES.has(stage.status)) { const start = runningStartRef.current.get(stage.id); - if (start) return formatDurationSecs(Math.floor((Date.now() - start) / 1000)); + if (start) return formatDurationSecs(Math.floor((now - start) / 1000)); return "0s"; } return stage.duration; @@ -100,7 +98,7 @@ export function StageSidebar({ stages, runId, selectedStageId, activeLink }: Sta }`} > - {stage.name} + {formatStageLabel(stage)} {stageDuration(stage)} diff --git a/apps/fabro-web/app/data/runs.ts b/apps/fabro-web/app/data/runs.ts index 3922b7cd3..b397c5ecc 100644 --- a/apps/fabro-web/app/data/runs.ts +++ b/apps/fabro-web/app/data/runs.ts @@ -1,8 +1,10 @@ import { formatElapsedSecs, formatDurationSecs } from "../lib/format"; -import type { - RunListItem, - RunStatus as ApiRunStatus, - RunSummary, +import { + BoardColumn, + type BoardColumn as ApiBoardColumn, + type RunListItem, + type RunStatus as ApiRunStatus, + type RunSummary, } from "@qltysh/fabro-api-client"; export type CiStatus = "passing" | "failing" | "pending"; @@ -37,9 +39,19 @@ export interface RunItem { sourceDirectory?: string; } -export type ColumnStatus = "initializing" | "running" | "blocked" | "succeeded" | "failed"; +export type ColumnStatus = ApiBoardColumn; + +export const columnStatuses = [ + BoardColumn.QUEUED, + BoardColumn.INITIALIZING, + BoardColumn.RUNNING, + BoardColumn.BLOCKED, + BoardColumn.SUCCEEDED, + BoardColumn.FAILED, +] as const satisfies readonly ColumnStatus[]; export const columnStatusDisplay: Record = { + queued: { label: "Queued", dot: "bg-fg-muted", text: "text-fg-muted" }, initializing: { label: "Initializing", dot: "bg-amber", text: "text-amber" }, running: { label: "Running", dot: "bg-teal-500", text: "text-teal-500" }, blocked: { label: "Blocked", dot: "bg-amber", text: "text-amber" }, @@ -113,6 +125,7 @@ export function columnForStatus(status: ApiRunStatus | null | undefined): Column switch (status?.kind) { case "submitted": case "queued": + return "queued"; case "starting": return "initializing"; case "running": diff --git a/apps/fabro-web/app/lib/api-client.ts b/apps/fabro-web/app/lib/api-client.ts index bd990361b..87792ed69 100644 --- a/apps/fabro-web/app/lib/api-client.ts +++ b/apps/fabro-web/app/lib/api-client.ts @@ -217,6 +217,69 @@ export async function apiPaginatedFetcher( } } +function stageEventsPagePath(key: string, sinceSeq: number, limit: number): string { + const url = new URL(apiPath(key), "http://fabro.local"); + url.searchParams.set("since_seq", String(sinceSeq)); + url.searchParams.set("limit", String(limit)); + return `${url.pathname}${url.search}`; +} + +/** + * Cursor-paginated fetcher for `/runs/{id}/stages/{stageId}/events`. + * + * Loops from `since_seq=1` with a 1000-event page size, advancing the cursor + * to `highestSeq + 1` until the server reports `meta.has_more === false`. + * The empty-page guard mirrors `apiPaginatedFetcher`: if the server claims + * `has_more` but returns no rows we exit and `console.warn` to surface the + * server invariant violation without spinning the UI. + */ +export async function fetchAllStageEvents( + key: string, +): Promise { + const PAGE_LIMIT = 1000; + const MAX_PAGES = 50; + const data: TItem[] = []; + let sinceSeq = 1; + let pagesLoaded = 0; + + while (true) { + const response = await apiRequest(stageEventsPagePath(key, sinceSeq, PAGE_LIMIT)); + if (!response.ok) { + throw await apiErrorFromResponse(response); + } + const page = (await response.json()) as PaginatedEnvelope; + pagesLoaded += 1; + + if (page.data.length === 0) { + if (page.meta.has_more) { + console.warn( + `Stage events fetch for ${key} returned an empty page with has_more=true; stopping at ${data.length} items to avoid spinning.`, + ); + } + return data; + } + + data.push(...page.data); + if (!page.meta.has_more) return data; + + if (pagesLoaded >= MAX_PAGES) { + console.warn( + `Stopped stage events fetch for ${key} after ${pagesLoaded} pages and ${data.length} items because the safety cap was reached.`, + ); + return data; + } + + const highestSeq = page.data.reduce((max, event) => Math.max(max, event.seq), sinceSeq - 1); + if (highestSeq < sinceSeq) { + console.warn( + `Stage events fetch for ${key} returned a non-advancing page at since_seq=${sinceSeq}; stopping at ${data.length} items to avoid spinning.`, + ); + return data; + } + sinceSeq = highestSeq + 1; + } +} + export async function apiJsonMutation( key: string, { arg }: { arg: TArg }, @@ -233,4 +296,4 @@ export async function apiJsonMutation( } if (response.status === 204) return undefined as TResponse; return response.json() as Promise; -} +} \ No newline at end of file diff --git a/apps/fabro-web/app/lib/board-events.test.tsx b/apps/fabro-web/app/lib/board-events.test.tsx index 853d300b9..497bb6984 100644 --- a/apps/fabro-web/app/lib/board-events.test.tsx +++ b/apps/fabro-web/app/lib/board-events.test.tsx @@ -4,7 +4,12 @@ import { shouldRefreshBoardForEvent, subscribeToBoardEvents, } from "./board-events"; +import { + createCrossTabSseCoordinator, + type BroadcastChannelLike, +} from "./cross-tab-sse"; import { queryKeys } from "./query-keys"; +import type { EventSourceLike } from "./sse"; type MessageHandler = ((event: { data: string }) => void) | null; @@ -21,6 +26,14 @@ class FakeEventSource { } } +class FakeBroadcastChannel implements BroadcastChannelLike { + onmessage: ((event: { data: unknown }) => void) | null = null; + + postMessage() {} + + close() {} +} + describe("shouldRefreshBoardForEvent", () => { test("refreshes board for run and interview status changes only", () => { expect(shouldRefreshBoardForEvent("run.running")).toBe(true); @@ -31,22 +44,28 @@ describe("shouldRefreshBoardForEvent", () => { }); describe("subscribeToBoardEvents", () => { - test("shares one source and invalidates the board runs key", () => { + test("coordinated mode shares one global source and invalidates the board runs key", async () => { const source = new FakeEventSource(); const created: string[] = []; const keys: string[] = []; + const coordinator = createCoordinator((url) => { + created.push(url); + return source; + }); const mutate = (key: string) => { keys.push(key); return Promise.resolve(); }; - const firstCleanup = subscribeToBoardEvents(mutate, (url) => { - created.push(url); - return source; - }, { debounceMs: 0 }); + const firstCleanup = subscribeToBoardEvents(mutate, () => { + throw new Error("source should be created by coordinator"); + }, { debounceMs: 0, coordinator }); const secondCleanup = subscribeToBoardEvents(mutate, () => { throw new Error("source should be reused"); - }, { debounceMs: 0 }); + }, { debounceMs: 0, coordinator }); + + await waitFor(() => created.length === 1); + keys.length = 0; source.emit({ event: "run.running" }); @@ -57,5 +76,68 @@ describe("subscribeToBoardEvents", () => { expect(source.closed).toBe(false); secondCleanup(); expect(source.closed).toBe(true); + coordinator.close(); + }); + + test("fallback mode preserves the existing shared board EventSource", () => { + const source = new FakeEventSource(); + const created: string[] = []; + const keys: string[] = []; + const coordinator = createFallbackCoordinator(); + const mutate = (key: string) => { + keys.push(key); + return Promise.resolve(); + }; + + const firstCleanup = subscribeToBoardEvents(mutate, (url) => { + created.push(url); + return source; + }, { debounceMs: 0, coordinator }); + const secondCleanup = subscribeToBoardEvents(mutate, () => { + throw new Error("source should be reused"); + }, { debounceMs: 0, coordinator }); + + source.emit({ event: "run.running" }); + + expect(created).toEqual(["/api/v1/attach"]); + expect(keys).toEqual([queryKeys.boards.runs()]); + + firstCleanup(); + expect(source.closed).toBe(false); + secondCleanup(); + expect(source.closed).toBe(true); + coordinator.close(); }); }); + +function createCoordinator(eventSourceFactory: (url: string) => EventSourceLike) { + return createCrossTabSseCoordinator({ + tabId: "board-test", + channelFactory: () => new FakeBroadcastChannel(), + eventSourceFactory, + addVisibilityChangeListener: () => () => {}, + addPagehideListener: () => () => {}, + timing: { + heartbeatMs: 10, + leaderStaleMs: 50, + electionJitterMs: 0, + }, + }); +} + +function createFallbackCoordinator() { + return createCrossTabSseCoordinator({ + channelFactory: () => { + throw new Error("BroadcastChannel unavailable"); + }, + }); +} + +async function waitFor(condition: () => boolean, timeoutMs = 200) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (condition()) return; + await new Promise((resolve) => setTimeout(resolve, 2)); + } + throw new Error("condition did not become true before timeout"); +} diff --git a/apps/fabro-web/app/lib/board-events.ts b/apps/fabro-web/app/lib/board-events.ts index ef30009b1..dc01765ee 100644 --- a/apps/fabro-web/app/lib/board-events.ts +++ b/apps/fabro-web/app/lib/board-events.ts @@ -1,6 +1,10 @@ import { useEffect } from "react"; import { useSWRConfig } from "swr"; +import { + subscribeToCrossTabSse, + type CrossTabSseCoordinator, +} from "./cross-tab-sse"; import { queryKeys } from "./query-keys"; import { createBrowserEventSource, @@ -11,6 +15,11 @@ import { type SharedEventSubscription, } from "./sse"; +interface BoardEventOptions { + debounceMs?: number; + coordinator?: CrossTabSseCoordinator; +} + const BOARD_STATUS_EVENTS = new Set([ "run.submitted", "run.queued", @@ -41,23 +50,36 @@ export function shouldRefreshBoardForEvent(event: string) { export function subscribeToBoardEvents( mutate: MutateFn, eventSourceFactory: (url: string) => EventSourceLike = createBrowserEventSource, - { debounceMs = 500 }: { debounceMs?: number } = {}, + { debounceMs = 500, coordinator }: BoardEventOptions = {}, ): () => void { - return subscribeToSharedEventSource({ - subscriptions, + return subscribeToCrossTabSse({ + coordinator, subscriptionKey: BOARD_SUBSCRIPTION_KEY, - url: queryKeys.system.attach(), mutate, - eventSourceFactory, debounceMs, - resolveInvalidation: (payload) => ({ - keys: payload.event && shouldRefreshBoardForEvent(payload.event) - ? [queryKeys.boards.runs()] - : [], - }), + resyncKeys: () => [queryKeys.boards.runs()], + resolveInvalidation: boardInvalidation, + fallbackSubscribe: () => + subscribeToSharedEventSource({ + subscriptions, + subscriptionKey: BOARD_SUBSCRIPTION_KEY, + url: queryKeys.system.attach(), + mutate, + eventSourceFactory, + debounceMs, + resolveInvalidation: boardInvalidation, + }), }); } +function boardInvalidation(payload: EventPayload) { + return { + keys: payload.event && shouldRefreshBoardForEvent(payload.event) + ? [queryKeys.boards.runs()] + : [], + }; +} + export function useBoardEvents() { const { mutate } = useSWRConfig(); diff --git a/apps/fabro-web/app/lib/cross-tab-sse.test.ts b/apps/fabro-web/app/lib/cross-tab-sse.test.ts new file mode 100644 index 000000000..76fc0b16e --- /dev/null +++ b/apps/fabro-web/app/lib/cross-tab-sse.test.ts @@ -0,0 +1,736 @@ +import { afterEach, describe, expect, test } from "bun:test"; + +import { + CROSS_TAB_SSE_CHANNEL, + createCrossTabSseCoordinator, + subscribeToCrossTabSse, + type BroadcastChannelLike, + type CrossTabSseCoordinator, + type CrossTabSseMessage, +} from "./cross-tab-sse"; +import type { EventPayload, MutateFn } from "./sse"; + +type MessageHandler = ((event: { data: string }) => void) | null; +type TabVisibility = "visible" | "hidden"; + +const TEST_TIMING = { + heartbeatMs: 10, + leaderStaleMs: 35, + electionJitterMs: 5, +}; + +class FakeEventSource { + onmessage: MessageHandler = null; + closed = false; + + constructor( + readonly url: string, + readonly owner: string, + ) {} + + emit(payload: unknown) { + this.onmessage?.({ data: JSON.stringify(payload) }); + } + + close() { + this.closed = true; + } +} + +class FakeBroadcastChannel implements BroadcastChannelLike { + static channels = new Set(); + static muted = false; + static throwOnTypes = new Set(); + + onmessage: ((event: { data: unknown }) => void) | null = null; + closed = false; + + constructor(readonly name: string) { + FakeBroadcastChannel.channels.add(this); + } + + postMessage(message: CrossTabSseMessage) { + if (FakeBroadcastChannel.throwOnTypes.has(message.type)) { + throw new Error(`postMessage failed for ${message.type}`); + } + if (FakeBroadcastChannel.muted) return; + const recipients = [...FakeBroadcastChannel.channels].filter( + (channel) => channel !== this && !channel.closed && channel.name === this.name, + ); + queueMicrotask(() => { + for (const channel of recipients) { + if (channel.closed) continue; + channel.onmessage?.({ data: { ...message } }); + } + }); + } + + static broadcastExternal(message: CrossTabSseMessage) { + queueMicrotask(() => { + for (const channel of FakeBroadcastChannel.channels) { + if (channel.closed) continue; + channel.onmessage?.({ data: { ...message } }); + } + }); + } + + close() { + this.closed = true; + FakeBroadcastChannel.channels.delete(this); + } + + static reset() { + for (const channel of FakeBroadcastChannel.channels) { + channel.closed = true; + } + FakeBroadcastChannel.channels.clear(); + FakeBroadcastChannel.muted = false; + FakeBroadcastChannel.throwOnTypes.clear(); + } +} + +class Harness { + readonly sources: FakeEventSource[] = []; + readonly coordinators = new Map(); + readonly visibility = new Map(); + readonly visibilityHandlers = new Map void>(); + now = 1000; + + createTab(tabId: string, visibility: TabVisibility = "visible") { + this.visibility.set(tabId, visibility); + const coordinator = createCrossTabSseCoordinator({ + tabId, + channelFactory: (name) => new FakeBroadcastChannel(name), + eventSourceFactory: (url) => { + const source = new FakeEventSource(url, tabId); + this.sources.push(source); + return source; + }, + getVisibility: () => this.visibility.get(tabId) ?? "visible", + addVisibilityChangeListener: (handler) => { + this.visibilityHandlers.set(tabId, handler); + return () => this.visibilityHandlers.delete(tabId); + }, + addPagehideListener: () => () => {}, + now: () => this.now, + timing: TEST_TIMING, + }); + this.coordinators.set(tabId, coordinator); + return coordinator; + } + + setVisibility(tabId: string, visibility: TabVisibility) { + this.visibility.set(tabId, visibility); + this.visibilityHandlers.get(tabId)?.(); + } + + openSources() { + return this.sources.filter((source) => !source.closed); + } + + close() { + for (const coordinator of this.coordinators.values()) { + coordinator.close(); + } + } +} + +const harnesses: Harness[] = []; + +afterEach(() => { + for (const harness of harnesses.splice(0)) { + harness.close(); + } + FakeBroadcastChannel.reset(); +}); + +describe("subscribeToCrossTabSse", () => { + test("opens one leader-owned global EventSource and keeps followers passive", async () => { + const harness = newHarness(); + const cleanups = ["a", "b", "c"].map((tabId) => { + const coordinator = harness.createTab(tabId); + return subscribeForRunEvent(coordinator, []); + }); + + await waitFor(() => harness.openSources().length === 1); + + expect(harness.openSources().map((source) => source.url)).toEqual(["/api/v1/attach"]); + expect([...FakeBroadcastChannel.channels].every((channel) => channel.name === CROSS_TAB_SSE_CHANNEL)).toBe(true); + + cleanups.forEach((cleanup) => cleanup()); + }); + + test("leader broadcasts events to all local subscribers", async () => { + const harness = newHarness(); + const keysByTab = new Map(); + + for (const tabId of ["a", "b", "c"]) { + keysByTab.set(tabId, []); + subscribeForRunEvent(harness.createTab(tabId), keysByTab.get(tabId)!); + } + + await waitFor(() => harness.openSources().length === 1); + clearRecordedKeys(keysByTab); + + harness.openSources()[0].emit(runEvent({ id: "evt-1", runId: "run-1", seq: 1 })); + await waitFor(() => [...keysByTab.values()].every((keys) => keys.length === 1)); + + expect(keysByTab.get("a")).toEqual(["event"]); + expect(keysByTab.get("b")).toEqual(["event"]); + expect(keysByTab.get("c")).toEqual(["event"]); + }); + + test("board and run subscriptions coexist on the same global stream", async () => { + const harness = newHarness(); + const coordinator = harness.createTab("a"); + const boardKeys: string[] = []; + const runKeys: string[] = []; + + subscribeForEvent(coordinator, { + subscriptionKey: "board", + keys: boardKeys, + resolveInvalidation: (payload) => ({ + keys: payload.event === "run.running" ? ["board"] : [], + }), + resyncKeys: () => ["board-resync"], + }); + subscribeForEvent(coordinator, { + subscriptionKey: "run:run-1", + keys: runKeys, + resolveInvalidation: (payload) => ({ + keys: payload.event === "run.running" && payload.run_id === "run-1" ? ["run"] : [], + }), + resyncKeys: () => ["run-resync"], + }); + + await waitFor(() => harness.openSources().length === 1); + boardKeys.length = 0; + runKeys.length = 0; + + harness.openSources()[0].emit(runEvent({ id: "evt-coexist", runId: "run-1", seq: 1 })); + + expect(boardKeys).toEqual(["board"]); + expect(runKeys).toEqual(["run"]); + expect(harness.openSources().map((source) => source.url)).toEqual(["/api/v1/attach"]); + }); + + test("dedupes duplicate event ids until TTL or max-size eviction", async () => { + const harness = newHarness(); + const keys: string[] = []; + subscribeForRunEvent(harness.createTab("a"), keys); + + await waitFor(() => harness.openSources().length === 1); + keys.length = 0; + + const source = harness.openSources()[0]; + source.emit(runEvent({ id: "evt-dup", runId: "run-1", seq: 1 })); + source.emit(runEvent({ id: "evt-dup", runId: "run-1", seq: 1 })); + + expect(keys).toEqual(["event"]); + + harness.now += 5 * 60 * 1000 + 1; + source.emit(runEvent({ id: "evt-dup", runId: "run-1", seq: 1 })); + expect(keys).toEqual(["event", "event"]); + + keys.length = 0; + for (let i = 0; i < 1001; i += 1) { + source.emit(runEvent({ id: `evt-${i}`, runId: "run-1", seq: i + 2 })); + } + source.emit(runEvent({ id: "evt-0", runId: "run-1", seq: 2 })); + expect(keys).toHaveLength(1002); + }); + + test("visible followers take over from a fresh hidden leader and resync", async () => { + const harness = newHarness(); + const hiddenKeys: string[] = []; + const visibleKeys: string[] = []; + + subscribeForRunEvent(harness.createTab("z", "hidden"), hiddenKeys); + await waitFor(() => harness.openSources().length === 1); + const hiddenSource = harness.openSources()[0]; + + subscribeForRunEvent(harness.createTab("a", "visible"), visibleKeys); + + await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "a"); + + expect(hiddenSource.closed).toBe(true); + expect(visibleKeys).toContain("resync"); + }); + + test("visible candidates racing for the same hidden leader resolve lexically", async () => { + const harness = newHarness(); + + subscribeForRunEvent(harness.createTab("z", "hidden"), []); + await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "z"); + + subscribeForRunEvent(harness.createTab("b", "visible"), []); + subscribeForRunEvent(harness.createTab("a", "visible"), []); + + await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner !== "z"); + expect(harness.openSources().map((source) => source.owner)).toEqual(["a"]); + }); + + test("a lower lexical follower does not preempt a fresh visible leader", async () => { + const harness = newHarness(); + + subscribeForRunEvent(harness.createTab("z", "visible"), []); + await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "z"); + + subscribeForRunEvent(harness.createTab("a", "visible"), []); + await sleep(TEST_TIMING.electionJitterMs * 4); + + expect(harness.openSources().map((source) => source.owner)).toEqual(["z"]); + }); + + test("stale leader detection opens a new leader source and resyncs followers", async () => { + const harness = newHarness(); + const followerKeys: string[] = []; + + subscribeForRunEvent(harness.createTab("a"), []); + subscribeForRunEvent(harness.createTab("b"), followerKeys); + await waitFor(() => harness.openSources().length === 1); + + const staleLeader = harness.openSources()[0]; + harness.coordinators.get(staleLeader.owner)?.close(); + harness.now += TEST_TIMING.leaderStaleMs + TEST_TIMING.heartbeatMs + 1; + + await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner !== staleLeader.owner); + + expect(followerKeys).toContain("resync"); + }); + + test("simultaneous stale leader elections resolve to the lexical winner", async () => { + const harness = newHarness(); + + subscribeForRunEvent(harness.createTab("z"), []); + await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "z"); + subscribeForRunEvent(harness.createTab("b"), []); + subscribeForRunEvent(harness.createTab("a"), []); + await sleep(TEST_TIMING.heartbeatMs * 2); + + harness.coordinators.get("z")?.close(); + harness.now += TEST_TIMING.leaderStaleMs + TEST_TIMING.heartbeatMs + 1; + + await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "a"); + }); + + test("hidden leader ignores candidates for old observed leadership", async () => { + const harness = newHarness(); + + subscribeForRunEvent(harness.createTab("z", "hidden"), []); + await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "z"); + const hiddenSource = harness.openSources()[0]; + + FakeBroadcastChannel.broadcastExternal({ + type: "candidate", + version: 1, + tabId: "ghost", + sentAt: harness.now, + candidateId: "ghost", + candidateGeneration: 1, + visibility: "visible", + observedLeaderId: "z", + observedGeneration: 0, + reason: "hidden-leader", + }); + await sleep(TEST_TIMING.electionJitterMs * 2); + + expect(hiddenSource.closed).toBe(false); + expect(harness.openSources().map((source) => source.owner)).toEqual(["z"]); + }); + + test("prunes candidate records from older generations", async () => { + const harness = newHarness(); + subscribeForRunEvent(harness.createTab("z", "hidden"), []); + await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "z"); + + const coordinator = harness.createTab("a", "visible"); + subscribeForRunEvent(coordinator, []); + await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "a"); + + FakeBroadcastChannel.broadcastExternal({ + type: "candidate", + version: 1, + tabId: "old-candidate", + sentAt: harness.now, + candidateId: "old-candidate", + candidateGeneration: 1, + visibility: "visible", + observedLeaderId: "previous-leader", + observedGeneration: 0, + reason: "stale-leader", + }); + await sleep(TEST_TIMING.electionJitterMs * 2); + + expect(candidateGenerations(coordinator)).not.toContain(1); + }); + + test("same-generation split brain converges to the higher-priority visible leader", async () => { + const harness = newHarness(); + FakeBroadcastChannel.muted = true; + + subscribeForRunEvent(harness.createTab("b"), []); + subscribeForRunEvent(harness.createTab("a"), []); + await waitFor(() => harness.openSources().length === 2); + + FakeBroadcastChannel.muted = false; + await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "a"); + }); + + test("old leader events are ignored after takeover", async () => { + const harness = newHarness(); + const keys: string[] = []; + + subscribeForRunEvent(harness.createTab("z", "hidden"), []); + await waitFor(() => harness.openSources().length === 1); + const oldSource = harness.openSources()[0]; + + subscribeForRunEvent(harness.createTab("a", "visible"), keys); + await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "a"); + keys.length = 0; + + oldSource.emit(runEvent({ id: "evt-old", runId: "run-1", seq: 1 })); + expect(keys).toEqual([]); + }); + + test("old leader heartbeats are ignored after takeover", async () => { + const harness = newHarness(); + + subscribeForRunEvent(harness.createTab("z", "hidden"), []); + await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "z"); + + subscribeForRunEvent(harness.createTab("a", "visible"), []); + await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "a"); + + FakeBroadcastChannel.broadcastExternal({ + type: "heartbeat", + version: 1, + tabId: "z", + sentAt: harness.now, + leaderId: "z", + generation: 1, + visibility: "hidden", + }); + await sleep(TEST_TIMING.heartbeatMs * 2); + + expect(harness.openSources().map((source) => source.owner)).toEqual(["a"]); + }); + + test("last unsubscribe closes the leader source and releases leadership", async () => { + const harness = newHarness(); + const cleanup = subscribeForRunEvent(harness.createTab("a"), []); + + await waitFor(() => harness.openSources().length === 1); + const source = harness.openSources()[0]; + + cleanup(); + + expect(source.closed).toBe(true); + expect(harness.openSources()).toEqual([]); + }); + + test("missing BroadcastChannel uses subscriber fallback", () => { + const coordinator = createCrossTabSseCoordinator({ + channelFactory: () => { + throw new Error("no channel"); + }, + }); + let fallbackStarted = 0; + let fallbackStopped = 0; + + const cleanup = subscribeToCrossTabSse({ + coordinator, + subscriptionKey: "fallback", + mutate: (() => Promise.resolve()) as MutateFn, + resolveInvalidation: () => ({ keys: [] }), + resyncKeys: () => [], + fallbackSubscribe: () => { + fallbackStarted += 1; + return () => { + fallbackStopped += 1; + }; + }, + debounceMs: 0, + }); + + cleanup(); + + expect(fallbackStarted).toBe(1); + expect(fallbackStopped).toBe(1); + }); + + test("postMessage failure after initialization degrades to fallback without coordinated resync", async () => { + const harness = newHarness(); + const coordinator = harness.createTab("a"); + const keys: string[] = []; + let fallbackStarted = 0; + let fallbackStopped = 0; + + FakeBroadcastChannel.throwOnTypes.add("leader-changed"); + const cleanup = subscribeToCrossTabSse({ + coordinator, + subscriptionKey: "throwing-channel", + mutate: ((key: string) => { + keys.push(key); + return Promise.resolve(); + }) as MutateFn, + resolveInvalidation: () => ({ keys: ["event"] }), + resyncKeys: () => ["resync"], + fallbackSubscribe: () => { + fallbackStarted += 1; + return () => { + fallbackStopped += 1; + }; + }, + debounceMs: 0, + }); + + await waitFor(() => fallbackStarted === 1); + + expect(harness.openSources()).toEqual([]); + expect(keys).toEqual([]); + + cleanup(); + expect(fallbackStopped).toBe(1); + }); + + test("close resets coordination availability after an initial channel failure", async () => { + let channelUnavailable = true; + const sources: FakeEventSource[] = []; + const coordinator = createCrossTabSseCoordinator({ + tabId: "recovering", + channelFactory: (name) => { + if (channelUnavailable) throw new Error("channel unavailable"); + return new FakeBroadcastChannel(name); + }, + eventSourceFactory: (url) => { + const source = new FakeEventSource(url, "recovering"); + sources.push(source); + return source; + }, + addVisibilityChangeListener: () => () => {}, + addPagehideListener: () => () => {}, + timing: TEST_TIMING, + }); + let firstFallbackStarted = 0; + let secondFallbackStarted = 0; + + const firstCleanup = subscribeWithFallback(coordinator, { + fallbackSubscribe: () => { + firstFallbackStarted += 1; + return () => {}; + }, + }); + firstCleanup(); + coordinator.close(); + + channelUnavailable = false; + const secondCleanup = subscribeWithFallback(coordinator, { + fallbackSubscribe: () => { + secondFallbackStarted += 1; + return () => {}; + }, + }); + + await waitFor(() => sources.some((source) => !source.closed)); + + expect(firstFallbackStarted).toBe(1); + expect(secondFallbackStarted).toBe(0); + expect(sources.filter((source) => !source.closed).map((source) => source.url)).toEqual(["/api/v1/attach"]); + + secondCleanup(); + coordinator.close(); + }); + + test("last unsubscribe retries coordination after an initial channel failure", async () => { + let channelUnavailable = true; + const sources: FakeEventSource[] = []; + const coordinator = createCrossTabSseCoordinator({ + tabId: "retry-after-unsubscribe", + channelFactory: (name) => { + if (channelUnavailable) throw new Error("channel unavailable"); + return new FakeBroadcastChannel(name); + }, + eventSourceFactory: (url) => { + const source = new FakeEventSource(url, "retry-after-unsubscribe"); + sources.push(source); + return source; + }, + addVisibilityChangeListener: () => () => {}, + addPagehideListener: () => () => {}, + timing: TEST_TIMING, + }); + let fallbackStarted = 0; + + const firstCleanup = subscribeWithFallback(coordinator, { + fallbackSubscribe: () => { + fallbackStarted += 1; + return () => {}; + }, + }); + firstCleanup(); + + channelUnavailable = false; + const secondCleanup = subscribeWithFallback(coordinator, { + fallbackSubscribe: () => { + fallbackStarted += 1; + return () => {}; + }, + }); + + await waitFor(() => sources.some((source) => !source.closed)); + + expect(fallbackStarted).toBe(1); + expect(sources.filter((source) => !source.closed).map((source) => source.url)).toEqual(["/api/v1/attach"]); + + secondCleanup(); + coordinator.close(); + }); + + test("close stops fallback subscriptions added after degradation", async () => { + const harness = newHarness(); + const coordinator = harness.createTab("a"); + let fallbackStarted = 0; + let fallbackStopped = 0; + + FakeBroadcastChannel.throwOnTypes.add("leader-changed"); + subscribeWithFallback(coordinator, { + subscriptionKey: "before-degrade", + fallbackSubscribe: () => { + fallbackStarted += 1; + return () => { + fallbackStopped += 1; + }; + }, + }); + await waitFor(() => fallbackStarted === 1); + + FakeBroadcastChannel.throwOnTypes.clear(); + subscribeWithFallback(coordinator, { + subscriptionKey: "after-degrade", + fallbackSubscribe: () => { + fallbackStarted += 1; + return () => { + fallbackStopped += 1; + }; + }, + }); + expect(fallbackStarted).toBe(2); + + coordinator.close(); + + expect(fallbackStopped).toBe(2); + }); +}); + +function newHarness() { + const harness = new Harness(); + harnesses.push(harness); + return harness; +} + +function subscribeForRunEvent(coordinator: CrossTabSseCoordinator, keys: string[]) { + return subscribeForEvent(coordinator, { + subscriptionKey: "run-feed", + keys, + resolveInvalidation: (payload) => ({ + keys: payload.event === "run.running" ? ["event"] : [], + }), + resyncKeys: () => ["resync"], + }); +} + +function subscribeForEvent( + coordinator: CrossTabSseCoordinator, + { + subscriptionKey, + keys, + resolveInvalidation, + resyncKeys, + }: { + subscriptionKey: string; + keys: string[]; + resolveInvalidation: (payload: EventPayload) => { keys: string[] }; + resyncKeys: () => string[]; + }, +) { + return subscribeToCrossTabSse({ + coordinator, + subscriptionKey, + mutate: ((key: string) => { + keys.push(key); + return Promise.resolve(); + }) as MutateFn, + resolveInvalidation, + resyncKeys, + fallbackSubscribe: () => { + throw new Error("fallback should not be used"); + }, + debounceMs: 0, + }); +} + +function subscribeWithFallback( + coordinator: CrossTabSseCoordinator, + { + subscriptionKey = "fallback-test", + fallbackSubscribe, + }: { + subscriptionKey?: string; + fallbackSubscribe: () => () => void; + }, +) { + return subscribeToCrossTabSse({ + coordinator, + subscriptionKey, + mutate: (() => Promise.resolve()) as MutateFn, + resolveInvalidation: () => ({ keys: [] }), + resyncKeys: () => [], + fallbackSubscribe, + debounceMs: 0, + }); +} + +function candidateGenerations(coordinator: CrossTabSseCoordinator): number[] { + const inspectable = coordinator as unknown as { + candidates: Map; + }; + return [...inspectable.candidates.values()].map((candidate) => candidate.candidateGeneration); +} + +function runEvent({ + id, + runId, + seq, +}: { + id: string; + runId: string; + seq: number; +}) { + return { + id, + seq, + run_id: runId, + event: "run.running", + ts: "2026-05-04T12:00:00.000Z", + }; +} + +async function waitFor(condition: () => boolean, timeoutMs = 500) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (condition()) return; + await sleep(2); + } + throw new Error("condition did not become true before timeout"); +} + +function sleep(ms: number) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function clearRecordedKeys(keysByTab: Map) { + for (const keys of keysByTab.values()) { + keys.length = 0; + } +} diff --git a/apps/fabro-web/app/lib/cross-tab-sse.ts b/apps/fabro-web/app/lib/cross-tab-sse.ts new file mode 100644 index 000000000..6975e6948 --- /dev/null +++ b/apps/fabro-web/app/lib/cross-tab-sse.ts @@ -0,0 +1,1175 @@ +import { queryKeys } from "./query-keys"; +import { + createBrowserEventSource, + type EventInvalidation, + type EventPayload, + type EventSourceLike, + type MutateFn, +} from "./sse"; +import { getNumber, getString, isRecord, type UnknownRecord } from "./unknown"; + +export const CROSS_TAB_SSE_CHANNEL = "fabro:sse:v1"; +export const HEARTBEAT_MS = 1000; +export const LEADER_STALE_MS = 4000; +export const ELECTION_JITTER_MS = 150; + +const MESSAGE_VERSION = 1 as const; +const EVENT_DEDUPE_TTL_MS = 5 * 60 * 1000; +const EVENT_DEDUPE_MAX = 1000; + +type TabVisibility = "visible" | "hidden"; +type CandidateReason = "hidden-leader" | "stale-leader" | "release" | "no-leader"; + +interface BaseMessage { + type: string; + version: typeof MESSAGE_VERSION; + tabId: string; + sentAt: number; +} + +interface HelloMessage extends BaseMessage { + type: "hello"; +} + +interface HeartbeatMessage extends BaseMessage { + type: "heartbeat"; + leaderId: string; + generation: number; + visibility: TabVisibility; +} + +interface CandidateMessage extends BaseMessage { + type: "candidate"; + candidateId: string; + candidateGeneration: number; + visibility: TabVisibility; + observedLeaderId: string | null; + observedGeneration: number; + reason: CandidateReason; +} + +interface LeaderChangedMessage extends BaseMessage { + type: "leader-changed"; + leaderId: string; + generation: number; + visibility: TabVisibility; +} + +interface ReleaseMessage extends BaseMessage { + type: "release"; + leaderId: string; + generation: number; +} + +interface ResyncMessage extends BaseMessage { + type: "resync"; + leaderId: string | null; + generation: number; + reason: CandidateReason; +} + +interface EventMessage extends BaseMessage { + type: "event"; + leaderId: string; + generation: number; + payload: EventPayload; +} + +export type CrossTabSseMessage = + | HelloMessage + | HeartbeatMessage + | CandidateMessage + | LeaderChangedMessage + | ReleaseMessage + | ResyncMessage + | EventMessage; + +export interface BroadcastChannelLike { + onmessage: ((event: { data: unknown }) => void) | null; + postMessage(message: CrossTabSseMessage): void; + close(): void; +} + +interface TimingOptions { + heartbeatMs: number; + leaderStaleMs: number; + electionJitterMs: number; +} + +export interface CrossTabSseCoordinatorOptions { + tabId?: string; + channelFactory?: (name: string) => BroadcastChannelLike; + eventSourceFactory?: (url: string) => EventSourceLike; + getVisibility?: () => TabVisibility; + addVisibilityChangeListener?: (handler: () => void) => () => void; + addPagehideListener?: (handler: () => void) => () => void; + now?: () => number; + timing?: Partial; +} + +interface SubscribeOptions { + subscriptionKey: string; + mutate: MutateFn; + resolveInvalidation: (payload: TPayload) => EventInvalidation; + resyncKeys: () => string[]; + fallbackSubscribe: () => () => void; + debounceMs?: number; +} + +export interface SubscribeToCrossTabSseOptions + extends SubscribeOptions { + coordinator?: CrossTabSseCoordinator; +} + +interface FallbackEntry { + count: number; + subscribe: () => () => void; + cleanup?: () => void; +} + +interface LocalSubscription { + refcount: number; + mutators: Map; + fallbacks: Map; + pendingKeys: Set; + debounceTimer: ReturnType | null; + debounceMs: number; + resolveInvalidation: (payload: EventPayload) => EventInvalidation; + resyncKeys: () => string[]; +} + +interface LeaderState { + leaderId: string; + generation: number; + visibility: TabVisibility; + lastSeen: number; +} + +class RecentEventCache { + private readonly seen = new Map(); + + constructor( + private readonly maxSize: number, + private readonly ttlMs: number, + ) {} + + remember(key: string | undefined, now: number): boolean { + if (!key) return true; + this.evictExpired(now); + if (this.seen.has(key)) return false; + + this.seen.set(key, now); + while (this.seen.size > this.maxSize) { + const oldest = this.seen.keys().next().value; + if (oldest === undefined) break; + this.seen.delete(oldest); + } + return true; + } + + private evictExpired(now: number) { + for (const [key, seenAt] of this.seen) { + if (now - seenAt <= this.ttlMs) return; + this.seen.delete(key); + } + } +} + +export class CrossTabSseCoordinator { + readonly tabId: string; + + private readonly channelFactory: (name: string) => BroadcastChannelLike; + private readonly getVisibility: () => TabVisibility; + private readonly addVisibilityChangeListener: (handler: () => void) => () => void; + private readonly addPagehideListener: (handler: () => void) => () => void; + private readonly now: () => number; + private readonly timing: TimingOptions; + private readonly recentEvents = new RecentEventCache(EVENT_DEDUPE_MAX, EVENT_DEDUPE_TTL_MS); + private readonly subscriptions = new Map(); + private readonly candidates = new Map(); + + private sourceFactory: (url: string) => EventSourceLike; + private channel: BroadcastChannelLike | null = null; + private source: EventSourceLike | null = null; + private initialized = false; + private coordinationUnavailable = false; + private fallbackMode = false; + private degradingToFallback = false; + private isLeader = false; + private leader: LeaderState | null = null; + private generation = 0; + private ownCandidate: CandidateMessage | null = null; + private candidateTimer: ReturnType | null = null; + private noLeaderTimer: ReturnType | null = null; + private heartbeatTimer: ReturnType | null = null; + private leaderCheckTimer: ReturnType | null = null; + private removeVisibilityListener: (() => void) | null = null; + private removePagehideListener: (() => void) | null = null; + + constructor(options: CrossTabSseCoordinatorOptions = {}) { + this.tabId = options.tabId ?? createTabId(); + this.channelFactory = options.channelFactory ?? createBrowserBroadcastChannel; + this.sourceFactory = options.eventSourceFactory ?? createBrowserEventSource; + this.getVisibility = options.getVisibility ?? getBrowserVisibility; + this.addVisibilityChangeListener = + options.addVisibilityChangeListener ?? addBrowserVisibilityChangeListener; + this.addPagehideListener = options.addPagehideListener ?? addBrowserPagehideListener; + this.now = options.now ?? Date.now; + this.timing = { + heartbeatMs: options.timing?.heartbeatMs ?? HEARTBEAT_MS, + leaderStaleMs: options.timing?.leaderStaleMs ?? LEADER_STALE_MS, + electionJitterMs: options.timing?.electionJitterMs ?? ELECTION_JITTER_MS, + }; + } + + subscribe(options: SubscribeOptions): () => void { + const subscription = this.addLocalSubscription(options); + + if (this.coordinationUnavailable) { + this.fallbackMode = true; + this.startFallbacksFor(subscription); + } else if (!this.initialized && !this.initialize()) { + this.coordinationUnavailable = true; + this.fallbackMode = true; + this.startFallbacksFor(subscription); + } else if (this.fallbackMode) { + this.startFallbacksFor(subscription); + } else { + this.ensureLeadershipProgress(); + } + + let active = true; + return () => { + if (!active) return; + active = false; + this.removeLocalSubscription(options.subscriptionKey, options.mutate); + }; + } + + close() { + this.releaseLeadership({ broadcast: false, resync: false }); + this.clearCandidate(); + this.clearNoLeaderTimer(); + this.shutdownTimersAndChannel(); + this.closeFallbacks(); + this.clearSubscriptionTimers(); + this.subscriptions.clear(); + this.candidates.clear(); + this.resetIdleState(); + } + + private initialize(): boolean { + try { + this.channel = this.channelFactory(CROSS_TAB_SSE_CHANNEL); + } catch { + this.channel = null; + return false; + } + + this.channel.onmessage = (event) => this.handleMessage(event.data); + this.initialized = true; + this.removeVisibilityListener = this.addVisibilityChangeListener(() => { + this.handleVisibilityChange(); + }); + this.removePagehideListener = this.addPagehideListener(() => { + this.handlePagehide(); + }); + this.leaderCheckTimer = setInterval(() => { + this.checkLeaderFreshness(); + }, this.timing.heartbeatMs); + + return this.post({ type: "hello", version: MESSAGE_VERSION, tabId: this.tabId, sentAt: this.now() }); + } + + private addLocalSubscription( + options: SubscribeOptions, + ): LocalSubscription { + let subscription = this.subscriptions.get(options.subscriptionKey); + if (!subscription) { + subscription = { + refcount: 0, + mutators: new Map(), + fallbacks: new Map(), + pendingKeys: new Set(), + debounceTimer: null, + debounceMs: options.debounceMs ?? 300, + resolveInvalidation: options.resolveInvalidation as (payload: EventPayload) => EventInvalidation, + resyncKeys: options.resyncKeys, + }; + this.subscriptions.set(options.subscriptionKey, subscription); + } else { + subscription.resolveInvalidation = + options.resolveInvalidation as (payload: EventPayload) => EventInvalidation; + subscription.resyncKeys = options.resyncKeys; + subscription.debounceMs = options.debounceMs ?? subscription.debounceMs; + } + + subscription.refcount += 1; + subscription.mutators.set( + options.mutate, + (subscription.mutators.get(options.mutate) ?? 0) + 1, + ); + + const fallback = subscription.fallbacks.get(options.mutate); + if (fallback) { + fallback.count += 1; + fallback.subscribe = options.fallbackSubscribe; + } else { + subscription.fallbacks.set(options.mutate, { + count: 1, + subscribe: options.fallbackSubscribe, + }); + } + + return subscription; + } + + private removeLocalSubscription(subscriptionKey: string, mutate: MutateFn) { + const subscription = this.subscriptions.get(subscriptionKey); + if (!subscription) return; + + const mutateCount = subscription.mutators.get(mutate) ?? 0; + if (mutateCount <= 1) { + subscription.mutators.delete(mutate); + } else { + subscription.mutators.set(mutate, mutateCount - 1); + } + + const fallback = subscription.fallbacks.get(mutate); + if (fallback) { + fallback.count -= 1; + if (fallback.count <= 0) { + fallback.cleanup?.(); + subscription.fallbacks.delete(mutate); + } + } + + subscription.refcount -= 1; + if (subscription.refcount <= 0) { + this.clearSubscriptionTimer(subscription); + this.subscriptions.delete(subscriptionKey); + } + + if (this.subscriptions.size === 0) { + this.releaseLeadership({ broadcast: true, resync: false }); + this.clearCandidate(); + this.clearNoLeaderTimer(); + this.shutdownTimersAndChannel(); + this.resetIdleState(); + } + } + + private handleMessage(data: unknown) { + const message = parseMessage(data); + if (!message || message.tabId === this.tabId) return; + + switch (message.type) { + case "hello": + if (this.isLeader) this.sendHeartbeat(); + break; + case "heartbeat": + this.handleLeaderAnnouncement(message, { resyncOnChange: true }); + break; + case "candidate": + this.handleCandidate(message); + break; + case "leader-changed": + this.handleLeaderAnnouncement(message, { resyncOnChange: true }); + break; + case "release": + this.handleRelease(message); + break; + case "resync": + this.handleResync(message); + break; + case "event": + this.handleBroadcastEvent(message); + break; + } + } + + private handleLeaderAnnouncement( + message: HeartbeatMessage | LeaderChangedMessage, + { resyncOnChange }: { resyncOnChange: boolean }, + ) { + const incoming: LeaderState = { + leaderId: message.leaderId, + generation: message.generation, + visibility: message.visibility, + lastSeen: this.now(), + }; + + if (this.isLeader && incoming.leaderId !== this.tabId) { + const own: LeaderState = { + leaderId: this.tabId, + generation: this.generation, + visibility: this.currentVisibility(), + lastSeen: this.now(), + }; + if ( + incoming.generation > own.generation || + (incoming.generation === own.generation && leaderHasHigherPriority(incoming, own)) + ) { + this.releaseLeadership({ broadcast: false, resync: true }); + } else { + return; + } + } + + const previous = this.leader; + if (this.ownCandidate && incoming.generation === this.ownCandidate.candidateGeneration) { + const sawIncomingCandidate = this.candidates.has( + `${incoming.generation}:${incoming.leaderId}`, + ); + const shouldAcceptFreshVisibleLeader = + this.ownCandidate.reason === "no-leader" && + incoming.visibility === "visible" && + !sawIncomingCandidate; + + if ( + !shouldAcceptFreshVisibleLeader && + !leaderHasHigherPriority(incoming, leaderStateForCandidate(this.ownCandidate, this.now())) + ) { + return; + } + } + + if (!this.shouldAcceptLeader(incoming)) return; + + this.leader = incoming; + this.clearNoLeaderTimer(); + this.generation = Math.max(this.generation, incoming.generation); + this.pruneStaleCandidates(); + if (this.ownCandidate && incoming.generation >= this.ownCandidate.candidateGeneration) { + this.clearCandidate(); + } + + const changed = + !previous || + previous.leaderId !== incoming.leaderId || + previous.generation !== incoming.generation; + + if (changed && resyncOnChange) { + this.resyncAll(); + } + + if (incoming.visibility === "hidden" && this.currentVisibility() === "visible") { + this.enterCandidacy("hidden-leader", incoming); + } + } + + private handleCandidate(message: CandidateMessage) { + this.candidates.set(candidateKey(message), message); + this.pruneStaleCandidates(); + + if ( + this.isLeader && + message.observedLeaderId === this.tabId && + message.observedGeneration >= this.generation + ) { + this.releaseLeadership({ broadcast: true, resync: true }); + } + + if ( + this.ownCandidate && + message.candidateGeneration === this.ownCandidate.candidateGeneration && + candidateHasHigherPriority(message, this.ownCandidate) + ) { + this.clearCandidate(); + } + } + + private handleRelease(message: ReleaseMessage) { + const current = this.leader; + if ( + current && + message.leaderId === current.leaderId && + message.generation >= current.generation + ) { + this.leader = null; + this.generation = Math.max(this.generation, message.generation); + this.pruneStaleCandidates(); + this.enterCandidacy("release", { + leaderId: message.leaderId, + generation: message.generation, + visibility: "hidden", + lastSeen: this.now(), + }); + } + } + + private handleResync(message: ResyncMessage) { + if (this.leader && message.generation < this.leader.generation) return; + this.resyncAll(); + } + + private handleBroadcastEvent(message: EventMessage) { + if (!this.isCurrentLeader(message.leaderId, message.generation)) return; + if (!this.recentEvents.remember(eventDedupeKey(message.payload), this.now())) return; + this.dispatchPayload(message.payload); + } + + private ensureLeadershipProgress() { + if (this.subscriptions.size === 0 || this.isLeader || this.ownCandidate) return; + + if (!this.leader) { + this.scheduleNoLeaderCandidacy(); + return; + } + + if (this.leader.visibility === "hidden" && this.currentVisibility() === "visible") { + this.enterCandidacy("hidden-leader", this.leader); + } + } + + private checkLeaderFreshness() { + if (this.subscriptions.size === 0 || this.fallbackMode) return; + if (this.isLeader) return; + + const current = this.leader; + if (!current) { + this.scheduleNoLeaderCandidacy(); + return; + } + + if (!this.leaderIsFresh(current)) { + this.leader = null; + this.generation = Math.max(this.generation, current.generation); + this.pruneStaleCandidates(); + this.resyncAll(); + this.enterCandidacy("stale-leader", current); + return; + } + + if (current.visibility === "hidden" && this.currentVisibility() === "visible") { + this.enterCandidacy("hidden-leader", current); + } + } + + private enterCandidacy(reason: CandidateReason, observedLeader: LeaderState | null = this.leader) { + if (this.subscriptions.size === 0 || this.fallbackMode) return; + + if ( + reason === "no-leader" && + this.leader && + this.leader.visibility === "visible" && + this.leaderIsFresh(this.leader) + ) { + return; + } + + const observedGeneration = observedLeader?.generation ?? this.generation; + const candidateGeneration = observedGeneration + 1; + if ( + this.ownCandidate && + this.ownCandidate.candidateGeneration >= candidateGeneration + ) { + return; + } + + this.clearCandidate(); + this.clearNoLeaderTimer(); + const candidate: CandidateMessage = { + type: "candidate", + version: MESSAGE_VERSION, + tabId: this.tabId, + sentAt: this.now(), + candidateId: this.tabId, + candidateGeneration, + visibility: this.currentVisibility(), + observedLeaderId: observedLeader?.leaderId ?? null, + observedGeneration, + reason, + }; + + this.ownCandidate = candidate; + this.candidates.set(candidateKey(candidate), candidate); + if (!this.post(candidate)) return; + this.candidateTimer = setTimeout(() => { + this.completeCandidacy(candidate); + }, this.timing.electionJitterMs); + } + + private completeCandidacy(candidate: CandidateMessage) { + if (this.ownCandidate !== candidate || this.fallbackMode) return; + + for (const other of this.candidates.values()) { + if ( + other.candidateGeneration === candidate.candidateGeneration && + candidateHasHigherPriority(other, candidate) + ) { + this.clearCandidate(); + return; + } + } + + if (this.leader && this.leaderIsFresh(this.leader)) { + if (this.leader.generation > candidate.candidateGeneration) { + this.clearCandidate(); + return; + } + if ( + this.leader.generation === candidate.candidateGeneration && + leaderHasHigherPriority(this.leader, leaderStateForCandidate(candidate, this.now())) + ) { + this.clearCandidate(); + return; + } + } + + this.becomeLeader(candidate.candidateGeneration); + } + + private becomeLeader(generation: number) { + this.clearCandidate(); + this.closeSource(); + this.isLeader = true; + this.generation = generation; + this.pruneStaleCandidates(); + this.leader = { + leaderId: this.tabId, + generation, + visibility: this.currentVisibility(), + lastSeen: this.now(), + }; + + const source = this.sourceFactory(queryKeys.system.attach()); + this.source = source; + source.onmessage = (message) => { + this.handleLeaderEventSourceMessage(message.data); + }; + + const announced = this.post({ + type: "leader-changed", + version: MESSAGE_VERSION, + tabId: this.tabId, + sentAt: this.now(), + leaderId: this.tabId, + generation, + visibility: this.currentVisibility(), + }); + if (!announced) return; + this.startHeartbeat(); + this.resyncAll(); + } + + private handleLeaderEventSourceMessage(data: string) { + if (!this.isLeader) return; + + let payload: EventPayload; + try { + payload = JSON.parse(data) as EventPayload; + } catch { + return; + } + + if (!this.recentEvents.remember(eventDedupeKey(payload), this.now())) return; + this.dispatchPayload(payload); + this.post({ + type: "event", + version: MESSAGE_VERSION, + tabId: this.tabId, + sentAt: this.now(), + leaderId: this.tabId, + generation: this.generation, + payload, + }); + } + + private dispatchPayload(payload: EventPayload) { + for (const subscription of this.subscriptions.values()) { + const invalidation = subscription.resolveInvalidation(payload); + this.queueInvalidations(subscription, invalidation.keys, { + immediate: invalidation.immediate, + }); + } + } + + private queueInvalidations( + subscription: LocalSubscription, + keys: string[], + { immediate = false }: { immediate?: boolean } = {}, + ) { + if (keys.length === 0) return; + for (const key of keys) { + subscription.pendingKeys.add(key); + } + + if (immediate || subscription.debounceMs <= 0) { + this.flushInvalidations(subscription); + return; + } + + if (subscription.debounceTimer) { + clearTimeout(subscription.debounceTimer); + } + subscription.debounceTimer = setTimeout(() => { + subscription.debounceTimer = null; + this.flushInvalidations(subscription); + }, subscription.debounceMs); + } + + private flushInvalidations(subscription: LocalSubscription) { + if (subscription.pendingKeys.size === 0) return; + const keys = [...subscription.pendingKeys]; + subscription.pendingKeys.clear(); + + for (const mutator of subscription.mutators.keys()) { + for (const key of keys) { + void mutator(key); + } + } + } + + private resyncAll() { + for (const subscription of this.subscriptions.values()) { + this.queueInvalidations(subscription, subscription.resyncKeys(), { immediate: true }); + } + } + + private handleVisibilityChange() { + if (this.isLeader) { + this.sendHeartbeat(); + } + + if (this.currentVisibility() === "visible") { + this.resyncAll(); + if (this.leader?.visibility === "hidden") { + this.enterCandidacy("hidden-leader", this.leader); + } + } + } + + private handlePagehide() { + this.releaseLeadership({ broadcast: true, resync: false }); + this.clearCandidate(); + this.clearNoLeaderTimer(); + } + + private startHeartbeat() { + if (this.heartbeatTimer) { + clearInterval(this.heartbeatTimer); + } + if (!this.sendHeartbeat()) return; + this.heartbeatTimer = setInterval(() => { + this.sendHeartbeat(); + }, this.timing.heartbeatMs); + } + + private sendHeartbeat(): boolean { + if (!this.isLeader) return false; + const visibility = this.currentVisibility(); + this.leader = { + leaderId: this.tabId, + generation: this.generation, + visibility, + lastSeen: this.now(), + }; + return this.post({ + type: "heartbeat", + version: MESSAGE_VERSION, + tabId: this.tabId, + sentAt: this.now(), + leaderId: this.tabId, + generation: this.generation, + visibility, + }); + } + + private releaseLeadership({ + broadcast, + resync, + }: { + broadcast: boolean; + resync: boolean; + }) { + if (!this.isLeader && !this.source) return; + + const generation = this.generation; + this.closeSource(); + this.isLeader = false; + if (this.heartbeatTimer) { + clearInterval(this.heartbeatTimer); + this.heartbeatTimer = null; + } + this.leader = null; + + if (broadcast) { + const released = this.post({ + type: "release", + version: MESSAGE_VERSION, + tabId: this.tabId, + sentAt: this.now(), + leaderId: this.tabId, + generation, + }); + if (!released) return; + this.post({ + type: "resync", + version: MESSAGE_VERSION, + tabId: this.tabId, + sentAt: this.now(), + leaderId: this.tabId, + generation, + reason: "release", + }); + } + if (resync) this.resyncAll(); + } + + private closeSource() { + if (!this.source) return; + this.source.close(); + this.source = null; + } + + private clearCandidate() { + if (this.candidateTimer) { + clearTimeout(this.candidateTimer); + this.candidateTimer = null; + } + this.ownCandidate = null; + } + + private pruneStaleCandidates() { + for (const [key, candidate] of this.candidates) { + if (candidate.candidateGeneration < this.generation) { + this.candidates.delete(key); + } + } + } + + private scheduleNoLeaderCandidacy() { + if ( + this.noLeaderTimer || + this.ownCandidate || + this.isLeader || + this.leader || + this.subscriptions.size === 0 || + this.fallbackMode + ) { + return; + } + + this.noLeaderTimer = setTimeout(() => { + this.noLeaderTimer = null; + if (!this.leader && !this.isLeader) { + this.enterCandidacy("no-leader"); + } + }, this.timing.electionJitterMs); + } + + private clearNoLeaderTimer() { + if (!this.noLeaderTimer) return; + clearTimeout(this.noLeaderTimer); + this.noLeaderTimer = null; + } + + private shutdownTimersAndChannel() { + this.clearNoLeaderTimer(); + if (this.heartbeatTimer) { + clearInterval(this.heartbeatTimer); + this.heartbeatTimer = null; + } + if (this.leaderCheckTimer) { + clearInterval(this.leaderCheckTimer); + this.leaderCheckTimer = null; + } + this.removeVisibilityListener?.(); + this.removeVisibilityListener = null; + this.removePagehideListener?.(); + this.removePagehideListener = null; + if (this.channel) { + this.channel.close(); + this.channel = null; + } + } + + private post(message: CrossTabSseMessage): boolean { + if (!this.channel) { + this.degradeToFallback(); + return false; + } + try { + this.channel.postMessage(message); + return true; + } catch { + this.degradeToFallback(); + return false; + } + } + + private degradeToFallback() { + if (this.degradingToFallback || this.fallbackMode) return; + this.degradingToFallback = true; + this.releaseLeadership({ broadcast: false, resync: false }); + this.clearCandidate(); + this.shutdownTimersAndChannel(); + this.initialized = false; + this.fallbackMode = true; + this.coordinationUnavailable = true; + for (const subscription of this.subscriptions.values()) { + this.startFallbacksFor(subscription); + } + this.degradingToFallback = false; + } + + private startFallbacksFor(subscription: LocalSubscription) { + for (const fallback of subscription.fallbacks.values()) { + if (!fallback.cleanup) { + fallback.cleanup = fallback.subscribe(); + } + } + } + + private closeFallbacks() { + for (const subscription of this.subscriptions.values()) { + for (const fallback of subscription.fallbacks.values()) { + fallback.cleanup?.(); + fallback.cleanup = undefined; + } + } + } + + private clearSubscriptionTimers() { + for (const subscription of this.subscriptions.values()) { + this.clearSubscriptionTimer(subscription); + } + } + + private clearSubscriptionTimer(subscription: LocalSubscription) { + if (!subscription.debounceTimer) return; + clearTimeout(subscription.debounceTimer); + subscription.debounceTimer = null; + } + + private resetIdleState() { + this.leader = null; + this.initialized = false; + this.coordinationUnavailable = false; + this.fallbackMode = false; + } + + private shouldAcceptLeader(incoming: LeaderState): boolean { + if (!this.leader) return true; + if (incoming.generation > this.leader.generation) return true; + if (incoming.generation < this.leader.generation) return false; + if (incoming.leaderId === this.leader.leaderId) return true; + return leaderHasHigherPriority(incoming, this.leader); + } + + private isCurrentLeader(leaderId: string, generation: number): boolean { + return Boolean( + this.leader && + this.leader.leaderId === leaderId && + this.leader.generation === generation, + ); + } + + private currentVisibility(): TabVisibility { + return this.getVisibility() === "hidden" ? "hidden" : "visible"; + } + + private leaderIsFresh(leader: LeaderState): boolean { + return this.now() - leader.lastSeen <= this.timing.leaderStaleMs; + } +} + +const defaultCoordinator = new CrossTabSseCoordinator(); + +export function createCrossTabSseCoordinator(options: CrossTabSseCoordinatorOptions = {}) { + return new CrossTabSseCoordinator(options); +} + +export function subscribeToCrossTabSse({ + coordinator = defaultCoordinator, + ...options +}: SubscribeToCrossTabSseOptions): () => void { + return coordinator.subscribe(options); +} + +function createBrowserBroadcastChannel(name: string): BroadcastChannelLike { + if (typeof BroadcastChannel === "undefined") { + throw new Error("BroadcastChannel is unavailable"); + } + return new BroadcastChannel(name); +} + +function createTabId(): string { + if (typeof crypto !== "undefined" && typeof crypto.randomUUID === "function") { + return crypto.randomUUID(); + } + return `tab-${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`; +} + +function getBrowserVisibility(): TabVisibility { + if (typeof document === "undefined") return "visible"; + return document.visibilityState === "visible" ? "visible" : "hidden"; +} + +function addBrowserVisibilityChangeListener(handler: () => void): () => void { + if (typeof document === "undefined") return () => {}; + document.addEventListener("visibilitychange", handler); + return () => document.removeEventListener("visibilitychange", handler); +} + +function addBrowserPagehideListener(handler: () => void): () => void { + if (typeof window === "undefined") return () => {}; + window.addEventListener("pagehide", handler); + return () => window.removeEventListener("pagehide", handler); +} + +function parseMessage(data: unknown): CrossTabSseMessage | undefined { + if (!isRecord(data)) return undefined; + if (data.version !== MESSAGE_VERSION) return undefined; + const type = getString(data, "type"); + const tabId = getString(data, "tabId"); + const sentAt = getNumber(data, "sentAt"); + if (!type || !tabId || sentAt === undefined) return undefined; + const base = { type, version: MESSAGE_VERSION, tabId, sentAt }; + + switch (type) { + case "hello": + return { ...base, type: "hello" }; + case "heartbeat": { + const triple = parseLeaderTriple(data); + return triple && { ...base, type: "heartbeat", ...triple }; + } + case "candidate": + return parseCandidate(data, base); + case "leader-changed": { + const triple = parseLeaderTriple(data); + return triple && { ...base, type: "leader-changed", ...triple }; + } + case "release": { + const pair = parseLeaderPair(data); + return pair && { ...base, type: "release", ...pair }; + } + case "resync": + return parseResync(data, base); + case "event": + return parseEvent(data, base); + default: + return undefined; + } +} + +interface ParsedBase { + version: typeof MESSAGE_VERSION; + tabId: string; + sentAt: number; +} + +function parseLeaderPair(data: unknown): { leaderId: string; generation: number } | undefined { + const leaderId = getString(data, "leaderId"); + const generation = getNumber(data, "generation"); + if (!leaderId || generation === undefined) return undefined; + return { leaderId, generation }; +} + +function parseLeaderTriple( + data: unknown, +): { leaderId: string; generation: number; visibility: TabVisibility } | undefined { + const pair = parseLeaderPair(data); + const visibility = getString(data, "visibility"); + if (!pair || !isVisibility(visibility)) return undefined; + return { ...pair, visibility }; +} + +function parseCandidate(data: UnknownRecord, base: ParsedBase): CandidateMessage | undefined { + const candidateId = getString(data, "candidateId"); + const candidateGeneration = getNumber(data, "candidateGeneration"); + const visibility = getString(data, "visibility"); + const observedGeneration = getNumber(data, "observedGeneration"); + const reason = getString(data, "reason"); + const observedLeaderRaw = data.observedLeaderId; + const observedLeaderId = + typeof observedLeaderRaw === "string" ? observedLeaderRaw : observedLeaderRaw === null ? null : undefined; + if ( + !candidateId || + candidateGeneration === undefined || + !isVisibility(visibility) || + observedLeaderId === undefined || + observedGeneration === undefined || + !isCandidateReason(reason) + ) { + return undefined; + } + return { + ...base, + type: "candidate", + candidateId, + candidateGeneration, + visibility, + observedLeaderId, + observedGeneration, + reason, + }; +} + +function parseResync(data: UnknownRecord, base: ParsedBase): ResyncMessage | undefined { + const generation = getNumber(data, "generation"); + const reason = getString(data, "reason"); + const leaderRaw = data.leaderId; + const leaderId = typeof leaderRaw === "string" ? leaderRaw : leaderRaw === null ? null : undefined; + if (generation === undefined || !isCandidateReason(reason) || leaderId === undefined) { + return undefined; + } + return { ...base, type: "resync", leaderId, generation, reason }; +} + +function parseEvent(data: UnknownRecord, base: ParsedBase): EventMessage | undefined { + const pair = parseLeaderPair(data); + if (!pair || !isRecord(data.payload)) return undefined; + return { ...base, type: "event", ...pair, payload: data.payload }; +} + +function isVisibility(value: unknown): value is TabVisibility { + return value === "visible" || value === "hidden"; +} + +function isCandidateReason(value: unknown): value is CandidateReason { + return ( + value === "hidden-leader" || + value === "stale-leader" || + value === "release" || + value === "no-leader" + ); +} + +function candidateHasHigherPriority(candidate: CandidateMessage, other: CandidateMessage): boolean { + if (candidate.visibility !== other.visibility) return candidate.visibility === "visible"; + return candidate.candidateId < other.candidateId; +} + +function leaderHasHigherPriority(candidate: LeaderState, other: LeaderState): boolean { + if (candidate.visibility !== other.visibility) return candidate.visibility === "visible"; + return candidate.leaderId < other.leaderId; +} + +function leaderStateForCandidate(candidate: CandidateMessage, lastSeen: number): LeaderState { + return { + leaderId: candidate.candidateId, + generation: candidate.candidateGeneration, + visibility: candidate.visibility, + lastSeen, + }; +} + +function candidateKey(candidate: CandidateMessage): string { + return `${candidate.candidateGeneration}:${candidate.candidateId}`; +} + +function eventDedupeKey(payload: EventPayload): string | undefined { + if (typeof payload.id === "string" && payload.id.length > 0) { + return payload.id; + } + + const runId = typeof payload.run_id === "string" ? payload.run_id : undefined; + const seq = typeof payload.seq === "number" ? payload.seq : undefined; + const event = typeof payload.event === "string" ? payload.event : undefined; + if (runId && seq != null && event) { + return `${runId}:${seq}:${event}`; + } + return undefined; +} diff --git a/apps/fabro-web/app/lib/queries.ts b/apps/fabro-web/app/lib/queries.ts index 4de23a73e..880cf97b6 100644 --- a/apps/fabro-web/app/lib/queries.ts +++ b/apps/fabro-web/app/lib/queries.ts @@ -1,12 +1,11 @@ import useSWR, { type SWRConfiguration } from "swr"; import type { ApiQuestion, + EventEnvelope, PaginatedBoardRunList, - PaginatedEventList, PaginatedRunFileList, PaginatedRunList, PaginatedRunStageList, - PaginatedStageTurnList, CommandLogResponse, CommandOutputStream, RunBilling, @@ -24,6 +23,7 @@ import { apiNullableTextFetcher, apiPaginatedFetcher, apiTextFetcher, + fetchAllStageEvents, type PaginatedEnvelope, } from "./api-client"; import { queryKeys } from "./query-keys"; @@ -34,6 +34,9 @@ const immutableOptions: SWRConfiguration = { revalidateOnReconnect: false, }; +type BoardRunsEnvelope = PaginatedEnvelope & + Pick; + export function useAuthConfig() { return useSWR<{ methods: string[] }>(queryKeys.auth.config(), apiFetcher, immutableOptions); } @@ -61,11 +64,7 @@ export function useSystemInfo() { } export function useBoardsRuns() { - return useSWR< - PaginatedEnvelope & { - columns: { id: string; name: string }[]; - } - >(queryKeys.boards.runs(), apiPaginatedFetcher); + return useSWR(queryKeys.boards.runs(), apiPaginatedFetcher); } export function useRun(id: string | undefined) { @@ -141,21 +140,10 @@ export function useRunQuestions(id: string | undefined, enabled: boolean) { ); } -export function useRunStageTurns( - id: string | undefined, - stageId: string | undefined, - enabled = true, -) { - return useSWR( - id && stageId && enabled ? queryKeys.runs.stageTurns(id, stageId) : null, - apiNullableFetcher, - ); -} - -export function useRunEventsList(id: string | undefined, enabled = true) { - return useSWR( - id && enabled ? queryKeys.runs.events(id, 1000) : null, - apiNullableFetcher, +export function useRunStageEvents(id: string | undefined, stageId: string | undefined) { + return useSWR( + id && stageId ? queryKeys.runs.stageEvents(id, stageId) : null, + fetchAllStageEvents, ); } @@ -206,4 +194,4 @@ export function useServerSettings() { return useSWR(queryKeys.settings.server(), apiFetcher, immutableOptions); } -export { apiTextFetcher }; +export { apiTextFetcher }; \ No newline at end of file diff --git a/apps/fabro-web/app/lib/query-keys.test.ts b/apps/fabro-web/app/lib/query-keys.test.ts index 8f036ac3e..a06acaedb 100644 --- a/apps/fabro-web/app/lib/query-keys.test.ts +++ b/apps/fabro-web/app/lib/query-keys.test.ts @@ -11,6 +11,9 @@ describe("queryKeys", () => { expect(queryKeys.runs.stageLog("run 1", "build step@2", "stderr", 12, 34)).toBe( "/api/v1/runs/run%201/stages/build%20step%402/logs/stderr?offset=12&limit=34", ); + expect(queryKeys.runs.stageEvents("run 1", "build step", 7, 25)).toBe( + "/api/v1/runs/run%201/stages/build%20step/events?since_seq=7&limit=25", + ); }); test("event-mapped keys match query hook resources", () => { @@ -19,11 +22,31 @@ describe("queryKeys", () => { ]); expect(queryKeysForRunEvent("run-1", "stage.completed", "stage-1")).toEqual([ queryKeys.runs.stages("run-1"), + queryKeys.runs.billing("run-1"), queryKeys.runs.events("run-1", 1000), queryKeys.runs.graph("run-1", "LR"), queryKeys.runs.graph("run-1", "TB"), queryKeys.runs.detail("run-1"), - queryKeys.runs.stageTurns("run-1", "stage-1"), + queryKeys.runs.stageEvents("run-1", "stage-1"), ]); }); + + test("agent activity events invalidate the per-stage events key", () => { + for (const event of [ + "stage.prompt", + "agent.message", + "agent.tool.started", + "agent.tool.completed", + "command.started", + "command.completed", + ]) { + expect(queryKeysForRunEvent("run-1", event, "stage-1")).toEqual([ + queryKeys.runs.stageEvents("run-1", "stage-1"), + ]); + } + }); + + test("agent activity events without a node_id invalidate nothing", () => { + expect(queryKeysForRunEvent("run-1", "agent.message")).toEqual([]); + }); }); diff --git a/apps/fabro-web/app/lib/query-keys.ts b/apps/fabro-web/app/lib/query-keys.ts index 7d3e847e7..402739897 100644 --- a/apps/fabro-web/app/lib/query-keys.ts +++ b/apps/fabro-web/app/lib/query-keys.ts @@ -44,8 +44,11 @@ export const queryKeys = { }), events: (id: string, limit = 1000) => withQuery(`/api/v1/runs/${pathSegment(id)}/events`, { limit }), - stageTurns: (id: string, stageId: string) => - `/api/v1/runs/${pathSegment(id)}/stages/${pathSegment(stageId)}/turns`, + stageEvents: (id: string, stageId: string, sinceSeq?: number, limit?: number) => + withQuery(`/api/v1/runs/${pathSegment(id)}/stages/${pathSegment(stageId)}/events`, { + since_seq: sinceSeq, + limit, + }), stageLog: ( id: string, stageId: string, @@ -75,4 +78,4 @@ export const queryKeys = { settings: { server: () => "/api/v1/settings", }, -}; +}; \ No newline at end of file diff --git a/apps/fabro-web/app/lib/run-events.test.tsx b/apps/fabro-web/app/lib/run-events.test.tsx index 14acf4f23..432791352 100644 --- a/apps/fabro-web/app/lib/run-events.test.tsx +++ b/apps/fabro-web/app/lib/run-events.test.tsx @@ -4,7 +4,12 @@ import { queryKeysForRunEvent, subscribeToRunEvents, } from "./run-events"; +import { + createCrossTabSseCoordinator, + type BroadcastChannelLike, +} from "./cross-tab-sse"; import { queryKeys } from "./query-keys"; +import type { EventSourceLike } from "./sse"; type MessageHandler = ((event: { data: string }) => void) | null; @@ -25,6 +30,14 @@ class FakeEventSource { } } +class FakeBroadcastChannel implements BroadcastChannelLike { + onmessage: ((event: { data: unknown }) => void) | null = null; + + postMessage() {} + + close() {} +} + describe("queryKeysForRunEvent", () => { test("terminal events invalidate run-scoped resources", () => { expect(queryKeysForRunEvent("run-1", "run.completed")).toEqual([ @@ -36,13 +49,91 @@ describe("queryKeysForRunEvent", () => { queryKeys.runs.graph("run-1", "TB"), ]); }); + + test("stage.retrying invalidates stages, billing, events, graph, detail, and stage events", () => { + expect(queryKeysForRunEvent("run-1", "stage.retrying", "verify@2")).toEqual([ + queryKeys.runs.stages("run-1"), + queryKeys.runs.billing("run-1"), + queryKeys.runs.events("run-1", 1000), + queryKeys.runs.graph("run-1", "LR"), + queryKeys.runs.graph("run-1", "TB"), + queryKeys.runs.detail("run-1"), + queryKeys.runs.stageEvents("run-1", "verify@2"), + ]); + }); }); describe("subscribeToRunEvents", () => { - test("refcounts shared sources and keeps mutators active until final unsubscribe", () => { + test("coordinated mode uses the global attach stream and filters by run_id", async () => { const source = new FakeEventSource(); const created: string[] = []; const keys: string[] = []; + const coordinator = createCoordinator((url) => { + created.push(url); + return source; + }); + + const cleanup = subscribeToRunEvents( + "run-coordinated", + (key) => { + keys.push(key); + return Promise.resolve(); + }, + () => { + throw new Error("source should be created by coordinator"); + }, + { debounceMs: 0, coordinator }, + ); + + await waitFor(() => created.length === 1); + keys.length = 0; + + source.emit({ event: "checkpoint.completed", run_id: "other-run" }); + source.emit({ event: "checkpoint.completed", run_id: "run-coordinated" }); + + expect(created).toEqual(["/api/v1/attach"]); + expect(keys).toEqual([queryKeys.runs.files("run-coordinated")]); + + cleanup(); + coordinator.close(); + }); + + test("coordinated terminal events invalidate without closing the global stream", async () => { + const source = new FakeEventSource(); + const keys: string[] = []; + const coordinator = createCoordinator(() => source); + const cleanup = subscribeToRunEvents( + "run-terminal", + (key) => { + keys.push(key); + return Promise.resolve(); + }, + () => source, + { debounceMs: 0, coordinator }, + ); + + await waitFor(() => source.onmessage !== null); + keys.length = 0; + + source.emit({ event: "run.failed", run_id: "run-terminal" }); + expect(source.closed).toBe(false); + expect(keys).toContain(queryKeys.runs.files("run-terminal")); + expect(keys).toContain(queryKeys.runs.billing("run-terminal")); + + keys.length = 0; + source.emit({ event: "run.archived", run_id: "run-terminal" }); + expect(source.closed).toBe(false); + expect(keys).toEqual([queryKeys.runs.detail("run-terminal")]); + + cleanup(); + coordinator.close(); + }); + + test("fallback refcounts run-scoped sources and keeps mutators active until final unsubscribe", () => { + const source = new FakeEventSource(); + const created: string[] = []; + const keys: string[] = []; + const coordinator = createFallbackCoordinator(); const mutate = (key: string) => { keys.push(key); return Promise.resolve(); @@ -51,10 +142,10 @@ describe("subscribeToRunEvents", () => { const firstCleanup = subscribeToRunEvents("run-refcount", mutate, (url) => { created.push(url); return source; - }, { debounceMs: 0 }); + }, { debounceMs: 0, coordinator }); const secondCleanup = subscribeToRunEvents("run-refcount", mutate, () => { throw new Error("source should be reused"); - }, { debounceMs: 0 }); + }, { debounceMs: 0, coordinator }); expect(created).toEqual(["/api/v1/runs/run-refcount/attach"]); @@ -66,11 +157,13 @@ describe("subscribeToRunEvents", () => { secondCleanup(); expect(source.closed).toBe(true); + coordinator.close(); }); - test("terminal events close the source after invalidating keys", () => { + test("fallback terminal events close the source after invalidating keys", () => { const source = new FakeEventSource(); const keys: string[] = []; + const coordinator = createFallbackCoordinator(); const cleanup = subscribeToRunEvents( "run-terminal", (key) => { @@ -78,7 +171,7 @@ describe("subscribeToRunEvents", () => { return Promise.resolve(); }, () => source, - { debounceMs: 0 }, + { debounceMs: 0, coordinator }, ); source.emit({ event: "run.failed" }); @@ -88,13 +181,72 @@ describe("subscribeToRunEvents", () => { expect(keys).toContain(queryKeys.runs.billing("run-terminal")); cleanup(); + coordinator.close(); }); - test("malformed events are ignored and StrictMode-style cleanup does not underflow", () => { + test("envelope with suffixed stage_id invalidates stageEvents(runId, stageId)", async () => { + const source = new FakeEventSource(); + const keys: string[] = []; + const coordinator = createCoordinator(() => source); + const cleanup = subscribeToRunEvents( + "run-stage", + (key) => { + keys.push(key); + return Promise.resolve(); + }, + () => source, + { debounceMs: 0, coordinator }, + ); + + await waitFor(() => source.onmessage !== null); + source.emit({ + event: "stage.retrying", + run_id: "run-stage", + stage_id: "verify@2", + node_id: "verify", + }); + + expect(keys).toContain(queryKeys.runs.stageEvents("run-stage", "verify@2")); + expect(keys).toContain(queryKeys.runs.stages("run-stage")); + expect(keys).toContain(queryKeys.runs.events("run-stage", 1000)); + expect(keys).toContain(queryKeys.runs.graph("run-stage", "LR")); + expect(keys).toContain(queryKeys.runs.detail("run-stage")); + expect(keys).not.toContain(queryKeys.runs.stageEvents("run-stage", "verify")); + + cleanup(); + coordinator.close(); + }); + + test("falls back to node_id when an event has no stage_id", async () => { + const source = new FakeEventSource(); + const keys: string[] = []; + const coordinator = createCoordinator(() => source); + const cleanup = subscribeToRunEvents( + "run-stage-node", + (key) => { + keys.push(key); + return Promise.resolve(); + }, + () => source, + { debounceMs: 0, coordinator }, + ); + + await waitFor(() => source.onmessage !== null); + source.emit({ event: "stage.started", run_id: "run-stage-node", node_id: "verify" }); + + expect(keys).toContain(queryKeys.runs.stageEvents("run-stage-node", "verify")); + expect(keys).toContain(queryKeys.runs.stages("run-stage-node")); + + cleanup(); + coordinator.close(); + }); + + test("fallback malformed events are ignored and StrictMode-style cleanup does not underflow", () => { const firstSource = new FakeEventSource(); const secondSource = new FakeEventSource(); const sources = [firstSource, secondSource]; const keys: string[] = []; + const coordinator = createFallbackCoordinator(); const firstCleanup = subscribeToRunEvents( "run-strict", @@ -103,7 +255,7 @@ describe("subscribeToRunEvents", () => { return Promise.resolve(); }, () => sources.shift()!, - { debounceMs: 0 }, + { debounceMs: 0, coordinator }, ); firstSource.emitRaw("{broken"); firstCleanup(); @@ -115,12 +267,45 @@ describe("subscribeToRunEvents", () => { return Promise.resolve(); }, () => sources.shift()!, - { debounceMs: 0 }, + { debounceMs: 0, coordinator }, ); secondCleanup(); expect(keys).toEqual([]); expect(firstSource.closed).toBe(true); expect(secondSource.closed).toBe(true); + coordinator.close(); }); }); + +function createCoordinator(eventSourceFactory: (url: string) => EventSourceLike) { + return createCrossTabSseCoordinator({ + tabId: "run-test", + channelFactory: () => new FakeBroadcastChannel(), + eventSourceFactory, + addVisibilityChangeListener: () => () => {}, + addPagehideListener: () => () => {}, + timing: { + heartbeatMs: 10, + leaderStaleMs: 50, + electionJitterMs: 0, + }, + }); +} + +function createFallbackCoordinator() { + return createCrossTabSseCoordinator({ + channelFactory: () => { + throw new Error("BroadcastChannel unavailable"); + }, + }); +} + +async function waitFor(condition: () => boolean, timeoutMs = 200) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (condition()) return; + await new Promise((resolve) => setTimeout(resolve, 2)); + } + throw new Error("condition did not become true before timeout"); +} diff --git a/apps/fabro-web/app/lib/run-events.ts b/apps/fabro-web/app/lib/run-events.ts index 924264af2..63855296e 100644 --- a/apps/fabro-web/app/lib/run-events.ts +++ b/apps/fabro-web/app/lib/run-events.ts @@ -1,6 +1,10 @@ import { useEffect } from "react"; import { useSWRConfig } from "swr"; +import { + subscribeToCrossTabSse, + type CrossTabSseCoordinator, +} from "./cross-tab-sse"; import { queryKeys } from "./query-keys"; import { createBrowserEventSource, @@ -13,10 +17,17 @@ import { interface RunEventPayload extends EventPayload { event?: string; + run_id?: string; node_id?: string; + stage_id?: string; properties?: Record; } +interface RunEventOptions { + debounceMs?: number; + coordinator?: CrossTabSseCoordinator; +} + const subscriptions = new Map(); const TERMINAL_EVENTS = new Set(["run.completed", "run.failed"]); @@ -32,8 +43,31 @@ const RUN_SUMMARY_EVENTS = new Set([ "run.archived", "run.unarchived", ]); -const STAGE_EVENTS = new Set(["stage.started", "stage.completed", "stage.failed"]); -const COMMAND_EVENTS = new Set(["command.started", "command.completed"]); +const STAGE_EVENTS = new Set([ + "stage.started", + "stage.completed", + "stage.failed", + "stage.retrying", +]); +// Single source of truth: every event type the `eventsToActivity` reducer in +// `routes/run-stages.tsx` consumes. When any of these arrive for a stage we +// currently view, the stage-events SWR key for that stage must be invalidated +// so the panel refetches. The reducer imports this list so the switch stays +// in sync with the invalidation set; if the reducer grows a new case, this +// list is the single edit point. +// +// The lifecycle `STAGE_EVENTS` set is kept separate because it also fans out +// to run-scoped invalidations (stages list, graph, detail). +export const STAGE_ACTIVITY_EVENT_TYPES = [ + "stage.prompt", + "agent.message", + "agent.tool.started", + "agent.tool.completed", + "command.started", + "command.completed", +] as const; +export type StageActivityEventType = (typeof STAGE_ACTIVITY_EVENT_TYPES)[number]; +const STAGE_ACTIVITY_EVENTS = new Set(STAGE_ACTIVITY_EVENT_TYPES); const INTERVIEW_EVENTS = new Set([ "interview.started", "interview.completed", @@ -75,26 +109,20 @@ export function queryKeysForRunEvent( if (STAGE_EVENTS.has(event)) { const keys = [ queryKeys.runs.stages(runId), + queryKeys.runs.billing(runId), queryKeys.runs.events(runId, 1000), queryKeys.runs.graph(runId, "LR"), queryKeys.runs.graph(runId, "TB"), queryKeys.runs.detail(runId), ]; if (stageId) { - keys.push(queryKeys.runs.stageTurns(runId, stageId)); + keys.push(queryKeys.runs.stageEvents(runId, stageId)); } return keys; } - if (COMMAND_EVENTS.has(event)) { - const keys = [ - queryKeys.runs.stages(runId), - queryKeys.runs.events(runId, 1000), - ]; - if (stageId) { - keys.push(queryKeys.runs.stageTurns(runId, stageId)); - } - return keys; + if (STAGE_ACTIVITY_EVENTS.has(event)) { + return stageId ? [queryKeys.runs.stageEvents(runId, stageId)] : []; } return []; @@ -104,32 +132,59 @@ export function subscribeToRunEvents( runId: string, mutate: MutateFn, eventSourceFactory: (url: string) => EventSourceLike = createBrowserEventSource, - { debounceMs = 300 }: { debounceMs?: number } = {}, + { debounceMs = 300, coordinator }: RunEventOptions = {}, ): () => void { - return subscribeToSharedEventSource({ - subscriptions, - subscriptionKey: runId, - url: queryKeys.runs.attach(runId), + return subscribeToCrossTabSse({ + coordinator, + subscriptionKey: `run:${runId}`, mutate, - eventSourceFactory, debounceMs, + resyncKeys: () => resyncKeysForRun(runId), resolveInvalidation: (payload) => { - const event = payload.event; - if (!event) return { keys: [] }; - - const stageId = stageIdFromPayload(payload); - const keys = queryKeysForRunEvent(runId, event, stageId); - const terminal = TERMINAL_EVENTS.has(event); - return { - keys, - close: terminal, - immediate: terminal, - }; + if (payload.run_id !== runId) return { keys: [] }; + return runInvalidation(runId, payload); }, + fallbackSubscribe: () => + subscribeToSharedEventSource({ + subscriptions, + subscriptionKey: runId, + url: queryKeys.runs.attach(runId), + mutate, + eventSourceFactory, + debounceMs, + resolveInvalidation: (payload) => { + const result = runInvalidation(runId, payload); + return { ...result, close: result.immediate }; + }, + }), }); } +function runInvalidation(runId: string, payload: RunEventPayload) { + const event = payload.event; + if (!event) return { keys: [], immediate: false }; + + const stageId = stageIdFromPayload(payload); + const keys = queryKeysForRunEvent(runId, event, stageId); + const terminal = TERMINAL_EVENTS.has(event); + return { keys, immediate: terminal }; +} + +function resyncKeysForRun(runId: string) { + return [ + queryKeys.runs.detail(runId), + queryKeys.runs.files(runId), + queryKeys.runs.billing(runId), + queryKeys.runs.stages(runId), + queryKeys.runs.events(runId, 1000), + queryKeys.runs.graph(runId, "LR"), + queryKeys.runs.graph(runId, "TB"), + queryKeys.runs.questions(runId, 25, 0), + ]; +} + function stageIdFromPayload(payload: RunEventPayload): string | undefined { + if (typeof payload.stage_id === "string") return payload.stage_id; if (typeof payload.node_id === "string") return payload.node_id; const nodeId = payload.properties?.node_id; return typeof nodeId === "string" ? nodeId : undefined; diff --git a/apps/fabro-web/app/lib/stage-sidebar.test.ts b/apps/fabro-web/app/lib/stage-sidebar.test.ts new file mode 100644 index 000000000..9e8891645 --- /dev/null +++ b/apps/fabro-web/app/lib/stage-sidebar.test.ts @@ -0,0 +1,172 @@ +import { describe, expect, test } from "bun:test"; +import type { PaginatedRunStageList, StageState } from "@qltysh/fabro-api-client"; + +import type { Stage } from "../components/stage-sidebar"; +import { aggregateGraphNodeStatus, formatStageLabel, mapRunStagesToSidebarStages } from "./stage-sidebar"; + +function makeStage(nodeId: string, visit: number, status: StageState): Stage { + return { + id: `${nodeId}@${visit}`, + name: nodeId, + nodeId, + visit, + status, + duration: "--", + }; +} + +describe("mapRunStagesToSidebarStages", () => { + test("maps two visits of the same node to distinct sidebar entries", () => { + const stages: PaginatedRunStageList = { + data: [ + { + id: "apply-changes@1", + name: "Apply Changes", + status: "succeeded", + duration_secs: 12.5, + node_id: "apply", + visit: 1, + }, + { + id: "apply-changes@2", + name: "Apply Changes", + status: "running", + node_id: "apply", + visit: 2, + }, + ], + meta: { has_more: false }, + }; + + const result = mapRunStagesToSidebarStages(stages); + expect(result).toHaveLength(2); + + expect(result[0].id).toBe("apply-changes@1"); + expect(result[0].nodeId).toBe("apply"); + expect(result[0].visit).toBe(1); + expect(formatStageLabel(result[0])).toBe("Apply Changes"); + + expect(result[1].id).toBe("apply-changes@2"); + expect(result[1].nodeId).toBe("apply"); + expect(result[1].visit).toBe(2); + expect(formatStageLabel(result[1])).toBe("Apply Changes (2)"); + }); + + test("filters by node_id (suffixed start@1 / exit@1 are still hidden)", () => { + const stages: PaginatedRunStageList = { + data: [ + { + id: "start@1", + name: "start", + status: "succeeded", + node_id: "start", + visit: 1, + }, + { + id: "verify@1", + name: "verify", + status: "succeeded", + node_id: "verify", + visit: 1, + }, + { + id: "exit@1", + name: "exit", + status: "succeeded", + node_id: "exit", + visit: 1, + }, + ], + meta: { has_more: false }, + }; + + const result = mapRunStagesToSidebarStages(stages); + expect(result.map((s) => s.id)).toEqual(["verify@1"]); + }); + + test("missing duration renders as '--'", () => { + const stages: PaginatedRunStageList = { + data: [ + { + id: "verify@1", + name: "verify", + status: "running", + node_id: "verify", + visit: 1, + }, + ], + meta: { has_more: false }, + }; + + expect(mapRunStagesToSidebarStages(stages)[0].duration).toBe("--"); + }); +}); + +describe("aggregateGraphNodeStatus", () => { + test("(failed, running) renders as running and clicks open the latest visit", () => { + const result = aggregateGraphNodeStatus([ + makeStage("verify", 1, "failed"), + makeStage("verify", 2, "running"), + ]); + expect(result.get("verify")).toEqual({ + displayStatus: "running", + latestStageId: "verify@2", + }); + }); + + test("(failed, succeeded) renders as succeeded — failure-then-fix shows healed", () => { + const result = aggregateGraphNodeStatus([ + makeStage("verify", 1, "failed"), + makeStage("verify", 2, "succeeded"), + ]); + expect(result.get("verify")).toEqual({ + displayStatus: "succeeded", + latestStageId: "verify@2", + }); + }); + + test("(succeeded, failed) renders as failed and clicks open the latest visit", () => { + const result = aggregateGraphNodeStatus([ + makeStage("verify", 1, "succeeded"), + makeStage("verify", 2, "failed"), + ]); + expect(result.get("verify")).toEqual({ + displayStatus: "failed", + latestStageId: "verify@2", + }); + }); + + test("(running, retrying) — latest active wins", () => { + const result = aggregateGraphNodeStatus([ + makeStage("verify", 1, "running"), + makeStage("verify", 2, "retrying"), + ]); + expect(result.get("verify")).toEqual({ + displayStatus: "retrying", + latestStageId: "verify@2", + }); + }); + + test("orders by visit even when input is shuffled", () => { + const result = aggregateGraphNodeStatus([ + makeStage("verify", 2, "running"), + makeStage("verify", 1, "failed"), + ]); + expect(result.get("verify")?.latestStageId).toBe("verify@2"); + }); + + test("single visit per node is unaffected", () => { + const result = aggregateGraphNodeStatus([ + makeStage("plan", 1, "succeeded"), + makeStage("apply", 1, "running"), + ]); + expect(result.get("plan")).toEqual({ + displayStatus: "succeeded", + latestStageId: "plan@1", + }); + expect(result.get("apply")).toEqual({ + displayStatus: "running", + latestStageId: "apply@1", + }); + }); +}); diff --git a/apps/fabro-web/app/lib/stage-sidebar.ts b/apps/fabro-web/app/lib/stage-sidebar.ts index 747c6a70e..84a5de2de 100644 --- a/apps/fabro-web/app/lib/stage-sidebar.ts +++ b/apps/fabro-web/app/lib/stage-sidebar.ts @@ -1,27 +1,83 @@ -import type { PaginatedRunStageList, StageState } from "@qltysh/fabro-api-client"; +import { StageState } from "@qltysh/fabro-api-client"; +import type { PaginatedRunStageList } from "@qltysh/fabro-api-client"; import type { Stage } from "../components/stage-sidebar"; import { isVisibleStage } from "../data/runs"; import { formatDurationSecs } from "./format"; -export const ACTIVE_STAGE_STATES: ReadonlySet = new Set(["running", "retrying"]); -export const SUCCEEDED_STAGE_STATES: ReadonlySet = new Set([ - "succeeded", - "partially_succeeded", +export const ACTIVE_STAGE_STATES: ReadonlySet = new Set([ + StageState.RUNNING, + StageState.RETRYING, ]); +export const IN_FLIGHT_STAGE_STATES: ReadonlySet = new Set([ + StageState.PENDING, + StageState.RUNNING, + StageState.RETRYING, +]); +export const SUCCEEDED_STAGE_STATES: ReadonlySet = new Set([ + StageState.SUCCEEDED, + StageState.PARTIALLY_SUCCEEDED, +]); + +/** + * Display label for a stage. Suffixes `(N)` for visits > 1 so a looped node + * (e.g. `verify`) renders as `verify`, `verify (2)`, `verify (3)` in the + * sidebar and stage header. + */ +export function formatStageLabel(stage: { name: string; visit: number }): string { + return stage.visit > 1 ? `${stage.name} (${stage.visit})` : stage.name; +} export function mapRunStagesToSidebarStages( stagesResult: PaginatedRunStageList | null | undefined, ): Stage[] { return (stagesResult?.data ?? []) - .filter((stage) => isVisibleStage(stage.id)) + .filter((stage) => isVisibleStage(stage.node_id)) .map((stage) => ({ id: stage.id, name: stage.name, - dotId: stage.dot_id ?? stage.id, + nodeId: stage.node_id, + visit: stage.visit, status: stage.status, duration: stage.duration_secs != null ? formatDurationSecs(stage.duration_secs) : "--", })); } + +/** + * Aggregate per-node display state for the workflow graph. + * + * Status policy: if any visit is active (running/retrying), the node renders + * that active state (latest active visit wins). Otherwise the node renders + * the latest visit's terminal state. The click target is always the latest + * visit's stageId. + */ +export function aggregateGraphNodeStatus(stages: readonly Stage[]): Map< + string, + { displayStatus: StageState; latestStageId: string } +> { + // Single pass per nodeId: track the visit with the highest `visit` overall + // (drives click target + terminal status) and the highest-visit *active* + // stage (drives display when any visit is in flight). + const latest = new Map(); + const latestActive = new Map(); + for (const stage of stages) { + const prevLatest = latest.get(stage.nodeId); + if (!prevLatest || stage.visit > prevLatest.visit) { + latest.set(stage.nodeId, stage); + } + if (ACTIVE_STAGE_STATES.has(stage.status)) { + const prevActive = latestActive.get(stage.nodeId); + if (!prevActive || stage.visit > prevActive.visit) { + latestActive.set(stage.nodeId, stage); + } + } + } + const result = new Map(); + for (const [nodeId, latestStage] of latest) { + const display = latestActive.get(nodeId) ?? latestStage; + result.set(nodeId, { displayStatus: display.status, latestStageId: latestStage.id }); + } + return result; +} diff --git a/apps/fabro-web/app/lib/time.ts b/apps/fabro-web/app/lib/time.ts index 1d8a818d4..fb6f9b87f 100644 --- a/apps/fabro-web/app/lib/time.ts +++ b/apps/fabro-web/app/lib/time.ts @@ -1,3 +1,21 @@ +import { useEffect, useState } from "react"; + +/** + * Re-renders the calling component every `intervalMs` milliseconds while + * `active` is true, returning the current `Date.now()` value at each tick. + * Returns the captured value when paused, so renders are stable. + */ +export function useTickingNow(active: boolean, intervalMs = 1000): number { + const [now, setNow] = useState(() => Date.now()); + useEffect(() => { + if (!active) return; + setNow(Date.now()); + const interval = setInterval(() => setNow(Date.now()), intervalMs); + return () => clearInterval(interval); + }, [active, intervalMs]); + return now; +} + function relativeTime(seconds: number, past: boolean): string { if (seconds < 60) return past ? "just now" : "in <1m"; const minutes = Math.floor(seconds / 60); @@ -21,4 +39,3 @@ export function timeAgo(iso: string): string { export function timeUntil(iso: string): string { return relativeTime(Math.floor((new Date(iso).getTime() - Date.now()) / 1000), false); } - diff --git a/apps/fabro-web/app/routes/run-billing.test.tsx b/apps/fabro-web/app/routes/run-billing.test.tsx index ed9e3040c..c35ab52cb 100644 --- a/apps/fabro-web/app/routes/run-billing.test.tsx +++ b/apps/fabro-web/app/routes/run-billing.test.tsx @@ -75,12 +75,14 @@ describe("RunBilling", () => { model: null, billing: zeroBilling(), runtime_secs: 0, + state: "succeeded", }, { stage: { id: "command", name: "command" }, model: null, billing: zeroBilling(), runtime_secs: 61, + state: "succeeded", }, ], totals: { @@ -96,7 +98,7 @@ describe("RunBilling", () => { expect(text).toMatch(/—\s*\/\s*—/); expect(text).toContain("1m 1s"); expect(text).not.toContain("By model"); - expect(text).not.toContain("No completed stages yet"); + expect(text).not.toContain("No stages yet"); }); test("renders mixed LLM and non-LLM rows while counting only LLM rows by model", () => { @@ -108,6 +110,7 @@ describe("RunBilling", () => { model: null, billing: zeroBilling(), runtime_secs: 0, + state: "succeeded", }, { stage: { id: "agent", name: "agent" }, @@ -119,6 +122,7 @@ describe("RunBilling", () => { total_usd_micros: 240000, }), runtime_secs: 42, + state: "succeeded", }, ], totals: { @@ -155,11 +159,61 @@ describe("RunBilling", () => { expect(textFromInstance(byModelFooterCells[1])).toBe("1"); }); - test("keeps the empty state for runs with no completed stages", () => { + test("keeps the empty state for runs with no stages", () => { const renderer = renderBilling(billing()); const text = textFromNode(renderer.toJSON()); - expect(text).toContain("No completed stages yet"); - expect(text).toContain("Stages will appear once the run produces completed nodes."); + expect(text).toContain("No stages yet"); + expect(text).toContain("Stages will appear as soon as the run starts executing."); + }); + + test("renders an in-flight row with live runtime and includes its elapsed time in the footer", () => { + const originalNow = Date.now; + // Pin "now" to 30s after the in-flight row started. + const startedAt = "2026-04-29T12:00:00.000Z"; + const fakeNow = new Date("2026-04-29T12:00:30.000Z").getTime(); + Date.now = () => fakeNow; + + try { + const renderer = renderBilling( + billing({ + stages: [ + { + stage: { id: "in-flight", name: "in-flight" }, + model: null, + // Server reports 0 runtime / no billing; the row is still being executed. + billing: zeroBilling(), + runtime_secs: 0, + started_at: startedAt, + state: "running", + }, + ], + // Server total is 0 because the in-flight row hasn't been finalized. + totals: { + runtime_secs: 0, + ...zeroBilling(), + }, + }), + ); + + const text = textFromNode(renderer.toJSON()); + // Empty-state must NOT show — the table should appear as soon as the + // first stage starts. + expect(text).not.toContain("No stages yet"); + expect(text).toContain("in-flight"); + + // Both the row's runtime cell and the footer total should reflect + // ~30s elapsed since started_at. + expect(text).toContain("30s"); + + const footers = renderer.root.findAll((node) => node.type === "tfoot"); + const footerCells = footers[0].findAll((node) => node.type === "td"); + // The Run time column in the footer is index 3 (Total / [empty Model] / + // Tokens / Run time / Billing). + const footerRuntime = textFromInstance(footerCells[3]); + expect(footerRuntime).toContain("30s"); + } finally { + Date.now = originalNow; + } }); }); diff --git a/apps/fabro-web/app/routes/run-billing.tsx b/apps/fabro-web/app/routes/run-billing.tsx index 6bb6ab763..be0d52386 100644 --- a/apps/fabro-web/app/routes/run-billing.tsx +++ b/apps/fabro-web/app/routes/run-billing.tsx @@ -1,7 +1,11 @@ +import { useMemo } from "react"; + import { EmptyState } from "../components/state"; import { formatDurationSecs } from "../lib/format"; import { useRunBilling } from "../lib/queries"; -import type { RunBilling } from "@qltysh/fabro-api-client"; +import { IN_FLIGHT_STAGE_STATES } from "../lib/stage-sidebar"; +import { useTickingNow } from "../lib/time"; +import type { RunBilling, RunBillingStage } from "@qltysh/fabro-api-client"; const EMPTY_VALUE = "—"; @@ -14,78 +18,103 @@ function formatUsdMicros(usdMicros?: number | null) { return usdMicros == null ? EMPTY_VALUE : `$${(usdMicros / 1_000_000).toFixed(2)}`; } -function mapBilling(billing: RunBilling | undefined) { - if (!billing) { - return { - stages: [], - totalRuntime: formatDurationSecs(0), - totalUsdMicros: undefined, - totalInput: null, - totalOutput: null, - modelBreakdown: [], - modelStageCount: 0, - }; - } +function isInFlight(stage: RunBillingStage): boolean { + return stage.state != null && IN_FLIGHT_STAGE_STATES.has(stage.state); +} - const stages = billing.stages.map((stage) => { - const hasModel = stage.model != null; - return { - stage: stage.stage.name, - model: stage.model?.id ?? null, - inputTokens: hasModel ? stage.billing.input_tokens : null, - outputTokens: hasModel - ? stage.billing.output_tokens + stage.billing.reasoning_tokens - : null, - runtime: formatDurationSecs(stage.runtime_secs), - totalUsdMicros: stage.billing.total_usd_micros, - }; - }); - const totalRuntime = formatDurationSecs(billing.totals.runtime_secs); - const hasLlmStages = billing.by_model.length > 0; - const totalInput = hasLlmStages ? billing.totals.input_tokens : null; - const totalOutput = hasLlmStages - ? billing.totals.output_tokens + billing.totals.reasoning_tokens - : null; - const totalUsdMicros = billing.totals.total_usd_micros; - const modelBreakdown = billing.by_model - .map((entry) => ({ - model: entry.model.id, - stages: entry.stages, - inputTokens: entry.billing.input_tokens, - outputTokens: entry.billing.output_tokens + entry.billing.reasoning_tokens, - totalUsdMicros: entry.billing.total_usd_micros, - })) - .sort((a, b) => (b.totalUsdMicros ?? -1) - (a.totalUsdMicros ?? -1)); - const modelStageCount = modelBreakdown.reduce((sum, row) => sum + row.stages, 0); +interface MappedStageRow { + stage: string; + model: string | null; + inputTokens: number | null; + outputTokens: number | null; + runtimeSecs: number; + totalUsdMicros: number | null | undefined; +} + +function liveRuntimeSecs(stage: RunBillingStage, now: number): number { + if (stage.started_at) { + const startedMs = new Date(stage.started_at).getTime(); + if (Number.isFinite(startedMs)) { + return Math.max(0, (now - startedMs) / 1000); + } + } + return stage.runtime_secs; +} + +function mapStageRow(stage: RunBillingStage, runtimeSecs: number): MappedStageRow { + const hasModel = stage.model != null; return { - stages, - totalRuntime, - totalUsdMicros, - totalInput, - totalOutput, - modelBreakdown, - modelStageCount, + stage: stage.stage.name, + model: stage.model?.id ?? null, + inputTokens: hasModel ? stage.billing.input_tokens : null, + outputTokens: hasModel + ? stage.billing.output_tokens + stage.billing.reasoning_tokens + : null, + runtimeSecs, + totalUsdMicros: stage.billing.total_usd_micros, }; } export default function RunBilling({ params }: { params: { id: string } }) { const billingQuery = useRunBilling(params.id); - const { - stages, - totalRuntime, - totalUsdMicros, - totalInput, - totalOutput, - modelBreakdown, - modelStageCount, - } = mapBilling(billingQuery.data); + const billing = billingQuery.data; + const hasInFlight = billing?.stages.some(isInFlight) ?? false; - if (!stages.length) { + // Tick once per second only while a stage is in-flight. + const now = useTickingNow(hasInFlight); + + // Completed rows don't depend on `now`; memoize them by `billing` so we + // don't reallocate them every tick. + const completedRows = useMemo(() => { + if (!billing) return []; + return billing.stages.map((stage) => mapStageRow(stage, stage.runtime_secs)); + }, [billing]); + + // The model breakdown is server-derived and stable across ticks too. + const modelBreakdown = useMemo(() => { + if (!billing) return []; + return billing.by_model + .map((entry) => ({ + model: entry.model.id, + stages: entry.stages, + inputTokens: entry.billing.input_tokens, + outputTokens: entry.billing.output_tokens + entry.billing.reasoning_tokens, + totalUsdMicros: entry.billing.total_usd_micros, + })) + .sort((a, b) => (b.totalUsdMicros ?? -1) - (a.totalUsdMicros ?? -1)); + }, [billing]); + + // Re-derive only the in-flight rows on each tick; everything else stays put. + const rows = useMemo(() => { + if (!billing) return []; + if (!hasInFlight) return completedRows; + return billing.stages.map((stage, idx) => + isInFlight(stage) + ? mapStageRow(stage, liveRuntimeSecs(stage, now)) + : completedRows[idx], + ); + }, [billing, completedRows, hasInFlight, now]); + + // While ticking, sum the displayed row runtimes so the footer updates in + // lock-step. Otherwise trust the server's authoritative total. + const totalRuntimeSecs = hasInFlight + ? rows.reduce((sum, row) => sum + row.runtimeSecs, 0) + : (billing?.totals.runtime_secs ?? 0); + + const hasLlmStages = (billing?.by_model.length ?? 0) > 0; + const totalInput = hasLlmStages ? (billing?.totals.input_tokens ?? null) : null; + const totalOutput = hasLlmStages && billing + ? billing.totals.output_tokens + billing.totals.reasoning_tokens + : null; + const totalUsdMicros = billing?.totals.total_usd_micros; + const modelStageCount = modelBreakdown.reduce((sum, row) => sum + row.stages, 0); + + if (!rows.length) { return (
); @@ -105,7 +134,7 @@ export default function RunBilling({ params }: { params: { id: string } }) { - {stages.map((row) => ( + {rows.map((row) => ( {row.stage} @@ -115,7 +144,9 @@ export default function RunBilling({ params }: { params: { id: string } }) { {formatTokens(row.inputTokens)} /{" "} {formatTokens(row.outputTokens)} - {row.runtime} + + {formatDurationSecs(row.runtimeSecs)} + {formatUsdMicros(row.totalUsdMicros)} @@ -131,7 +162,7 @@ export default function RunBilling({ params }: { params: { id: string } }) { {formatTokens(totalOutput)} - {totalRuntime} + {formatDurationSecs(totalRuntimeSecs)} {formatUsdMicros(totalUsdMicros)} diff --git a/apps/fabro-web/app/routes/run-overview.tsx b/apps/fabro-web/app/routes/run-overview.tsx index bf0c0dd93..48066eae0 100644 --- a/apps/fabro-web/app/routes/run-overview.tsx +++ b/apps/fabro-web/app/routes/run-overview.tsx @@ -3,7 +3,6 @@ import { useNavigate, useParams } from "react-router"; import { graphTheme } from "../lib/graph-theme"; import { useRun, useRunGraph, useRunStages } from "../lib/queries"; import { StageSidebar } from "../components/stage-sidebar"; -import type { Stage } from "../components/stage-sidebar"; import { GRAPH_DEFAULT_ZOOM_INDEX, GRAPH_ZOOM_STEPS, @@ -13,6 +12,7 @@ import { EmptyState } from "../components/state"; import { ACTIVE_STAGE_STATES, SUCCEEDED_STAGE_STATES, + aggregateGraphNodeStatus, mapRunStagesToSidebarStages, } from "../lib/stage-sidebar"; @@ -54,27 +54,27 @@ export default function RunOverview() { const inner = innerRef.current; if (!inner || !graphSvg) return; - let cancelled = false; - (async () => { - if (cancelled) return; inner.innerHTML = graphSvg; const svg = inner.querySelector("svg"); if (!svg) return; svgRef.current = svg; const gt = graphTheme; - const runningDotIds = new Set( - stages.filter((s: Stage) => ACTIVE_STAGE_STATES.has(s.status)).map((s: Stage) => s.dotId ?? s.id), - ); - const failedDotIds = new Set( - stages.filter((s: Stage) => s.status === "failed").map((s: Stage) => s.dotId ?? s.id), - ); - const completedDotIds = new Set( - stages.filter((s: Stage) => SUCCEEDED_STAGE_STATES.has(s.status)).map((s: Stage) => s.dotId ?? s.id), - ); - const dotIdToStageId = new Map( - stages.map((s: Stage) => [s.dotId ?? s.id, s.id]), - ); + const aggregated = aggregateGraphNodeStatus(stages); + const runningDotIds = new Set(); + const failedDotIds = new Set(); + const completedDotIds = new Set(); + const dotIdToStageId = new Map(); + for (const [nodeId, { displayStatus, latestStageId }] of aggregated) { + dotIdToStageId.set(nodeId, latestStageId); + if (ACTIVE_STAGE_STATES.has(displayStatus)) { + runningDotIds.add(nodeId); + } else if (displayStatus === "failed") { + failedDotIds.add(nodeId); + } else if (SUCCEEDED_STAGE_STATES.has(displayStatus)) { + completedDotIds.add(nodeId); + } + } const ns = "http://www.w3.org/2000/svg"; for (const group of svg.querySelectorAll(".node")) { @@ -148,8 +148,6 @@ export default function RunOverview() { } } } - })(); - return () => { cancelled = true; }; }, [stages, graphSvg, id, navigate, terminalOutcome]); const onPointerDown = useCallback((e: React.PointerEvent) => { diff --git a/apps/fabro-web/app/routes/run-stages.test.ts b/apps/fabro-web/app/routes/run-stages.test.ts index d10b5b2d2..52f10ed05 100644 --- a/apps/fabro-web/app/routes/run-stages.test.ts +++ b/apps/fabro-web/app/routes/run-stages.test.ts @@ -1,6 +1,7 @@ import { describe, expect, test } from "bun:test"; +import type { EventEnvelope } from "@qltysh/fabro-api-client"; -import { isSafeMarkdownHref } from "./run-stages"; +import { eventsToActivity, isSafeMarkdownHref } from "./run-stages"; describe("isSafeMarkdownHref", () => { test("rejects protocol-relative URLs", () => { @@ -15,3 +16,191 @@ describe("isSafeMarkdownHref", () => { expect(isSafeMarkdownHref("mailto:test@example.com")).toBe(true); }); }); + +function envelope(seq: number, partial: Partial): EventEnvelope { + return { + seq, + id: `evt-${seq}`, + ts: "2026-04-09T12:00:00Z", + run_id: "run-1", + event: "stage.prompt", + ...partial, + } as EventEnvelope; +} + +describe("eventsToActivity", () => { + test("filters events by stage_id (verify@1 vs verify@2 do not cross-contaminate)", () => { + const events: EventEnvelope[] = [ + envelope(1, { + event: "stage.prompt", + stage_id: "verify@1", + node_id: "verify", + properties: { text: "first visit prompt" }, + }), + envelope(2, { + event: "stage.prompt", + stage_id: "verify@2", + node_id: "verify", + properties: { text: "second visit prompt" }, + }), + envelope(3, { + event: "agent.message", + stage_id: "verify@1", + node_id: "verify", + properties: { text: "first visit reply" }, + }), + envelope(4, { + event: "agent.message", + stage_id: "verify@2", + node_id: "verify", + properties: { text: "second visit reply" }, + }), + ]; + + const firstVisit = eventsToActivity(events, "verify@1"); + expect(firstVisit).toEqual([ + { kind: "system", content: "first visit prompt" }, + { kind: "assistant", content: "first visit reply" }, + ]); + + const secondVisit = eventsToActivity(events, "verify@2"); + expect(secondVisit).toEqual([ + { kind: "system", content: "second visit prompt" }, + { kind: "assistant", content: "second visit reply" }, + ]); + }); + + test("pairs command.started + command.completed into a single command turn", () => { + const events: EventEnvelope[] = [ + envelope(1, { + event: "command.started", + node_id: "fmt", + properties: { script: "cargo fmt", language: "shell" }, + }), + envelope(2, { + event: "command.completed", + node_id: "fmt", + properties: { + stdout: "ok", + stderr: "", + exit_code: 0, + duration_ms: 12, + termination: "exited", + }, + }), + ]; + + const turns = eventsToActivity(events, "fmt"); + expect(turns).toHaveLength(1); + expect(turns[0]).toMatchObject({ + kind: "command", + stageId: "fmt", + script: "cargo fmt", + language: "shell", + stdout: "ok", + exitCode: 0, + running: false, + }); + }); + + test("command turn carries the requested stage_id, no @1 fallback", () => { + const events: EventEnvelope[] = [ + envelope(1, { + event: "command.started", + stage_id: "verify@2", + node_id: "verify", + properties: { script: "echo hi", language: "shell" }, + }), + envelope(2, { + event: "command.completed", + stage_id: "verify@2", + node_id: "verify", + properties: { + stdout: "hi", + stderr: "", + exit_code: 0, + duration_ms: 5, + termination: "exited", + }, + }), + ]; + + const turns = eventsToActivity(events, "verify@2"); + expect(turns).toHaveLength(1); + const turn = turns[0]; + expect(turn.kind).toBe("command"); + if (turn.kind === "command") { + expect(turn.stageId).toBe("verify@2"); + expect(turn.script).toBe("echo hi"); + expect(turn.running).toBe(false); + } + }); + + test("pairs agent.tool.started + agent.tool.completed into a single tool turn", () => { + const events: EventEnvelope[] = [ + envelope(1, { + event: "agent.tool.started", + node_id: "detect-drift", + properties: { + tool_call_id: "call-1", + tool_name: "read_file", + arguments: { path: "config.toml" }, + }, + }), + envelope(2, { + event: "agent.tool.completed", + node_id: "detect-drift", + properties: { + tool_call_id: "call-1", + tool_name: "read_file", + output: "[redis]", + is_error: false, + }, + }), + ]; + + const turns = eventsToActivity(events, "detect-drift"); + expect(turns).toHaveLength(1); + expect(turns[0].kind).toBe("tool"); + if (turns[0].kind === "tool") { + expect(turns[0].tools).toHaveLength(1); + expect(turns[0].tools[0]).toMatchObject({ + id: "call-1", + toolName: "read_file", + result: "[redis]", + isError: false, + }); + } + }); + + test("ignores unknown event types and events for other stages", () => { + const events: EventEnvelope[] = [ + envelope(1, { + event: "stage.started", + node_id: "detect-drift", + properties: {}, + }), + envelope(2, { + event: "agent.message", + node_id: "detect-drift", + properties: { text: "signal" }, + }), + envelope(3, { + event: "run.running", + node_id: "detect-drift", + properties: {}, + }), + envelope(4, { + event: "agent.message", + node_id: "other-stage", + properties: { text: "wrong stage" }, + }), + ]; + + const turns = eventsToActivity(events, "detect-drift"); + expect(turns).toHaveLength(1); + if (turns[0].kind === "assistant") { + expect(turns[0].content).toBe("signal"); + } + }); +}); diff --git a/apps/fabro-web/app/routes/run-stages.tsx b/apps/fabro-web/app/routes/run-stages.tsx index 6d179d701..2b01fc5f3 100644 --- a/apps/fabro-web/app/routes/run-stages.tsx +++ b/apps/fabro-web/app/routes/run-stages.tsx @@ -40,16 +40,15 @@ import type { Stage } from "../components/stage-sidebar"; import { EmptyState } from "../components/state"; import { CopyButton } from "../components/ui"; import { formatDurationSecs } from "../lib/format"; -import { fetchRunCommandLog, useRunEventsList, useRunStageTurns, useRunStages } from "../lib/queries"; -import { mapRunStagesToSidebarStages } from "../lib/stage-sidebar"; +import { useTickingNow } from "../lib/time"; +import { fetchRunCommandLog, useRunStageEvents, useRunStages } from "../lib/queries"; +import { STAGE_ACTIVITY_EVENT_TYPES, type StageActivityEventType } from "../lib/run-events"; +import { ACTIVE_STAGE_STATES, formatStageLabel, mapRunStagesToSidebarStages } from "../lib/stage-sidebar"; import { getNumber, getString, type UnknownRecord } from "../lib/unknown"; import { CommandOutputStream, CommandTermination, type EventEnvelope, - type StageTurn as ApiStageTurn, - type PaginatedStageTurnList, - type PaginatedEventList, } from "@qltysh/fabro-api-client"; export const handle = { wide: true }; @@ -68,17 +67,40 @@ function readTermination(props: UnknownRecord): CommandTermination { return CommandTermination.EXITED; } -function turnsFromEvents(events: EventEnvelope[], stageId: string): TurnType[] { - const stageEvents = events.filter((e) => e.node_id === stageId); +const STAGE_ACTIVITY_EVENT_SET = new Set(STAGE_ACTIVITY_EVENT_TYPES); + +function assertNever(value: never): never { + throw new Error(`Unhandled stage activity event type: ${value}`); +} + +function activityEventStageId(event: EventEnvelope): string | undefined { + if (typeof event.stage_id === "string") return event.stage_id; + if (typeof event.node_id === "string") return event.node_id; + return getString(event.properties ?? {}, "node_id"); +} + +export function eventsToActivity(events: EventEnvelope[], stageId: string): TurnType[] { const turns: TurnType[] = []; // Collect tool pairs: started → completed const pendingTools = new Map(); // Track pending command for pairing started → completed let pendingCommand: { stageId: string; script: string; language: string } | undefined; - for (const e of stageEvents) { + for (const e of events) { + const eventName = e.event; + if ( + activityEventStageId(e) !== stageId || + !eventName || + !STAGE_ACTIVITY_EVENT_SET.has(eventName) + ) { + continue; + } + // Exhaustive switch over StageActivityEventType: adding a new variant to + // STAGE_ACTIVITY_EVENT_TYPES forces a TS error here until the case is + // handled, keeping the SWR invalidation set and the reducer in sync. + const eventType = eventName as StageActivityEventType; const props = e.properties ?? {}; - switch (e.event) { + switch (eventType) { case "stage.prompt": turns.push({ kind: "system", content: getString(props, "text") ?? e.text ?? "" }); break; @@ -114,7 +136,7 @@ function turnsFromEvents(events: EventEnvelope[], stageId: string): TurnType[] { } case "command.started": { pendingCommand = { - stageId: e.stage_id ?? `${stageId}@1`, + stageId, script: getString(props, "script") ?? "", language: getString(props, "language") ?? "shell", }; @@ -123,7 +145,7 @@ function turnsFromEvents(events: EventEnvelope[], stageId: string): TurnType[] { case "command.completed": { turns.push({ kind: "command", - stageId: pendingCommand?.stageId ?? e.stage_id ?? `${stageId}@1`, + stageId: pendingCommand?.stageId ?? stageId, script: pendingCommand?.script ?? "", language: pendingCommand?.language ?? "shell", stdout: getString(props, "stdout") ?? "", @@ -136,6 +158,8 @@ function turnsFromEvents(events: EventEnvelope[], stageId: string): TurnType[] { pendingCommand = undefined; break; } + default: + assertNever(eventType); } } @@ -153,41 +177,6 @@ function turnsFromEvents(events: EventEnvelope[], stageId: string): TurnType[] { return turns; } -function mapApiStageTurn(t: ApiStageTurn): TurnType { - switch (t.kind) { - case "tool": - return { - kind: "tool", - tools: (t.tools ?? []).map((tu) => ({ - id: tu.id, - toolName: tu.tool_name, - input: tu.input, - result: tu.result, - isError: tu.is_error, - durationMs: tu.duration_ms, - })), - }; - case "system": - case "assistant": - return { kind: t.kind, content: t.content ?? "" }; - } -} - -function mapTurns( - turnsResult: PaginatedStageTurnList | null | undefined, - eventsResult: PaginatedEventList | null | undefined, - selectedStageId: string | undefined, -): TurnType[] { - if (!selectedStageId) return []; - if (turnsResult?.data?.length) { - return turnsResult.data.map(mapApiStageTurn); - } - if (eventsResult?.data) { - return turnsFromEvents(eventsResult.data, selectedStageId); - } - return []; -} - function Markdown({ content }: { content: string }) { const html = useMemo(() => markedSafe.parse(content, { async: false }) as string, [content]); return ( @@ -575,7 +564,6 @@ function RunningStageDuration({ const [startedAt, setStartedAt] = useState(() => isRunning ? Date.now() : null, ); - const [, setTick] = useState(0); useEffect(() => { setStartedAt((current) => { @@ -584,14 +572,10 @@ function RunningStageDuration({ }); }, [isRunning]); - useEffect(() => { - if (!isRunning) return; - const interval = setInterval(() => setTick((tick) => tick + 1), 1000); - return () => clearInterval(interval); - }, [isRunning]); + const now = useTickingNow(isRunning); if (isRunning && startedAt) { - return formatDurationSecs(Math.floor((Date.now() - startedAt) / 1000)); + return formatDurationSecs(Math.floor((now - startedAt) / 1000)); } return duration; } @@ -605,16 +589,16 @@ export default function RunStages() { ); const selectedStage = stages.find((s: Stage) => s.id === stageId) ?? stages[0]; - const turnsQuery = useRunStageTurns(id, selectedStage?.id); - const hasStageTurns = (turnsQuery.data?.data.length ?? 0) > 0; - const shouldLoadEventFallback = - !!selectedStage?.id && !turnsQuery.isLoading && !turnsQuery.error && !hasStageTurns; - const eventsQuery = useRunEventsList(id, shouldLoadEventFallback); + const selectedStageId = selectedStage?.id; + const stageEventsQuery = useRunStageEvents(id, selectedStageId); const turns = useMemo( - () => mapTurns(turnsQuery.data, eventsQuery.data, selectedStage?.id), - [eventsQuery.data, selectedStage?.id, turnsQuery.data], + () => + selectedStageId + ? eventsToActivity(stageEventsQuery.data ?? [], selectedStageId) + : [], + [stageEventsQuery.data, selectedStageId], ); - const isRunning = selectedStage?.status === "running"; + const isActive = selectedStage ? ACTIVE_STAGE_STATES.has(selectedStage.status) : false; if (!id || !stages.length) { return ( @@ -636,11 +620,13 @@ export default function RunStages() {
- -

{selectedStage.name}

+ +

+ {formatStageLabel(selectedStage)} +

diff --git a/apps/fabro-web/app/routes/runs.test.tsx b/apps/fabro-web/app/routes/runs.test.tsx index 948900769..3cd9dec0a 100644 --- a/apps/fabro-web/app/routes/runs.test.tsx +++ b/apps/fabro-web/app/routes/runs.test.tsx @@ -25,6 +25,7 @@ describe("runs route board mapping", () => { test("keeps blocked runs in the blocked lane and preserves question text", () => { const columns = buildBoardColumns({ columns: [ + { id: "queued", name: "Queued" }, { id: "initializing", name: "Initializing" }, { id: "running", name: "Running" }, { id: "blocked", name: "Blocked" }, diff --git a/apps/fabro-web/app/routes/runs.tsx b/apps/fabro-web/app/routes/runs.tsx index f46c19396..78db03496 100644 --- a/apps/fabro-web/app/routes/runs.tsx +++ b/apps/fabro-web/app/routes/runs.tsx @@ -18,7 +18,7 @@ import { arrayMove, } from "@dnd-kit/sortable"; import { CSS } from "@dnd-kit/utilities"; -import { ciConfig, columnStatusDisplay, deriveCiStatus, mapRunListItem } from "../data/runs"; +import { ciConfig, columnStatusDisplay, columnStatuses, deriveCiStatus, mapRunListItem } from "../data/runs"; import type { CiStatus, CheckRun, CheckStatus, RunItem, RunWithStatus, ColumnStatus } from "../data/runs"; import { EmptyState } from "../components/state"; import { shouldRefreshBoardForEvent, useBoardEvents } from "../lib/board-events"; @@ -38,6 +38,7 @@ interface ColumnStyle { } const columnStyles: Record = { + queued: { iconType: "branch", actions: [] }, initializing: { iconType: "branch", actions: [] }, running: { iconType: "branch", actions: ["Watch", "Steer"] }, blocked: { iconType: "branch", actions: ["Answer Question"] }, @@ -49,7 +50,7 @@ const defaultColumnStyle: ColumnStyle = { iconType: "branch", actions: [] }; const defaultColumnColors = { dot: "bg-fg-muted", text: "text-fg-muted" }; interface BoardRunsResponse { - columns: { id: string; name: string }[]; + columns: PaginatedBoardRunList["columns"]; data: PaginatedBoardRunList["data"]; meta: PaginatedBoardRunList["meta"]; } @@ -64,16 +65,8 @@ type Column = { items: RunItem[]; }; -const SKELETON_STATUSES: ColumnStatus[] = [ - "initializing", - "running", - "blocked", - "succeeded", - "failed", -]; - function buildSkeletonColumns(): Column[] { - return SKELETON_STATUSES.map((id) => { + return columnStatuses.map((id) => { const colors = columnStatusDisplay[id]; return { id, @@ -98,7 +91,7 @@ export function buildBoardColumns(response: BoardRunsResponse): Column[] { } return response.columns.map((col) => { - const id = col.id as ColumnStatus; + const id = col.id; const colors = columnStatusDisplay[id] ?? defaultColumnColors; return { id, @@ -754,6 +747,9 @@ export default function Runs() { (sum, col) => sum + col.items.length, 0, ); + const visibleColumns = filteredColumns.filter( + (col) => col.id !== "queued" || col.items.length > 0, + ); return ( @@ -815,7 +811,7 @@ export default function Runs() { {view === "columns" ? ( <>
- {filteredColumns.map((col) => ( + {visibleColumns.map((col) => (
@@ -838,7 +834,7 @@ export default function Runs() { ) : ( <>
- {filteredColumns.map((col) => { + {visibleColumns.map((col) => { const isCollapsed = collapsed.has(col.id); return (
diff --git a/docs/public/api-reference/fabro-api.yaml b/docs/public/api-reference/fabro-api.yaml index d1b8bc7f3..0b499e7fa 100644 --- a/docs/public/api-reference/fabro-api.yaml +++ b/docs/public/api-reference/fabro-api.yaml @@ -1906,26 +1906,26 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" - /api/v1/runs/{id}/stages/{stageId}/turns: + /api/v1/runs/{id}/stages/{stageId}/events: get: - operationId: listStageTurns + operationId: listStageEvents tags: [Run Internals] - summary: List Stage Turns - description: Returns a paginated list of conversation turns within a specific stage, including system prompts, assistant responses, and tool invocations. + summary: List Stage Events + description: Returns a paginated JSON list of stored run events scoped to a single stage visit. parameters: - $ref: "#/components/parameters/RunId" - $ref: "#/components/parameters/StageId" - - $ref: "#/components/parameters/PageLimit" - - $ref: "#/components/parameters/PageOffset" + - $ref: "#/components/parameters/SinceSeq" + - $ref: "#/components/parameters/EventLimit" responses: "200": - description: Paginated list of conversation turns + description: Paginated list of stage events content: application/json: schema: - $ref: "#/components/schemas/PaginatedStageTurnList" + $ref: "#/components/schemas/PaginatedEventList" "404": - description: Run or stage not found + description: Run not found. headers: x-request-id: $ref: "#/components/headers/XRequestId" @@ -3856,20 +3856,6 @@ components: meta: $ref: "#/components/schemas/PaginationMeta" - PaginatedStageTurnList: - description: Paginated list of stage turns. - type: object - required: - - data - - meta - properties: - data: - type: array - items: - $ref: "#/components/schemas/StageTurn" - meta: - $ref: "#/components/schemas/PaginationMeta" - PaginatedApiQuestionList: description: Paginated list of pending questions. type: object @@ -5328,6 +5314,20 @@ components: oneOf: - $ref: "#/components/schemas/CommandTermination" - type: "null" + started_at: + type: ["string", "null"] + format: date-time + description: Wall-clock time the latest attempt of this stage started, if known. + duration_ms: + type: ["integer", "null"] + format: uint64 + minimum: 0 + description: Wall-clock duration of the stage's latest terminal attempt, if known. + state: + oneOf: + - $ref: "#/components/schemas/StageState" + - type: "null" + description: Lifecycle state of the stage projection. InterviewOption: description: Option stored with an interview question in the event log. @@ -5656,6 +5656,7 @@ components: description: Board column status for a run in the list view. type: string enum: + - queued - initializing - running - blocked @@ -5669,7 +5670,7 @@ components: - name properties: id: - type: string + $ref: "#/components/schemas/BoardColumn" name: type: string @@ -6159,7 +6160,7 @@ components: example: 3501.0 BillingStageRef: - description: Reference to a billing stage. + description: Reference to a workflow node in a billing stage row. type: object required: - id @@ -6319,11 +6320,13 @@ components: - id - name - status + - node_id + - visit properties: id: type: string - description: Unique stage identifier within the run. - example: propose-changes + description: StageId in "node_id@visit" form, e.g. verify@2. + example: verify@2 name: type: string description: Human-readable stage name. @@ -6334,107 +6337,21 @@ components: type: number description: Time spent in this stage, in seconds. example: 154.0 - dot_id: + node_id: type: string - description: Node identifier in the Graphviz graph source. - example: propose - - ToolUse: - description: A single tool invocation with its input, result, and execution metadata. - type: object - required: - - id - - tool_name - - input - - result - - is_error - properties: - id: - type: string - description: Unique identifier for this tool invocation. Enables correlation in parallel tool use. - example: toolu_01A09q90qw90lq917835lq9 - tool_name: - type: string - description: Name of the tool that was invoked. - example: read_file - input: - type: string - description: JSON-encoded input passed to the tool. - example: '{ "path": "src/routes/auth.ts" }' - result: - type: string - description: Output returned by the tool. Contains the error message when is_error is true. - example: 'import { Router } from "express";' - is_error: - type: boolean - description: Whether the tool invocation failed. When true, the result field contains the error message. - example: false - duration_ms: + description: Node id in the workflow graph; multiple stages with different visits share the same node_id. + example: verify + visit: type: integer - description: Wall-clock execution time of the tool invocation in milliseconds. - example: 142 - - StageTurn: - description: A single turn in a stage conversation — a system prompt, assistant response, or tool invocation block. - discriminator: - propertyName: kind - mapping: - system: "#/components/schemas/SystemStageTurn" - assistant: "#/components/schemas/AssistantStageTurn" - tool: "#/components/schemas/ToolStageTurn" - oneOf: - - $ref: "#/components/schemas/SystemStageTurn" - - $ref: "#/components/schemas/AssistantStageTurn" - - $ref: "#/components/schemas/ToolStageTurn" - - SystemStageTurn: - description: A system prompt turn that sets the stage's instructions. - type: object - required: - - kind - - content - properties: - kind: - type: string - enum: [system] - content: - type: string - description: System prompt text. - example: You are a drift detection agent. Compare the production and staging environments. - - AssistantStageTurn: - description: An assistant response turn within a stage. - type: object - required: - - kind - - content - properties: - kind: - type: string - enum: [assistant] - content: - type: string - description: Assistant response text. - example: I'll start by loading the environment configurations for both production and staging. - - ToolStageTurn: - description: A tool invocation turn containing one or more tool calls. - type: object - required: - - kind - - tools - properties: - kind: - type: string - enum: [tool] - content: - type: string - description: Text accompanying the tool invocations, or null when the turn contains only tool calls. - tools: - type: array - description: Tool invocations executed in this turn. - items: - $ref: "#/components/schemas/ToolUse" + format: uint32 + minimum: 1 + description: 1-based visit count; bumped each time the workflow re-enters this node. + example: 2 + started_at: + type: ["string", "null"] + format: date-time + description: Wall-clock time the latest attempt of this stage started, if known. + example: "2026-04-29T12:34:56Z" # ── File Diff Schemas ────────────────────────────────────────────── @@ -6607,7 +6524,7 @@ components: # ── Billing Schemas ────────────────────────────────────────────────── RunBillingStage: - description: Token counts and billed totals for a single stage within a run. + description: Token counts and billed totals for one workflow node within a run. Rows are grouped by node; billing and runtime sum every visit of that node. type: object required: - stage @@ -6618,7 +6535,7 @@ components: stage: $ref: "#/components/schemas/BillingStageRef" model: - description: Model used for this stage; null for non-LLM stages. + description: Latest usage-bearing visit model for this node; null when no visit used an LLM model. oneOf: - $ref: "#/components/schemas/ModelReference" - type: "null" @@ -6626,8 +6543,18 @@ components: $ref: "#/components/schemas/BilledTokenCounts" runtime_secs: type: number - description: Wall-clock runtime in seconds. + description: Wall-clock runtime in seconds, summed across every visit of this node. example: 154.0 + started_at: + type: ["string", "null"] + format: date-time + description: Wall-clock time the latest attempt of this stage started, if known. + example: "2026-04-29T12:34:56Z" + state: + oneOf: + - $ref: "#/components/schemas/StageState" + - type: "null" + description: Lifecycle state of the stage. Use to detect in-flight rows for client-side runtime ticking. RunBillingTotals: description: Aggregate billing totals across all stages of a run. @@ -6687,7 +6614,7 @@ components: $ref: "#/components/schemas/ModelReference" stages: type: integer - description: Number of stages that used this model. + description: Number of usage-bearing stage visits that used this model. example: 2 billing: $ref: "#/components/schemas/BilledTokenCounts" @@ -6702,7 +6629,7 @@ components: properties: stages: type: array - description: Per-stage billing breakdown. + description: Per-node billing breakdown. Each row sums billing and runtime across all visits of that node. items: $ref: "#/components/schemas/RunBillingStage" totals: diff --git a/docs/superpowers/plans/2026-05-04-cross-tab-sse-coordination.md b/docs/superpowers/plans/2026-05-04-cross-tab-sse-coordination.md new file mode 100644 index 000000000..115ff9326 --- /dev/null +++ b/docs/superpowers/plans/2026-05-04-cross-tab-sse-coordination.md @@ -0,0 +1,106 @@ +# Cross-Tab SSE Coordination Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Coordinate Fabro web SSE subscriptions across tabs so one browser profile/origin opens at most one UI-owned `/api/v1/attach` EventSource in steady state when `BroadcastChannel` is available; brief overlap during election/takeover is tolerated and deduped. + +**Architecture:** Add a browser-side SSE coordinator that elects one tab as leader, has that leader own the global EventSource, and broadcasts parsed run events to follower tabs over `BroadcastChannel`. Existing board and run-detail invalidation logic becomes a consumer of that global event feed, with the current per-tab SSE behavior preserved as a compatibility fallback. + +**Tech Stack:** React, SWR, browser `BroadcastChannel`, browser `EventSource`, Bun tests, existing Fabro web API query keys. + +--- + +## Summary + +Build a browser-side SSE coordinator so Fabro web opens at most one `/api/v1/attach` EventSource per origin/browser profile in steady state when `BroadcastChannel` is available. Temporary duplicate leaders may exist during election/takeover, but event dedupe and generation checks make the overlap harmless and short-lived. The global stream becomes a shared cache-invalidation feed for both the runs board and run detail pages. No server API, OpenAPI, or Rust streaming contract changes are part of v1. + +This supersedes the earlier web SSE limitation documented in `docs/plans/2026-04-19-002-feat-web-ui-lifecycle-actions-plan.md`: the old shared hook was code reuse only; this plan adds actual socket deduplication. + +## Implementation Changes + +- Add `apps/fabro-web/app/lib/cross-tab-sse.ts`. + - Export `subscribeToCrossTabSse(...)` with the same invalidation style as `subscribeToSharedEventSource`, a `resyncKeys` callback for gap recovery, and a `fallbackSubscribe` callback used when cross-tab coordination is unavailable. + - Use `BroadcastChannel` name `fabro:sse:v1`. + - Generate `tabId` with `crypto.randomUUID()` and a safe random fallback. + - Open one leader-owned `EventSource` to `queryKeys.system.attach()` (`/api/v1/attach`) in steady state. + - Leader dispatches each parsed `EventEnvelope` locally and broadcasts it to follower tabs. + - Followers do not open EventSource while a valid visible leader heartbeat exists; lower lexical `tabId` does not preempt a healthy visible leader. + +- Implement leader election in the cross-tab module. + - Constants: `HEARTBEAT_MS = 1000`, `LEADER_STALE_MS = 4000`, `ELECTION_JITTER_MS = 150`. + - Messages: `hello`, `heartbeat`, `candidate`, `leader-changed`, `release`, `resync`, `event`. + - Define a typed message union. Every message includes `type`, `version: 1`, `tabId`, and `sentAt`. + - `heartbeat`: `{ type, version, tabId, sentAt, leaderId, generation, visibility }`. + - `candidate`: `{ type, version, tabId, sentAt, candidateId: tabId, candidateGeneration, visibility, observedLeaderId, observedGeneration, reason }`, where `reason` is `"hidden-leader" | "stale-leader" | "release" | "no-leader"`. + - `leader-changed`: `{ type, version, tabId, sentAt, leaderId, generation, visibility }`. + - `release`: `{ type, version, tabId, sentAt, leaderId, generation }`. + - `resync`: `{ type, version, tabId, sentAt, leaderId, generation, reason }`. + - `event`: `{ type, version, tabId, sentAt, leaderId, generation, payload }`. + - Use the candidacy phase for all leadership changes: hidden-leader takeover, stale-leader recovery, leader release, and no-leader startup. + - A candidate sets `candidateGeneration = observedGeneration + 1`, broadcasts `candidate`, waits jitter, and opens EventSource only if no higher-priority candidate for the same `candidateGeneration` appears. + - Candidate priority is election-scoped: visible candidates outrank hidden candidates; for equal visibility, lower lexical `candidateId` wins. This priority resolves elections and same-generation split brain only; it is not a reason to preempt a fresh visible leader. + - When a visible follower observes a fresh hidden leader heartbeat, it enters candidacy with `reason: "hidden-leader"`. + - When tabs detect a stale leader, leader release, or no known leader, they enter the same candidacy flow with the matching `reason`. + - If two visible candidates race for the same observed leader/generation, the lower lexical `candidateId` wins. + - Current leaders release when they observe a candidate whose `observedLeaderId` matches their `leaderId` and whose `observedGeneration` is current or newer. + - If same-generation split brain still occurs, lower-priority leaders release when they observe a same-generation higher-priority leader heartbeat or `leader-changed`. + - Each new leader uses `candidateGeneration`, broadcasts `leader-changed`, and followers ignore heartbeats/events from non-current leaders or stale generations. + - Hidden leader keeps the stream only when no visible candidate takes over. + - On `pagehide`/last local unsubscribe, a leader closes EventSource and broadcasts `release`. + - Brief split brain is tolerated; dedupe events by `payload.id`, falling back to `${run_id}:${seq}:${event}`. + - Keep dedupe bounded with a recent-event cache: max 1000 IDs and 5 minute TTL. Evict oldest entries when the max is exceeded and prune expired entries during event handling. Duplicate invalidations after eviction are acceptable; unbounded growth is not. + +- Migrate consumers. + - `apps/fabro-web/app/lib/board-events.ts`: subscribe through the cross-tab global stream; keep existing board event allowlist. + - `apps/fabro-web/app/lib/run-events.ts`: subscribe through the same global stream, filter by `payload.run_id === runId`, and reuse `queryKeysForRunEvent`. + - In coordinated mode, run detail pages stay subscribed while mounted, including terminal runs, so post-terminal archive/unarchive changes can reconcile live. + - Do not close the global stream on `run.completed` / `run.failed`; terminal events only invalidate run-scoped keys. + - Keep `subscribeToSharedEventSource` in `apps/fabro-web/app/lib/sse.ts` for fallback and existing local sharing behavior. + +- Gap and fallback behavior. + - If `BroadcastChannel` is unavailable or throws, call each subscriber's `fallbackSubscribe`. + - Board fallback uses the existing global `/api/v1/attach` path. + - Run detail fallback preserves the existing run-scoped `/api/v1/runs/:id/attach` path, so the old terminal-tab stale limitation remains only in fallback mode. + - Do not add replay to `/api/v1/attach`. + - On leader takeover, stale leader timeout, leader release, and new leader generation, broadcast `resync` or `leader-changed` so every tab with active local subscriptions runs its own `resyncKeys`. + - On `visibilitychange` back to visible without leadership change, run only that tab's local `resyncKeys`; do not broadcast cross-tab resync. + - Board `resyncKeys`: `queryKeys.boards.runs()`. + - Run `resyncKeys`: detail, files, billing, stages, events, LR graph, TB graph, and questions for that run. + +## Tests + +- Add `apps/fabro-web/app/lib/cross-tab-sse.test.ts` with fake `BroadcastChannel`, fake `EventSource`, and fake timers. + - One leader opens `/api/v1/attach`; followers open no EventSource. + - Leader broadcasts an event and all local subscribers receive invalidations. + - Run subscribers ignore events for other `run_id` values. + - Board and run subscriptions can coexist on the same global stream. + - Temporary duplicate leaders are allowed only during election/takeover and converge back to one leader. + - With fresh hidden-leader heartbeats, a visible tab broadcasts candidacy, hidden leader closes, visible tab opens `/api/v1/attach`, and followers resync. + - Two visible candidates racing for the same hidden leader resolve to the lower lexical `candidateId`. + - Two tabs detect the same stale leader simultaneously; only the winning candidate opens `/api/v1/attach` after jitter. + - Same-generation split brain converges to one leader by visibility, then lexical `tabId`. + - A fresh visible leader exists; a new visible follower with a lower lexical `tabId` joins and does not take leadership. + - A hidden leader does not release for a candidate that references an old `observedLeaderId` or stale `observedGeneration`. + - Stale heartbeat triggers takeover and every active tab calls its own `resyncKeys`. + - A follower tab calls its own board/run `resyncKeys` after another tab becomes leader. + - Duplicate event IDs are ignored. + - The recent-event dedupe cache evicts by TTL and max-size bound; duplicate invalidation may recur only after eviction. + - Stale heartbeat/event messages from an old leader/generation are ignored after takeover. + - Last unsubscribe closes leader EventSource and sends `release`. + - Missing/broken BroadcastChannel uses the per-tab fallback. + +- Update existing tests. + - `apps/fabro-web/app/lib/board-events.test.tsx`: assert coordinated mode uses `/api/v1/attach` once and fallback preserves current behavior. + - `apps/fabro-web/app/lib/run-events.test.tsx`: assert coordinated mode filters by `run_id`; fallback preserves current `/runs/:id/attach` behavior; terminal run events do not close the global coordinator; terminal detail tabs still receive archive/unarchive invalidations while mounted in coordinated mode. + +- Verification commands. + - `cd apps/fabro-web && bun test app/lib/cross-tab-sse.test.ts app/lib/board-events.test.tsx app/lib/run-events.test.tsx` + - `cd apps/fabro-web && bun run typecheck` + - Optional manual check: open 8 run-detail tabs for active runs; after election settles and with BroadcastChannel available, Chrome should show one active `/api/v1/attach` EventStream across the participating tabs and no UI-created `/api/v1/runs/:id/attach` streams. + +## Assumptions + +- Treat SSE as live cache invalidation, not an exact gapless event log. +- Preserve current run-specific attach endpoint for CLI, API clients, and fallback only. +- Do not touch unrelated dirty worktree files; intentionally replace or adapt the earlier interrupted SSE test edits as part of the new test suite. +- No docs or public API updates are required for v1 because this is an internal web transport change. diff --git a/lib/crates/fabro-api/tests/run_billing_stage_round_trip.rs b/lib/crates/fabro-api/tests/run_billing_stage_round_trip.rs index c160a840a..52fb4ece1 100644 --- a/lib/crates/fabro-api/tests/run_billing_stage_round_trip.rs +++ b/lib/crates/fabro-api/tests/run_billing_stage_round_trip.rs @@ -1,4 +1,5 @@ use fabro_api::types::RunBillingStage; +use fabro_types::StageState; use serde_json::json; #[test] @@ -28,3 +29,59 @@ fn run_billing_stage_model_accepts_required_null() { assert!(encoded.get("model").is_some()); assert!(encoded["model"].is_null()); } + +#[test] +fn run_billing_stage_round_trips_terminal_row_with_started_at_and_state() { + let value = json!({ + "stage": { + "id": "build", + "name": "build" + }, + "model": { "id": "claude-sonnet-4-5" }, + "billing": { + "input_tokens": 12, + "output_tokens": 34, + "total_tokens": 46, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "runtime_secs": 5.5, + "started_at": "2026-04-29T12:34:56Z", + "state": "succeeded" + }); + + let stage: RunBillingStage = + serde_json::from_value(value.clone()).expect("terminal stage row should deserialize"); + assert!(stage.started_at.is_some()); + assert_eq!(stage.state, Some(StageState::Succeeded)); + assert_eq!(serde_json::to_value(stage).unwrap(), value); +} + +#[test] +fn run_billing_stage_round_trips_in_flight_row() { + let value = json!({ + "stage": { + "id": "build", + "name": "build" + }, + "model": null, + "billing": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "runtime_secs": 1.25, + "started_at": "2026-04-29T12:34:56Z", + "state": "running" + }); + + let stage: RunBillingStage = + serde_json::from_value(value.clone()).expect("in-flight stage row should deserialize"); + assert!(stage.model.is_none()); + assert_eq!(stage.state, Some(StageState::Running)); + assert_eq!(serde_json::to_value(stage).unwrap(), value); +} diff --git a/lib/crates/fabro-api/tests/stage_projection_round_trip.rs b/lib/crates/fabro-api/tests/stage_projection_round_trip.rs index 0523197cf..ad4dcb579 100644 --- a/lib/crates/fabro-api/tests/stage_projection_round_trip.rs +++ b/lib/crates/fabro-api/tests/stage_projection_round_trip.rs @@ -28,7 +28,10 @@ fn stage_projection_round_trips_representative_json() { "parallel_results": [{ "branch": 0, "status": "succeeded" }], "stdout": "ok", "stderr": "", - "termination": "exited" + "termination": "exited", + "started_at": "2026-04-29T12:34:00Z", + "duration_ms": 56000, + "state": "succeeded" }); let state: StageProjection = serde_json::from_value(value.clone()).unwrap(); diff --git a/lib/crates/fabro-cli/src/commands/auth/status.rs b/lib/crates/fabro-cli/src/commands/auth/status.rs index db30eb115..2002fa1cd 100644 --- a/lib/crates/fabro-cli/src/commands/auth/status.rs +++ b/lib/crates/fabro-cli/src/commands/auth/status.rs @@ -1,5 +1,5 @@ use anyhow::Result; -use chrono::{DateTime, Utc}; +use chrono::{DateTime, SecondsFormat, Utc}; use fabro_client::{AuthEntry, AuthStore, OAuthEntry}; use serde::Serialize; @@ -108,12 +108,12 @@ pub(super) fn status_command(args: &AuthStatusArgs, ctx: &CommandContext) -> Res fabro_util::printerr!( printer, " Access expires: {}", - access_token_expires_at.to_rfc3339() + access_token_expires_at.to_rfc3339_opts(SecondsFormat::Secs, true) ); fabro_util::printerr!( printer, " Refresh expires: {}", - refresh_token_expires_at.to_rfc3339() + refresh_token_expires_at.to_rfc3339_opts(SecondsFormat::Secs, true) ); } StatusRow::DevToken { @@ -122,7 +122,11 @@ pub(super) fn status_command(args: &AuthStatusArgs, ctx: &CommandContext) -> Res } => { fabro_util::printerr!(printer, "{server}"); fabro_util::printerr!(printer, " Auth: dev-token"); - fabro_util::printerr!(printer, " Logged in: {}", logged_in_at.to_rfc3339()); + fabro_util::printerr!( + printer, + " Logged in: {}", + logged_in_at.to_rfc3339_opts(SecondsFormat::Secs, true) + ); } } } diff --git a/lib/crates/fabro-cli/src/commands/run/logs.rs b/lib/crates/fabro-cli/src/commands/run/logs.rs index 8f01378cc..7d7e9063b 100644 --- a/lib/crates/fabro-cli/src/commands/run/logs.rs +++ b/lib/crates/fabro-cli/src/commands/run/logs.rs @@ -14,7 +14,7 @@ use std::time::Duration; use anyhow::{Context, Result, bail}; use chrono::{DateTime, Utc}; use fabro_redact::redact_jsonl_line; -use fabro_types::run_event::is_metadata_snapshot_compat_notice_code; +use fabro_types::RunNoticeCode; use fabro_util::json::normalize_json_value; use fabro_util::terminal::Styles; use tokio::time; @@ -801,7 +801,9 @@ fn format_event_pretty_value(envelope: &serde_json::Value, styles: &Styles) -> O } fn is_metadata_snapshot_compat_notice(envelope: &serde_json::Value) -> bool { - prop_str_field(envelope, "code").is_some_and(is_metadata_snapshot_compat_notice_code) + prop_str_field(envelope, "code") + .and_then(|code| code.parse::().ok()) + .is_some_and(RunNoticeCode::is_metadata_snapshot_compat) } fn str_field<'a>(value: &'a serde_json::Value, key: &str) -> Option<&'a str> { @@ -1150,14 +1152,27 @@ mod tests { #[test] fn pretty_run_notice_warn() { let styles = no_color_styles(); - let line = r#"{"ts":"2026-01-01T14:25:00Z","event":"run.notice","properties":{"level":"warn","code":"sandbox_cleanup_failed","message":"sandbox cleanup failed: boom"}}"#; - let result = format_event_pretty(line, &styles).unwrap(); + let code = RunNoticeCode::SandboxCleanupFailed.to_string(); + let line = serde_json::json!({ + "ts": "2026-01-01T14:25:00Z", + "event": "run.notice", + "properties": { + "level": "warn", + "code": code, + "message": "sandbox cleanup failed: boom", + }, + }) + .to_string(); + let result = format_event_pretty(&line, &styles).unwrap(); assert!(result.contains("Warning:"), "got: {result}"); assert!( result.contains("sandbox cleanup failed: boom"), "got: {result}" ); - assert!(result.contains("[sandbox_cleanup_failed]"), "got: {result}"); + assert!( + result.contains(&format!("[{}]", RunNoticeCode::SandboxCleanupFailed)), + "got: {result}" + ); } #[test] @@ -1235,13 +1250,31 @@ mod tests { fn pretty_stream_suppresses_metadata_compat_notice_only() { let styles = no_color_styles(); let failed = r#"{"ts":"2026-01-01T14:25:00Z","event":"metadata.snapshot.failed","properties":{"phase":"checkpoint","branch":"fabro/meta","duration_ms":900,"failure_kind":"write","error":"write failed"}}"#; - let compat_notice = r#"{"ts":"2026-01-01T14:25:01Z","event":"run.notice","properties":{"level":"warn","code":"checkpoint_metadata_write_failed","message":"legacy metadata warning"}}"#; - let degraded_notice = r#"{"ts":"2026-01-01T14:25:02Z","event":"run.notice","properties":{"level":"warn","code":"checkpoint_metadata_degraded","message":"metadata snapshots disabled"}}"#; + let compat_notice = serde_json::json!({ + "ts": "2026-01-01T14:25:01Z", + "event": "run.notice", + "properties": { + "level": "warn", + "code": RunNoticeCode::CheckpointMetadataWriteFailed, + "message": "legacy metadata warning", + }, + }) + .to_string(); + let degraded_notice = serde_json::json!({ + "ts": "2026-01-01T14:25:02Z", + "event": "run.notice", + "properties": { + "level": "warn", + "code": RunNoticeCode::CheckpointMetadataDegraded, + "message": "metadata snapshots disabled", + }, + }) + .to_string(); let mut state = PrettyEventState::default(); assert!(format_event_pretty_streamed(failed, &styles, &mut state).is_some()); - assert!(format_event_pretty_streamed(compat_notice, &styles, &mut state).is_none()); - let degraded = format_event_pretty_streamed(degraded_notice, &styles, &mut state).unwrap(); + assert!(format_event_pretty_streamed(&compat_notice, &styles, &mut state).is_none()); + let degraded = format_event_pretty_streamed(°raded_notice, &styles, &mut state).unwrap(); assert!( degraded.contains("metadata snapshots disabled"), "got: {degraded}" diff --git a/lib/crates/fabro-cli/src/commands/run/run_progress/event.rs b/lib/crates/fabro-cli/src/commands/run/run_progress/event.rs index 9e40f0d8b..ec56dce46 100644 --- a/lib/crates/fabro-cli/src/commands/run/run_progress/event.rs +++ b/lib/crates/fabro-cli/src/commands/run/run_progress/event.rs @@ -527,7 +527,7 @@ fn display_value(value: &Value) -> Option { mod tests { use fabro_agent::AgentEvent; use fabro_types::{MetadataSnapshotFailureKind, MetadataSnapshotPhase, fixtures}; - use fabro_workflow::event::{Event, to_run_event}; + use fabro_workflow::event::{Event, RunNoticeCode, to_run_event}; use super::*; @@ -804,10 +804,11 @@ mod tests { fn round_trip_run_notice() { let event = Event::RunNotice { level: RunNoticeLevel::Warn, - code: "sandbox_cleanup_failed".into(), + code: RunNoticeCode::SandboxCleanupFailed.to_string(), message: "sandbox cleanup failed".into(), exec_output_tail: None, }; + let expected_code = RunNoticeCode::SandboxCleanupFailed.to_string(); let stored = to_run_event(&fixtures::RUN_1, &event); let parsed = from_run_event(&stored).unwrap(); @@ -817,7 +818,7 @@ mod tests { level: RunNoticeLevel::Warn, code, message, - } if code == "sandbox_cleanup_failed" && message == "sandbox cleanup failed" + } if code == expected_code && message == "sandbox cleanup failed" )); } diff --git a/lib/crates/fabro-cli/src/commands/run/run_progress/mod.rs b/lib/crates/fabro-cli/src/commands/run/run_progress/mod.rs index 2004ac0ba..7e7df7f2d 100644 --- a/lib/crates/fabro-cli/src/commands/run/run_progress/mod.rs +++ b/lib/crates/fabro-cli/src/commands/run/run_progress/mod.rs @@ -3,8 +3,7 @@ reason = "sync CLI run-progress renderer: writes to std::io::stderr directly" )] -use fabro_types::RunEvent; -use fabro_types::run_event::is_metadata_snapshot_compat_notice_code; +use fabro_types::{RunEvent, RunNoticeCode}; mod event; mod info_display; @@ -444,7 +443,10 @@ impl ProgressUI { message, } => { if self.saw_metadata_snapshot_failure - && is_metadata_snapshot_compat_notice_code(&code) + && code + .parse::() + .ok() + .is_some_and(RunNoticeCode::is_metadata_snapshot_compat) { return; } @@ -1208,7 +1210,7 @@ mod tests { emit(&mut ui, Event::RunNotice { level: RunNoticeLevel::Warn, - code: "sandbox_cleanup_failed".into(), + code: RunNoticeCode::SandboxCleanupFailed.to_string(), message: "sandbox cleanup failed".into(), exec_output_tail: None, }); @@ -1279,13 +1281,13 @@ mod tests { }); emit(&mut ui, Event::RunNotice { level: RunNoticeLevel::Warn, - code: "checkpoint_metadata_write_failed".into(), + code: RunNoticeCode::CheckpointMetadataWriteFailed.to_string(), message: "legacy metadata warning".into(), exec_output_tail: None, }); emit(&mut ui, Event::RunNotice { level: RunNoticeLevel::Warn, - code: "checkpoint_metadata_degraded".into(), + code: RunNoticeCode::CheckpointMetadataDegraded.to_string(), message: "metadata snapshots are disabled for this run".into(), exec_output_tail: None, }); diff --git a/lib/crates/fabro-cli/tests/it/cmd/attach.rs b/lib/crates/fabro-cli/tests/it/cmd/attach.rs index e703e3207..7c6728f39 100644 --- a/lib/crates/fabro-cli/tests/it/cmd/attach.rs +++ b/lib/crates/fabro-cli/tests/it/cmd/attach.rs @@ -160,6 +160,7 @@ fn attach_replays_completed_detached_run() { ----- stdout ----- ----- stderr ----- Web UI: http://localhost:3000/runs/[ULID] + Warning: Worktree mode `always` requested but no Git repository was found; running without a worktree. [worktree_skipped_no_git] Sandbox: local (ready in [TIME]) ✓ Start [TIME] ✓ Run Tests [TIME] @@ -267,6 +268,7 @@ fn attach_before_completion_streams_to_finished_state() { ----- stdout ----- ----- stderr ----- Web UI: http://localhost:3000/runs/[ULID] + Warning: Worktree mode `always` requested but no Git repository was found; running without a worktree. [worktree_skipped_no_git] Sandbox: local (ready in [TIME]) ✓ start [DURATION] ✓ wait [DURATION] @@ -699,6 +701,21 @@ fn attach_json_errors_without_prompting_for_human_input() { "run_id": "[ULID]", "ts": "[TIMESTAMP]" }, + { + "actor": { + "kind": "worker", + "run_id": "[ULID]" + }, + "event": "run.notice", + "id": "[EVENT_ID]", + "properties": { + "code": "worktree_skipped_no_git", + "level": "warn", + "message": "Worktree mode `always` requested but no Git repository was found; running without a worktree." + }, + "run_id": "[ULID]", + "ts": "[TIMESTAMP]" + }, { "actor": { "kind": "worker", diff --git a/lib/crates/fabro-cli/tests/it/cmd/dump.rs b/lib/crates/fabro-cli/tests/it/cmd/dump.rs index 12b13ee9a..c541b59eb 100644 --- a/lib/crates/fabro-cli/tests/it/cmd/dump.rs +++ b/lib/crates/fabro-cli/tests/it/cmd/dump.rs @@ -278,9 +278,9 @@ fn dump_exports_completed_run_snapshot() { "); assert_snapshot!(dump_file_summary(&output_dir), @" - checkpoints/0013.json - checkpoints/0017.json - checkpoints/0021.json + checkpoints/0014.json + checkpoints/0018.json + checkpoints/0022.json events.jsonl graph.fabro run.json diff --git a/lib/crates/fabro-cli/tests/it/cmd/run.rs b/lib/crates/fabro-cli/tests/it/cmd/run.rs index 19d876cfc..2aa614979 100644 --- a/lib/crates/fabro-cli/tests/it/cmd/run.rs +++ b/lib/crates/fabro-cli/tests/it/cmd/run.rs @@ -684,6 +684,7 @@ fn dry_run_simple() { Run: [ULID] Web UI: http://localhost:3000/runs/[ULID] + Warning: Worktree mode `always` requested but no Git repository was found; running without a worktree. [worktree_skipped_no_git] Sandbox: local (ready in [TIME]) ✓ Start [TIME] ✓ Run Tests [TIME] diff --git a/lib/crates/fabro-cli/tests/it/workflow/dry_run_examples.rs b/lib/crates/fabro-cli/tests/it/workflow/dry_run_examples.rs index bb8a50c85..c3f8d836c 100644 --- a/lib/crates/fabro-cli/tests/it/workflow/dry_run_examples.rs +++ b/lib/crates/fabro-cli/tests/it/workflow/dry_run_examples.rs @@ -21,6 +21,7 @@ fn dry_run_branching() { warning [node: implement]: Node 'implement' has goal_gate=true but no retry_target or fallback_retry_target (goal_gate_has_retry) Run: [ULID] Web UI: http://localhost:3000/runs/[ULID] + Warning: Worktree mode `always` requested but no Git repository was found; running without a worktree. [worktree_skipped_no_git] Sandbox: local (ready in [TIME]) ✓ Start [TIME] ✓ Plan [TIME] @@ -57,6 +58,7 @@ fn dry_run_conditions() { Run: [ULID] Web UI: http://localhost:3000/runs/[ULID] + Warning: Worktree mode `always` requested but no Git repository was found; running without a worktree. [worktree_skipped_no_git] Sandbox: local (ready in [TIME]) ✓ start [TIME] ✓ Decide [TIME] @@ -91,6 +93,7 @@ fn dry_run_parallel() { Run: [ULID] Web UI: http://localhost:3000/runs/[ULID] + Warning: Worktree mode `always` requested but no Git repository was found; running without a worktree. [worktree_skipped_no_git] Sandbox: local (ready in [TIME]) ✓ start [TIME] ✓ Fork Work [TIME] @@ -126,6 +129,7 @@ fn dry_run_styled() { Run: [ULID] Web UI: http://localhost:3000/runs/[ULID] + Warning: Worktree mode `always` requested but no Git repository was found; running without a worktree. [worktree_skipped_no_git] Sandbox: local (ready in [TIME]) ✓ start [TIME] ✓ Plan [TIME] @@ -161,6 +165,7 @@ fn dry_run_legacy_tool() { Run: [ULID] Web UI: http://localhost:3000/runs/[ULID] + Warning: Worktree mode `always` requested but no Git repository was found; running without a worktree. [worktree_skipped_no_git] Sandbox: local (ready in [TIME]) ✓ Start [TIME] ✓ Echo [TIME] diff --git a/lib/crates/fabro-dump/src/lib.rs b/lib/crates/fabro-dump/src/lib.rs index 69a670565..5b2da9403 100644 --- a/lib/crates/fabro-dump/src/lib.rs +++ b/lib/crates/fabro-dump/src/lib.rs @@ -66,18 +66,13 @@ impl RunDump { entries.push(RunDumpEntry::text("graph.fabro", graph_source.clone())); } - let mut stages: Vec<_> = state.iter_stages().collect(); + let stages: Vec<_> = state.iter_stages().collect(); if stages.len() > MAX_STAGES_IN_DUMP { bail!( "run dump supports at most {MAX_STAGES_IN_DUMP} stages with the current path prefix width (got {})", stages.len() ); } - stages.sort_by(|(left_id, left), (right_id, right)| { - left.first_event_seq - .cmp(&right.first_event_seq) - .then_with(|| left_id.cmp(right_id)) - }); let mut stage_ranks = HashMap::new(); for (index, (stage_id, _)) in stages.iter().enumerate() { diff --git a/lib/crates/fabro-sandbox/src/local.rs b/lib/crates/fabro-sandbox/src/local.rs index 44734e9f1..a8b49e215 100644 --- a/lib/crates/fabro-sandbox/src/local.rs +++ b/lib/crates/fabro-sandbox/src/local.rs @@ -127,6 +127,19 @@ fn process_env_vars() -> Vec<(String, String)> { std::env::vars().collect() } +async fn drain_pipe(mut pipe: Option, stream: CommandOutputStream) -> String +where + R: AsyncRead + Unpin, +{ + let mut buf = String::new(); + if let Some(ref mut reader) = pipe { + if let Err(err) = reader.read_to_string(&mut buf).await { + tracing::warn!(error = %err, ?stream, "Failed to drain child output"); + } + } + buf +} + #[async_trait] impl Sandbox for LocalSandbox { async fn read_file( @@ -278,22 +291,12 @@ impl Sandbox for LocalSandbox { // it writes more than the OS pipe buffer (~64 KB) the write() syscall // blocks until the parent drains the pipe, but the parent is blocked // on child.wait(). - let mut stdout_pipe = child.stdout.take(); - let mut stderr_pipe = child.stderr.take(); - let stdout_task = tokio::spawn(async move { - let mut buf = String::new(); - if let Some(ref mut r) = stdout_pipe { - let _ = r.read_to_string(&mut buf).await; - } - buf - }); - let stderr_task = tokio::spawn(async move { - let mut buf = String::new(); - if let Some(ref mut r) = stderr_pipe { - let _ = r.read_to_string(&mut buf).await; - } - buf - }); + let stdout_pipe = child.stdout.take(); + let stderr_pipe = child.stderr.take(); + let stdout_task = + tokio::spawn(async move { drain_pipe(stdout_pipe, CommandOutputStream::Stdout).await }); + let stderr_task = + tokio::spawn(async move { drain_pipe(stderr_pipe, CommandOutputStream::Stderr).await }); let (termination, exit_code) = tokio::select! { status_result = child.wait() => { @@ -714,7 +717,12 @@ where )] mod tests { use std::collections::HashMap; + use std::io; use std::path::PathBuf; + use std::pin::Pin; + use std::task::{Context as TaskContext, Poll}; + + use tokio::io::ReadBuf; use super::*; @@ -724,6 +732,25 @@ mod tests { dir } + #[tokio::test] + async fn drain_pipe_returns_empty_buffer_after_read_failure() { + struct FailingReader; + + impl AsyncRead for FailingReader { + fn poll_read( + self: Pin<&mut Self>, + _cx: &mut TaskContext<'_>, + _buf: &mut ReadBuf<'_>, + ) -> Poll> { + Poll::Ready(Err(io::Error::other("simulated read failure"))) + } + } + + let output = drain_pipe(Some(FailingReader), CommandOutputStream::Stdout).await; + + assert!(output.is_empty()); + } + #[tokio::test] async fn read_file_with_line_numbers() { let dir = temp_dir(); diff --git a/lib/crates/fabro-server/src/demo/mod.rs b/lib/crates/fabro-server/src/demo/mod.rs index c9751cf83..539731c82 100644 --- a/lib/crates/fabro-server/src/demo/mod.rs +++ b/lib/crates/fabro-server/src/demo/mod.rs @@ -15,15 +15,16 @@ use axum::http::StatusCode; use axum::response::sse::{Event, Sse}; use axum::response::{IntoResponse, Response}; use fabro_api::types::{ - CreateSecretRequest, DeleteSecretRequest, DiffFile, DiffStats, FileDiff, FileDiffChangeKind, - PaginatedRunFileList, RunArtifactListResponse, RunFilesMeta, + CreateSecretRequest, DeleteSecretRequest, DiffFile, DiffStats, EventEnvelope, FileDiff, + FileDiffChangeKind, PaginatedEventList, PaginatedRunFileList, PaginationMeta, + RunArtifactListResponse, RunFilesMeta, }; use serde_json::json; use crate::error::ApiError; use crate::principal_middleware::RequiredUser; use crate::run_selector::{ResolveRunError, resolve_run_by_selector}; -use crate::server::{AppState, PaginationParams}; +use crate::server::{AppState, EventListParams, PaginationParams, parse_stage_id_path}; fn paginated_response( items: Vec, @@ -133,13 +134,39 @@ pub(crate) async fn get_run_stages( paginated_response(runs::stages(), &pagination) } -pub(crate) async fn get_stage_turns( +pub(crate) async fn get_stage_events( _auth: RequiredUser, State(_state): State>, - Path((_id, _stage_id)): Path<(String, String)>, - Query(pagination): Query, + Path((_id, stage_id)): Path<(String, String)>, + Query(params): Query, ) -> Response { - paginated_response(runs::turns(), &pagination) + let stage_id = match parse_stage_id_path(&stage_id) { + Ok(stage_id) => stage_id, + Err(response) => return response, + }; + let since_seq = params.since_seq(); + let limit = params.limit(); + let mut matches: Vec = runs::stage_events() + .into_iter() + .filter(|envelope| { + envelope.seq >= since_seq + && (envelope.event.stage_id.as_ref() == Some(&stage_id) + || (envelope.event.stage_id.is_none() + && stage_id.visit() == 1 + && envelope.event.node_id.as_deref() == Some(stage_id.node_id()))) + }) + .take(limit + 1) + .collect(); + let has_more = matches.len() > limit; + matches.truncate(limit); + ( + StatusCode::OK, + Json(PaginatedEventList { + data: matches, + meta: PaginationMeta { has_more }, + }), + ) + .into_response() } pub(crate) async fn list_run_artifacts_stub( @@ -784,9 +811,10 @@ mod runs { RunNamespace, RunPrepareSettings, RunSandboxSettings, }; use fabro_types::settings::{InterpString, ProjectNamespace, WorkflowNamespace}; - use fabro_types::{RunId, WorkflowSettings}; + use fabro_types::{RunId, StageId, WorkflowSettings}; use super::ts; + use crate::server::run_stage_from_stage_id; fn labels(entries: &[(&str, &str)]) -> HashMap { entries @@ -795,8 +823,8 @@ mod runs { .collect() } - fn demo_run_ids() -> &'static [RunId; 6] { - static IDS: OnceLock<[RunId; 6]> = OnceLock::new(); + fn demo_run_ids() -> &'static [RunId; 7] { + static IDS: OnceLock<[RunId; 7]> = OnceLock::new(); IDS.get_or_init(|| { [ RunId::with_timestamp(ts("2026-03-06T14:30:00Z"), 1), @@ -805,6 +833,7 @@ mod runs { RunId::with_timestamp(ts("2026-03-04T10:00:00Z"), 4), RunId::with_timestamp(ts("2026-03-03T16:45:00Z"), 5), RunId::with_timestamp(ts("2026-02-28T14:00:00Z"), 6), + RunId::with_timestamp(ts("2026-03-06T14:35:00Z"), 7), ] }) } @@ -974,23 +1003,27 @@ mod runs { pub(super) fn columns() -> Vec { vec![ BoardColumnDefinition { - id: "initializing".into(), + id: BoardColumn::Queued, + name: "Queued".into(), + }, + BoardColumnDefinition { + id: BoardColumn::Initializing, name: "Initializing".into(), }, BoardColumnDefinition { - id: "running".into(), + id: BoardColumn::Running, name: "Running".into(), }, BoardColumnDefinition { - id: "blocked".into(), + id: BoardColumn::Blocked, name: "Blocked".into(), }, BoardColumnDefinition { - id: "succeeded".into(), + id: BoardColumn::Succeeded, name: "Succeeded".into(), }, BoardColumnDefinition { - id: "failed".into(), + id: BoardColumn::Failed, name: "Failed".into(), }, ] @@ -1082,6 +1115,20 @@ mod runs { Some(720000), &[("release", "preview")], ), + summary( + 7, + "api-server", + "implement", + "Implement", + "Add audit log retention policy", + "queued", + "2026-03-06T14:35:00Z", + None, + None, + None, + None, + &[("owner", "platform")], + ), ] } @@ -1150,54 +1197,164 @@ mod runs { None, None, ), + board_item( + take_summary(&mut summaries, demo_run_id(7)), + BoardColumn::Queued, + None, + None, + None, + ), ] } pub(super) fn stages() -> Vec { vec![ - RunStage { - id: "detect-drift".into(), - name: "Detect Drift".into(), - status: StageState::Succeeded, - duration_secs: Some(72.0), - dot_id: Some("detect".into()), - }, - RunStage { - id: "propose-changes".into(), - name: "Propose Changes".into(), - status: StageState::Succeeded, - duration_secs: Some(154.0), - dot_id: Some("propose".into()), - }, - RunStage { - id: "review-changes".into(), - name: "Review Changes".into(), - status: StageState::Succeeded, - duration_secs: Some(45.0), - dot_id: Some("review".into()), - }, - RunStage { - id: "apply-changes".into(), - name: "Apply Changes".into(), - status: StageState::Running, - duration_secs: Some(118.0), - dot_id: Some("apply".into()), - }, + run_stage_from_stage_id( + &StageId::new("detect-drift", 1), + "Detect Drift", + StageState::Succeeded, + Some(72.0), + None, + ), + run_stage_from_stage_id( + &StageId::new("propose-changes", 1), + "Propose Changes", + StageState::Succeeded, + Some(154.0), + None, + ), + run_stage_from_stage_id( + &StageId::new("review-changes", 1), + "Review Changes", + StageState::Succeeded, + Some(45.0), + None, + ), + run_stage_from_stage_id( + &StageId::new("apply-changes", 1), + "Apply Changes", + StageState::Succeeded, + Some(118.0), + None, + ), + run_stage_from_stage_id( + &StageId::new("apply-changes", 2), + "Apply Changes", + StageState::Running, + None, + None, + ), ] } - pub(super) fn turns() -> Vec { + pub(super) fn stage_events() -> Vec { + use fabro_model::BilledTokenCounts; + use fabro_types::run_event::agent::{ + AgentMessageProps, AgentToolCompletedProps, AgentToolStartedProps, + }; + use fabro_types::run_event::stage::StagePromptProps; + use fabro_types::{EventBody, EventEnvelope, RunEvent}; + + let run_id = demo_run_id(1); + let node_id = "detect-drift"; + let stage_id = fabro_types::StageId::new(node_id, 1); + let ts = ts("2026-03-06T14:30:00Z"); + + let make_envelope = |seq: u32, id: &str, body: EventBody| EventEnvelope { + seq, + event: RunEvent { + id: id.into(), + ts, + run_id, + node_id: Some(node_id.into()), + node_label: Some("Detect Drift".into()), + stage_id: Some(stage_id.clone()), + parallel_group_id: None, + parallel_branch_id: None, + session_id: None, + parent_session_id: None, + tool_call_id: None, + actor: None, + body, + }, + }; + vec![ - StageTurn::SystemStageTurn(SystemStageTurn { kind: SystemStageTurnKind::System, content: "You are a drift detection agent. Compare the production and staging environments and identify any configuration or code drift.".into() }), - StageTurn::AssistantStageTurn(AssistantStageTurn { kind: AssistantStageTurnKind::Assistant, content: "I'll start by loading the environment configurations for both production and staging to compare them.".into() }), - StageTurn::ToolStageTurn(ToolStageTurn { - kind: ToolStageTurnKind::Tool, content: None, - tools: vec![ - ToolUse { id: "toolu_01".into(), tool_name: "read_file".into(), input: r#"{ "path": "environments/production/config.toml" }"#.into(), result: "[redis]\nhost = \"redis-prod.internal\"\nport = 6379".into(), is_error: false, duration_ms: Some(45) }, - ToolUse { id: "toolu_02".into(), tool_name: "read_file".into(), input: r#"{ "path": "environments/staging/config.toml" }"#.into(), result: "[redis]\nhost = \"redis-staging.internal\"\nport = 6379".into(), is_error: false, duration_ms: Some(38) }, - ], - }), - StageTurn::AssistantStageTurn(AssistantStageTurn { kind: AssistantStageTurnKind::Assistant, content: "I've detected drift in 3 resources between production and staging:\n\n1. **redis.max_connections** — production has 200, staging has 100\n2. **redis.tls** — enabled in production, disabled in staging\n3. **iam.session_duration** — production uses 3600s, staging uses 1800s".into() }), + make_envelope( + 1, + "evt-detect-drift-1", + EventBody::StagePrompt(StagePromptProps { + visit: 1, + text: "You are a drift detection agent. Compare the production and staging environments and identify any configuration or code drift.".into(), + mode: None, + provider: None, + model: None, + }), + ), + make_envelope( + 2, + "evt-detect-drift-2", + EventBody::AgentMessage(AgentMessageProps { + text: "I'll start by loading the environment configurations for both production and staging to compare them.".into(), + model: "Opus 4.6".into(), + billing: BilledTokenCounts::default(), + tool_call_count: 0, + visit: 1, + }), + ), + make_envelope( + 3, + "evt-detect-drift-3", + EventBody::AgentToolStarted(AgentToolStartedProps { + tool_name: "read_file".into(), + tool_call_id: "toolu_01".into(), + arguments: serde_json::json!({ "path": "environments/production/config.toml" }), + visit: 1, + }), + ), + make_envelope( + 4, + "evt-detect-drift-4", + EventBody::AgentToolCompleted(AgentToolCompletedProps { + tool_name: "read_file".into(), + tool_call_id: "toolu_01".into(), + output: serde_json::json!("[redis]\nhost = \"redis-prod.internal\"\nport = 6379"), + is_error: false, + visit: 1, + }), + ), + make_envelope( + 5, + "evt-detect-drift-5", + EventBody::AgentToolStarted(AgentToolStartedProps { + tool_name: "read_file".into(), + tool_call_id: "toolu_02".into(), + arguments: serde_json::json!({ "path": "environments/staging/config.toml" }), + visit: 1, + }), + ), + make_envelope( + 6, + "evt-detect-drift-6", + EventBody::AgentToolCompleted(AgentToolCompletedProps { + tool_name: "read_file".into(), + tool_call_id: "toolu_02".into(), + output: serde_json::json!("[redis]\nhost = \"redis-staging.internal\"\nport = 6379"), + is_error: false, + visit: 1, + }), + ), + make_envelope( + 7, + "evt-detect-drift-7", + EventBody::AgentMessage(AgentMessageProps { + text: "I've detected drift in 3 resources between production and staging:\n\n1. **redis.max_connections** — production has 200, staging has 100\n2. **redis.tls** — enabled in production, disabled in staging\n3. **iam.session_duration** — production uses 3600s, staging uses 1800s".into(), + model: "Opus 4.6".into(), + billing: BilledTokenCounts::default(), + tool_call_count: 0, + visit: 1, + }), + ), ] } @@ -1222,6 +1379,8 @@ mod runs { total_usd_micros: Some(480_000), }, runtime_secs: 72.0, + started_at: None, + state: Some(StageState::Succeeded), }, RunBillingStage { stage: BillingStageRef { @@ -1241,6 +1400,8 @@ mod runs { total_usd_micros: Some(720_000), }, runtime_secs: 154.0, + started_at: None, + state: Some(StageState::Succeeded), }, RunBillingStage { stage: BillingStageRef { @@ -1260,6 +1421,8 @@ mod runs { total_usd_micros: Some(190_000), }, runtime_secs: 45.0, + started_at: None, + state: Some(StageState::Succeeded), }, RunBillingStage { stage: BillingStageRef { @@ -1279,6 +1442,8 @@ mod runs { total_usd_micros: Some(870_000), }, runtime_secs: 118.0, + started_at: None, + state: Some(StageState::Running), }, ], totals: RunBillingTotals { diff --git a/lib/crates/fabro-server/src/principal_middleware.rs b/lib/crates/fabro-server/src/principal_middleware.rs index e8971831c..822d87dc6 100644 --- a/lib/crates/fabro-server/src/principal_middleware.rs +++ b/lib/crates/fabro-server/src/principal_middleware.rs @@ -52,6 +52,7 @@ pub(crate) struct RequestAuth(pub(crate) AuthContextSlot); pub(crate) struct RequiredUser(pub(crate) UserPrincipal); pub(crate) struct RequireRunScoped(pub(crate) RunId); pub(crate) struct RequireRunBlob(pub(crate) RunId, pub(crate) RunBlobId); +pub(crate) struct RequireRunStageScoped(pub(crate) RunId, pub(crate) String); pub(crate) struct RequireStageArtifact(pub(crate) RunId, pub(crate) StageId); pub(crate) struct RequireCommandLog( pub(crate) RunId, @@ -203,6 +204,23 @@ impl FromRequestParts> for RequireRunBlob { } } +impl FromRequestParts> for RequireRunStageScoped { + type Rejection = Response; + + async fn from_request_parts( + parts: &mut Parts, + state: &Arc, + ) -> Result { + let Path((id, stage_id)): Path<(String, String)> = Path::from_request_parts(parts, state) + .await + .map_err(IntoResponse::into_response)?; + let run_id = parse_run_id_path(&id)?; + require_worker_or_user_for_run(&auth_slot_from_parts(parts), &run_id) + .map_err(IntoResponse::into_response)?; + Ok(Self(run_id, stage_id)) + } +} + impl FromRequestParts> for RequireStageArtifact { type Rejection = Response; diff --git a/lib/crates/fabro-server/src/server.rs b/lib/crates/fabro-server/src/server.rs index 85abdaaf7..c0508a8e3 100644 --- a/lib/crates/fabro-server/src/server.rs +++ b/lib/crates/fabro-server/src/server.rs @@ -54,7 +54,7 @@ use fabro_llm::types::{ ContentPart, FinishReason, Message as LlmMessage, Request as LlmRequest, Role, ToolChoice, ToolDefinition, }; -use fabro_model::{BilledModelUsage, BilledTokenCounts, Catalog, ModelTestMode, Provider}; +use fabro_model::{BilledTokenCounts, Catalog, ModelTestMode, Provider}; use fabro_redact::redact_jsonl_line; use fabro_sandbox::daytona::{self, DaytonaSandbox}; use fabro_sandbox::reconnect::reconnect; @@ -126,7 +126,7 @@ use crate::ip_allowlist::{IpAllowlistConfig, ip_allowlist_middleware}; use crate::jwt_auth::{self, AuthMode}; use crate::principal_middleware::{ AuthContextSlot, RequestAuth, RequestAuthContext, RequireRunBlob, RequireRunScoped, - RequireStageArtifact, RequiredUser, principal_middleware, + RequireRunStageScoped, RequireStageArtifact, RequiredUser, principal_middleware, }; use crate::request_id::{self, RequestId}; use crate::run_files::{FilesInFlight, new_files_in_flight}; @@ -139,6 +139,7 @@ use crate::{ mod handler; +pub(crate) use handler::events::EventListParams; #[cfg(test)] pub(in crate::server) use handler::events::filtered_global_events; pub(crate) use handler::graph::render_graph_bytes; @@ -537,17 +538,50 @@ pub(crate) struct ResolvedAppStateSettings { pub(crate) manifest_run_settings: std::result::Result, } -fn accumulate_model_billing(entry: &mut ModelBillingTotals, usage: &BilledModelUsage) { - let tokens = usage.tokens(); - entry.stages += 1; - entry.billing.input_tokens += tokens.input_tokens; - entry.billing.output_tokens += tokens.output_tokens; - entry.billing.reasoning_tokens += tokens.reasoning_tokens; - entry.billing.cache_read_tokens += tokens.cache_read_tokens; - entry.billing.cache_write_tokens += tokens.cache_write_tokens; - entry.billing.total_tokens += tokens.total_tokens(); - if let Some(value) = usage.total_usd_micros { - *entry.billing.total_usd_micros.get_or_insert(0) += value; +fn accumulate_billed_token_counts(target: &mut BilledTokenCounts, source: &BilledTokenCounts) { + target.input_tokens += source.input_tokens; + target.output_tokens += source.output_tokens; + target.reasoning_tokens += source.reasoning_tokens; + target.cache_read_tokens += source.cache_read_tokens; + target.cache_write_tokens += source.cache_write_tokens; + target.total_tokens += source.total_tokens; + if let Some(value) = source.total_usd_micros { + *target.total_usd_micros.get_or_insert(0) += value; + } +} + +fn accumulate_billing_rollup( + accumulator: &mut BillingAccumulator, + rollup: &fabro_workflow::ProjectionBillingRollup, +) { + accumulator.total_runs += 1; + accumulator.total_runtime_secs += rollup.runtime_ms as f64 / 1000.0; + for model in &rollup.by_model { + let entry = accumulator + .by_model + .entry(model.model_id.clone()) + .or_default(); + entry.stages += model.stages; + accumulate_billed_token_counts(&mut entry.billing, &model.billing); + } +} + +pub(crate) fn run_stage_from_stage_id( + stage_id: &StageId, + name: impl Into, + status: StageState, + duration_secs: Option, + started_at: Option>, +) -> RunStage { + RunStage { + id: stage_id.to_string(), + name: name.into(), + status, + duration_secs, + node_id: stage_id.node_id().to_string(), + visit: std::num::NonZeroU32::new(stage_id.visit()) + .expect("StageId stores a non-zero visit"), + started_at, } } @@ -2777,9 +2811,9 @@ async fn execute_run_in_process(state: Arc, run_id: RunId) { } } - // Save final checkpoint - let checkpoint = match run_store.state().await { - Ok(state) => state.checkpoint, + // Save final projection + let final_projection = match run_store.state().await { + Ok(state) => Some(state), Err(err) => { tracing::warn!(run_id = %run_id, error = %err, "Failed to load run state from store"); None @@ -2787,32 +2821,17 @@ async fn execute_run_in_process(state: Arc, run_id: RunId) { }; // Accumulate aggregate usage after execution completes. - if let Some(ref cp) = checkpoint { - let stage_durations = match run_store.list_events().await { - Ok(events) => fabro_workflow::extract_stage_durations_from_events(&events), - Err(err) => { - tracing::warn!(run_id = %run_id, error = %err, "Failed to load run events from store"); - HashMap::default() - } - }; - let mut agg = state - .aggregate_billing - .lock() - .expect("aggregate_billing lock poisoned"); - agg.total_runs += 1; - let mut run_runtime: f64 = 0.0; - for (node_id, outcome) in &cp.node_outcomes { - if let Some(usage) = &outcome.usage { - let entry = agg - .by_model - .entry(usage.model_id().to_string()) - .or_default(); - accumulate_model_billing(entry, usage); - } - let duration_ms = stage_durations.get(node_id).copied().unwrap_or(0); - run_runtime += duration_ms as f64 / 1000.0; + if let Some(ref projection) = final_projection { + if projection.checkpoint.is_some() { + let mut agg = state + .aggregate_billing + .lock() + .expect("aggregate_billing lock poisoned"); + accumulate_billing_rollup( + &mut agg, + &fabro_workflow::billing_rollup_from_projection(projection), + ); } - agg.total_runtime_secs += run_runtime; } let mut runs = state.runs.lock().expect("runs lock poisoned"); @@ -2861,7 +2880,9 @@ async fn execute_run_in_process(state: Arc, run_id: RunId) { }; } } - managed_run.checkpoint = checkpoint; + managed_run.checkpoint = final_projection + .as_ref() + .and_then(|projection| projection.checkpoint.clone()); managed_run.run_dir = Some(run_dir); clear_live_run_state(managed_run); } @@ -3104,32 +3125,15 @@ async fn execute_run_subprocess(state: Arc, run_id: RunId) { } }; - if let Some(ref checkpoint) = final_state.checkpoint { - let stage_durations = match run_store.list_events().await { - Ok(events) => fabro_workflow::extract_stage_durations_from_events(&events), - Err(err) => { - tracing::warn!(run_id = %run_id, error = %err, "Failed to load run events from store"); - HashMap::default() - } - }; + if final_state.checkpoint.is_some() { let mut agg = state .aggregate_billing .lock() .expect("aggregate_billing lock poisoned"); - agg.total_runs += 1; - let mut run_runtime: f64 = 0.0; - for (node_id, outcome) in &checkpoint.node_outcomes { - if let Some(usage) = &outcome.usage { - let entry = agg - .by_model - .entry(usage.model_id().to_string()) - .or_default(); - accumulate_model_billing(entry, usage); - } - let duration_ms = stage_durations.get(node_id).copied().unwrap_or(0); - run_runtime += duration_ms as f64 / 1000.0; - } - agg.total_runtime_secs += run_runtime; + accumulate_billing_rollup( + &mut agg, + &fabro_workflow::billing_rollup_from_projection(&final_state), + ); } let mut runs = state.runs.lock().expect("runs lock poisoned"); diff --git a/lib/crates/fabro-server/src/server/handler/billing.rs b/lib/crates/fabro-server/src/server/handler/billing.rs index dfe7d5176..5c000869c 100644 --- a/lib/crates/fabro-server/src/server/handler/billing.rs +++ b/lib/crates/fabro-server/src/server/handler/billing.rs @@ -1,13 +1,14 @@ +use std::collections::HashMap; use std::sync::Arc; -use fabro_types::EventBody; +use chrono::{DateTime, Utc}; +use fabro_types::{RunProjection, StageProjection, StageState}; use super::super::{ - ApiError, AppState, BilledTokenCounts, BillingByModel, BillingStageRef, EventEnvelope, HashMap, - IntoResponse, Json, ListResponse, ModelBillingTotals, ModelReference, PaginationParams, Path, - Query, RequiredUser, Response, Router, RunBilling, RunBillingStage, RunBillingTotals, RunId, - RunStage, RunStatus, StageState, State, StatusCode, accumulate_model_billing, get, - parse_run_id_path, + ApiError, AppState, BillingByModel, BillingStageRef, IntoResponse, Json, ListResponse, + ModelReference, PaginationParams, Path, Query, RequiredUser, Response, Router, RunBilling, + RunBillingStage, RunBillingTotals, RunId, State, StatusCode, get, parse_run_id_path, + run_stage_from_stage_id, }; pub(super) fn routes() -> Router> { @@ -16,25 +17,6 @@ pub(super) fn routes() -> Router> { .route("/runs/{id}/billing", get(get_run_billing)) } -fn active_stage_state_from_events(events: &[EventEnvelope], node_id: &str) -> StageState { - let latest = events.iter().rev().find(|envelope| { - envelope.event.node_id.as_deref() == Some(node_id) - && matches!( - &envelope.event.body, - EventBody::StageRetrying(_) - | EventBody::StageStarted(_) - | EventBody::StageCompleted(_) - | EventBody::StageFailed(_) - ) - }); - - if latest.is_some_and(|e| matches!(&e.event.body, EventBody::StageRetrying(_))) { - StageState::Retrying - } else { - StageState::Running - } -} - async fn list_run_stages( _auth: RequiredUser, State(state): State>, @@ -46,81 +28,30 @@ async fn list_run_stages( Err(response) => return response, }; - // Try live run first. - let (checkpoint, run_is_active) = { - let runs = state.runs.lock().expect("runs lock poisoned"); - match runs.get(&id) { - Some(managed_run) => { - let active = !matches!( - managed_run.status, - RunStatus::Succeeded { .. } | RunStatus::Failed { .. } | RunStatus::Dead - ); - (managed_run.checkpoint.clone(), active) - } - None => (None, false), + let Ok(run_store) = state.store.open_run_reader(&id).await else { + return ApiError::not_found("Run not found.").into_response(); + }; + let projection = match run_store.state().await { + Ok(state) => state, + Err(err) => { + return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) + .into_response(); } }; - // Fall back to stored run. - let (checkpoint, run_is_active) = if checkpoint.is_some() { - (checkpoint, run_is_active) - } else { - match state.store.open_run_reader(&id).await { - Ok(run_store) => match run_store.state().await { - Ok(run_state) => { - let active = run_state.status.is_some_and(|status| !status.is_terminal()); - (run_state.checkpoint, active) - } - Err(_) => (None, false), - }, - Err(_) => return ApiError::not_found("Run not found.").into_response(), - } - }; - - let Some(checkpoint) = checkpoint else { - return ( - StatusCode::OK, - Json(ListResponse::new(Vec::::new())), - ) - .into_response(); - }; - - let events = match state.store.open_run_reader(&id).await { - Ok(run_store) => run_store.list_events().await.unwrap_or_default(), - Err(_) => Vec::new(), - }; - let stage_durations = fabro_workflow::extract_stage_durations_from_events(&events); - - let mut stages = Vec::new(); - for node_id in &checkpoint.completed_nodes { - let duration_ms = stage_durations.get(node_id).copied().unwrap_or(0); - let status = match checkpoint.node_outcomes.get(node_id) { - Some(outcome) => StageState::from(outcome.status), - None => StageState::Succeeded, - }; - stages.push(RunStage { - id: node_id.clone(), - name: node_id.clone(), - status, - duration_secs: Some(duration_ms as f64 / 1000.0), - dot_id: Some(node_id.clone()), - }); - } - - // Add next node as running if the run is still active. - // The checkpoint's current_node is the last *completed* stage; next_node_id - // is the stage that is currently executing. - if let Some(next_id) = &checkpoint.next_node_id { - if run_is_active && next_id != "exit" && !checkpoint.completed_nodes.contains(next_id) { - stages.push(RunStage { - id: next_id.clone(), - name: next_id.clone(), - status: active_stage_state_from_events(&events, next_id), - duration_secs: None, - dot_id: Some(next_id.clone()), - }); - } - } + let now = Utc::now(); + let stages = projection + .iter_stages() + .map(|(stage_id, stage)| { + run_stage_from_stage_id( + stage_id, + stage_id.node_id().to_string(), + stage.effective_state(), + stage.runtime_secs(now), + stage.started_at, + ) + }) + .collect::>(); (StatusCode::OK, Json(ListResponse::new(stages))).into_response() } @@ -137,91 +68,53 @@ async fn get_run_billing( } }; - let checkpoint = match run_store.state().await { - Ok(state) => state.checkpoint, + let projection = match run_store.state().await { + Ok(state) => state, Err(err) => { return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) .into_response(); } }; - let Some(checkpoint) = checkpoint else { - let empty = RunBilling { - by_model: Vec::new(), - stages: Vec::new(), - totals: RunBillingTotals { - cache_read_tokens: 0, - cache_write_tokens: 0, - input_tokens: 0, - output_tokens: 0, - reasoning_tokens: 0, - runtime_secs: 0.0, - total_tokens: 0, - total_usd_micros: None, + let rollup = fabro_workflow::billing_rollup_from_projection(&projection); + let by_model = rollup + .by_model + .iter() + .map(|model| BillingByModel { + billing: model.billing.clone(), + model: ModelReference { + id: model.model_id.clone(), }, - }; - return (StatusCode::OK, Json(empty)).into_response(); - }; + stages: model.stages, + }) + .collect::>(); - let stage_durations = match run_store.list_events().await { - Ok(events) => fabro_workflow::extract_stage_durations_from_events(&events), - Err(err) => { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) - .into_response(); - } - }; - - let mut by_model_totals = HashMap::::new(); - let mut billed_usages = Vec::new(); - let mut runtime_secs = 0.0_f64; - let mut stages = Vec::new(); - - for node_id in &checkpoint.completed_nodes { - let duration_ms = stage_durations.get(node_id).copied().unwrap_or(0); - runtime_secs += duration_ms as f64 / 1000.0; - - let usage = checkpoint - .node_outcomes - .get(node_id) - .and_then(|outcome| outcome.usage.as_ref()); - - let (billing, model) = if let Some(usage) = usage { - billed_usages.push(usage.clone()); - let tokens = usage.tokens(); - let billing = BilledTokenCounts { - cache_read_tokens: tokens.cache_read_tokens, - cache_write_tokens: tokens.cache_write_tokens, - input_tokens: tokens.input_tokens, - output_tokens: tokens.output_tokens, - reasoning_tokens: tokens.reasoning_tokens, - total_tokens: tokens.total_tokens(), - total_usd_micros: usage.total_usd_micros, - }; - let model_id = usage.model_id().to_string(); - accumulate_model_billing(by_model_totals.entry(model_id.clone()).or_default(), usage); - (billing, Some(ModelReference { id: model_id })) - } else { - (BilledTokenCounts::default(), None) - }; - - stages.push(RunBillingStage { - billing, - model, - runtime_secs: duration_ms as f64 / 1000.0, - stage: BillingStageRef { - id: node_id.clone(), - name: node_id.clone(), - }, - }); - } - - let totals = BilledTokenCounts::from_billed_usage(&billed_usages); - let by_model = by_model_totals + let rollup_by_node = rollup + .stages + .iter() + .map(|stage| (stage.node_id.as_str(), stage)) + .collect::>(); + let live_rows = live_billing_rows(&projection, Utc::now()); + let runtime_secs = live_rows.iter().map(|row| row.runtime_secs).sum::(); + let stages = live_rows .into_iter() - .map(|(model, totals)| BillingByModel { - billing: totals.billing, - model: ModelReference { id: model }, - stages: totals.stages, + .map(|row| { + let rollup_stage = rollup_by_node.get(row.node_id.as_str()); + RunBillingStage { + billing: rollup_stage + .map(|stage| stage.billing.clone()) + .unwrap_or_default(), + model: rollup_stage + .and_then(|stage| stage.model_id.as_ref()) + .map(|id| ModelReference { id: id.clone() }), + runtime_secs: row.runtime_secs, + stage: BillingStageRef { + id: row.node_id.clone(), + name: row.node_id, + }, + started_at: row.started_at, + state: row.state, + } }) .collect::>(); @@ -229,16 +122,80 @@ async fn get_run_billing( by_model, stages, totals: RunBillingTotals { - cache_read_tokens: totals.cache_read_tokens, - cache_write_tokens: totals.cache_write_tokens, - input_tokens: totals.input_tokens, - output_tokens: totals.output_tokens, - reasoning_tokens: totals.reasoning_tokens, + cache_read_tokens: rollup.totals.cache_read_tokens, + cache_write_tokens: rollup.totals.cache_write_tokens, + input_tokens: rollup.totals.input_tokens, + output_tokens: rollup.totals.output_tokens, + reasoning_tokens: rollup.totals.reasoning_tokens, runtime_secs, - total_tokens: totals.total_tokens, - total_usd_micros: totals.total_usd_micros, + total_tokens: rollup.totals.total_tokens, + total_usd_micros: rollup.totals.total_usd_micros, }, }; (StatusCode::OK, Json(response)).into_response() } + +struct LiveBillingRow { + node_id: String, + runtime_secs: f64, + started_at: Option>, + state: Option, + latest_visit: u32, +} + +fn live_billing_rows(projection: &RunProjection, now: DateTime) -> Vec { + let mut row_indices = HashMap::::new(); + let mut rows = Vec::::new(); + + for (stage_id, stage) in projection.iter_stages() { + let node_id = stage_id.node_id(); + if is_exit_stage(projection, node_id) || !stage_has_billing_row(stage) { + continue; + } + + let index = *row_indices.entry(node_id.to_string()).or_insert_with(|| { + let index = rows.len(); + rows.push(LiveBillingRow { + node_id: node_id.to_string(), + runtime_secs: 0.0, + started_at: None, + state: None, + latest_visit: 0, + }); + index + }); + let row = &mut rows[index]; + row.runtime_secs += billing_runtime_secs(stage, now).unwrap_or(0.0); + + if stage_id.visit() >= row.latest_visit { + row.latest_visit = stage_id.visit(); + row.started_at = stage.started_at; + row.state = Some(stage.effective_state()); + } + } + + rows +} + +fn billing_runtime_secs(stage: &StageProjection, now: DateTime) -> Option { + stage + .duration_ms + .map(|ms| ms as f64 / 1000.0) + .or_else(|| stage.runtime_secs(now)) +} + +fn stage_has_billing_row(stage: &StageProjection) -> bool { + stage.completion.is_some() + || stage.duration_ms.is_some() + || stage.usage.is_some() + || stage.started_at.is_some() + || stage.state.is_some() +} + +fn is_exit_stage(projection: &RunProjection, node_id: &str) -> bool { + projection + .spec() + .and_then(|spec| spec.graph().nodes.get(node_id)) + .is_some_and(|node| node.handler_type() == Some("exit")) +} diff --git a/lib/crates/fabro-server/src/server/handler/events.rs b/lib/crates/fabro-server/src/server/handler/events.rs index ff3889cd5..fce4a4d11 100644 --- a/lib/crates/fabro-server/src/server/handler/events.rs +++ b/lib/crates/fabro-server/src/server/handler/events.rs @@ -3,9 +3,10 @@ use std::sync::Arc; use super::super::{ ApiError, AppState, AppendEventResponse, BroadcastStream, Event, EventBody, EventEnvelope, EventPayload, HashSet, IntoResponse, Json, KeepAlive, PaginatedEventList, PaginationMeta, Path, - Query, RequireRunScoped, RequiredUser, Response, Router, RunEvent, RunId, RunStatus, Sse, - State, StatusCode, StreamExt, UnboundedReceiverStream, broadcast, get, mpsc, parse_run_id_path, - redact_jsonl_line, reject_if_archived, update_live_run_from_event, + Query, RequireRunScoped, RequireRunStageScoped, RequiredUser, Response, Router, RunEvent, + RunId, RunStatus, Sse, State, StatusCode, StreamExt, UnboundedReceiverStream, broadcast, get, + mpsc, parse_run_id_path, parse_stage_id_path, redact_jsonl_line, reject_if_archived, + update_live_run_from_event, }; pub(super) fn routes() -> Router> { @@ -15,11 +16,15 @@ pub(super) fn routes() -> Router> { "/runs/{id}/events", get(list_run_events).post(append_run_event), ) + .route( + "/runs/{id}/stages/{stageId}/events", + get(list_run_stage_events), + ) .route("/runs/{id}/attach", get(attach_run_events)) } #[derive(serde::Deserialize)] -struct EventListParams { +pub(crate) struct EventListParams { #[serde(default)] since_seq: Option, #[serde(default)] @@ -27,11 +32,11 @@ struct EventListParams { } impl EventListParams { - fn since_seq(&self) -> u32 { + pub(crate) fn since_seq(&self) -> u32 { self.since_seq.unwrap_or(1).max(1) } - fn limit(&self) -> usize { + pub(crate) fn limit(&self) -> usize { self.limit.unwrap_or(100).clamp(1, 1000) } } @@ -200,6 +205,39 @@ async fn list_run_events( } } +async fn list_run_stage_events( + RequireRunStageScoped(id, stage_id): RequireRunStageScoped, + State(state): State>, + Query(params): Query, +) -> Response { + let stage_id = match parse_stage_id_path(&stage_id) { + Ok(stage_id) => stage_id, + Err(response) => return response, + }; + let since_seq = params.since_seq(); + let limit = params.limit(); + match state.store.open_run_reader(&id).await { + Ok(run_store) => match run_store + .list_events_for_stage_from_with_limit(&stage_id, since_seq, limit) + .await + { + Ok(mut events) => { + let has_more = events.len() > limit; + events.truncate(limit); + Json(PaginatedEventList { + data: events, + meta: PaginationMeta { has_more }, + }) + .into_response() + } + Err(err) => { + ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() + } + }, + Err(_) => ApiError::not_found("Run not found.").into_response(), + } +} + async fn attach_run_events( _auth: RequiredUser, State(state): State>, @@ -342,3 +380,264 @@ fn denied_lifecycle_event_name(body: &EventBody) -> Option<&'static str> { _ => None, } } + +#[cfg(test)] +mod stage_events_tests { + use axum::body::{Body, to_bytes}; + use axum::http::{Request, StatusCode, header}; + use fabro_store::EventPayload; + use fabro_types::RunId; + use serde_json::json; + use tower::ServiceExt; + + use crate::test_support::{build_test_router, test_app_state}; + + fn req_get(uri: &str) -> Request { + Request::builder() + .method("GET") + .uri(uri) + .body(Body::empty()) + .expect("stage events GET request should build") + } + + fn make_event(run_id: &RunId, idx: u32, node_id: Option<&str>) -> EventPayload { + make_event_with_stage_id(run_id, idx, node_id, None) + } + + fn make_event_with_stage_id( + run_id: &RunId, + idx: u32, + node_id: Option<&str>, + stage_id: Option<&str>, + ) -> EventPayload { + let mut value = json!({ + "id": format!("evt-{idx}"), + "ts": "2026-04-09T12:00:00Z", + "run_id": run_id.to_string(), + "event": "stage.prompt", + "properties": { + "visit": 1, + "text": format!("prompt {idx}"), + }, + }); + if let Some(node) = node_id { + value + .as_object_mut() + .unwrap() + .insert("node_id".into(), json!(node)); + } + if let Some(stage_id) = stage_id { + value + .as_object_mut() + .unwrap() + .insert("stage_id".into(), json!(stage_id)); + } + EventPayload::new(value, run_id).expect("event payload should validate") + } + + async fn body_json(response: axum::response::Response) -> serde_json::Value { + let bytes = to_bytes(response.into_body(), usize::MAX) + .await + .expect("response body should fit in memory"); + serde_json::from_slice(&bytes).expect("response body should be valid JSON") + } + + async fn seed_run_with_mixed_events() -> (RunId, axum::Router) { + let state = test_app_state(); + let app = build_test_router(state.clone()); + let run_id = RunId::new(); + let run_store = state + .store_ref() + .create_run(&run_id) + .await + .expect("test run should be creatable"); + + // Seed 200 unrelated 'beta' events first so any node-blind + // truncation would lose the sparse 'alpha' tail. Then 3 'alpha' + // events past seq 100, plus a couple with no node_id at all. + for idx in 1..=200_u32 { + run_store + .append_event(&make_event(&run_id, idx, Some("beta"))) + .await + .expect("append should succeed"); + } + run_store + .append_event(&make_event(&run_id, 201, None)) + .await + .expect("append should succeed"); + for idx in 202..=204_u32 { + run_store + .append_event(&make_event(&run_id, idx, Some("alpha"))) + .await + .expect("append should succeed"); + } + + (run_id, app) + } + + #[tokio::test] + async fn returns_only_matching_node_events_in_seq_order() { + let (run_id, app) = seed_run_with_mixed_events().await; + let response = app + .oneshot(req_get(&format!( + "/api/v1/runs/{run_id}/stages/alpha@1/events" + ))) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let body = body_json(response).await; + let data = body["data"].as_array().expect("data is array"); + let seqs: Vec = data.iter().map(|e| e["seq"].as_u64().unwrap()).collect(); + assert_eq!(seqs, vec![202, 203, 204]); + assert_eq!(body["meta"]["has_more"], false); + } + + #[tokio::test] + async fn since_seq_filters_to_events_with_seq_at_least_k() { + let (run_id, app) = seed_run_with_mixed_events().await; + let response = app + .oneshot(req_get(&format!( + "/api/v1/runs/{run_id}/stages/alpha@1/events?since_seq=203" + ))) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let body = body_json(response).await; + let seqs: Vec = body["data"] + .as_array() + .unwrap() + .iter() + .map(|e| e["seq"].as_u64().unwrap()) + .collect(); + assert_eq!(seqs, vec![203, 204]); + } + + #[tokio::test] + async fn limit_one_returns_first_envelope_with_has_more_true() { + let (run_id, app) = seed_run_with_mixed_events().await; + let response = app + .oneshot(req_get(&format!( + "/api/v1/runs/{run_id}/stages/alpha@1/events?limit=1" + ))) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let body = body_json(response).await; + let data = body["data"].as_array().unwrap(); + assert_eq!(data.len(), 1); + assert_eq!(data[0]["seq"].as_u64().unwrap(), 202); + assert_eq!(body["meta"]["has_more"], true); + } + + #[tokio::test] + async fn unknown_stage_in_existing_run_returns_empty_list_with_no_more() { + let (run_id, app) = seed_run_with_mixed_events().await; + let response = app + .oneshot(req_get(&format!( + "/api/v1/runs/{run_id}/stages/unknown-stage@1/events" + ))) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let body = body_json(response).await; + assert_eq!(body["data"].as_array().unwrap().len(), 0); + assert_eq!(body["meta"]["has_more"], false); + } + + #[tokio::test] + async fn missing_run_returns_404_with_run_not_found() { + let app = build_test_router(test_app_state()); + // A syntactically valid RunId that the store has never seen, so + // `parse_run_id_path` succeeds but `open_run_reader` fails — that + // exercises the handler's not-found branch rather than the path + // parser's 400 branch. + let absent = RunId::new(); + let response = app + .oneshot(req_get(&format!( + "/api/v1/runs/{absent}/stages/alpha@1/events" + ))) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::NOT_FOUND); + + let body = body_json(response).await; + let detail = body["errors"][0]["detail"] + .as_str() + .expect("error detail string"); + assert!( + detail.contains("Run not found."), + "unexpected error body: {body}" + ); + } + + #[tokio::test] + async fn unauthenticated_request_is_rejected() { + let state = test_app_state(); + // Bypass `build_test_router`'s auto-injected bearer token by + // building the raw router directly. The principal middleware sees + // a missing Authorization header and the extractor enforces auth. + let app = crate::server::build_router(state, crate::test_support::test_auth_mode()); + let run_id = RunId::new(); + + let request = Request::builder() + .method("GET") + .uri(format!("/api/v1/runs/{run_id}/stages/alpha@1/events")) + .header(header::ACCEPT, "application/json") + .body(Body::empty()) + .unwrap(); + let response = app.oneshot(request).await.unwrap(); + + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + } + + #[tokio::test] + async fn returns_only_requested_visit_when_stage_id_is_present() { + let state = test_app_state(); + let app = build_test_router(state.clone()); + let run_id = RunId::new(); + let run_store = state + .store_ref() + .create_run(&run_id) + .await + .expect("test run should be creatable"); + run_store + .append_event(&make_event_with_stage_id( + &run_id, + 1, + Some("verify"), + Some("verify@1"), + )) + .await + .expect("append should succeed"); + run_store + .append_event(&make_event_with_stage_id( + &run_id, + 2, + Some("verify"), + Some("verify@2"), + )) + .await + .expect("append should succeed"); + + let response = app + .oneshot(req_get(&format!( + "/api/v1/runs/{run_id}/stages/verify@2/events" + ))) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let body = body_json(response).await; + let seqs: Vec = body["data"] + .as_array() + .unwrap() + .iter() + .map(|e| e["seq"].as_u64().unwrap()) + .collect(); + assert_eq!(seqs, vec![2]); + } +} diff --git a/lib/crates/fabro-server/src/server/handler/mod.rs b/lib/crates/fabro-server/src/server/handler/mod.rs index d2625395b..79a3d8246 100644 --- a/lib/crates/fabro-server/src/server/handler/mod.rs +++ b/lib/crates/fabro-server/src/server/handler/mod.rs @@ -57,8 +57,8 @@ pub(super) fn demo_routes() -> Router> { .route("/runs/{id}/artifacts", get(demo::list_run_artifacts_stub)) .route("/runs/{id}/files", get(demo::list_run_files_stub)) .route( - "/runs/{id}/stages/{stageId}/turns", - get(demo::get_stage_turns), + "/runs/{id}/stages/{stageId}/events", + get(demo::get_stage_events), ) .route( "/runs/{id}/stages/{stageId}/artifacts", @@ -113,7 +113,6 @@ pub(super) fn demo_routes() -> Router> { pub(super) fn real_routes() -> Router> { Router::new() - .route("/runs/{id}/stages/{stageId}/turns", get(not_implemented)) .route("/runs/{id}/steer", post(not_implemented)) .route("/workflows", get(not_implemented)) .route("/workflows/{name}", get(not_implemented)) diff --git a/lib/crates/fabro-server/src/server/handler/runs.rs b/lib/crates/fabro-server/src/server/handler/runs.rs index ff03b55e5..9f31597ad 100644 --- a/lib/crates/fabro-server/src/server/handler/runs.rs +++ b/lib/crates/fabro-server/src/server/handler/runs.rs @@ -10,7 +10,9 @@ use axum::{Json, Router}; use base64::Engine as _; use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; use bytes::Bytes; -use fabro_api::types::{RunManifest, RunStatusResponse, SubmitAnswerRequest}; +use fabro_api::types::{ + BoardColumn, BoardColumnDefinition, RunManifest, RunStatusResponse, SubmitAnswerRequest, +}; use fabro_config::Storage; use fabro_interview::AnswerSubmission; use fabro_types::{ @@ -82,25 +84,45 @@ impl ListRunsParams { } } -fn board_column(status: RunStatus) -> Option<&'static str> { +fn board_column(status: RunStatus) -> Option { match status { - RunStatus::Submitted | RunStatus::Queued | RunStatus::Starting => Some("initializing"), - RunStatus::Running | RunStatus::Paused { .. } => Some("running"), - RunStatus::Blocked { .. } => Some("blocked"), - RunStatus::Succeeded { .. } => Some("succeeded"), - RunStatus::Failed { .. } | RunStatus::Dead => Some("failed"), + RunStatus::Submitted | RunStatus::Queued => Some(BoardColumn::Queued), + RunStatus::Starting => Some(BoardColumn::Initializing), + RunStatus::Running | RunStatus::Paused { .. } => Some(BoardColumn::Running), + RunStatus::Blocked { .. } => Some(BoardColumn::Blocked), + RunStatus::Succeeded { .. } => Some(BoardColumn::Succeeded), + RunStatus::Failed { .. } | RunStatus::Dead => Some(BoardColumn::Failed), RunStatus::Removing | RunStatus::Archived { .. } => None, } } -pub(crate) fn board_columns() -> serde_json::Value { - serde_json::json!([ - {"id": "initializing", "name": "Initializing"}, - {"id": "running", "name": "Running"}, - {"id": "blocked", "name": "Blocked"}, - {"id": "succeeded", "name": "Succeeded"}, - {"id": "failed", "name": "Failed"}, - ]) +pub(crate) fn board_columns() -> Vec { + vec![ + BoardColumnDefinition { + id: BoardColumn::Queued, + name: "Queued".into(), + }, + BoardColumnDefinition { + id: BoardColumn::Initializing, + name: "Initializing".into(), + }, + BoardColumnDefinition { + id: BoardColumn::Running, + name: "Running".into(), + }, + BoardColumnDefinition { + id: BoardColumn::Blocked, + name: "Blocked".into(), + }, + BoardColumnDefinition { + id: BoardColumn::Succeeded, + name: "Succeeded".into(), + }, + BoardColumnDefinition { + id: BoardColumn::Failed, + name: "Failed".into(), + }, + ] } async fn board_run_metadata( diff --git a/lib/crates/fabro-server/src/server/tests.rs b/lib/crates/fabro-server/src/server/tests.rs index 6078c362f..aadc30c8e 100644 --- a/lib/crates/fabro-server/src/server/tests.rs +++ b/lib/crates/fabro-server/src/server/tests.rs @@ -18,8 +18,8 @@ use fabro_model::Provider; use fabro_types::settings::ServerAuthMethod; use fabro_types::{ AttrValue, AuthMethod, CommandTermination, FailureCategory, FailureDetail, Graph, - InterviewQuestionRecord, Outcome, QuestionType, RunBlobId, RunId, RunSpec, StageOutcome, - SystemActorKind, fixtures, + InterviewQuestionRecord, Outcome, QuestionType, RunBlobId, RunId, RunSpec, SystemActorKind, + fixtures, }; use fabro_util::check_report::CheckStatus; use httpmock::Method::{GET, POST}; @@ -2112,6 +2112,30 @@ async fn create_durable_run_with_events( } } +/// Append a stage lifecycle event with an explicit `StageScope`, so the +/// stored envelope carries the full `stage_id` (`node_id@visit`). The bare +/// [`workflow_event::append_event`] helper only writes `node_id` because +/// stage lifecycle variants don't carry visit in their payload — production +/// always emits via `Emitter::emit_scoped`. +async fn append_scoped_stage_event( + state: &Arc, + run_id: RunId, + node_id: &str, + visit: u32, + event: &workflow_event::Event, +) { + let scope = fabro_workflow::event::StageScope { + node_id: node_id.to_string(), + visit, + parallel_group_id: None, + parallel_branch_id: None, + }; + let stored = fabro_workflow::event::to_run_event_at(&run_id, event, Utc::now(), Some(&scope)); + let payload = fabro_workflow::event::build_redacted_event_payload(&stored, &run_id).unwrap(); + let run_store = state.store.open_run(&run_id).await.unwrap(); + run_store.append_event(&payload).await.unwrap(); +} + fn stage_status<'a>(body: &'a serde_json::Value, id: &str) -> &'a str { body["data"] .as_array() @@ -2134,7 +2158,58 @@ async fn list_run_stages_projects_retrying_until_completion() { }, workflow_event::Event::RunStarting, workflow_event::Event::RunRunning, - workflow_event::Event::StageStarted { + ]) + .await; + append_scoped_stage_event( + &state, + run_id, + "setup", + 1, + &workflow_event::Event::StageStarted { + node_id: "setup".to_string(), + name: "Setup".to_string(), + index: 0, + handler_type: "command".to_string(), + attempt: 1, + max_attempts: 1, + }, + ) + .await; + append_scoped_stage_event( + &state, + run_id, + "setup", + 1, + &workflow_event::Event::StageCompleted { + node_id: "setup".to_string(), + name: "Setup".to_string(), + index: 0, + duration_ms: 5, + status: "succeeded".to_string(), + preferred_label: None, + suggested_next_ids: Vec::new(), + billing: None, + failure: None, + notes: None, + files_touched: Vec::new(), + context_updates: None, + jump_to_node: None, + context_values: None, + node_visits: None, + loop_failure_signatures: None, + restart_failure_signatures: None, + response: None, + attempt: 1, + max_attempts: 1, + }, + ) + .await; + append_scoped_stage_event( + &state, + run_id, + "work", + 1, + &workflow_event::Event::StageStarted { node_id: "work".to_string(), name: "Work".to_string(), index: 1, @@ -2142,16 +2217,31 @@ async fn list_run_stages_projects_retrying_until_completion() { attempt: 1, max_attempts: 3, }, - workflow_event::Event::StageFailed { + ) + .await; + append_scoped_stage_event( + &state, + run_id, + "work", + 1, + &workflow_event::Event::StageFailed { node_id: "work".to_string(), name: "Work".to_string(), index: 1, failure: FailureDetail::new("try again", FailureCategory::TransientInfra), will_retry: true, duration_ms: 10, + billing: None, actor: None, }, - workflow_event::Event::StageRetrying { + ) + .await; + append_scoped_stage_event( + &state, + run_id, + "work", + 1, + &workflow_event::Event::StageRetrying { node_id: "work".to_string(), name: "Work".to_string(), index: 1, @@ -2159,41 +2249,9 @@ async fn list_run_stages_projects_retrying_until_completion() { max_attempts: 3, delay_ms: 100, }, - ]) + ) .await; - let mut node_outcomes = HashMap::new(); - node_outcomes.insert("setup".to_string(), Outcome::success()); - let mut checkpoint = Checkpoint { - timestamp: Utc::now(), - current_node: "setup".to_string(), - completed_nodes: vec!["setup".to_string()], - node_retries: HashMap::new(), - context_values: HashMap::new(), - node_outcomes, - next_node_id: Some("work".to_string()), - git_commit_sha: None, - loop_failure_signatures: HashMap::new(), - restart_failure_signatures: HashMap::new(), - node_visits: HashMap::new(), - }; - - let run_dir = std::env::temp_dir().join(format!("fabro-server-test-{run_id}")); - std::fs::create_dir_all(&run_dir).unwrap(); - let mut managed = managed_run( - MINIMAL_DOT.to_string(), - RunStatus::Running, - Utc::now(), - run_dir, - RunExecutionMode::Start, - ); - managed.checkpoint = Some(checkpoint.clone()); - state - .runs - .lock() - .expect("runs lock poisoned") - .insert(run_id, managed); - let response = app .clone() .oneshot( @@ -2206,29 +2264,14 @@ async fn list_run_stages_projects_retrying_until_completion() { .await .unwrap(); let body = response_json!(response, StatusCode::OK).await; - assert_eq!(stage_status(&body, "setup"), "succeeded"); - assert_eq!(stage_status(&body, "work"), "retrying"); + assert_eq!(stage_status(&body, "setup@1"), "succeeded"); + assert_eq!(stage_status(&body, "work@1"), "retrying"); - let mut work_outcome = Outcome::success(); - work_outcome.status = StageOutcome::PartiallySucceeded; - checkpoint.completed_nodes.push("work".to_string()); - checkpoint - .node_outcomes - .insert("work".to_string(), work_outcome); - checkpoint.current_node = "work".to_string(); - checkpoint.next_node_id = Some("exit".to_string()); - state - .runs - .lock() - .expect("runs lock poisoned") - .get_mut(&run_id) - .unwrap() - .checkpoint = Some(checkpoint); - - let run_store = state.store.open_run(&run_id).await.unwrap(); - workflow_event::append_event( - &run_store, - &run_id, + append_scoped_stage_event( + &state, + run_id, + "work", + 1, &workflow_event::Event::StageCompleted { node_id: "work".to_string(), name: "Work".to_string(), @@ -2252,8 +2295,7 @@ async fn list_run_stages_projects_retrying_until_completion() { max_attempts: 3, }, ) - .await - .unwrap(); + .await; let response = app .oneshot( @@ -2266,7 +2308,766 @@ async fn list_run_stages_projects_retrying_until_completion() { .await .unwrap(); let body = response_json!(response, StatusCode::OK).await; - assert_eq!(stage_status(&body, "work"), "partially_succeeded"); + assert_eq!(stage_status(&body, "work@1"), "partially_succeeded"); +} + +fn stage_entry<'a>(body: &'a serde_json::Value, id: &str) -> &'a serde_json::Value { + body["data"] + .as_array() + .unwrap() + .iter() + .find(|stage| stage["id"] == id) + .unwrap_or_else(|| panic!("stage {id} not found in {body:#?}")) +} + +fn test_billed_usage( + model_id: &str, + input_tokens: i64, + output_tokens: i64, +) -> fabro_model::BilledModelUsage { + serde_json::from_value(json!({ + "input": { + "usage": { + "model": { + "provider": "openai", + "model_id": model_id + }, + "tokens": { + "input_tokens": input_tokens, + "output_tokens": output_tokens + } + }, + "facts": { + "provider": "open_ai" + } + }, + "total_usd_micros": input_tokens + output_tokens + })) + .unwrap() +} + +#[tokio::test] +async fn list_run_stages_distinguishes_visits() { + let state = test_app_state_with_isolated_storage(); + let app = crate::test_support::build_test_router(Arc::clone(&state)); + let run_id = RunId::new(); + + create_durable_run_with_events(&state, run_id, &[ + workflow_event::Event::RunSubmitted { + definition_blob: None, + }, + workflow_event::Event::RunStarting, + workflow_event::Event::RunRunning, + ]) + .await; + + // First visit of `verify` — failed. + append_scoped_stage_event( + &state, + run_id, + "verify", + 1, + &workflow_event::Event::StageStarted { + node_id: "verify".to_string(), + name: "Verify".to_string(), + index: 1, + handler_type: "command".to_string(), + attempt: 1, + max_attempts: 1, + }, + ) + .await; + append_scoped_stage_event( + &state, + run_id, + "verify", + 1, + &workflow_event::Event::StageCompleted { + node_id: "verify".to_string(), + name: "Verify".to_string(), + index: 1, + duration_ms: 1500, + status: "failed".to_string(), + preferred_label: None, + suggested_next_ids: Vec::new(), + billing: None, + failure: None, + notes: None, + files_touched: Vec::new(), + context_updates: None, + jump_to_node: None, + context_values: None, + node_visits: None, + loop_failure_signatures: None, + restart_failure_signatures: None, + response: None, + attempt: 1, + max_attempts: 1, + }, + ) + .await; + + // Second visit of `verify` — running. + append_scoped_stage_event( + &state, + run_id, + "verify", + 2, + &workflow_event::Event::StageStarted { + node_id: "verify".to_string(), + name: "Verify".to_string(), + index: 1, + handler_type: "command".to_string(), + attempt: 1, + max_attempts: 1, + }, + ) + .await; + + let response = app + .clone() + .oneshot( + Request::builder() + .method("GET") + .uri(api(&format!("/runs/{run_id}/stages"))) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let body = response_json!(response, StatusCode::OK).await; + + let data = body["data"].as_array().unwrap(); + let verify_entries: Vec<_> = data.iter().filter(|s| s["node_id"] == "verify").collect(); + assert_eq!(verify_entries.len(), 2, "expected two verify visits"); + + let first = stage_entry(&body, "verify@1"); + assert_eq!(first["node_id"], "verify"); + assert_eq!(first["visit"], 1); + assert_eq!(first["status"], "failed"); + assert_eq!(first["duration_secs"], 1.5); + + let second = stage_entry(&body, "verify@2"); + assert_eq!(second["node_id"], "verify"); + assert_eq!(second["visit"], 2); + assert_eq!(second["status"], "running"); + + // Old `dot_id` field must be gone. + assert!(first.get("dot_id").is_none(), "dot_id should be removed"); +} + +/// `checkpoint.completed_nodes` records every visit, so a looped node appears +/// once per re-entry. Billing must dedup so a retried node renders as one row +/// and `runtime_secs` is summed across all visits exactly once. +#[tokio::test] +async fn run_billing_dedups_retried_nodes_and_sums_their_durations() { + let state = test_app_state_with_isolated_storage(); + let app = crate::test_support::build_test_router(Arc::clone(&state)); + let run_id = RunId::new(); + + create_durable_run_with_events(&state, run_id, &[ + workflow_event::Event::RunSubmitted { + definition_blob: None, + }, + workflow_event::Event::RunStarting, + workflow_event::Event::RunRunning, + ]) + .await; + + // Visit 1 of `verify` — completed in 1.5s. + append_scoped_stage_event( + &state, + run_id, + "verify", + 1, + &workflow_event::Event::StageCompleted { + node_id: "verify".to_string(), + name: "Verify".to_string(), + index: 1, + duration_ms: 1500, + status: "failed".to_string(), + preferred_label: None, + suggested_next_ids: Vec::new(), + billing: None, + failure: None, + notes: None, + files_touched: Vec::new(), + context_updates: None, + jump_to_node: None, + context_values: None, + node_visits: None, + loop_failure_signatures: None, + restart_failure_signatures: None, + response: None, + attempt: 1, + max_attempts: 1, + }, + ) + .await; + + // Visit 2 of `verify` — completed in 0.8s. + append_scoped_stage_event( + &state, + run_id, + "verify", + 2, + &workflow_event::Event::StageCompleted { + node_id: "verify".to_string(), + name: "Verify".to_string(), + index: 1, + duration_ms: 800, + status: "succeeded".to_string(), + preferred_label: None, + suggested_next_ids: Vec::new(), + billing: None, + failure: None, + notes: None, + files_touched: Vec::new(), + context_updates: None, + jump_to_node: None, + context_values: None, + node_visits: None, + loop_failure_signatures: None, + restart_failure_signatures: None, + response: None, + attempt: 1, + max_attempts: 1, + }, + ) + .await; + + // Checkpoint records `verify` twice (once per visit) — this is what makes + // the dedup necessary. + let run_store = state.store.open_run(&run_id).await.unwrap(); + workflow_event::append_event( + &run_store, + &run_id, + &workflow_event::Event::CheckpointCompleted { + node_id: "verify".to_string(), + status: "running".to_string(), + current_node: "verify".to_string(), + completed_nodes: vec!["verify".to_string(), "verify".to_string()], + node_retries: std::collections::BTreeMap::new(), + context_values: std::collections::BTreeMap::new(), + node_outcomes: std::collections::BTreeMap::from([( + "verify".to_string(), + Outcome::default(), + )]), + next_node_id: Some("done".to_string()), + git_commit_sha: None, + loop_failure_signatures: std::collections::BTreeMap::new(), + restart_failure_signatures: std::collections::BTreeMap::new(), + node_visits: std::collections::BTreeMap::from([("verify".to_string(), 2usize)]), + diff: None, + }, + ) + .await + .unwrap(); + + let response = app + .clone() + .oneshot( + Request::builder() + .method("GET") + .uri(api(&format!("/runs/{run_id}/billing"))) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let body = response_json!(response, StatusCode::OK).await; + + let stages = body["stages"].as_array().unwrap(); + assert_eq!( + stages.len(), + 1, + "expected one row for the retried verify node" + ); + assert_eq!(stages[0]["stage"]["id"], "verify"); + // Duration on the row is the sum across visits (1.5s + 0.8s = 2.3s). + assert!( + (stages[0]["runtime_secs"].as_f64().unwrap() - 2.3).abs() < f64::EPSILON, + "row runtime_secs should sum visits, got {}", + stages[0]["runtime_secs"] + ); + + // Totals must not double-count: a single 2.3s, not 4.6s. + assert!( + (body["totals"]["runtime_secs"].as_f64().unwrap() - 2.3).abs() < f64::EPSILON, + "totals.runtime_secs should sum visits exactly once, got {}", + body["totals"]["runtime_secs"] + ); +} + +#[tokio::test] +async fn run_billing_sums_usage_across_retry_visits_and_uses_latest_model() { + let state = test_app_state_with_isolated_storage(); + let app = crate::test_support::build_test_router(Arc::clone(&state)); + let run_id = RunId::new(); + let failed_usage = test_billed_usage("gpt-old", 100, 10); + let success_usage = test_billed_usage("gpt-new", 200, 20); + + create_durable_run_with_events(&state, run_id, &[ + workflow_event::Event::RunSubmitted { + definition_blob: None, + }, + workflow_event::Event::RunStarting, + workflow_event::Event::RunRunning, + ]) + .await; + + append_scoped_stage_event( + &state, + run_id, + "verify", + 1, + &workflow_event::Event::StageFailed { + node_id: "verify".to_string(), + name: "Verify".to_string(), + index: 1, + failure: FailureDetail::new("try again", FailureCategory::TransientInfra), + will_retry: true, + duration_ms: 1200, + billing: Some(failed_usage), + actor: None, + }, + ) + .await; + append_scoped_stage_event( + &state, + run_id, + "verify", + 2, + &workflow_event::Event::StageCompleted { + node_id: "verify".to_string(), + name: "Verify".to_string(), + index: 1, + duration_ms: 800, + status: "succeeded".to_string(), + preferred_label: None, + suggested_next_ids: Vec::new(), + billing: Some(success_usage.clone()), + failure: None, + notes: None, + files_touched: Vec::new(), + context_updates: None, + jump_to_node: None, + context_values: None, + node_visits: None, + loop_failure_signatures: None, + restart_failure_signatures: None, + response: None, + attempt: 2, + max_attempts: 2, + }, + ) + .await; + + let mut latest_outcome: Outcome> = Outcome::success(); + latest_outcome.usage = Some(success_usage); + latest_outcome.duration_ms = Some(800); + let run_store = state.store.open_run(&run_id).await.unwrap(); + workflow_event::append_event( + &run_store, + &run_id, + &workflow_event::Event::CheckpointCompleted { + node_id: "verify".to_string(), + status: "running".to_string(), + current_node: "verify".to_string(), + completed_nodes: vec!["verify".to_string(), "verify".to_string()], + node_retries: std::collections::BTreeMap::from([("verify".to_string(), 2)]), + context_values: std::collections::BTreeMap::new(), + node_outcomes: std::collections::BTreeMap::from([( + "verify".to_string(), + latest_outcome, + )]), + next_node_id: None, + git_commit_sha: None, + loop_failure_signatures: std::collections::BTreeMap::new(), + restart_failure_signatures: std::collections::BTreeMap::new(), + node_visits: std::collections::BTreeMap::from([("verify".to_string(), 2usize)]), + diff: None, + }, + ) + .await + .unwrap(); + + let response = app + .clone() + .oneshot( + Request::builder() + .method("GET") + .uri(api(&format!("/runs/{run_id}/billing"))) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let body = response_json!(response, StatusCode::OK).await; + + let stages = body["stages"].as_array().unwrap(); + assert_eq!(stages.len(), 1); + assert_eq!(stages[0]["stage"]["id"], "verify"); + assert_eq!(stages[0]["model"]["id"], "gpt-new"); + assert_eq!(stages[0]["billing"]["input_tokens"], 300); + assert_eq!(stages[0]["billing"]["output_tokens"], 30); + assert_eq!(stages[0]["billing"]["total_usd_micros"], 330); + assert!((stages[0]["runtime_secs"].as_f64().unwrap() - 2.0).abs() < f64::EPSILON); + + assert_eq!(body["totals"]["input_tokens"], 300); + assert_eq!(body["totals"]["output_tokens"], 30); + assert_eq!(body["totals"]["total_usd_micros"], 330); + assert!((body["totals"]["runtime_secs"].as_f64().unwrap() - 2.0).abs() < f64::EPSILON); + + let by_model = body["by_model"].as_array().unwrap(); + assert_eq!(by_model.len(), 2); + let old_model = by_model + .iter() + .find(|entry| entry["model"]["id"] == "gpt-old") + .unwrap(); + let new_model = by_model + .iter() + .find(|entry| entry["model"]["id"] == "gpt-new") + .unwrap(); + assert_eq!(old_model["stages"], 1); + assert_eq!(old_model["billing"]["input_tokens"], 100); + assert_eq!(new_model["stages"], 1); + assert_eq!(new_model["billing"]["input_tokens"], 200); +} + +#[tokio::test] +async fn list_run_stages_shows_retrying_after_failed_event() { + let state = test_app_state_with_isolated_storage(); + let app = crate::test_support::build_test_router(Arc::clone(&state)); + let run_id = RunId::new(); + + create_durable_run_with_events(&state, run_id, &[ + workflow_event::Event::RunSubmitted { + definition_blob: None, + }, + workflow_event::Event::RunStarting, + workflow_event::Event::RunRunning, + ]) + .await; + + append_scoped_stage_event( + &state, + run_id, + "work", + 1, + &workflow_event::Event::StageStarted { + node_id: "work".to_string(), + name: "Work".to_string(), + index: 0, + handler_type: "command".to_string(), + attempt: 1, + max_attempts: 3, + }, + ) + .await; + append_scoped_stage_event( + &state, + run_id, + "work", + 1, + &workflow_event::Event::StageFailed { + node_id: "work".to_string(), + name: "Work".to_string(), + index: 0, + failure: FailureDetail::new("flake", FailureCategory::TransientInfra), + will_retry: true, + duration_ms: 5, + billing: None, + actor: None, + }, + ) + .await; + append_scoped_stage_event( + &state, + run_id, + "work", + 1, + &workflow_event::Event::StageRetrying { + node_id: "work".to_string(), + name: "Work".to_string(), + index: 0, + attempt: 2, + max_attempts: 3, + delay_ms: 50, + }, + ) + .await; + + let response = app + .clone() + .oneshot( + Request::builder() + .method("GET") + .uri(api(&format!("/runs/{run_id}/stages"))) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let body = response_json!(response, StatusCode::OK).await; + assert_eq!(stage_status(&body, "work@1"), "retrying"); +} + +#[tokio::test] +async fn list_run_stages_shows_retrying_when_failed_will_retry() { + let state = test_app_state_with_isolated_storage(); + let app = crate::test_support::build_test_router(Arc::clone(&state)); + let run_id = RunId::new(); + + create_durable_run_with_events(&state, run_id, &[ + workflow_event::Event::RunSubmitted { + definition_blob: None, + }, + workflow_event::Event::RunStarting, + workflow_event::Event::RunRunning, + ]) + .await; + + append_scoped_stage_event( + &state, + run_id, + "work", + 1, + &workflow_event::Event::StageStarted { + node_id: "work".to_string(), + name: "Work".to_string(), + index: 0, + handler_type: "command".to_string(), + attempt: 1, + max_attempts: 3, + }, + ) + .await; + // Only StageFailed, no StageRetrying yet — should still render retrying + // because props.will_retry is true. + append_scoped_stage_event( + &state, + run_id, + "work", + 1, + &workflow_event::Event::StageFailed { + node_id: "work".to_string(), + name: "Work".to_string(), + index: 0, + failure: FailureDetail::new("flake", FailureCategory::TransientInfra), + will_retry: true, + duration_ms: 5, + billing: None, + actor: None, + }, + ) + .await; + + let response = app + .clone() + .oneshot( + Request::builder() + .method("GET") + .uri(api(&format!("/runs/{run_id}/stages"))) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let body = response_json!(response, StatusCode::OK).await; + assert_eq!(stage_status(&body, "work@1"), "retrying"); +} + +#[tokio::test] +async fn run_billing_retried_node_then_succeeded_emits_one_row_with_final_attempt_duration() { + let state = test_app_state_with_isolated_storage(); + let app = crate::test_support::build_test_router(Arc::clone(&state)); + let run_id = RunId::new(); + + create_durable_run_with_events(&state, run_id, &[ + workflow_event::Event::RunSubmitted { + definition_blob: None, + }, + workflow_event::Event::RunStarting, + workflow_event::Event::RunRunning, + workflow_event::Event::StageStarted { + node_id: "work".to_string(), + name: "Work".to_string(), + index: 0, + handler_type: "command".to_string(), + attempt: 1, + max_attempts: 3, + }, + workflow_event::Event::StageFailed { + node_id: "work".to_string(), + name: "Work".to_string(), + index: 0, + failure: FailureDetail::new("transient", FailureCategory::TransientInfra), + will_retry: true, + duration_ms: 10, + billing: None, + actor: None, + }, + workflow_event::Event::StageRetrying { + node_id: "work".to_string(), + name: "Work".to_string(), + index: 0, + attempt: 2, + max_attempts: 3, + delay_ms: 0, + }, + workflow_event::Event::StageStarted { + node_id: "work".to_string(), + name: "Work".to_string(), + index: 0, + handler_type: "command".to_string(), + attempt: 2, + max_attempts: 3, + }, + workflow_event::Event::StageCompleted { + node_id: "work".to_string(), + name: "Work".to_string(), + index: 0, + duration_ms: 25, + status: "succeeded".to_string(), + preferred_label: None, + suggested_next_ids: Vec::new(), + billing: None, + failure: None, + notes: None, + files_touched: Vec::new(), + context_updates: None, + jump_to_node: None, + context_values: None, + node_visits: None, + loop_failure_signatures: None, + restart_failure_signatures: None, + response: None, + attempt: 2, + max_attempts: 3, + }, + ]) + .await; + + let response = app + .oneshot( + Request::builder() + .method("GET") + .uri(api(&format!("/runs/{run_id}/billing"))) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let body = response_json!(response, StatusCode::OK).await; + let stages = body["stages"].as_array().unwrap(); + assert_eq!(stages.len(), 1, "retry collapses to one row per node_id"); + let row = &stages[0]; + assert_eq!(row["stage"]["id"], "work"); + assert_eq!( + row["state"], "succeeded", + "final state mirrors the latest StageCompleted" + ); + let runtime = row["runtime_secs"].as_f64().unwrap(); + assert!( + (runtime - 0.025).abs() < f64::EPSILON, + "runtime should equal final attempt's 25ms, got {runtime}" + ); +} + +fn revisit_test_started(node_id: &str) -> workflow_event::Event { + workflow_event::Event::StageStarted { + node_id: node_id.to_string(), + name: node_id.to_string(), + index: 0, + handler_type: "command".to_string(), + attempt: 1, + max_attempts: 1, + } +} + +fn revisit_test_completed_with_visit( + node_id: &str, + duration_ms: u64, + visit: usize, +) -> workflow_event::Event { + let mut node_visits = std::collections::BTreeMap::new(); + node_visits.insert(node_id.to_string(), visit); + workflow_event::Event::StageCompleted { + node_id: node_id.to_string(), + name: node_id.to_string(), + index: 0, + duration_ms, + status: "succeeded".to_string(), + preferred_label: None, + suggested_next_ids: Vec::new(), + billing: None, + failure: None, + notes: None, + files_touched: Vec::new(), + context_updates: None, + jump_to_node: None, + context_values: None, + node_visits: Some(node_visits), + loop_failure_signatures: None, + restart_failure_signatures: None, + response: None, + attempt: 1, + max_attempts: 1, + } +} + +#[tokio::test] +async fn run_billing_revisited_node_collapses_to_two_rows_with_summed_visit_duration() { + let state = test_app_state_with_isolated_storage(); + let app = crate::test_support::build_test_router(Arc::clone(&state)); + let run_id = RunId::new(); + + create_durable_run_with_events(&state, run_id, &[ + workflow_event::Event::RunSubmitted { + definition_blob: None, + }, + workflow_event::Event::RunStarting, + workflow_event::Event::RunRunning, + // A → B → A loop. Per-visit `node_visits` payload steers the reducer + // to attribute each StageCompleted to the right visit. + revisit_test_started("a"), + revisit_test_completed_with_visit("a", 1, 1), + revisit_test_started("b"), + revisit_test_completed_with_visit("b", 2, 1), + revisit_test_started("a"), + revisit_test_completed_with_visit("a", 99, 2), + ]) + .await; + + let response = app + .oneshot( + Request::builder() + .method("GET") + .uri(api(&format!("/runs/{run_id}/billing"))) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let body = response_json!(response, StatusCode::OK).await; + let stages = body["stages"].as_array().unwrap(); + assert_eq!(stages.len(), 2, "two distinct node_ids → two rows"); + assert_eq!( + stages[0]["stage"]["id"], "a", + "A appeared first → A's row first" + ); + assert_eq!(stages[1]["stage"]["id"], "b"); + let a_runtime = stages[0]["runtime_secs"].as_f64().unwrap(); + assert!( + (a_runtime - 0.1).abs() < f64::EPSILON, + "A should sum both visit durations (1ms + 99ms), got {a_runtime}" + ); + let b_runtime = stages[1]["runtime_secs"].as_f64().unwrap(); + assert!( + (b_runtime - 0.002).abs() < f64::EPSILON, + "B should carry its single visit's duration (2ms), got {b_runtime}" + ); } async fn append_raw_run_event( @@ -3665,7 +4466,13 @@ async fn create_run_pull_request_creates_and_persists_record() { .header("authorization", "Bearer openai-key"); then.status(200) .header("content-type", "application/json") - .json_body(openai_responses_payload("Narrative from mock.")); + .json_body(openai_responses_payload( + &serde_json::to_string(&json!({ + "title": "Mock title", + "body": "Narrative from mock.", + })) + .unwrap(), + )); }) .await; let openai_base_url = llm.url("/v1"); @@ -5829,6 +6636,62 @@ async fn get_aggregate_billing_returns_zeros_initially() { assert!(body["by_model"].as_array().unwrap().is_empty()); } +#[test] +fn aggregate_billing_counts_projection_rollup_usage_visits() { + let mut accumulator = BillingAccumulator::default(); + let rollup = fabro_workflow::ProjectionBillingRollup { + stages: Vec::new(), + totals: BilledTokenCounts { + input_tokens: 300, + output_tokens: 30, + total_tokens: 330, + reasoning_tokens: 0, + cache_read_tokens: 0, + cache_write_tokens: 0, + total_usd_micros: Some(330), + }, + by_model: vec![ + fabro_workflow::ProjectionBillingByModel { + model_id: "gpt-old".to_string(), + stages: 1, + billing: BilledTokenCounts { + input_tokens: 100, + output_tokens: 10, + total_tokens: 110, + reasoning_tokens: 0, + cache_read_tokens: 0, + cache_write_tokens: 0, + total_usd_micros: Some(110), + }, + }, + fabro_workflow::ProjectionBillingByModel { + model_id: "gpt-new".to_string(), + stages: 1, + billing: BilledTokenCounts { + input_tokens: 200, + output_tokens: 20, + total_tokens: 220, + reasoning_tokens: 0, + cache_read_tokens: 0, + cache_write_tokens: 0, + total_usd_micros: Some(220), + }, + }, + ], + runtime_ms: 2000, + billed_visit_count: 2, + }; + + accumulate_billing_rollup(&mut accumulator, &rollup); + + assert_eq!(accumulator.total_runs, 1); + assert_eq!(accumulator.total_runtime_secs, 2.0); + assert_eq!(accumulator.by_model["gpt-old"].stages, 1); + assert_eq!(accumulator.by_model["gpt-old"].billing.input_tokens, 100); + assert_eq!(accumulator.by_model["gpt-new"].stages, 1); + assert_eq!(accumulator.by_model["gpt-new"].billing.input_tokens, 200); +} + #[tokio::test] async fn post_runs_returns_submitted_status() { let state = test_app_state(); @@ -6125,7 +6988,7 @@ async fn pause_run_sets_pending_control_on_board_response() { assert_eq!(body["pending_control"].as_str(), Some("pause")); // Verify the run appears on the board (store has Submitted status → - // "initializing" column) + // "queued" column) let req = Request::builder() .method("GET") .uri(api("/boards/runs")) @@ -6140,7 +7003,7 @@ async fn pause_run_sets_pending_control_on_board_response() { .find(|item| item["run_id"].as_str() == Some(run_id_str.as_str())) .expect("board item should exist"); assert!(item["status"].is_object()); - assert_eq!(item["column"].as_str(), Some("initializing")); + assert_eq!(item["column"].as_str(), Some("queued")); assert_eq!(item["pending_control"].as_str(), Some("pause")); } @@ -6615,8 +7478,8 @@ async fn queue_position_reported_for_queued_runs() { let first_run_id = create_and_start_run(&app, MINIMAL_DOT).await; let second_run_id = create_and_start_run(&app, MINIMAL_DOT).await; - // Queued runs are excluded from the board, so verify queue positions - // via the in-memory state directly. + // Queue position is tracked in memory even when queued runs are also + // visible on the board. let runs = state.runs.lock().expect("runs lock poisoned"); let positions = compute_queue_positions(&runs); let first_id = first_run_id.parse::().unwrap(); diff --git a/lib/crates/fabro-server/tests/it/event_pagination.rs b/lib/crates/fabro-server/tests/it/event_pagination.rs new file mode 100644 index 000000000..cd6ceec94 --- /dev/null +++ b/lib/crates/fabro-server/tests/it/event_pagination.rs @@ -0,0 +1,75 @@ +//! Cursor-pagination tests for the per-stage events endpoint (demo mode). +//! +//! The stage-events route uses `since_seq=` + `limit=` (cursor-based) instead +//! of the offset-based `page[limit]/page[offset]` pagination used by other +//! list endpoints, so it gets its own test rather than living in the generic +//! offset-shape matrix. + +#![allow( + clippy::absolute_paths, + reason = "This test module prefers explicit type paths over extra imports." +)] + +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use tower::ServiceExt; + +use super::helpers::{response_json, test_app_state}; + +async fn get_json(app: &axum::Router, uri: &str) -> serde_json::Value { + let req = Request::builder() + .method("GET") + .uri(uri) + .header("x-fabro-demo", "1") + .body(Body::empty()) + .expect("event pagination request should build"); + let response = app.clone().oneshot(req).await.unwrap(); + response_json(response, StatusCode::OK, format!("GET {uri}")).await +} + +#[tokio::test] +async fn demo_stage_events_default_returns_all_fixture_events_with_no_more() { + let app = fabro_server::test_support::build_test_router(test_app_state()); + + let body = get_json(&app, "/api/v1/runs/run-1/stages/detect-drift@1/events").await; + let data = body["data"].as_array().expect("data is an array"); + + assert_eq!(data.len(), 7, "all seven fixture events should be returned"); + assert_eq!(body["meta"]["has_more"], false); +} + +#[tokio::test] +async fn demo_stage_events_limit_one_signals_has_more() { + let app = fabro_server::test_support::build_test_router(test_app_state()); + + let body = get_json( + &app, + "/api/v1/runs/run-1/stages/detect-drift@1/events?limit=1", + ) + .await; + let data = body["data"].as_array().expect("data is an array"); + + assert_eq!(data.len(), 1); + assert_eq!(body["meta"]["has_more"], true); +} + +#[tokio::test] +async fn demo_stage_events_since_seq_filters_out_earlier_events() { + let app = fabro_server::test_support::build_test_router(test_app_state()); + + // The fixture seqs are 1..=7. since_seq=4 should skip the first three. + let body = get_json( + &app, + "/api/v1/runs/run-1/stages/detect-drift@1/events?since_seq=4", + ) + .await; + let data = body["data"].as_array().expect("data is an array"); + + assert_eq!(data.len(), 4); + let seqs: Vec = data + .iter() + .map(|envelope| envelope["seq"].as_u64().expect("seq is a number")) + .collect(); + assert_eq!(seqs, vec![4, 5, 6, 7]); + assert_eq!(body["meta"]["has_more"], false); +} diff --git a/lib/crates/fabro-server/tests/it/main.rs b/lib/crates/fabro-server/tests/it/main.rs index 9e5ddab9f..865bb8132 100644 --- a/lib/crates/fabro-server/tests/it/main.rs +++ b/lib/crates/fabro-server/tests/it/main.rs @@ -4,6 +4,7 @@ )] mod api; +mod event_pagination; mod helpers; mod openapi_conformance; mod pagination; diff --git a/lib/crates/fabro-server/tests/it/pagination.rs b/lib/crates/fabro-server/tests/it/pagination.rs index 64174fd78..f773ac207 100644 --- a/lib/crates/fabro-server/tests/it/pagination.rs +++ b/lib/crates/fabro-server/tests/it/pagination.rs @@ -56,10 +56,6 @@ const ENDPOINTS: &[PaginatedEndpoint] = &[ path: "/api/v1/models", name: "listModels", }, - PaginatedEndpoint { - path: "/api/v1/runs/run-1/stages/detect-drift/turns", - name: "listStageTurns", - }, PaginatedEndpoint { path: "/api/v1/runs/run-1/questions", name: "listRunQuestions", diff --git a/lib/crates/fabro-store/src/artifact_store.rs b/lib/crates/fabro-store/src/artifact_store.rs index 4cd4f85e1..5be00d792 100644 --- a/lib/crates/fabro-store/src/artifact_store.rs +++ b/lib/crates/fabro-store/src/artifact_store.rs @@ -297,8 +297,13 @@ fn decode_artifact_location( )) })?; let (retry, filename) = decode_retry_and_filename(location, &mut parts)?; + let stage_id = StageId::try_new(node_id, visit).map_err(|err| { + Error::Other(format!( + "artifact location {location} has an invalid stage id: {err}" + )) + })?; Ok(NodeArtifact { - node: StageId::new(node_id, visit), + node: stage_id, retry, filename, size, diff --git a/lib/crates/fabro-store/src/run_state.rs b/lib/crates/fabro-store/src/run_state.rs index d54eec315..16b70e0cc 100644 --- a/lib/crates/fabro-store/src/run_state.rs +++ b/lib/crates/fabro-store/src/run_state.rs @@ -10,7 +10,8 @@ use fabro_types::{ BilledModelUsage, Checkpoint, CommandTermination, Conclusion, EventBody, FailureSignature, InterviewQuestionRecord, Outcome, PendingInterviewRecord, PullRequestRecord, RunControlAction, RunEvent, RunId, RunProjection, RunSpec, RunStatus, RunSummary, SandboxRecord, StageCompletion, - StageOutcome, StageProjection, StartRecord, TerminalStatus, first_event_seq, + StageId, StageOutcome, StageProjection, StageState, StartRecord, TerminalStatus, + first_event_seq, }; use fabro_util::error::render_with_causes; use serde_json::Value; @@ -290,59 +291,76 @@ impl RunProjectionReducer for RunProjection { let Some(stage_id) = stored.stage_id.as_ref() else { return Ok(()); }; - self.stage_entry( + let stage = self.stage_entry( stage_id.node_id(), stage_id.visit(), first_event_seq(event.seq), ); + stage.begin_attempt(ts); + } + EventBody::StageRetrying(_) => { + let Some(stage) = stage_at_stored_or_current_visit(self, stored, event.seq) else { + return Ok(()); + }; + stage.state = Some(StageState::Retrying); } EventBody::StagePrompt(props) => { - let Some(stage) = stage_at_visit(self, stored, props.visit, event.seq) else { + let Some(stage) = stage_at_stored_or_visit(self, stored, props.visit, event.seq) + else { return Ok(()); }; stage.prompt = Some(props.text.clone()); stage.provider_used = provider_used_from_prompt(props); } EventBody::PromptCompleted(props) => { - let Some(stage) = stage_at_current_visit(self, stored, event.seq) else { + let Some(stage) = stage_at_stored_or_current_visit(self, stored, event.seq) else { return Ok(()); }; stage.response = Some(props.response.clone()); } EventBody::StageCompleted(props) => { - let Some(node_id) = stored.node_id.as_deref() else { - return Ok(()); - }; - let visit = stage_visit(node_id, props.node_visits.as_ref(), self).unwrap_or(1); let response = props.response.clone(); let outcome = stage_outcome_from_props(props); let completion = stage_completion_from_outcome(&outcome, ts); - let stage = self.stage_entry(node_id, visit, first_event_seq(event.seq)); + let Some(stage) = + stage_at_completed_visit(self, stored, props.node_visits.as_ref(), event.seq) + else { + return Ok(()); + }; stage.response = response; stage.completion = Some(completion); + stage.duration_ms = Some(props.duration_ms); + stage.usage.clone_from(&props.billing); + stage.state = Some(StageState::from(outcome.status)); } EventBody::StageFailed(props) => { let failure_reason = props.failure.as_ref().map(|detail| detail.message.clone()); - let Some(stage) = stage_at_current_visit(self, stored, event.seq) else { + let Some(stage) = stage_at_stored_or_current_visit(self, stored, event.seq) else { return Ok(()); }; + let outcome = StageOutcome::Failed { + retry_requested: props.will_retry, + }; stage.completion = Some(StageCompletion { - outcome: StageOutcome::Failed { - retry_requested: false, - }, + outcome, notes: None, failure_reason, timestamp: ts, }); + stage.duration_ms = Some(props.duration_ms); + stage.usage.clone_from(&props.billing); + stage.state = Some(StageState::from(outcome)); } EventBody::AgentSessionStarted(props) => { - let Some(stage) = stage_at_visit(self, stored, props.visit, event.seq) else { + let Some(stage) = stage_at_stored_or_visit(self, stored, props.visit, event.seq) + else { return Ok(()); }; stage.provider_used = Some(provider_used_from_agent_session_started(props)); } EventBody::AgentCliStarted(props) => { - let Some(stage) = stage_at_visit(self, stored, props.visit, event.seq) else { + let Some(stage) = stage_at_stored_or_visit(self, stored, props.visit, event.seq) + else { return Ok(()); }; stage.provider_used = Some(provider_used_from_agent_cli_started(props)); @@ -351,7 +369,7 @@ impl RunProjectionReducer for RunProjection { let script_invocation = serde_json::to_value(props).map_err(|err| { Error::InvalidEvent(format!("invalid command.started payload: {err}")) })?; - let Some(stage) = stage_at_current_visit(self, stored, event.seq) else { + let Some(stage) = stage_at_stored_or_current_visit(self, stored, event.seq) else { return Ok(()); }; stage.script_invocation = Some(script_invocation); @@ -360,7 +378,7 @@ impl RunProjectionReducer for RunProjection { let script_timing = serde_json::to_value(props).map_err(|err| { Error::InvalidEvent(format!("invalid command.completed payload: {err}")) })?; - let Some(stage) = stage_at_current_visit(self, stored, event.seq) else { + let Some(stage) = stage_at_stored_or_current_visit(self, stored, event.seq) else { return Ok(()); }; stage.stdout = Some(props.stdout.clone()); @@ -412,7 +430,7 @@ impl RunProjectionReducer for RunProjection { let parallel_results = serde_json::to_value(&props.results).map_err(|err| { Error::InvalidEvent(format!("invalid parallel.completed payload: {err}")) })?; - let Some(stage) = stage_at_current_visit(self, stored, event.seq) else { + let Some(stage) = stage_at_stored_or_current_visit(self, stored, event.seq) else { return Ok(()); }; stage.parallel_results = Some(parallel_results); @@ -430,6 +448,9 @@ fn stage_at_visit<'a>( visit: u32, seq: u32, ) -> Option<&'a mut StageProjection> { + if visit == 0 { + return None; + } let node_id = stored.node_id.as_deref()?; Some(state.stage_entry(node_id, visit, first_event_seq(seq))) } @@ -444,6 +465,51 @@ fn stage_at_current_visit<'a>( Some(state.stage_entry(node_id, visit, first_event_seq(seq))) } +fn stage_at_stored_stage_id<'a>( + state: &'a mut RunProjection, + stage_id: &StageId, + seq: u32, +) -> &'a mut StageProjection { + state.stage_entry(stage_id.node_id(), stage_id.visit(), first_event_seq(seq)) +} + +fn stage_at_stored_or_visit<'a>( + state: &'a mut RunProjection, + stored: &RunEvent, + visit: u32, + seq: u32, +) -> Option<&'a mut StageProjection> { + if let Some(stage_id) = stored.stage_id.as_ref() { + return Some(stage_at_stored_stage_id(state, stage_id, seq)); + } + stage_at_visit(state, stored, visit, seq) +} + +fn stage_at_stored_or_current_visit<'a>( + state: &'a mut RunProjection, + stored: &RunEvent, + seq: u32, +) -> Option<&'a mut StageProjection> { + if let Some(stage_id) = stored.stage_id.as_ref() { + return Some(stage_at_stored_stage_id(state, stage_id, seq)); + } + stage_at_current_visit(state, stored, seq) +} + +fn stage_at_completed_visit<'a>( + state: &'a mut RunProjection, + stored: &RunEvent, + node_visits: Option<&BTreeMap>, + seq: u32, +) -> Option<&'a mut StageProjection> { + if let Some(stage_id) = stored.stage_id.as_ref() { + return Some(stage_at_stored_stage_id(state, stage_id, seq)); + } + let node_id = stored.node_id.as_deref()?; + let visit = stage_visit(node_id, node_visits, state).unwrap_or(1); + Some(state.stage_entry(node_id, visit, first_event_seq(seq))) +} + pub(crate) fn build_summary(state: &RunProjection, run_id: &RunId) -> RunSummary { let workflow_name = state.spec.as_ref().map(|spec| { if spec.graph.name.is_empty() { @@ -565,6 +631,7 @@ fn stage_visit( node_visits .and_then(|visits| visits.get(node_id).copied()) .and_then(|visit| u32::try_from(visit).ok()) + .filter(|visit| *visit > 0) .or_else(|| state.current_visit_for(node_id)) } @@ -666,12 +733,14 @@ mod tests { use fabro_types::run_event::{ AgentCliCancelledProps, AgentCliCompletedProps, AgentCliTimedOutProps, CheckpointCompletedProps, InterviewCompletedProps, InterviewOption, InterviewStartedProps, - RunControlEffectProps, StagePromptProps, StageStartedProps, + RunControlEffectProps, StageCompletedProps, StageFailedProps, StagePromptProps, + StageRetryingProps, StageStartedProps, }; use fabro_types::{ - BlockedReason, Checkpoint, CommandTermination, EventBody, FailureReason, Outcome, - QuestionType, RunBlobId, RunControlAction, RunEvent, RunStatus, StageOutcome, - SuccessReason, TerminalStatus, WorkflowSettings, first_event_seq, fixtures, + BilledModelUsage, BlockedReason, Checkpoint, CommandTermination, EventBody, + FailureCategory, FailureDetail, FailureReason, Outcome, QuestionType, RunBlobId, + RunControlAction, RunEvent, RunStatus, StageOutcome, StageState, SuccessReason, + TerminalStatus, WorkflowSettings, first_event_seq, fixtures, }; use serde_json::json; @@ -704,6 +773,32 @@ mod tests { event } + fn test_usage(model_id: &str, input_tokens: i64, output_tokens: i64) -> BilledModelUsage { + serde_json::from_value(json!({ + "input": { + "usage": { + "model": { + "provider": "openai", + "model_id": model_id + }, + "tokens": { + "input_tokens": input_tokens, + "output_tokens": output_tokens + } + }, + "facts": { + "provider": "open_ai" + } + }, + "total_usd_micros": input_tokens + output_tokens + })) + .unwrap() + } + + fn usage_json(usage: &BilledModelUsage) -> serde_json::Value { + serde_json::to_value(usage).unwrap() + } + fn test_raw_event( seq: u32, event: &str, @@ -1014,6 +1109,210 @@ mod tests { ); } + #[test] + fn stage_completed_event_captures_duration_and_usage_per_visit() { + let mut state = RunProjection::default(); + let usage = test_usage("gpt-5.2", 123, 45); + + state + .apply_event(&test_event( + 3, + EventBody::StageCompleted(StageCompletedProps { + index: 0, + duration_ms: 789, + status: StageOutcome::Succeeded, + preferred_label: None, + suggested_next_ids: Vec::new(), + billing: Some(usage.clone()), + failure: None, + notes: None, + files_touched: Vec::new(), + context_updates: None, + jump_to_node: None, + context_values: None, + node_visits: None, + loop_failure_signatures: None, + restart_failure_signatures: None, + response: Some("done".to_string()), + attempt: 1, + max_attempts: 1, + }), + Some("build"), + )) + .unwrap(); + + let stage = state.stage(&StageId::new("build", 1)).unwrap(); + assert_eq!(stage.duration_ms, Some(789)); + assert_eq!(stage.usage.as_ref(), Some(&usage)); + } + + #[test] + fn stage_failed_event_captures_duration_and_usage_per_visit() { + let mut state = RunProjection::default(); + let stage_id = StageId::new("build", 1); + let usage = test_usage("gpt-5.2", 321, 54); + + state + .apply_event(&test_stage_event( + 2, + EventBody::StageStarted(StageStartedProps { + index: 0, + handler_type: "agent".to_string(), + attempt: 1, + max_attempts: 1, + }), + stage_id.clone(), + )) + .unwrap(); + state + .apply_event(&test_raw_event( + 3, + "stage.failed", + &json!({ + "index": 0, + "failure": { + "message": "provider failed", + "failure_class": "transient_infra" + }, + "will_retry": false, + "duration_ms": 654, + "billing": usage_json(&usage) + }), + Some("build"), + )) + .unwrap(); + + let stage = state.stage(&stage_id).unwrap(); + assert_eq!(stage.duration_ms, Some(654)); + assert_eq!(stage.usage.as_ref(), Some(&usage)); + } + + #[test] + fn two_visits_of_one_node_retain_distinct_usage() { + let mut state = RunProjection::default(); + let first_usage = test_usage("gpt-5.2", 100, 10); + let second_usage = test_usage("gpt-5.2", 200, 20); + + for (seq, visit, duration_ms, usage) in [ + (3, 1usize, 111, first_usage.clone()), + (4, 2usize, 222, second_usage.clone()), + ] { + state + .apply_event(&test_event( + seq, + EventBody::StageCompleted(StageCompletedProps { + index: 0, + duration_ms, + status: StageOutcome::Succeeded, + preferred_label: None, + suggested_next_ids: Vec::new(), + billing: Some(usage), + failure: None, + notes: None, + files_touched: Vec::new(), + context_updates: None, + jump_to_node: None, + context_values: None, + node_visits: Some(BTreeMap::from([("build".to_string(), visit)])), + loop_failure_signatures: None, + restart_failure_signatures: None, + response: None, + attempt: 1, + max_attempts: 1, + }), + Some("build"), + )) + .unwrap(); + } + + let first_stage = state.stage(&StageId::new("build", 1)).unwrap(); + let second_stage = state.stage(&StageId::new("build", 2)).unwrap(); + assert_eq!(first_stage.duration_ms, Some(111)); + assert_eq!(first_stage.usage.as_ref(), Some(&first_usage)); + assert_eq!(second_stage.duration_ms, Some(222)); + assert_eq!(second_stage.usage.as_ref(), Some(&second_usage)); + } + + #[test] + fn stage_completed_prefers_stored_stage_id_over_legacy_node_visits() { + let mut state = RunProjection::default(); + let usage = test_usage("gpt-5.2", 300, 30); + let scoped_stage_id = StageId::new("build", 2); + + state + .apply_event(&test_stage_event( + 3, + EventBody::StageCompleted(StageCompletedProps { + index: 0, + duration_ms: 333, + status: StageOutcome::Succeeded, + preferred_label: None, + suggested_next_ids: Vec::new(), + billing: Some(usage.clone()), + failure: None, + notes: None, + files_touched: Vec::new(), + context_updates: None, + jump_to_node: None, + context_values: None, + node_visits: Some(BTreeMap::from([("build".to_string(), 1usize)])), + loop_failure_signatures: None, + restart_failure_signatures: None, + response: Some("done".to_string()), + attempt: 1, + max_attempts: 1, + }), + scoped_stage_id.clone(), + )) + .unwrap(); + + assert!( + state.stage(&StageId::new("build", 1)).is_none(), + "legacy node_visits must not override stored stage_id" + ); + let stage = state.stage(&scoped_stage_id).unwrap(); + assert_eq!(stage.duration_ms, Some(333)); + assert_eq!(stage.usage.as_ref(), Some(&usage)); + assert_eq!(stage.response.as_deref(), Some("done")); + } + + #[test] + fn stage_failed_prefers_stored_stage_id_and_preserves_retry_request() { + let mut state = RunProjection::default(); + let usage = test_usage("gpt-5.2", 400, 40); + let scoped_stage_id = StageId::new("build", 2); + + state + .apply_event(&test_stage_event( + 3, + EventBody::StageFailed(StageFailedProps { + index: 0, + failure: Some(fabro_types::FailureDetail::new( + "try again", + fabro_types::FailureCategory::TransientInfra, + )), + will_retry: true, + duration_ms: 444, + billing: Some(usage.clone()), + }), + scoped_stage_id.clone(), + )) + .unwrap(); + + assert!( + state.stage(&StageId::new("build", 1)).is_none(), + "current-visit fallback must not override stored stage_id" + ); + let stage = state.stage(&scoped_stage_id).unwrap(); + assert_eq!(stage.duration_ms, Some(444)); + assert_eq!(stage.usage.as_ref(), Some(&usage)); + let completion = stage.completion.as_ref().unwrap(); + assert_eq!(completion.outcome, StageOutcome::Failed { + retry_requested: true, + }); + assert_eq!(completion.failure_reason.as_deref(), Some("try again")); + } + #[test] fn checkpoint_completed_creates_projection_entry_for_skipped_stage() { let mut state = RunProjection::default(); @@ -1641,4 +1940,224 @@ mod tests { ); assert_eq!(state.status_updated_at, updated_at); } + + fn started_props() -> StageStartedProps { + StageStartedProps { + index: 0, + handler_type: "agent".to_string(), + attempt: 1, + max_attempts: 3, + } + } + + fn failed_props(duration_ms: u64, will_retry: bool) -> StageFailedProps { + StageFailedProps { + index: 0, + failure: Some(FailureDetail::new("boom", FailureCategory::TransientInfra)), + will_retry, + duration_ms, + billing: None, + } + } + + fn retrying_props() -> StageRetryingProps { + StageRetryingProps { + index: 0, + attempt: 2, + max_attempts: 3, + delay_ms: 0, + } + } + + fn completed_props(duration_ms: u64, status: StageOutcome) -> StageCompletedProps { + StageCompletedProps { + index: 0, + duration_ms, + status, + preferred_label: None, + suggested_next_ids: Vec::new(), + billing: None, + failure: None, + notes: None, + files_touched: Vec::new(), + context_updates: None, + jump_to_node: None, + context_values: None, + node_visits: None, + loop_failure_signatures: None, + restart_failure_signatures: None, + response: None, + attempt: 1, + max_attempts: 3, + } + } + + fn billed_usage() -> BilledModelUsage { + serde_json::from_value(json!({ + "input": { + "usage": { + "model": { + "provider": "openai", + "model_id": "gpt-test" + }, + "tokens": { + "input_tokens": 10, + "output_tokens": 5, + "reasoning_tokens": 2, + "cache_read_tokens": 3, + "cache_write_tokens": 4 + } + }, + "facts": { "provider": "open_ai" } + }, + "total_usd_micros": 123 + })) + .expect("billing fixture should deserialize") + } + + #[test] + fn stage_started_records_started_at_and_running_state() { + let mut state = RunProjection::default(); + let stage_id = StageId::new("build", 1); + + state + .apply_event(&test_stage_event( + 3, + EventBody::StageStarted(started_props()), + stage_id.clone(), + )) + .unwrap(); + + let stage = state.stage(&stage_id).unwrap(); + assert_eq!(stage.state, Some(StageState::Running)); + assert!(stage.started_at.is_some()); + assert_eq!(stage.effective_state(), StageState::Running); + } + + #[test] + fn stage_completed_records_duration_usage_and_terminal_state() { + let mut state = RunProjection::default(); + let stage_id = StageId::new("build", 1); + let usage = billed_usage(); + + state + .apply_event(&test_stage_event( + 1, + EventBody::StageStarted(started_props()), + stage_id.clone(), + )) + .unwrap(); + let mut props = completed_props(42, StageOutcome::Succeeded); + props.billing = Some(usage.clone()); + state + .apply_event(&test_event( + 2, + EventBody::StageCompleted(props), + Some("build"), + )) + .unwrap(); + + let stage = state.stage(&stage_id).unwrap(); + assert_eq!(stage.duration_ms, Some(42)); + assert_eq!(stage.usage.as_ref(), Some(&usage)); + assert_eq!(stage.state, Some(StageState::Succeeded)); + assert_eq!(stage.effective_state(), StageState::Succeeded); + } + + #[test] + fn stage_failed_records_duration_and_failed_state() { + let mut state = RunProjection::default(); + let stage_id = StageId::new("build", 1); + + state + .apply_event(&test_stage_event( + 1, + EventBody::StageStarted(started_props()), + stage_id.clone(), + )) + .unwrap(); + state + .apply_event(&test_event( + 2, + EventBody::StageFailed(failed_props(10, false)), + Some("build"), + )) + .unwrap(); + + let stage = state.stage(&stage_id).unwrap(); + assert_eq!(stage.duration_ms, Some(10)); + assert_eq!(stage.state, Some(StageState::Failed)); + } + + #[test] + fn stage_retrying_sets_retrying_state() { + let mut state = RunProjection::default(); + let stage_id = StageId::new("build", 1); + + state + .apply_event(&test_stage_event( + 1, + EventBody::StageStarted(started_props()), + stage_id.clone(), + )) + .unwrap(); + state + .apply_event(&test_event( + 2, + EventBody::StageFailed(failed_props(10, true)), + Some("build"), + )) + .unwrap(); + state + .apply_event(&test_event( + 3, + EventBody::StageRetrying(retrying_props()), + Some("build"), + )) + .unwrap(); + + let stage = state.stage(&stage_id).unwrap(); + assert_eq!(stage.state, Some(StageState::Retrying)); + } + + #[test] + fn stage_started_after_retrying_returns_to_running_and_resets_attempt_data() { + let mut state = RunProjection::default(); + let stage_id = StageId::new("build", 1); + + state + .apply_event(&test_stage_event( + 1, + EventBody::StageStarted(started_props()), + stage_id.clone(), + )) + .unwrap(); + state + .apply_event(&test_event( + 2, + EventBody::StageFailed(failed_props(10, true)), + Some("build"), + )) + .unwrap(); + state + .apply_event(&test_event( + 3, + EventBody::StageRetrying(retrying_props()), + Some("build"), + )) + .unwrap(); + state + .apply_event(&test_stage_event( + 4, + EventBody::StageStarted(started_props()), + stage_id.clone(), + )) + .unwrap(); + + let stage = state.stage(&stage_id).unwrap(); + assert_eq!(stage.state, Some(StageState::Running)); + // Prior attempt's terminal data must not leak into the new attempt. + assert!(stage.completion.is_none()); + assert_eq!(stage.duration_ms, None); + } } diff --git a/lib/crates/fabro-store/src/slate/run_store.rs b/lib/crates/fabro-store/src/slate/run_store.rs index a334281b9..f28e52341 100644 --- a/lib/crates/fabro-store/src/slate/run_store.rs +++ b/lib/crates/fabro-store/src/slate/run_store.rs @@ -12,7 +12,7 @@ use tokio_stream::wrappers::UnboundedReceiverStream; use super::blob_store::BlobStore; use crate::run_state::{EventProjectionCache, RunProjectionReducer, build_summary}; -use crate::{Error, EventEnvelope, EventPayload, Result, RunProjection, keys}; +use crate::{Error, EventEnvelope, EventPayload, Result, RunProjection, StageId, keys}; const DEFAULT_EVENT_TAIL_LIMIT: usize = 1024; #[derive(Clone)] @@ -213,6 +213,30 @@ impl RunDatabase { list_events_from_with_limit(&self.inner.db, &self.inner.run_id, start_seq, limit).await } + /// Returns up to `limit + 1` events for the given stage visit, + /// starting at `start_seq`. The `+1` lets callers compute `has_more`. + /// + /// Implementation note: scans the unbounded run-event prefix and + /// filters by stage identity *before* applying `limit`, so a stage with + /// matches sparsely scattered late in the event log still returns its + /// full slice (no premature truncation from a generic `limit`-bounded + /// scan). + pub async fn list_events_for_stage_from_with_limit( + &self, + stage_id: &StageId, + start_seq: u32, + limit: usize, + ) -> Result> { + list_events_for_stage_from_with_limit( + &self.inner.db, + &self.inner.run_id, + stage_id, + start_seq, + limit, + ) + .await + } + pub fn watch_events_from( &self, seq: u32, @@ -348,6 +372,74 @@ where Ok(events) } +async fn list_events_for_stage_from_with_limit( + db: &R, + run_id: &RunId, + stage_id: &StageId, + start_seq: u32, + limit: usize, +) -> Result> +where + R: DbRead + Sync, +{ + // Unbounded scan first: filtering by stage identity with a generic + // limit-bounded scan would silently drop matches whenever the stage's + // events are sparse late in the event log. + // + // We probe just the stage identity fields with a small partial deserialize and + // only run the full `RunEvent` parse on matches. Most events in a run + // belong to other nodes, so this avoids deserializing large payloads + // (`agent.tool.completed.output`, `agent.message.text`, …) we'd discard. + #[derive(serde::Deserialize)] + struct StageIdProbe<'a> { + #[serde(default, borrow)] + stage_id: Option<&'a str>, + #[serde(default, borrow)] + node_id: Option<&'a str>, + } + + let stage_id_string = stage_id.to_string(); + let max_events = limit.saturating_add(1); + let mut iter = db.scan_prefix(keys::run_events_prefix(run_id)).await?; + let mut events: Vec = Vec::new(); + while let Some(entry) = iter.next().await? { + let key = key_to_string(&entry.key)?; + let Some(seq) = keys::parse_event_seq(&key) else { + continue; + }; + if seq < start_seq { + continue; + } + let probe: StageIdProbe = serde_json::from_slice(&entry.value)?; + let matches_stage_id = probe.stage_id == Some(stage_id_string.as_str()); + let matches_legacy_node_id = probe.stage_id.is_none() + && stage_id.visit() == 1 + && probe.node_id == Some(stage_id.node_id()); + if !matches_stage_id && !matches_legacy_node_id { + continue; + } + let event: RunEvent = serde_json::from_slice(&entry.value)?; + let envelope = EventEnvelope { seq, event }; + if events.len() < max_events { + events.push(envelope); + continue; + } + + if let Some((max_index, max_seq)) = events + .iter() + .enumerate() + .max_by_key(|(_, existing)| existing.seq) + .map(|(index, existing)| (index, existing.seq)) + { + if seq < max_seq { + events[max_index] = envelope; + } + } + } + events.sort_by_key(|event| event.seq); + Ok(events) +} + async fn list_blobs(db: &R) -> Result> where R: DbRead + Sync, @@ -375,9 +467,12 @@ mod tests { use std::sync::Arc; use std::time::Duration; + use fabro_types::{RunId, StageId}; use object_store::memory::InMemory; + use serde_json::json; + + use crate::{Database, EventPayload}; - use crate::Database; #[tokio::test] async fn list_blobs_reads_global_cas_namespace() { let object_store = Arc::new(InMemory::new()); @@ -394,4 +489,190 @@ mod tests { assert_eq!(blob_ids, vec![first_id, second_id]); } + + fn stage_prompt_payload(run_id: &RunId, idx: u32, node_id: Option<&str>) -> EventPayload { + stage_prompt_payload_for_stage(run_id, idx, node_id, None) + } + + fn stage_prompt_payload_for_stage( + run_id: &RunId, + idx: u32, + node_id: Option<&str>, + stage_id: Option<&StageId>, + ) -> EventPayload { + let mut value = json!({ + "id": format!("evt-{idx}"), + "ts": "2026-04-09T12:00:00Z", + "run_id": run_id.to_string(), + "event": "stage.prompt", + "properties": { + "visit": 1, + "text": format!("prompt {idx}"), + }, + }); + if let Some(node_id) = node_id { + value + .as_object_mut() + .unwrap() + .insert("node_id".into(), json!(node_id)); + } + if let Some(stage_id) = stage_id { + value + .as_object_mut() + .unwrap() + .insert("stage_id".into(), json!(stage_id.to_string())); + } + EventPayload::new(value, run_id).unwrap() + } + + async fn fresh_run() -> super::RunDatabase { + let object_store = Arc::new(InMemory::new()); + let store = Database::new(object_store, "", Duration::from_millis(1), None); + let run_id: RunId = "01JT56VE4Z5NZ814GZN2JZD65A".parse().unwrap(); + store.create_run(&run_id).await.unwrap() + } + + #[tokio::test] + async fn list_events_for_stage_returns_only_matching_events_in_seq_order() { + let run = fresh_run().await; + let run_id = run.run_id(); + run.append_event(&stage_prompt_payload(&run_id, 1, Some("alpha"))) + .await + .unwrap(); + run.append_event(&stage_prompt_payload(&run_id, 2, Some("beta"))) + .await + .unwrap(); + run.append_event(&stage_prompt_payload(&run_id, 3, Some("alpha"))) + .await + .unwrap(); + + let events = run + .list_events_for_stage_from_with_limit(&StageId::new("alpha", 1), 1, 100) + .await + .unwrap(); + + let seqs: Vec = events.iter().map(|e| e.seq).collect(); + assert_eq!(seqs, vec![1, 3]); + } + + #[tokio::test] + async fn list_events_for_stage_skips_events_with_no_stage_identity() { + let run = fresh_run().await; + let run_id = run.run_id(); + run.append_event(&stage_prompt_payload(&run_id, 1, None)) + .await + .unwrap(); + run.append_event(&stage_prompt_payload(&run_id, 2, Some("alpha"))) + .await + .unwrap(); + + let events = run + .list_events_for_stage_from_with_limit(&StageId::new("alpha", 1), 1, 100) + .await + .unwrap(); + + let seqs: Vec = events.iter().map(|e| e.seq).collect(); + assert_eq!(seqs, vec![2]); + } + + #[tokio::test] + async fn list_events_for_stage_paginates_via_start_seq_on_filtered_slice() { + let run = fresh_run().await; + let run_id = run.run_id(); + for idx in 1..=5 { + let node = if idx % 2 == 0 { "beta" } else { "alpha" }; + run.append_event(&stage_prompt_payload(&run_id, idx, Some(node))) + .await + .unwrap(); + } + + // alpha events live at seqs 1, 3, 5. Start at seq=2 should skip seq=1. + let events = run + .list_events_for_stage_from_with_limit(&StageId::new("alpha", 1), 2, 100) + .await + .unwrap(); + + let seqs: Vec = events.iter().map(|e| e.seq).collect(); + assert_eq!(seqs, vec![3, 5]); + } + + #[tokio::test] + async fn list_events_for_stage_walks_past_unrelated_events_for_sparse_matches() { + let run = fresh_run().await; + let run_id = run.run_id(); + // 200 unrelated events first. + for idx in 1..=200 { + run.append_event(&stage_prompt_payload(&run_id, idx, Some("noise"))) + .await + .unwrap(); + } + // Then 3 sparse "alpha" events at the tail. + for idx in 201..=203 { + run.append_event(&stage_prompt_payload(&run_id, idx, Some("alpha"))) + .await + .unwrap(); + } + + // limit smaller than the number of unrelated events would have + // truncated the upstream scan if we had post-filtered. + let events = run + .list_events_for_stage_from_with_limit(&StageId::new("alpha", 1), 1, 5) + .await + .unwrap(); + + let seqs: Vec = events.iter().map(|e| e.seq).collect(); + assert_eq!(seqs, vec![201, 202, 203]); + } + + #[tokio::test] + async fn list_events_for_stage_returns_limit_plus_one_for_has_more_signal() { + let run = fresh_run().await; + let run_id = run.run_id(); + for idx in 1..=5 { + run.append_event(&stage_prompt_payload(&run_id, idx, Some("alpha"))) + .await + .unwrap(); + } + + let events = run + .list_events_for_stage_from_with_limit(&StageId::new("alpha", 1), 1, 2) + .await + .unwrap(); + + // With limit=2, we expect up to limit+1 = 3 envelopes so the + // caller can compute has_more. + assert_eq!(events.len(), 3); + } + + #[tokio::test] + async fn list_events_for_stage_prefers_stage_id_over_node_id() { + let run = fresh_run().await; + let run_id = run.run_id(); + let first_visit = StageId::new("verify", 1); + let second_visit = StageId::new("verify", 2); + run.append_event(&stage_prompt_payload_for_stage( + &run_id, + 1, + Some("verify"), + Some(&first_visit), + )) + .await + .unwrap(); + run.append_event(&stage_prompt_payload_for_stage( + &run_id, + 2, + Some("verify"), + Some(&second_visit), + )) + .await + .unwrap(); + + let events = run + .list_events_for_stage_from_with_limit(&second_visit, 1, 100) + .await + .unwrap(); + + let seqs: Vec = events.iter().map(|e| e.seq).collect(); + assert_eq!(seqs, vec![2]); + } } diff --git a/lib/crates/fabro-store/tests/serializable_projection.rs b/lib/crates/fabro-store/tests/serializable_projection.rs index 6d51065ba..8809acda2 100644 --- a/lib/crates/fabro-store/tests/serializable_projection.rs +++ b/lib/crates/fabro-store/tests/serializable_projection.rs @@ -5,8 +5,8 @@ use fabro_store::{RunProjection, SerializableProjection, StageId}; use fabro_types::graph::Graph; use fabro_types::run::RunSpec; use fabro_types::{ - Checkpoint, RunStatus, SandboxRecord, StageCompletion, StageOutcome, StartRecord, - TerminalStatus, WorkflowSettings, first_event_seq, fixtures, + BilledModelUsage, Checkpoint, RunStatus, SandboxRecord, StageCompletion, StageOutcome, + StartRecord, TerminalStatus, WorkflowSettings, first_event_seq, fixtures, }; use serde_json::json; @@ -52,6 +52,28 @@ fn sample_checkpoint() -> Checkpoint { } } +fn sample_usage() -> BilledModelUsage { + serde_json::from_value(json!({ + "input": { + "usage": { + "model": { + "provider": "openai", + "model_id": "gpt-5.2" + }, + "tokens": { + "input_tokens": 123, + "output_tokens": 45 + } + }, + "facts": { + "provider": "open_ai" + } + }, + "total_usd_micros": 168 + })) + .expect("sample usage should deserialize") +} + #[test] fn serializable_projection_round_trips_and_trims_bulky_node_fields() { let stage_id = StageId::new("build", 2); @@ -94,11 +116,17 @@ fn serializable_projection_round_trips_and_trims_bulky_node_fields() { stage.script_invocation = Some(json!({ "command": "cargo test" })); stage.script_timing = Some(json!({ "duration_ms": 10 })); stage.parallel_results = Some(json!([{ "stage": "fanout@1" }])); + stage.duration_ms = Some(1234); + stage.usage = Some(sample_usage()); stage.stdout = Some("stdout".to_string()); stage.stderr = Some("stderr".to_string()); let serialized = serde_json::to_value(SerializableProjection(&projection)) .expect("projection should serialize"); + assert!( + serialized["stages"]["build@2"].get("usage").is_none(), + "stage usage is server-internal and should not be serialized" + ); let round_tripped: RunProjection = serde_json::from_value(serialized).expect("serialized projection should deserialize"); let node = round_tripped.stage(&stage_id).expect("node should remain"); @@ -138,6 +166,8 @@ fn serializable_projection_round_trips_and_trims_bulky_node_fields() { node.parallel_results, Some(json!([{ "stage": "fanout@1" }])) ); + assert_eq!(node.duration_ms, Some(1234)); + assert_eq!(node.usage, None); } #[test] diff --git a/lib/crates/fabro-types/src/lib.rs b/lib/crates/fabro-types/src/lib.rs index c04702c1e..e5d3b65a5 100644 --- a/lib/crates/fabro-types/src/lib.rs +++ b/lib/crates/fabro-types/src/lib.rs @@ -72,7 +72,7 @@ pub use run::{ pub use run_blob_id::RunBlobId; pub use run_event::{ EventBody, ExecOutputTail, InterviewOption, MetadataSnapshotFailureKind, MetadataSnapshotPhase, - RunEvent, RunNoticeLevel, + RunEvent, RunNoticeCode, RunNoticeLevel, }; pub use run_id::{RunId, fixtures}; pub use run_projection::{PendingInterviewRecord, RunProjection, StageProjection, first_event_seq}; @@ -80,7 +80,7 @@ pub use run_summary::RunSummary; pub use sandbox_record::SandboxRecord; pub use secret::{SecretMetadata, SecretType}; pub use stage_completion::StageCompletion; -pub use stage_id::{ParallelBranchId, StageId}; +pub use stage_id::{InvalidStageVisit, ParallelBranchId, StageId}; pub use start::StartRecord; pub use status::{ BlockedReason, FailureReason, InvalidTransition, ParseFailureReasonError, diff --git a/lib/crates/fabro-types/src/outcome.rs b/lib/crates/fabro-types/src/outcome.rs index af34cd10b..8ef0c67dc 100644 --- a/lib/crates/fabro-types/src/outcome.rs +++ b/lib/crates/fabro-types/src/outcome.rs @@ -138,12 +138,32 @@ impl From for StageState { match outcome { StageOutcome::Succeeded => Self::Succeeded, StageOutcome::PartiallySucceeded => Self::PartiallySucceeded, - StageOutcome::Failed { .. } => Self::Failed, + StageOutcome::Failed { + retry_requested: true, + } => Self::Retrying, + StageOutcome::Failed { + retry_requested: false, + } => Self::Failed, StageOutcome::Skipped => Self::Skipped, } } } +#[cfg(test)] +mod stage_state_tests { + use super::{StageOutcome, StageState}; + + #[test] + fn retry_requested_failure_projects_as_retrying() { + assert_eq!( + StageState::from(StageOutcome::Failed { + retry_requested: true, + }), + StageState::Retrying + ); + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum FailureCategory { @@ -340,9 +360,10 @@ mod tests { StageState::from(StageOutcome::Failed { retry_requested: true, }), - StageState::Failed + StageState::Retrying ); assert!(StageState::Cancelled.is_terminal()); + assert!(!StageState::Retrying.is_terminal()); assert!(!StageState::Running.is_terminal()); } } diff --git a/lib/crates/fabro-types/src/run_event/infra.rs b/lib/crates/fabro-types/src/run_event/infra.rs index 8ee1bc32c..77689087b 100644 --- a/lib/crates/fabro-types/src/run_event/infra.rs +++ b/lib/crates/fabro-types/src/run_event/infra.rs @@ -1,17 +1,48 @@ use serde::{Deserialize, Serialize}; -/// Legacy `run.notice` codes paired with the new `metadata.snapshot.failed` -/// event for backward compatibility. Display layers suppress these so the -/// typed event renders without a duplicate raw warning. -pub const NOTICE_CODE_CHECKPOINT_METADATA_WRITE_FAILED: &str = "checkpoint_metadata_write_failed"; -pub const NOTICE_CODE_CHECKPOINT_METADATA_PUSH_FAILED: &str = "checkpoint_metadata_push_failed"; +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + Hash, + Serialize, + Deserialize, + strum::Display, + strum::EnumString, + strum::IntoStaticStr, +)] +#[serde(rename_all = "snake_case")] +#[strum(serialize_all = "snake_case")] +pub enum RunNoticeCode { + ArtifactCollectionFailed, + ArtifactOffloadFailed, + ArtifactSyncFailed, + ArtifactUploadFailed, + CheckpointMetadataDegraded, + CheckpointMetadataPushFailed, + CheckpointMetadataWriteFailed, + DirtyWorktree, + GitDiffFailed, + GitPushFailed, + GithubTokenFailed, + ParallelBaseCheckpointFailed, + PullRequestFailed, + SandboxCleanupFailed, + SandboxGitUnavailable, + SandboxPreserved, + WorktreeSkippedNoGit, +} -#[must_use] -pub fn is_metadata_snapshot_compat_notice_code(code: &str) -> bool { - matches!( - code, - NOTICE_CODE_CHECKPOINT_METADATA_WRITE_FAILED | NOTICE_CODE_CHECKPOINT_METADATA_PUSH_FAILED - ) +impl RunNoticeCode { + #[must_use] + pub fn is_metadata_snapshot_compat(self) -> bool { + matches!( + self, + Self::CheckpointMetadataWriteFailed | Self::CheckpointMetadataPushFailed + ) + } } #[derive( diff --git a/lib/crates/fabro-types/src/run_event/mod.rs b/lib/crates/fabro-types/src/run_event/mod.rs index 11f16c9ca..f84d943c1 100644 --- a/lib/crates/fabro-types/src/run_event/mod.rs +++ b/lib/crates/fabro-types/src/run_event/mod.rs @@ -1372,7 +1372,7 @@ mod tests { for body in [ EventBody::RunNotice(RunNoticeProps { level: RunNoticeLevel::Warn, - code: "git_diff_failed".to_string(), + code: RunNoticeCode::GitDiffFailed.to_string(), message: "git diff failed".to_string(), exec_output_tail: Some(tail.clone()), }), @@ -1408,7 +1408,7 @@ mod tests { for body in [ EventBody::RunNotice(RunNoticeProps { level: RunNoticeLevel::Warn, - code: "git_diff_failed".to_string(), + code: RunNoticeCode::GitDiffFailed.to_string(), message: "git diff failed".to_string(), exec_output_tail: None, }), diff --git a/lib/crates/fabro-types/src/run_event/stage.rs b/lib/crates/fabro-types/src/run_event/stage.rs index 9f1781d82..1b6609884 100644 --- a/lib/crates/fabro-types/src/run_event/stage.rs +++ b/lib/crates/fabro-types/src/run_event/stage.rs @@ -57,6 +57,8 @@ pub struct StageFailedProps { pub will_retry: bool, #[serde(default)] pub duration_ms: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub billing: Option, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] diff --git a/lib/crates/fabro-types/src/run_projection.rs b/lib/crates/fabro-types/src/run_projection.rs index de2e4a1a5..90da9b59b 100644 --- a/lib/crates/fabro-types/src/run_projection.rs +++ b/lib/crates/fabro-types/src/run_projection.rs @@ -4,9 +4,9 @@ use std::num::NonZeroU32; use chrono::{DateTime, Utc}; use crate::{ - Checkpoint, Conclusion, InterviewQuestionRecord, InvalidTransition, PullRequestRecord, Retro, - RunControlAction, RunId, RunSpec, RunStatus, SandboxRecord, StageCompletion, StageId, - StartRecord, + BilledModelUsage, Checkpoint, Conclusion, InterviewQuestionRecord, InvalidTransition, + PullRequestRecord, Retro, RunControlAction, RunId, RunSpec, RunStatus, SandboxRecord, + StageCompletion, StageId, StageState, StartRecord, }; #[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] @@ -61,6 +61,17 @@ pub struct StageProjection { pub live_streaming: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub termination: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub started_at: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub duration_ms: Option, + /// Server-internal billing usage for the latest attempt; not part of the + /// wire contract because `BilledModelUsage` is not modeled in OpenAPI. + /// Read only in-process by the billing handler. + #[serde(skip)] + pub usage: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub state: Option, } /// Convert a 1-based event sequence number into the `NonZeroU32` form used for @@ -78,6 +89,8 @@ impl StageProjection { prompt: None, response: None, completion: None, + duration_ms: None, + usage: None, provider_used: None, diff: None, script_invocation: None, @@ -90,8 +103,55 @@ impl StageProjection { streams_separated: None, live_streaming: None, termination: None, + started_at: None, + state: None, } } + + /// Effective lifecycle state derived from stored event data. + /// + /// Falls back to deriving from `completion` for projections that predate + /// the stored `state` field, so old serialized projections still work + /// without a backfill. + #[must_use] + pub fn effective_state(&self) -> StageState { + self.state.unwrap_or_else(|| match &self.completion { + Some(completion) => StageState::from(completion.outcome), + None => StageState::Running, + }) + } + + /// Live wall-clock runtime in seconds. + /// + /// While the stage is non-terminal (`Pending`, `Running`, or `Retrying`), + /// this returns the elapsed time since `started_at` so the UI can tick + /// client-side. Once terminal, the stored `duration_ms` is returned. This + /// also handles retries safely: a new `StageStarted` resets the state + /// back to `Running` and keeps the live computation correct even if a + /// previous attempt left a stale `duration_ms`. + #[must_use] + pub fn runtime_secs(&self, now: DateTime) -> Option { + let state = self.effective_state(); + if matches!( + state, + StageState::Running | StageState::Retrying | StageState::Pending + ) { + return self.started_at.map(|started| { + now.signed_duration_since(started).num_milliseconds().max(0) as f64 / 1000.0 + }); + } + self.duration_ms.map(|ms| ms as f64 / 1000.0) + } + + /// Begin a new attempt (or visit) for this stage: clear every + /// per-attempt field so prior-attempt data does not leak, then record + /// `started_at` and `state = Running`. Preserves `first_event_seq` + /// (identity / sort key). + pub fn begin_attempt(&mut self, started_at: DateTime) { + *self = Self::new(self.first_event_seq); + self.started_at = Some(started_at); + self.state = Some(StageState::Running); + } } impl RunProjection { @@ -99,12 +159,32 @@ impl RunProjection { self.stages.get(stage) } + /// Iterate stages in `first_event_seq` order (the chronological order in + /// which each stage's first lifecycle event was recorded). Internal + /// storage is a `HashMap`, so iteration would otherwise be + /// non-deterministic; every caller wants chronological order, so we sort + /// here once instead of asking each caller to remember. pub fn iter_stages(&self) -> impl Iterator { - self.stages.iter() + let mut entries: Vec<(&StageId, &StageProjection)> = self.stages.iter().collect(); + entries.sort_by(|(left_id, left_stage), (right_id, right_stage)| { + left_stage + .first_event_seq + .cmp(&right_stage.first_event_seq) + .then_with(|| left_id.cmp(right_id)) + }); + entries.into_iter() } + /// Mutable counterpart of [`iter_stages`]. Same chronological ordering. pub fn iter_stages_mut(&mut self) -> impl Iterator { - self.stages.iter_mut() + let mut entries: Vec<(&StageId, &mut StageProjection)> = self.stages.iter_mut().collect(); + entries.sort_by(|(left_id, left_stage), (right_id, right_stage)| { + left_stage + .first_event_seq + .cmp(&right_stage.first_event_seq) + .then_with(|| left_id.cmp(right_id)) + }); + entries.into_iter() } pub fn is_empty(&self) -> bool { @@ -186,3 +266,90 @@ impl RunProjection { } } } + +#[cfg(test)] +mod iter_stages_tests { + use std::num::NonZeroU32; + + use super::RunProjection; + + fn seq(n: u32) -> NonZeroU32 { + NonZeroU32::new(n).unwrap() + } + + #[test] + fn iter_stages_yields_chronological_order_across_nodes() { + let mut p = RunProjection::default(); + // Insert in non-monotonic seq order to exercise the sort. + p.stage_entry("c", 1, seq(30)); + p.stage_entry("a", 1, seq(10)); + p.stage_entry("b", 1, seq(20)); + + let order: Vec<&str> = p + .iter_stages() + .map(|(stage_id, _)| stage_id.node_id()) + .collect(); + assert_eq!(order, vec!["a", "b", "c"]); + } + + #[test] + fn iter_stages_orders_visits_within_a_node() { + let mut p = RunProjection::default(); + // Visit 2 inserted first; visit 1's earlier first_event_seq must still + // win the chronological ordering. + p.stage_entry("verify", 2, seq(50)); + p.stage_entry("verify", 1, seq(20)); + + let visits: Vec = p + .iter_stages() + .map(|(stage_id, _)| stage_id.visit()) + .collect(); + assert_eq!(visits, vec![1, 2]); + } + + #[test] + fn iter_stages_mut_yields_chronological_order() { + let mut p = RunProjection::default(); + p.stage_entry("c", 1, seq(30)); + p.stage_entry("a", 1, seq(10)); + p.stage_entry("b", 1, seq(20)); + + let order: Vec = p + .iter_stages_mut() + .map(|(stage_id, _)| stage_id.node_id().to_string()) + .collect(); + assert_eq!(order, vec!["a", "b", "c"]); + } + + #[test] + fn iter_stages_tie_breaks_same_first_event_seq_by_stage_id() { + for _ in 0..128 { + let mut p = RunProjection::default(); + p.stage_entry("verify", 2, seq(10)); + p.stage_entry("build", 1, seq(10)); + p.stage_entry("verify", 1, seq(10)); + + let order: Vec = p + .iter_stages() + .map(|(stage_id, _)| stage_id.to_string()) + .collect(); + assert_eq!(order, vec!["build@1", "verify@1", "verify@2"]); + } + } + + #[test] + fn iter_stages_mut_tie_breaks_same_first_event_seq_by_stage_id() { + for _ in 0..128 { + let mut p = RunProjection::default(); + p.stage_entry("verify", 2, seq(10)); + p.stage_entry("build", 1, seq(10)); + p.stage_entry("verify", 1, seq(10)); + + let order: Vec = p + .iter_stages_mut() + .map(|(stage_id, _)| stage_id.to_string()) + .collect(); + assert_eq!(order, vec!["build@1", "verify@1", "verify@2"]); + } + } +} diff --git a/lib/crates/fabro-types/src/stage_id.rs b/lib/crates/fabro-types/src/stage_id.rs index baae58864..91711cb79 100644 --- a/lib/crates/fabro-types/src/stage_id.rs +++ b/lib/crates/fabro-types/src/stage_id.rs @@ -1,4 +1,5 @@ use std::fmt; +use std::num::NonZeroU32; use std::str::FromStr; use serde::de::Error as _; @@ -7,16 +8,21 @@ use serde::{Deserialize, Deserializer, Serialize, Serializer}; #[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct StageId { node_id: String, - visit: u32, + visit: NonZeroU32, } impl StageId { #[must_use] pub fn new(node_id: impl Into, visit: u32) -> Self { - Self { + Self::try_new(node_id, visit).expect("stage id visit must be greater than zero") + } + + pub fn try_new(node_id: impl Into, visit: u32) -> Result { + let visit = NonZeroU32::new(visit).ok_or(InvalidStageVisit)?; + Ok(Self { node_id: node_id.into(), visit, - } + }) } #[must_use] @@ -26,7 +32,7 @@ impl StageId { #[must_use] pub fn visit(&self) -> u32 { - self.visit + self.visit.get() } } @@ -47,6 +53,17 @@ impl fmt::Display for ParseStageIdError { impl std::error::Error for ParseStageIdError {} +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct InvalidStageVisit; + +impl fmt::Display for InvalidStageVisit { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("stage id visit must be greater than zero") + } +} + +impl std::error::Error for InvalidStageVisit {} + impl FromStr for StageId { type Err = ParseStageIdError; @@ -67,7 +84,7 @@ impl FromStr for StageId { let visit = visit .parse() .map_err(|err| ParseStageIdError(format!("invalid stage id visit: {err}")))?; - Ok(Self::new(node_id, visit)) + Self::try_new(node_id, visit).map_err(|err| ParseStageIdError(err.to_string())) } } @@ -224,6 +241,18 @@ mod tests { assert!(err.to_string().starts_with("invalid stage id visit:")); } + #[test] + fn parse_rejects_zero_visit() { + let err = "code@0".parse::().unwrap_err(); + assert_eq!(err.to_string(), "stage id visit must be greater than zero"); + } + + #[test] + fn try_new_rejects_zero_visit() { + let err = StageId::try_new("code", 0).unwrap_err(); + assert_eq!(err.to_string(), "stage id visit must be greater than zero"); + } + #[test] fn parse_rejects_empty_node_id() { let err = "@3".parse::().unwrap_err(); diff --git a/lib/crates/fabro-workflow/src/billing_rollup.rs b/lib/crates/fabro-workflow/src/billing_rollup.rs new file mode 100644 index 000000000..03d5207d3 --- /dev/null +++ b/lib/crates/fabro-workflow/src/billing_rollup.rs @@ -0,0 +1,291 @@ +use std::collections::{BTreeMap, HashMap}; + +use fabro_types::{BilledModelUsage, BilledTokenCounts, RunProjection}; + +#[derive(Debug, Clone, PartialEq)] +pub struct ProjectionBillingStage { + pub node_id: String, + pub billing: BilledTokenCounts, + pub duration_ms: u64, + pub model_id: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ProjectionBillingByModel { + pub model_id: String, + pub stages: i64, + pub billing: BilledTokenCounts, +} + +#[derive(Debug, Clone, Default, PartialEq)] +pub struct ProjectionBillingRollup { + pub stages: Vec, + pub totals: BilledTokenCounts, + pub by_model: Vec, + pub runtime_ms: u64, + pub billed_visit_count: usize, +} + +impl ProjectionBillingRollup { + #[must_use] + pub fn billing_if_present(&self) -> Option { + (self.billed_visit_count > 0).then(|| self.totals.clone()) + } +} + +#[must_use] +pub fn billing_rollup_from_projection(projection: &RunProjection) -> ProjectionBillingRollup { + let mut stage_indices = HashMap::::new(); + let mut stages = Vec::::new(); + let mut by_model = BTreeMap::::new(); + let mut totals = BilledTokenCounts::default(); + let mut runtime_ms = 0_u64; + let mut billed_visit_count = 0_usize; + + for (stage_id, stage) in projection.iter_stages() { + if is_exit_stage(projection, stage_id.node_id()) { + continue; + } + if stage.completion.is_none() && stage.duration_ms.is_none() && stage.usage.is_none() { + continue; + } + + let node_id = stage_id.node_id(); + let index = *stage_indices.entry(node_id.to_string()).or_insert_with(|| { + let index = stages.len(); + stages.push(ProjectionBillingStage { + node_id: node_id.to_string(), + billing: BilledTokenCounts::default(), + duration_ms: 0, + model_id: None, + }); + index + }); + let row = &mut stages[index]; + + if let Some(duration_ms) = stage.duration_ms { + row.duration_ms = row.duration_ms.saturating_add(duration_ms); + runtime_ms = runtime_ms.saturating_add(duration_ms); + } + + if let Some(usage) = stage.usage.as_ref() { + billed_visit_count += 1; + row.model_id = Some(usage.model_id().to_string()); + accumulate_usage(&mut row.billing, usage); + accumulate_usage(&mut totals, usage); + + let model_id = usage.model_id().to_string(); + let model_entry = + by_model + .entry(model_id.clone()) + .or_insert_with(|| ProjectionBillingByModel { + model_id, + stages: 0, + billing: BilledTokenCounts::default(), + }); + model_entry.stages += 1; + accumulate_usage(&mut model_entry.billing, usage); + } + } + + ProjectionBillingRollup { + stages, + totals, + by_model: by_model.into_values().collect(), + runtime_ms, + billed_visit_count, + } +} + +fn is_exit_stage(projection: &RunProjection, node_id: &str) -> bool { + projection + .spec() + .and_then(|spec| spec.graph().nodes.get(node_id)) + .is_some_and(|node| node.handler_type() == Some("exit")) +} + +fn accumulate_usage(counts: &mut BilledTokenCounts, usage: &BilledModelUsage) { + let tokens = usage.tokens(); + counts.input_tokens += tokens.input_tokens; + counts.output_tokens += tokens.output_tokens; + counts.reasoning_tokens += tokens.reasoning_tokens; + counts.cache_read_tokens += tokens.cache_read_tokens; + counts.cache_write_tokens += tokens.cache_write_tokens; + counts.total_tokens += tokens.total_tokens(); + if let Some(value) = usage.total_usd_micros { + *counts.total_usd_micros.get_or_insert(0) += value; + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use fabro_types::{ + AttrValue, BilledModelUsage, Graph, Node, RunProjection, RunSpec, StageCompletion, + StageOutcome, WorkflowSettings, first_event_seq, fixtures, + }; + use serde_json::json; + + use super::billing_rollup_from_projection; + + fn test_usage(model_id: &str, input_tokens: i64, output_tokens: i64) -> BilledModelUsage { + serde_json::from_value(json!({ + "input": { + "usage": { + "model": { + "provider": "openai", + "model_id": model_id + }, + "tokens": { + "input_tokens": input_tokens, + "output_tokens": output_tokens + } + }, + "facts": { + "provider": "open_ai" + } + }, + "total_usd_micros": input_tokens + output_tokens + })) + .unwrap() + } + + #[test] + fn rollup_groups_stage_rows_by_node_and_sums_retry_visit_usage() { + let mut projection = RunProjection::default(); + let failed_usage = test_usage("gpt-old", 100, 10); + let success_usage = test_usage("gpt-new", 200, 20); + let first = projection.stage_entry("verify", 1, first_event_seq(1)); + first.duration_ms = Some(1200); + first.usage = Some(failed_usage); + first.completion = Some(StageCompletion { + outcome: StageOutcome::Failed { + retry_requested: true, + }, + notes: None, + failure_reason: Some("try again".to_string()), + timestamp: chrono::Utc::now(), + }); + let second = projection.stage_entry("verify", 2, first_event_seq(2)); + second.duration_ms = Some(800); + second.usage = Some(success_usage); + second.completion = Some(StageCompletion { + outcome: StageOutcome::Succeeded, + notes: None, + failure_reason: None, + timestamp: chrono::Utc::now(), + }); + + let rollup = billing_rollup_from_projection(&projection); + + assert_eq!(rollup.stages.len(), 1); + assert_eq!(rollup.stages[0].node_id, "verify"); + assert_eq!(rollup.stages[0].model_id.as_deref(), Some("gpt-new")); + assert_eq!(rollup.stages[0].duration_ms, 2000); + assert_eq!(rollup.stages[0].billing.input_tokens, 300); + assert_eq!(rollup.stages[0].billing.output_tokens, 30); + assert_eq!(rollup.stages[0].billing.total_usd_micros, Some(330)); + + assert_eq!(rollup.runtime_ms, 2000); + assert_eq!(rollup.totals.input_tokens, 300); + assert_eq!(rollup.totals.output_tokens, 30); + assert_eq!(rollup.totals.total_usd_micros, Some(330)); + assert_eq!(rollup.billed_visit_count, 2); + + assert_eq!(rollup.by_model.len(), 2); + assert_eq!(rollup.by_model[0].model_id, "gpt-new"); + assert_eq!(rollup.by_model[0].stages, 1); + assert_eq!(rollup.by_model[0].billing.input_tokens, 200); + assert_eq!(rollup.by_model[1].model_id, "gpt-old"); + assert_eq!(rollup.by_model[1].stages, 1); + assert_eq!(rollup.by_model[1].billing.input_tokens, 100); + } + + #[test] + fn rollup_includes_completed_non_llm_stage_rows_with_zero_billing() { + let mut projection = RunProjection::default(); + let stage = projection.stage_entry("start", 1, first_event_seq(1)); + stage.duration_ms = Some(25); + stage.completion = Some(StageCompletion { + outcome: StageOutcome::Succeeded, + notes: None, + failure_reason: None, + timestamp: chrono::Utc::now(), + }); + + let rollup = billing_rollup_from_projection(&projection); + + assert_eq!(rollup.stages.len(), 1); + assert_eq!(rollup.stages[0].node_id, "start"); + assert_eq!(rollup.stages[0].duration_ms, 25); + assert!(rollup.stages[0].model_id.is_none()); + assert_eq!(rollup.stages[0].billing.input_tokens, 0); + assert_eq!(rollup.runtime_ms, 25); + assert!(rollup.by_model.is_empty()); + assert!(rollup.billing_if_present().is_none()); + } + + #[test] + fn rollup_excludes_terminal_exit_stage_rows() { + let mut projection = RunProjection::default(); + projection.spec = Some(run_spec_with_exit_node()); + let start = projection.stage_entry("start", 1, first_event_seq(1)); + start.duration_ms = Some(25); + start.completion = Some(StageCompletion { + outcome: StageOutcome::Succeeded, + notes: None, + failure_reason: None, + timestamp: chrono::Utc::now(), + }); + let exit = projection.stage_entry("exit", 1, first_event_seq(2)); + exit.duration_ms = Some(7); + exit.completion = Some(StageCompletion { + outcome: StageOutcome::Succeeded, + notes: None, + failure_reason: None, + timestamp: chrono::Utc::now(), + }); + + let rollup = billing_rollup_from_projection(&projection); + + assert_eq!(rollup.stages.len(), 1); + assert_eq!(rollup.stages[0].node_id, "start"); + assert_eq!(rollup.runtime_ms, 25); + } + + fn run_spec_with_exit_node() -> RunSpec { + let mut graph = Graph::new("test"); + graph.nodes.insert("start".to_string(), { + let mut node = Node::new("start"); + node.attrs.insert( + "shape".to_string(), + AttrValue::String("Mdiamond".to_string()), + ); + node + }); + graph.nodes.insert("exit".to_string(), { + let mut node = Node::new("exit"); + node.attrs.insert( + "shape".to_string(), + AttrValue::String("Msquare".to_string()), + ); + node + }); + + RunSpec { + run_id: fixtures::RUN_1, + settings: WorkflowSettings::default(), + graph, + workflow_slug: None, + source_directory: None, + labels: HashMap::new(), + provenance: None, + manifest_blob: None, + definition_blob: None, + git: None, + fork_source_ref: None, + in_place: false, + } + } +} diff --git a/lib/crates/fabro-workflow/src/error.rs b/lib/crates/fabro-workflow/src/error.rs index 1d08b2777..6a3ff0d06 100644 --- a/lib/crates/fabro-workflow/src/error.rs +++ b/lib/crates/fabro-workflow/src/error.rs @@ -1838,6 +1838,7 @@ mod tests { failure: failure.clone(), will_retry: false, duration_ms: 0, + billing: None, actor: None, }; diff --git a/lib/crates/fabro-workflow/src/event.rs b/lib/crates/fabro-workflow/src/event.rs index f5120e62e..e64b2ae30 100644 --- a/lib/crates/fabro-workflow/src/event.rs +++ b/lib/crates/fabro-workflow/src/event.rs @@ -8,7 +8,7 @@ mod stored_fields; #[cfg(test)] mod test_support; -pub use fabro_types::{EventBody, RunNoticeLevel}; +pub use fabro_types::{EventBody, RunNoticeCode, RunNoticeLevel}; pub use self::convert::{to_run_event, to_run_event_at}; pub use self::emitter::Emitter; diff --git a/lib/crates/fabro-workflow/src/event/convert.rs b/lib/crates/fabro-workflow/src/event/convert.rs index 5f5f66ad8..4a5737fd5 100644 --- a/lib/crates/fabro-workflow/src/event/convert.rs +++ b/lib/crates/fabro-workflow/src/event/convert.rs @@ -290,12 +290,14 @@ fn event_body_from_event(event: &Event) -> EventBody { failure, will_retry, duration_ms, + billing, .. } => EventBody::StageFailed(fabro_types::StageFailedProps { index: *index, failure: Some(failure.clone()), will_retry: *will_retry, duration_ms: *duration_ms, + billing: billing.clone(), }), Event::StageRetrying { index, @@ -1187,8 +1189,8 @@ mod tests { use std::collections::BTreeMap; use ::fabro_types::{ - EventBody, FailureReason, ParallelBranchId, Principal, RunNoticeLevel, RunProvenance, - StageId, SystemActorKind, fixtures, run_event as fabro_types, + EventBody, FailureReason, ParallelBranchId, Principal, RunNoticeCode, RunNoticeLevel, + RunProvenance, StageId, SystemActorKind, fixtures, run_event as fabro_types, }; use chrono::Utc; use fabro_agent::{AgentEvent, SandboxEvent}; @@ -1198,7 +1200,7 @@ mod tests { use crate::error::Error; use crate::event::test_support::user_principal; use crate::event::{Event, StageScope}; - use crate::outcome::FailureDetail; + use crate::outcome::{BilledModelUsage, FailureDetail}; #[derive(Debug)] struct EventTestCause; @@ -1220,6 +1222,28 @@ mod tests { } } + fn test_usage(model_id: &str, input_tokens: i64, output_tokens: i64) -> BilledModelUsage { + serde_json::from_value(serde_json::json!({ + "input": { + "usage": { + "model": { + "provider": "openai", + "model_id": model_id + }, + "tokens": { + "input_tokens": input_tokens, + "output_tokens": output_tokens + } + }, + "facts": { + "provider": "open_ai" + } + }, + "total_usd_micros": input_tokens + output_tokens + })) + .unwrap() + } + #[test] fn run_event_stage_completed_places_node_fields_in_header() { let stored = to_run_event_at( @@ -1299,6 +1323,7 @@ mod tests { #[test] fn run_event_stage_failure_keeps_failure_detail() { + let usage = test_usage("gpt-5.2", 321, 54); let stored = to_run_event(&fixtures::RUN_3, &Event::StageFailed { node_id: "code".to_string(), name: "Code".to_string(), @@ -1309,6 +1334,7 @@ mod tests { ), will_retry: true, duration_ms: 5000, + billing: Some(usage.clone()), actor: None, }); @@ -1317,6 +1343,7 @@ mod tests { assert_eq!(properties["failure"]["message"], "lint failed"); assert_eq!(properties["failure"]["failure_class"], "deterministic"); assert_eq!(properties["will_retry"], true); + assert_eq!(properties["billing"], serde_json::to_value(&usage).unwrap()); } #[test] @@ -1636,7 +1663,7 @@ mod tests { fn run_notice_maps_exec_output_tail_to_props() { let stored = to_run_event(&fixtures::RUN_1, &Event::RunNotice { level: RunNoticeLevel::Warn, - code: "git_diff_failed".to_string(), + code: RunNoticeCode::GitDiffFailed.to_string(), message: "git diff failed".to_string(), exec_output_tail: Some(exec_tail()), }); diff --git a/lib/crates/fabro-workflow/src/event/emitter.rs b/lib/crates/fabro-workflow/src/event/emitter.rs index a1bba727a..dc50a56dd 100644 --- a/lib/crates/fabro-workflow/src/event/emitter.rs +++ b/lib/crates/fabro-workflow/src/event/emitter.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use std::sync::atomic::{AtomicI64, Ordering}; -use ::fabro_types::{ExecOutputTail, RunEvent, RunId, RunNoticeLevel}; +use ::fabro_types::{ExecOutputTail, RunEvent, RunId, RunNoticeCode, RunNoticeLevel}; use chrono::Utc; use fabro_agent::{WorktreeEvent, WorktreeEventCallback}; @@ -76,15 +76,10 @@ impl Emitter { self.emit_with_scope(event, Some(scope)); } - pub fn notice( - &self, - level: RunNoticeLevel, - code: impl Into, - message: impl Into, - ) { + pub fn notice(&self, level: RunNoticeLevel, code: RunNoticeCode, message: impl Into) { self.emit(&Event::RunNotice { level, - code: code.into(), + code: code.to_string(), message: message.into(), exec_output_tail: None, }); @@ -93,13 +88,13 @@ impl Emitter { pub fn notice_with_tail( &self, level: RunNoticeLevel, - code: impl Into, + code: RunNoticeCode, message: impl Into, exec_output_tail: Option, ) { self.emit(&Event::RunNotice { level, - code: code.into(), + code: code.to_string(), message: message.into(), exec_output_tail, }); diff --git a/lib/crates/fabro-workflow/src/event/events.rs b/lib/crates/fabro-workflow/src/event/events.rs index eb6956ec5..82cddd44f 100644 --- a/lib/crates/fabro-workflow/src/event/events.rs +++ b/lib/crates/fabro-workflow/src/event/events.rs @@ -206,6 +206,7 @@ pub enum Event { failure: FailureDetail, will_retry: bool, duration_ms: u64, + billing: Option, #[serde(default, skip_serializing_if = "Option::is_none")] actor: Option, }, diff --git a/lib/crates/fabro-workflow/src/handler/agent.rs b/lib/crates/fabro-workflow/src/handler/agent.rs index eaebea7ee..d66ef8ed6 100644 --- a/lib/crates/fabro-workflow/src/handler/agent.rs +++ b/lib/crates/fabro-workflow/src/handler/agent.rs @@ -54,6 +54,8 @@ pub trait CodergenBackend: Send + Sync { _node: &Node, _prompt: &str, _system_prompt: Option<&str>, + _emitter: &Arc, + _stage_scope: &StageScope, ) -> Result { Err(Error::Validation( "one_shot mode not supported by this backend".into(), diff --git a/lib/crates/fabro-workflow/src/handler/llm/api.rs b/lib/crates/fabro-workflow/src/handler/llm/api.rs index d7d11d3cb..cb99d2344 100644 --- a/lib/crates/fabro-workflow/src/handler/llm/api.rs +++ b/lib/crates/fabro-workflow/src/handler/llm/api.rs @@ -379,6 +379,8 @@ impl CodergenBackend for AgentApiBackend { node: &Node, prompt: &str, system_prompt: Option<&str>, + emitter: &Arc, + stage_scope: &StageScope, ) -> Result { let client = Client::from_source(self.source.as_ref()) .await @@ -452,14 +454,16 @@ impl CodergenBackend for AgentApiBackend { let mut found = None; for target in fallback_chain { - tracing::warn!( - stage = node.id.as_str(), - from_provider = from_provider.as_str(), - from_model = from_model.as_str(), - to_provider = target.provider.as_str(), - to_model = target.model.as_str(), - error = error_msg.as_str(), - "LLM provider failover (prompt)" + emitter.emit_scoped( + &Event::Failover { + stage: node.id.clone(), + from_provider: from_provider.clone(), + from_model: from_model.clone(), + to_provider: target.provider.clone(), + to_model: target.model.clone(), + error: error_msg.clone(), + }, + stage_scope, ); let max_tokens = node.max_tokens().or_else(|| { diff --git a/lib/crates/fabro-workflow/src/handler/llm/cli.rs b/lib/crates/fabro-workflow/src/handler/llm/cli.rs index b692f6300..90832063a 100644 --- a/lib/crates/fabro-workflow/src/handler/llm/cli.rs +++ b/lib/crates/fabro-workflow/src/handler/llm/cli.rs @@ -901,9 +901,13 @@ impl CodergenBackend for BackendRouter { node: &Node, prompt: &str, system_prompt: Option<&str>, + emitter: &Arc, + stage_scope: &StageScope, ) -> Result { // CLI backend doesn't support one_shot, always route to API - self.api_backend.one_shot(node, prompt, system_prompt).await + self.api_backend + .one_shot(node, prompt, system_prompt, emitter, stage_scope) + .await } } diff --git a/lib/crates/fabro-workflow/src/handler/parallel.rs b/lib/crates/fabro-workflow/src/handler/parallel.rs index 57323b82d..dcf6de747 100644 --- a/lib/crates/fabro-workflow/src/handler/parallel.rs +++ b/lib/crates/fabro-workflow/src/handler/parallel.rs @@ -12,7 +12,7 @@ use tokio::sync::Semaphore; use super::{EngineServices, Handler}; use crate::context::{Context, WorkflowContext, keys}; use crate::error::Error; -use crate::event::{Event, StageScope}; +use crate::event::{Event, RunNoticeCode, RunNoticeLevel, StageScope}; use crate::git::sanitize_ref_component; use crate::hook_context::set_hook_node; use crate::millis_u64; @@ -207,6 +207,12 @@ impl Handler for ParallelHandler { error = %fabro_sandbox::display_for_log(&e), "parallel base checkpoint failed" ); + services.run.emitter.notice_with_tail( + RunNoticeLevel::Warn, + RunNoticeCode::ParallelBaseCheckpointFailed, + format!("Could not checkpoint base state before parallel branches: {e}"), + fabro_sandbox::default_redacted_output_tail(&e), + ); None } } diff --git a/lib/crates/fabro-workflow/src/handler/prompt.rs b/lib/crates/fabro-workflow/src/handler/prompt.rs index 5d241923e..50dc04e06 100644 --- a/lib/crates/fabro-workflow/src/handler/prompt.rs +++ b/lib/crates/fabro-workflow/src/handler/prompt.rs @@ -113,7 +113,13 @@ impl Handler for PromptHandler { let (response_text, stage_usage, backend_files_touched) = if let Some(backend) = &self.backend { let result = backend - .one_shot(node, &prompt, system_prompt.as_deref()) + .one_shot( + node, + &prompt, + system_prompt.as_deref(), + &services.run.emitter, + &stage_scope, + ) .await; match result { Ok(CodergenResult::Full(outcome)) => return Ok(outcome), @@ -197,6 +203,7 @@ mod tests { use tokio_util::sync::CancellationToken; use super::*; + use crate::event::Emitter; fn make_services() -> EngineServices { EngineServices::test_default() @@ -221,7 +228,7 @@ mod tests { let mut services = EngineServices::test_default(); services.run = services .run - .with_emitter(Arc::new(crate::event::Emitter::new(fixtures::RUN_1))) + .with_emitter(Arc::new(Emitter::new(fixtures::RUN_1))) .with_run_store(run_store.clone().into()); let logger = crate::event::StoreProgressLogger::new(run_store.clone()); logger.register(services.run.emitter.as_ref()); @@ -277,7 +284,7 @@ mod tests { _prompt: &str, _context: &Context, _thread_id: Option<&str>, - _emitter: &Arc, + _emitter: &Arc, _sandbox: &Arc, _tool_hooks: Option>, _cancel_token: CancellationToken, @@ -290,6 +297,8 @@ mod tests { _node: &Node, _prompt: &str, _system_prompt: Option<&str>, + _emitter: &Arc, + _stage_scope: &StageScope, ) -> Result { Ok(CodergenResult::Text { text: "one-shot response".to_string(), @@ -338,7 +347,7 @@ mod tests { _prompt: &str, _context: &Context, _thread_id: Option<&str>, - _emitter: &Arc, + _emitter: &Arc, _sandbox: &Arc, _tool_hooks: Option>, _cancel_token: CancellationToken, @@ -351,6 +360,8 @@ mod tests { _node: &Node, _prompt: &str, _system_prompt: Option<&str>, + _emitter: &Arc, + _stage_scope: &StageScope, ) -> Result { Ok(CodergenResult::Text { text: "one-shot response".to_string(), @@ -396,7 +407,7 @@ mod tests { _prompt: &str, _context: &Context, _thread_id: Option<&str>, - _emitter: &Arc, + _emitter: &Arc, _sandbox: &Arc, _tool_hooks: Option>, _cancel_token: CancellationToken, @@ -409,6 +420,8 @@ mod tests { _node: &Node, prompt: &str, system_prompt: Option<&str>, + _emitter: &Arc, + _stage_scope: &StageScope, ) -> Result { *self.captured_prompt.lock().unwrap() = Some(prompt.to_string()); *self.captured_system_prompt.lock().unwrap() = Some(system_prompt.map(String::from)); diff --git a/lib/crates/fabro-workflow/src/lib.rs b/lib/crates/fabro-workflow/src/lib.rs index 9af074d97..5478e611d 100644 --- a/lib/crates/fabro-workflow/src/lib.rs +++ b/lib/crates/fabro-workflow/src/lib.rs @@ -20,6 +20,7 @@ use std::sync::Arc; use fabro_retro::retro::CompletedStage; use fabro_store::EventEnvelope; +use fabro_types::{EventBody, StageId}; /// Callback invoked when a workflow node starts executing. pub type OnNodeCallback = Option>; @@ -86,34 +87,190 @@ pub fn build_completed_stages(cp: &records::Checkpoint, run_failed: bool) -> Vec stages } -pub fn extract_stage_durations_from_events(events: &[EventEnvelope]) -> HashMap { +/// Extract the `duration_ms` from a `stage.completed` / `stage.failed` +/// event body, or `None` for any other variant. +fn stage_completion_duration_ms(body: &EventBody) -> Option { + match body { + EventBody::StageCompleted(props) => Some(props.duration_ms), + EventBody::StageFailed(props) => Some(props.duration_ms), + _ => None, + } +} + +/// Extract per-stage (node_id, visit) durations from `stage.completed` / +/// `stage.failed` events. Keys on the full [`StageId`] so multi-visit stages +/// (e.g. a looped `verify` node) keep distinct durations. +/// +/// This is the canonical primitive; [`total_stage_duration_by_node`] and +/// [`latest_stage_duration_by_node`] are explicit rollups built on top of it. +pub fn extract_stage_durations_by_stage_id(events: &[EventEnvelope]) -> HashMap { let mut durations = HashMap::new(); for envelope in events { - let event = &envelope.event; - let event_name = event.event_name(); - if event_name != "stage.completed" && event_name != "stage.failed" { - continue; - } - let Some(node_id) = event.node_id.as_deref() else { + let Some(duration_ms) = stage_completion_duration_ms(&envelope.event.body) else { continue; }; - let Some(duration_ms) = event - .properties() - .ok() - .and_then(|properties| properties.get("duration_ms").cloned()) - .and_then(|duration| duration.as_u64()) - else { + let Some(stage_id) = envelope.event.stage_id.as_ref() else { continue; }; - durations.insert(node_id.to_string(), duration_ms); + durations.insert(stage_id.clone(), duration_ms); } durations } +/// Total duration spent in each node, summed across every visit. Use for +/// billing/usage where a retried node should count its full time. +pub fn total_stage_duration_by_node(events: &[EventEnvelope]) -> HashMap { + let mut totals: HashMap = HashMap::new(); + for (stage_id, duration_ms) in extract_stage_durations_by_stage_id(events) { + *totals.entry(stage_id.node_id().to_string()).or_default() += duration_ms; + } + totals +} + +/// Duration of each node's most recent visit (the highest visit number). Use +/// for run summaries and retros where the table shows one row per node and +/// "the last attempt" is the right representative. +pub fn latest_stage_duration_by_node(events: &[EventEnvelope]) -> HashMap { + let mut entries: Vec<(StageId, u64)> = extract_stage_durations_by_stage_id(events) + .into_iter() + .collect(); + entries.sort_by_key(|(stage_id, _)| stage_id.visit()); + let mut latest = HashMap::new(); + for (stage_id, duration_ms) in entries { + latest.insert(stage_id.node_id().to_string(), duration_ms); + } + latest +} + +#[cfg(test)] +mod duration_tests { + use chrono::{TimeZone, Utc}; + use fabro_store::EventEnvelope; + use fabro_types::run_event::{StageCompletedProps, StageFailedProps}; + use fabro_types::{EventBody, RunEvent, StageId, StageOutcome, fixtures}; + + use super::{ + extract_stage_durations_by_stage_id, latest_stage_duration_by_node, + total_stage_duration_by_node, + }; + + fn completed_event(seq: u32, node: &str, visit: u32, duration_ms: u64) -> EventEnvelope { + let event = RunEvent { + id: format!("evt_{seq}"), + ts: Utc.with_ymd_and_hms(2026, 1, 1, 0, 0, 0).unwrap(), + run_id: fixtures::RUN_1, + node_id: Some(node.to_string()), + node_label: None, + stage_id: Some(StageId::new(node, visit)), + parallel_group_id: None, + parallel_branch_id: None, + session_id: None, + parent_session_id: None, + tool_call_id: None, + actor: None, + body: EventBody::StageCompleted(StageCompletedProps { + index: 0, + duration_ms, + status: StageOutcome::Succeeded, + preferred_label: None, + suggested_next_ids: vec![], + billing: None, + failure: None, + notes: None, + files_touched: vec![], + context_updates: None, + jump_to_node: None, + context_values: None, + node_visits: None, + loop_failure_signatures: None, + restart_failure_signatures: None, + response: None, + attempt: 1, + max_attempts: 1, + }), + }; + EventEnvelope { seq, event } + } + + fn failed_event(seq: u32, node: &str, visit: u32, duration_ms: u64) -> EventEnvelope { + let event = RunEvent { + id: format!("evt_{seq}"), + ts: Utc.with_ymd_and_hms(2026, 1, 1, 0, 0, 0).unwrap(), + run_id: fixtures::RUN_1, + node_id: Some(node.to_string()), + node_label: None, + stage_id: Some(StageId::new(node, visit)), + parallel_group_id: None, + parallel_branch_id: None, + session_id: None, + parent_session_id: None, + tool_call_id: None, + actor: None, + body: EventBody::StageFailed(StageFailedProps { + index: 0, + failure: None, + will_retry: true, + duration_ms, + billing: None, + }), + }; + EventEnvelope { seq, event } + } + + #[test] + fn extract_keys_durations_by_full_stage_id() { + let events = vec![ + completed_event(1, "verify", 1, 100), + completed_event(2, "verify", 2, 200), + ]; + let durations = extract_stage_durations_by_stage_id(&events); + assert_eq!( + durations.get(&StageId::new("verify", 1)).copied(), + Some(100) + ); + assert_eq!( + durations.get(&StageId::new("verify", 2)).copied(), + Some(200) + ); + } + + #[test] + fn total_sums_across_visits_per_node() { + let events = vec![ + completed_event(1, "verify", 1, 100), + completed_event(2, "verify", 2, 200), + completed_event(3, "build", 1, 50), + ]; + let totals = total_stage_duration_by_node(&events); + assert_eq!(totals.get("verify").copied(), Some(300)); + assert_eq!(totals.get("build").copied(), Some(50)); + } + + #[test] + fn latest_picks_highest_visit_regardless_of_input_order() { + // Visit 2 appears in the events vector before visit 1; the result + // must still reflect visit 2's duration (the latest visit). + let events = vec![ + completed_event(1, "verify", 2, 999), + completed_event(2, "verify", 1, 100), + ]; + let latest = latest_stage_duration_by_node(&events); + assert_eq!(latest.get("verify").copied(), Some(999)); + } + + #[test] + fn stage_failed_durations_are_included() { + let events = vec![failed_event(1, "verify", 1, 75)]; + let durations = extract_stage_durations_by_stage_id(&events); + assert_eq!(durations.get(&StageId::new("verify", 1)).copied(), Some(75)); + } +} + #[doc(hidden)] pub mod artifact; pub mod artifact_snapshot; pub mod artifact_upload; +pub mod billing_rollup; pub mod command_log; pub(crate) mod condition; pub mod context; @@ -142,6 +299,10 @@ pub mod run_control; pub(crate) mod run_dir; pub mod run_lookup; +pub use billing_rollup::{ + ProjectionBillingByModel, ProjectionBillingRollup, ProjectionBillingStage, + billing_rollup_from_projection, +}; pub use error::{Error, FailureCategory, FailureSignature, FailureSignatureExt, Result}; pub use manifest_path::ManifestPath; pub mod run_materialization; diff --git a/lib/crates/fabro-workflow/src/lifecycle/artifact.rs b/lib/crates/fabro-workflow/src/lifecycle/artifact.rs index f65fcee86..50d6ccded 100644 --- a/lib/crates/fabro-workflow/src/lifecycle/artifact.rs +++ b/lib/crates/fabro-workflow/src/lifecycle/artifact.rs @@ -16,7 +16,7 @@ use tokio::time::sleep; use crate::artifact::{normalize_durable_updates, offload_large_values, sync_artifacts_to_env}; use crate::artifact_snapshot::collect_artifacts; use crate::artifact_upload::ArtifactSink; -use crate::event::{Emitter, Event, RunNoticeLevel}; +use crate::event::{Emitter, Event, RunNoticeCode, RunNoticeLevel}; use crate::graph::{WorkflowGraph, WorkflowNode}; use crate::lifecycle::event::{stage_scope_for, stage_visit}; use crate::outcome::BilledModelUsage; @@ -125,7 +125,7 @@ impl RunLifecycle for ArtifactLifecycle { { self.emitter.notice( RunNoticeLevel::Warn, - "artifact_upload_failed", + RunNoticeCode::ArtifactUploadFailed, format!("[node: {node_id}] artifact upload failed: {err}"), ); return Ok(()); @@ -151,7 +151,7 @@ impl RunLifecycle for ArtifactLifecycle { Err(e) => { self.emitter.notice( RunNoticeLevel::Warn, - "artifact_collection_failed", + RunNoticeCode::ArtifactCollectionFailed, format!("[node: {node_id}] artifact collection failed: {e}"), ); } @@ -174,7 +174,7 @@ impl RunLifecycle for ArtifactLifecycle { { self.emitter.notice( RunNoticeLevel::Warn, - "artifact_offload_failed", + RunNoticeCode::ArtifactOffloadFailed, format!("[node: {node_id}] artifact offload failed: {e}"), ); } @@ -187,7 +187,7 @@ impl RunLifecycle for ArtifactLifecycle { { self.emitter.notice( RunNoticeLevel::Warn, - "artifact_sync_failed", + RunNoticeCode::ArtifactSyncFailed, format!("[node: {node_id}] artifact sync failed: {e}"), ); } diff --git a/lib/crates/fabro-workflow/src/lifecycle/event.rs b/lib/crates/fabro-workflow/src/lifecycle/event.rs index 482f0311a..82dbd0501 100644 --- a/lib/crates/fabro-workflow/src/lifecycle/event.rs +++ b/lib/crates/fabro-workflow/src/lifecycle/event.rs @@ -224,6 +224,7 @@ impl RunLifecycle for EventLifecycle { failure, will_retry: true, duration_ms, + billing: outcome.usage.clone(), actor, }, &scope, @@ -275,6 +276,7 @@ impl RunLifecycle for EventLifecycle { failure, will_retry: false, duration_ms, + billing: outcome.usage.clone(), actor, }, &scope, diff --git a/lib/crates/fabro-workflow/src/lifecycle/git.rs b/lib/crates/fabro-workflow/src/lifecycle/git.rs index e8d43d4fe..840894756 100644 --- a/lib/crates/fabro-workflow/src/lifecycle/git.rs +++ b/lib/crates/fabro-workflow/src/lifecycle/git.rs @@ -14,7 +14,7 @@ use fabro_util::error::collect_causes; use fabro_util::time::elapsed_ms; use crate::artifact; -use crate::event::{Emitter, Event, RunNoticeLevel, StageScope}; +use crate::event::{Emitter, Event, RunNoticeCode, RunNoticeLevel, StageScope}; use crate::graph::{WorkflowGraph, WorkflowNode}; use crate::lifecycle::event::stage_scope_for; use crate::outcome::BilledModelUsage; @@ -128,7 +128,10 @@ impl RunLifecycle for GitLifecycle { None, None, ); - self.emit_metadata_warning("checkpoint_metadata_write_failed", message); + self.emit_metadata_warning( + RunNoticeCode::CheckpointMetadataWriteFailed, + message, + ); } }, Err(err) => { @@ -145,7 +148,10 @@ impl RunLifecycle for GitLifecycle { None, None, ); - self.emit_metadata_warning("checkpoint_metadata_write_failed", message); + self.emit_metadata_warning( + RunNoticeCode::CheckpointMetadataWriteFailed, + message, + ); } } } @@ -217,7 +223,7 @@ impl RunLifecycle for GitLifecycle { Some(&scope), ); self.emit_metadata_warning( - "checkpoint_metadata_write_failed", + RunNoticeCode::CheckpointMetadataWriteFailed, message, ); None @@ -239,7 +245,10 @@ impl RunLifecycle for GitLifecycle { None, Some(&scope), ); - self.emit_metadata_warning("checkpoint_metadata_write_failed", message); + self.emit_metadata_warning( + RunNoticeCode::CheckpointMetadataWriteFailed, + message, + ); None } } @@ -292,6 +301,12 @@ impl RunLifecycle for GitLifecycle { error = %fabro_sandbox::display_for_log(&err), "git push from run lifecycle failed" ); + self.emitter.notice_with_tail( + RunNoticeLevel::Warn, + RunNoticeCode::GitPushFailed, + format!("Failed to push run branch {branch}: {err}"), + exec_output_tail.clone(), + ); (false, exec_output_tail) } }; @@ -321,7 +336,7 @@ impl RunLifecycle for GitLifecycle { fabro_sandbox::default_redacted_output_tail(&err); self.emitter.notice_with_tail( RunNoticeLevel::Warn, - "git_diff_failed", + RunNoticeCode::GitDiffFailed, format!("[node: {node_id}] git diff failed: {err}"), exec_output_tail, ); @@ -395,7 +410,10 @@ impl GitLifecycle { Some(snapshot.bytes), scope, ); - self.emit_metadata_warning("checkpoint_metadata_push_failed", message); + self.emit_metadata_warning( + RunNoticeCode::CheckpointMetadataPushFailed, + message, + ); } else { self.emit_metadata_snapshot_completed( phase, @@ -421,7 +439,7 @@ impl GitLifecycle { None, scope, ); - self.emit_metadata_warning("checkpoint_metadata_write_failed", message); + self.emit_metadata_warning(RunNoticeCode::CheckpointMetadataWriteFailed, message); None } } @@ -506,14 +524,9 @@ impl GitLifecycle { } } - fn emit_metadata_warning(&self, code: &str, message: String) { + fn emit_metadata_warning(&self, code: RunNoticeCode, message: String) { if self.metadata_runtime.mark_metadata_degraded() { - self.emitter.emit(&Event::RunNotice { - level: RunNoticeLevel::Warn, - code: code.to_string(), - message, - exec_output_tail: None, - }); + self.emitter.notice(RunNoticeLevel::Warn, code, message); } } } diff --git a/lib/crates/fabro-workflow/src/pipeline/finalize.rs b/lib/crates/fabro-workflow/src/pipeline/finalize.rs index 6e71ca4ca..a1cc84d19 100644 --- a/lib/crates/fabro-workflow/src/pipeline/finalize.rs +++ b/lib/crates/fabro-workflow/src/pipeline/finalize.rs @@ -10,7 +10,7 @@ use fabro_util::time::elapsed_ms; use super::types::{Concluded, FinalizeOptions, Retroed}; use crate::error::Error; -use crate::event::{Event, RunNoticeLevel}; +use crate::event::{Event, RunNoticeCode, RunNoticeLevel}; use crate::outcome::{Outcome, OutcomeExt, StageOutcome}; use crate::records::{Checkpoint, Conclusion, StageSummary}; use crate::run_metadata::MetadataSnapshot; @@ -19,6 +19,7 @@ use crate::run_status::{FailureReason, RunStatus, SuccessReason}; use crate::runtime_store::RunStoreHandle; use crate::sandbox_git::git_diff_with_timeout; use crate::services::RunServices; +use crate::{ProjectionBillingRollup, billing_rollup_from_projection}; pub fn classify_engine_result( engine_result: &Result, @@ -68,22 +69,22 @@ pub(crate) async fn build_conclusion_from_store( run_duration_ms: u64, final_git_commit_sha: Option, ) -> Conclusion { - let (state_result, events_result) = tokio::join!(run_store.state(), run_store.list_events()); - let projection = state_result.ok(); + let projection = run_store.state().await.ok(); let projection_order = projection .as_ref() .map(stage_projection_order) .unwrap_or_default(); + let projection_billing = projection + .as_ref() + .map(billing_rollup_from_projection) + .unwrap_or_default(); let checkpoint = projection .as_ref() .and_then(|state| state.checkpoint.as_ref()); - let stage_durations = events_result - .map(|events| crate::extract_stage_durations_from_events(&events)) - .unwrap_or_default(); build_conclusion_from_parts( checkpoint, - &stage_durations, + &projection_billing, &projection_order, status, failure_reason, @@ -94,7 +95,7 @@ pub(crate) async fn build_conclusion_from_store( fn build_conclusion_from_parts( checkpoint: Option<&Checkpoint>, - stage_durations: &HashMap, + projection_billing: &ProjectionBillingRollup, projection_order: &HashMap, status: StageOutcome, failure_reason: Option, @@ -105,6 +106,11 @@ fn build_conclusion_from_parts( // while the other checkpoint maps are keyed by node_id. Dedupe to one row // per node so the stages table matches the deduped billing total. let (stages, total_retries) = if let Some(cp) = checkpoint { + let billing_by_node = projection_billing + .stages + .iter() + .map(|stage| (stage.node_id.as_str(), stage)) + .collect::>(); let mut stage_rows = Vec::new(); let mut seen = std::collections::HashSet::new(); let mut retries_sum: u32 = 0; @@ -130,7 +136,6 @@ fn build_conclusion_from_parts( } for (original_checkpoint_order, node_id) in stage_order { - let outcome = cp.node_outcomes.get(node_id); let retries = cp .node_retries .get(node_id) @@ -138,14 +143,13 @@ fn build_conclusion_from_parts( .unwrap_or(1) .saturating_sub(1); retries_sum += retries; + let billing = billing_by_node.get(node_id); let summary = StageSummary { stage_id: node_id.to_string(), stage_label: node_id.to_string(), - duration_ms: stage_durations.get(node_id).copied().unwrap_or(0), - billing_usd_micros: outcome - .and_then(|o| o.usage.as_ref()) - .and_then(|usage| usage.total_usd_micros), + duration_ms: billing.map_or(0, |stage| stage.duration_ms), + billing_usd_micros: billing.and_then(|stage| stage.billing.total_usd_micros), retries, }; stage_rows.push(( @@ -176,7 +180,7 @@ fn build_conclusion_from_parts( failure_reason, final_git_commit_sha, stages, - billing: checkpoint.and_then(billing_from_checkpoint), + billing: projection_billing.billing_if_present(), total_retries, } } @@ -235,7 +239,11 @@ pub async fn write_finalize_commit( None, None, ); - emit_metadata_warning(services, "checkpoint_metadata_write_failed", message); + emit_metadata_warning( + services, + RunNoticeCode::CheckpointMetadataWriteFailed, + message, + ); return; } }; @@ -256,7 +264,11 @@ pub async fn write_finalize_commit( None, None, ); - emit_metadata_warning(services, "checkpoint_metadata_write_failed", message); + emit_metadata_warning( + services, + RunNoticeCode::CheckpointMetadataWriteFailed, + message, + ); return; } }; @@ -277,7 +289,11 @@ pub async fn write_finalize_commit( Some(snapshot.entry_count), Some(snapshot.bytes), ); - emit_metadata_warning(services, "checkpoint_metadata_push_failed", message); + emit_metadata_warning( + services, + RunNoticeCode::CheckpointMetadataPushFailed, + message, + ); } else { emit_metadata_snapshot_completed(services, phase, meta_branch, started, &snapshot); } @@ -296,7 +312,11 @@ pub async fn write_finalize_commit( None, None, ); - emit_metadata_warning(services, "checkpoint_metadata_write_failed", message); + emit_metadata_warning( + services, + RunNoticeCode::CheckpointMetadataWriteFailed, + message, + ); } } } @@ -359,7 +379,7 @@ fn emit_metadata_snapshot_failed( }); } -fn emit_metadata_warning(services: &RunServices, code: &str, message: String) { +fn emit_metadata_warning(services: &RunServices, code: RunNoticeCode, message: String) { if services.metadata_runtime.mark_metadata_degraded() { services.emitter.notice(RunNoticeLevel::Warn, code, message); } @@ -383,7 +403,7 @@ async fn compute_final_patch( Err(err) => { services.emitter.notice( RunNoticeLevel::Warn, - "git_diff_failed", + RunNoticeCode::GitDiffFailed, format!("final diff failed: {err}"), ); None @@ -391,15 +411,8 @@ async fn compute_final_patch( } } -/// Iterates `node_outcomes.values()` rather than `completed_nodes` to avoid -/// over-counting the last visit's usage on looping workflows. -pub(crate) fn billing_from_checkpoint(cp: &Checkpoint) -> Option { - let usage: Vec<_> = cp - .node_outcomes - .values() - .filter_map(|o| o.usage.clone()) - .collect(); - (!usage.is_empty()).then(|| BilledTokenCounts::from_billed_usage(&usage)) +pub(crate) fn billing_from_projection(projection: &RunProjection) -> Option { + billing_rollup_from_projection(projection).billing_if_present() } pub(crate) fn build_terminal_event( @@ -503,7 +516,6 @@ pub async fn finalize(retroed: Retroed, options: &FinalizeOptions) -> Result Result Result Result Result BilledModelUsage { + serde_json::from_value(serde_json::json!({ + "input": { + "usage": { + "model": { + "provider": "openai", + "model_id": model_id + }, + "tokens": { + "input_tokens": input_tokens, + "output_tokens": output_tokens + } + }, + "facts": { + "provider": "open_ai" + } + }, + "total_usd_micros": input_tokens + output_tokens + })) + .unwrap() + } + #[test] fn conclusion_stage_order_follows_projection_first_event_order() { let mut projection = RunProjection::default(); @@ -753,7 +794,7 @@ mod tests { let conclusion = build_conclusion_from_parts( Some(&checkpoint), - &HashMap::new(), + &ProjectionBillingRollup::default(), &projection_order, StageOutcome::Succeeded, None, @@ -788,7 +829,7 @@ mod tests { let conclusion = build_conclusion_from_parts( Some(&checkpoint), - &HashMap::new(), + &ProjectionBillingRollup::default(), &projection_order, StageOutcome::Succeeded, None, @@ -804,6 +845,66 @@ mod tests { assert_eq!(stage_ids, vec!["skipped", "finished"]); } + #[test] + fn conclusion_billing_sums_retry_visit_usage_from_projection() { + let mut projection = RunProjection::default(); + let failed_usage = test_usage("gpt-old", 100, 10); + let success_usage = test_usage("gpt-new", 200, 20); + let failed = projection.stage_entry("verify", 1, first_event_seq(1)); + failed.duration_ms = Some(1200); + failed.usage = Some(failed_usage); + failed.completion = Some(StageCompletion { + outcome: StageOutcome::Failed { + retry_requested: true, + }, + notes: None, + failure_reason: Some("try again".to_string()), + timestamp: chrono::Utc::now(), + }); + let succeeded = projection.stage_entry("verify", 2, first_event_seq(2)); + succeeded.duration_ms = Some(800); + succeeded.usage = Some(success_usage.clone()); + succeeded.completion = Some(StageCompletion { + outcome: StageOutcome::Succeeded, + notes: None, + failure_reason: None, + timestamp: chrono::Utc::now(), + }); + + let projection_order = stage_projection_order(&projection); + let projection_billing = billing_rollup_from_projection(&projection); + let mut latest_outcome = Outcome::success(); + latest_outcome.usage = Some(success_usage); + latest_outcome.duration_ms = Some(800); + let mut checkpoint = checkpoint_with( + vec!["verify", "verify"], + HashMap::from([("verify".to_string(), latest_outcome)]), + ); + checkpoint.node_retries.insert("verify".to_string(), 2); + + let conclusion = build_conclusion_from_parts( + Some(&checkpoint), + &projection_billing, + &projection_order, + StageOutcome::Succeeded, + None, + 10, + None, + ); + + assert_eq!(conclusion.billing.as_ref().unwrap().input_tokens, 300); + assert_eq!(conclusion.billing.as_ref().unwrap().output_tokens, 30); + assert_eq!( + conclusion.billing.as_ref().unwrap().total_usd_micros, + Some(330) + ); + assert_eq!(conclusion.stages.len(), 1); + assert_eq!(conclusion.stages[0].stage_id, "verify"); + assert_eq!(conclusion.stages[0].duration_ms, 2000); + assert_eq!(conclusion.stages[0].billing_usd_micros, Some(330)); + assert_eq!(conclusion.stages[0].retries, 1); + } + fn test_services( run_store: RunStoreHandle, emitter: Arc, diff --git a/lib/crates/fabro-workflow/src/pipeline/initialize.rs b/lib/crates/fabro-workflow/src/pipeline/initialize.rs index 6f035592e..81042bab9 100644 --- a/lib/crates/fabro-workflow/src/pipeline/initialize.rs +++ b/lib/crates/fabro-workflow/src/pipeline/initialize.rs @@ -27,7 +27,7 @@ use tokio::time::timeout as tokio_timeout; use super::types::{InitOptions, Initialized, LlmSpec, Persisted, SandboxEnvSpec}; use crate::devcontainer_bridge::{devcontainer_to_snapshot_config, run_devcontainer_lifecycle}; use crate::error::Error; -use crate::event::{Emitter, Event, RunNoticeLevel}; +use crate::event::{Emitter, Event, RunNoticeCode, RunNoticeLevel}; use crate::git::RUN_BRANCH_PREFIX; use crate::handler::llm::{AgentApiBackend, AgentCliBackend, BackendRouter}; use crate::handler::{HandlerRegistry, default_registry}; @@ -155,7 +155,7 @@ fn resolve_worktree_plan(options: &mut InitOptions) -> Option { if let Some(env_name) = env_name { options.emitter.notice( RunNoticeLevel::Warn, - "dirty_worktree", + RunNoticeCode::DirtyWorktree, format!("Uncommitted changes will not be included in the {env_name}."), ); } @@ -200,6 +200,14 @@ fn resolve_worktree_plan(options: &mut InitOptions) -> Option { }) } +fn worktree_skipped_notice(mode: Option) -> Option<(RunNoticeCode, &'static str)> { + matches!(mode, Some(WorktreeMode::Always)).then_some(( + RunNoticeCode::WorktreeSkippedNoGit, + "Worktree mode `always` requested but no Git repository was found; running without a \ + worktree.", + )) +} + fn git_setup_intent(run_options: &RunOptions) -> GitSetupIntent { if let Some(source) = run_options.fork_source_ref.as_ref() { GitSetupIntent::ForkFromCheckpoint { @@ -239,11 +247,14 @@ async fn build_sandbox_env( Ok(token) => { env.insert("GITHUB_TOKEN".to_string(), token); } - Err(e) => emitter.notice( - RunNoticeLevel::Warn, - "github_token_failed", - format!("Failed to mint GitHub token: {e}"), - ), + Err(e) => { + tracing::warn!(error = %e, "Failed to mint GitHub token"); + emitter.notice( + RunNoticeLevel::Warn, + RunNoticeCode::GithubTokenFailed, + format!("Failed to mint GitHub token: {e}"), + ); + } } } } @@ -516,6 +527,13 @@ pub async fn initialize( )) }; if worktree_plan.is_some() && !worktree_created { + if let Some((code, message)) = worktree_skipped_notice(options.worktree_mode) { + tracing::warn!( + worktree_mode = ?options.worktree_mode, + "worktree skipped: cwd is not a git repository" + ); + options.emitter.notice(RunNoticeLevel::Warn, code, message); + } options.run_options.git = None; } let cleanup_guard = scopeguard::guard(Arc::clone(&sandbox), |sandbox| { @@ -593,7 +611,8 @@ pub async fn initialize( .is_some(); if !has_run_branch { let intent = git_setup_intent(&options.run_options); - if sandbox.origin_url().is_some() { + let sandbox_has_origin = sandbox.origin_url().is_some(); + if sandbox_has_origin { sandbox_git .ensure_git_available(&*sandbox) .await @@ -619,7 +638,16 @@ pub async fn initialize( options.run_options.base_branch = info.base_branch; } } - Ok(None) => {} + Ok(None) => { + if sandbox_has_origin { + options.emitter.notice( + RunNoticeLevel::Warn, + RunNoticeCode::SandboxGitUnavailable, + "Sandbox could not set up Git despite a configured origin; running \ + without checkpointing or PR support.", + ); + } + } Err(e) => { return Err(Error::engine_with_source("Sandbox git setup failed", &e)); } @@ -700,7 +728,7 @@ pub async fn initialize( if metadata_runtime.mark_metadata_degraded() { options.emitter.notice( RunNoticeLevel::Warn, - "checkpoint_metadata_write_failed", + RunNoticeCode::CheckpointMetadataWriteFailed, message, ); } @@ -1008,6 +1036,17 @@ mod tests { assert!(options.run_options.git.is_none()); } + #[test] + fn worktree_skipped_notice_only_warns_for_always() { + assert!(worktree_skipped_notice(None).is_none()); + assert!(worktree_skipped_notice(Some(WorktreeMode::Clean)).is_none()); + assert!(worktree_skipped_notice(Some(WorktreeMode::Dirty)).is_none()); + assert!(worktree_skipped_notice(Some(WorktreeMode::Never)).is_none()); + + let (code, _) = worktree_skipped_notice(Some(WorktreeMode::Always)).unwrap(); + assert_eq!(code, RunNoticeCode::WorktreeSkippedNoGit); + } + #[tokio::test] async fn initialize_prepares_sandbox_and_uses_persisted_run_dir() { let temp = tempfile::tempdir().unwrap(); diff --git a/lib/crates/fabro-workflow/src/pipeline/mod.rs b/lib/crates/fabro-workflow/src/pipeline/mod.rs index 71aefbef9..424dd69bd 100644 --- a/lib/crates/fabro-workflow/src/pipeline/mod.rs +++ b/lib/crates/fabro-workflow/src/pipeline/mod.rs @@ -12,14 +12,15 @@ mod validate; pub use execute::execute; pub use fabro_types::PullRequestRecord; pub(crate) use finalize::{ - billing_from_checkpoint, build_conclusion_from_store, build_terminal_event, + billing_from_projection, build_conclusion_from_store, build_terminal_event, }; pub use finalize::{classify_engine_result, finalize, write_finalize_commit}; pub use initialize::initialize; pub use parse::parse; pub(crate) use persist::persist; pub use pull_request::{ - AutoMergeOptions, OpenPullRequestRequest, build_pr_body, maybe_open_pull_request, pull_request, + AutoMergeOptions, OpenPullRequestRequest, PrContent, build_pr_content, maybe_open_pull_request, + pull_request, }; pub use retro::{retro, run_retro}; pub use transform::transform; diff --git a/lib/crates/fabro-workflow/src/pipeline/prompts/pr_body.md b/lib/crates/fabro-workflow/src/pipeline/prompts/pr_body.md new file mode 100644 index 000000000..9175e62db --- /dev/null +++ b/lib/crates/fabro-workflow/src/pipeline/prompts/pr_body.md @@ -0,0 +1,54 @@ +You are writing a pull request title and description for a code change produced by an AI workflow. + +OUTPUT FORMAT +Return a JSON object with exactly two fields: +- "title": a one-line title, max 72 characters, no trailing period. +- "body": the markdown body as described below. + +DO NOT INCLUDE in the body +- A `#` or `##` title heading at the top -- the title goes in the `title` field. +- A "Retro" section, "Fabro Details" section, cost/duration table, or "Generated with" footer -- those are appended programmatically after your output. +- The full plan text -- the full plan is appended programmatically as a
block. +- Bare `#1`, `#2` list prefixes -- GitHub auto-links those as issue references. Use plain `1.`, `2.` instead. +- A test plan unless the testing approach is non-obvious. + +SIZE THE BODY TO THE CHANGE +First classify along two axes from the diff: +- Size: how many files changed, how large the diff is. +- Complexity: trivial (rename / typo / dep bump / config) vs. design decisions / new patterns / cross-cutting concerns. + +Then write at the matching depth: + +| Profile | Body shape | +|---|---| +| Small + simple (typo, config, dep bump) | 1-2 sentences, no headers, total under ~300 characters | +| Small + non-trivial (targeted bugfix, behavioral change) | Short "Problem / Fix" narrative, 3-5 sentences. No headers unless two distinct concerns. | +| Medium feature or refactor | Summary paragraph, then a section explaining what changed and why. Call out design decisions. | +| Large or architecturally significant | Full narrative: problem context, approach chosen (and why), key decisions, migration/rollback notes if relevant. | +| Performance improvement | Include before/after measurements if available. A markdown table works well here. | + +Brevity matters for small changes. A 3-line bugfix with a 20-line description signals miscalibration. When in doubt, shorter is better -- reviewers can read the diff. + +WRITING PRINCIPLES +- Lead with value: the first sentence tells the reviewer *why this PR exists*, not *what files changed*. +- Describe the net result, not the journey: skip intermediate failures, debugging steps, and refactors done during development. +- Trust the final diff: if the goal or plan disagree with the diff, the diff is authoritative. +- Explain the non-obvious: spend description space on what the diff doesn't show -- why this approach, what was rejected, what to look at first. +- Use structure when it earns its keep: no empty sections, no template headers without content. +- If the body uses any `##` heading, the opening summary must also be under a heading (e.g. `## Summary`); otherwise a bare paragraph is fine. + +PLAN SUMMARY +The full plan is attached separately as a
block, so do not restate it. Include a brief `### Plan Summary` with bullet points only when the change is medium or larger in the sizing matrix above. Skip it for small changes. + +VISUAL AIDS +Include a visual aid only when a reviewer would struggle to reconstruct the mental model from prose alone -- based on what changes structurally, not on PR size. Skip for trivial / mechanical changes, or when prose already communicates clearly. + +| PR changes... | Visual aid | +|---|---| +| 3+ interacting components or services | Mermaid component / interaction diagram | +| Multi-step workflow or pipeline with non-obvious sequencing | Mermaid flow diagram | +| 3+ behavioral modes or variants | Markdown comparison table | +| Before/after data or trade-offs | Markdown table | +| Data model changes with 3+ related entities | Mermaid ERD | + +Mermaid: prefer `TB` direction, <=10 nodes typical. Place inline at the point of relevance, not in a separate "Diagrams" section. diff --git a/lib/crates/fabro-workflow/src/pipeline/pull_request.rs b/lib/crates/fabro-workflow/src/pipeline/pull_request.rs index 5b9355ccf..37aec323e 100644 --- a/lib/crates/fabro-workflow/src/pipeline/pull_request.rs +++ b/lib/crates/fabro-workflow/src/pipeline/pull_request.rs @@ -1,33 +1,133 @@ -use std::sync::Arc; +use std::sync::{Arc, LazyLock}; use fabro_auth::CredentialSource; use fabro_github::{self as github_app, ssh_url_to_https}; use fabro_graphviz::parser; use fabro_llm::client::Client; -use fabro_llm::generate::{GenerateParams, generate}; +use fabro_llm::generate::{GenerateParams, generate_object}; +use fabro_model::Catalog; use fabro_retro::retro::Retro; use fabro_store::RunProjection; use fabro_types::PullRequestRecord; use fabro_types::settings::run::MergeStrategy; use fabro_util::text::strip_goal_decoration; -use tracing::{debug, info}; +use tracing::{debug, info, warn}; use super::types::{Concluded, Finalized, PullRequestOptions}; -use crate::event::{Event, RunNoticeLevel}; +use crate::event::{Event, RunNoticeCode, RunNoticeLevel}; use crate::outcome::{StageOutcome, format_cost as outcome_format_cost}; use crate::records::{Conclusion, RunSpec}; use crate::runtime_store::RunStoreHandle; +/// Maximum length of a PR title (Unicode scalar values). +const PR_TITLE_MAX_CHARS: usize = 72; + +/// Structured output schema for the LLM-generated PR title and body. +static PR_CONTENT_SCHEMA: LazyLock = LazyLock::new(|| { + serde_json::json!({ + "type": "object", + "properties": { + "title": { "type": "string" }, + "body": { "type": "string" } + }, + "required": ["title", "body"], + "additionalProperties": false + }) +}); + +/// Complete pull request content generated for a workflow run. +#[derive(Debug, serde::Deserialize)] +pub struct PrContent { + pub title: String, + pub body: String, +} + +/// System prompt that instructs the LLM how to write a Fabro PR title and +/// body. The trailing programmatic sections (Plan `
`, Retro, +/// Fabro Details, footer) are appended after the LLM body — the prompt +/// explicitly forbids the LLM from duplicating them. +const PR_BODY_SYSTEM_PROMPT: &str = include_str!("prompts/pr_body.md"); + +const DEFAULT_PR_TITLE: &str = "Update workflow output"; +const EMPTY_BODY_NOTICE: &str = "> _The LLM did not produce a description for this change. The diff and the appended details are the source of truth for review._"; + +/// Truncation budget for the LLM prompt's plan / diff sections. +#[derive(Debug, PartialEq, Eq)] +struct TruncationCaps { + plan: usize, + diff: usize, +} + +const DIFF_HARD_CAP: usize = 500_000; +const PLAN_HARD_CAP: usize = 100_000; +const DIFF_FRACTION_NUM: usize = 4; +const PLAN_FRACTION_NUM: usize = 1; +const FRACTION_DEN: usize = 10; +const UNKNOWN_MODEL_CTX: usize = 200_000; + +/// Resolve truncation caps based on the model's context window. Unknown +/// models use the baseline 200k context-window assumption. +fn truncation_caps(model: &str) -> TruncationCaps { + let ctx = Catalog::builtin() + .get(model) + .and_then(|m| usize::try_from(m.context_window()).ok()) + .unwrap_or(UNKNOWN_MODEL_CTX); + + truncation_caps_for_context_window(ctx) +} + +fn truncation_caps_for_context_window(ctx: usize) -> TruncationCaps { + TruncationCaps { + diff: ctx + .saturating_mul(DIFF_FRACTION_NUM) + .checked_div(FRACTION_DEN) + .unwrap_or(DIFF_HARD_CAP) + .min(DIFF_HARD_CAP), + plan: ctx + .saturating_mul(PLAN_FRACTION_NUM) + .checked_div(FRACTION_DEN) + .unwrap_or(PLAN_HARD_CAP) + .min(PLAN_HARD_CAP), + } +} + +/// Truncate `s` to at most `max` Unicode scalar values without splitting a +/// UTF-8 sequence. +fn truncate_chars(s: &str, max: usize) -> &str { + s.char_indices() + .nth(max) + .map_or(s, |(boundary, _)| &s[..boundary]) +} + +/// Truncate `s` to at most `max` Unicode scalar values, replacing the +/// trailing char with `…` when truncation occurs. +fn truncate_with_ellipsis(s: &str, max: usize) -> String { + if s.chars().count() > max { + let truncated: String = s.chars().take(max - 1).collect(); + format!("{truncated}\u{2026}") + } else { + s.to_string() + } +} + +/// Cap a PR title at [`PR_TITLE_MAX_CHARS`]. +fn enforce_title_cap(title: &str) -> String { + truncate_with_ellipsis(title, PR_TITLE_MAX_CHARS) +} + /// Derive a PR title from the workflow goal. /// -/// Uses the first line, truncated to 120 characters for readability. +/// Uses the first line, truncated to the same cap as LLM-generated titles. fn pr_title_from_goal(goal: &str) -> String { - let stripped = strip_goal_decoration(goal); - if stripped.chars().count() > 120 { - let truncated: String = stripped.chars().take(119).collect(); - format!("{truncated}…") + truncate_with_ellipsis(strip_goal_decoration(goal), PR_TITLE_MAX_CHARS) +} + +fn fallback_pr_title(goal: &str) -> String { + let title = pr_title_from_goal(goal); + if title.trim().is_empty() { + DEFAULT_PR_TITLE.to_string() } else { - stripped.to_string() + title } } @@ -286,70 +386,43 @@ async fn load_pull_request_diff(run_store: &RunStoreHandle) -> String { .unwrap_or_default() } -/// Build a complete PR body by combining LLM-generated narrative with -/// programmatic sections (plan, retro, fabro details). -pub async fn build_pr_body( +/// Build complete PR content by combining LLM-generated narrative with +/// deterministic fallbacks and programmatic sections. +pub async fn build_pr_content( diff: &str, goal: &str, model: &str, run_store: &RunStoreHandle, llm_source: &dyn CredentialSource, conclusion: Option<&Conclusion>, -) -> Result { + run_state: Option<&RunProjection>, +) -> Result { let client = Client::from_source(llm_source) .await .map_err(|e| format!("Failed to create LLM client: {e}"))?; - build_pr_body_with_client(diff, goal, model, run_store, conclusion, Arc::new(client)).await -} - -async fn build_pr_body_with_client( - diff: &str, - goal: &str, - model: &str, - run_store: &RunStoreHandle, - conclusion: Option<&Conclusion>, - client: Arc, -) -> Result { - build_pr_body_with_client_and_state(diff, goal, model, run_store, conclusion, client, None) - .await -} - -async fn build_pr_body_with_source_and_state( - diff: &str, - goal: &str, - model: &str, - run_store: &RunStoreHandle, - llm_source: &dyn CredentialSource, - conclusion: Option<&Conclusion>, - run_state: Option<&fabro_store::RunProjection>, -) -> Result { - let client = Client::from_source(llm_source) - .await - .map_err(|e| format!("Failed to create LLM client: {e}"))?; - - build_pr_body_with_client_and_state( + build_pr_content_with_client( diff, goal, model, run_store, conclusion, - Arc::new(client), run_state, + Arc::new(client), ) .await } -async fn build_pr_body_with_client_and_state( +async fn build_pr_content_with_client( diff: &str, goal: &str, model: &str, run_store: &RunStoreHandle, conclusion: Option<&Conclusion>, + run_state: Option<&RunProjection>, client: Arc, - run_state: Option<&fabro_store::RunProjection>, -) -> Result { - info!("Building PR body"); +) -> Result { + info!("Building PR content"); let loaded_run_state = if run_state.is_none() { run_store @@ -369,29 +442,11 @@ async fn build_pr_body_with_client_and_state( let run_spec = run_state.and_then(|state| state.spec.clone()); let dot_source = run_state.and_then(|state| state.graph_source.clone()); - // Build LLM prompt - let system = if plan_text.is_some() { - "Write a PR description with: (1) 2-3 concise paragraphs explaining the change, then (2) a '### Plan Summary' section with bullet points summarizing the plan. Do not include a title. Do not include the full plan.".to_string() - } else { - "Write a concise PR description in 2-3 paragraphs explaining the change. Do not include a title.".to_string() - }; - - // Truncate diff to fit context windows (~50k chars) - let max_diff_len = 50_000; - let truncated_diff = if diff.len() > max_diff_len { - &diff[..diff.floor_char_boundary(max_diff_len)] - } else { - diff - }; + let caps = truncation_caps(model); + let truncated_diff = truncate_chars(diff, caps.diff); let prompt = if let Some(ref plan) = plan_text { - // Truncate plan for LLM context (~20k chars) - let max_plan_len = 20_000; - let truncated_plan = if plan.len() > max_plan_len { - &plan[..plan.floor_char_boundary(max_plan_len)] - } else { - plan.as_str() - }; + let truncated_plan = truncate_chars(plan, caps.plan); format!( "Goal: {goal}\n\nPlan:\n```\n{truncated_plan}\n```\n\nDiff:\n```\n{truncated_diff}\n```" ) @@ -400,14 +455,32 @@ async fn build_pr_body_with_client_and_state( }; let params = GenerateParams::new(model, client) - .system(system) + .system(PR_BODY_SYSTEM_PROMPT) .prompt(prompt); - let result = generate(params) + let result = generate_object(params, PR_CONTENT_SCHEMA.clone()) .await .map_err(|e| format!("LLM generation failed: {e}"))?; - let llm_output = result.response.text(); + let output = result + .output + .ok_or_else(|| "LLM generation returned no structured output".to_string())?; + let generated: PrContent = serde_json::from_value(output) + .map_err(|e| format!("Failed to deserialize PR content: {e}"))?; + + let title = if generated.title.trim().is_empty() { + fallback_pr_title(goal) + } else { + generated.title.trim().to_string() + }; + let title = enforce_title_cap(&title); + + let llm_body = if generated.body.trim().is_empty() { + warn!(model = %model, "LLM generated empty PR body; using skeleton PR body"); + EMPTY_BODY_NOTICE.to_string() + } else { + generated.body + }; let retro_section = retro.as_ref().map(format_retro_section).unwrap_or_default(); let arc_details_section = conclusion @@ -416,15 +489,15 @@ async fn build_pr_body_with_client_and_state( .unwrap_or_default(); let body = assemble_pr_body( - &llm_output, + &llm_body, plan_text.as_deref(), &retro_section, &arc_details_section, ); - info!("PR body generated"); + info!("PR content generated"); - Ok(body) + Ok(PrContent { title, body }) } /// Auto-merge configuration for a pull request. @@ -446,7 +519,7 @@ pub struct OpenPullRequestRequest<'a> { pub run_store: &'a RunStoreHandle, pub llm_source: &'a dyn CredentialSource, pub conclusion: Option<&'a Conclusion>, - pub run_state: Option<&'a fabro_store::RunProjection>, + pub run_state: Option<&'a RunProjection>, } /// Optionally open a pull request after a successful workflow run. @@ -465,7 +538,7 @@ pub async fn maybe_open_pull_request( let (owner, repo) = github_app::parse_github_owner_repo(&https_url).map_err(|err| format!("{err:#}"))?; - let body = build_pr_body_with_source_and_state( + let content = build_pr_content( req.diff, req.goal, req.model, @@ -476,9 +549,8 @@ pub async fn maybe_open_pull_request( ) .await .map_err(|err| format!("{err:#}"))?; - let body = truncate_pr_body(&body); - - let title = pr_title_from_goal(req.goal); + let body = truncate_pr_body(&content.body); + let title = content.title; let created = github_app::create_pull_request( &req.github, @@ -603,7 +675,7 @@ pub async fn pull_request(concluded: Concluded, options: &PullRequestOptions) -> .emit(&Event::PullRequestFailed { error: e.clone() }); services.emitter.notice( RunNoticeLevel::Warn, - "pull_request_failed", + RunNoticeCode::PullRequestFailed, format!("PR creation failed: {e}"), ); } @@ -782,6 +854,16 @@ mod tests { }) } + /// JSON string the MockProvider/openai mock returns to simulate the + /// structured-output response for `(title, body)`. + fn pr_content_json(title: &str, body: &str) -> String { + serde_json::to_string(&serde_json::json!({ + "title": title, + "body": body, + })) + .unwrap() + } + fn make_test_conclusion() -> Conclusion { Conclusion { timestamp: Utc::now(), @@ -1123,20 +1205,25 @@ mod tests { } #[tokio::test] - async fn build_pr_body_uses_in_memory_conclusion() { + async fn build_pr_content_uses_in_memory_conclusion() { let store = test_store(); let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); - let body = build_pr_body_with_client( + let PrContent { title, body } = build_pr_content_with_client( "diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n", "Implement feature", "mock-model", &run_store.clone().into(), Some(&make_test_conclusion()), - explicit_client("mock", "Narrative from mock."), + None, + explicit_client( + "mock", + &pr_content_json("Mock title", "Narrative from mock."), + ), ) .await .unwrap(); + assert_eq!(title, "Mock title"); assert!(body.contains("Narrative from mock.")); assert!(body.contains("### Fabro Details")); assert!(body.contains("Ran 3 stages in 2m 30s for $0.42")); @@ -1144,7 +1231,7 @@ mod tests { } #[tokio::test] - async fn build_pr_body_uses_store_records_without_legacy_files() { + async fn build_pr_content_uses_store_records_without_legacy_files() { let store = test_store(); let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); @@ -1196,16 +1283,21 @@ mod tests { .await .unwrap(); - let body = build_pr_body_with_client( + let body = build_pr_content_with_client( "diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n", "Implement feature", "mock-model", &run_store.clone().into(), Some(&make_test_conclusion()), - explicit_client("mock", "Narrative from mock."), + None, + explicit_client( + "mock", + &pr_content_json("Mock title", "Narrative from mock."), + ), ) .await - .unwrap(); + .unwrap() + .body; assert!(body.contains("Narrative from mock.")); assert!(body.contains("### Retro")); @@ -1214,7 +1306,7 @@ mod tests { } #[tokio::test] - async fn build_pr_body_uses_plan_text_from_store_without_response_md() { + async fn build_pr_content_uses_plan_text_from_store_without_response_md() { let store = test_store(); let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); @@ -1283,42 +1375,52 @@ mod tests { .await .unwrap(); - let body = build_pr_body_with_client( + let body = build_pr_content_with_client( "diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n", "Implement feature", "mock-model", &run_store.clone().into(), Some(&make_test_conclusion()), - explicit_client("mock", "Narrative from mock."), + None, + explicit_client( + "mock", + &pr_content_json("Mock title", "Narrative from mock."), + ), ) .await - .unwrap(); + .unwrap() + .body; assert!(body.contains("Full plan")); assert!(body.contains("Plan from store")); } #[tokio::test] - async fn build_pr_body_uses_explicit_llm_client() { + async fn build_pr_content_uses_explicit_llm_client() { let store = test_store(); let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); - let body = build_pr_body_with_client( + let body = build_pr_content_with_client( "diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n", "Implement feature", "gpt-5.4", &run_store.clone().into(), Some(&make_test_conclusion()), - explicit_client("openai", "Narrative from explicit client."), + None, + explicit_client( + "openai", + &pr_content_json("Explicit title", "Narrative from explicit client."), + ), ) .await - .unwrap(); + .unwrap() + .body; assert!(body.contains("Narrative from explicit client.")); assert!(!body.contains("Narrative from mock.")); } #[tokio::test] - async fn build_pr_body_uses_vault_only_openai_codex_source() { + async fn build_pr_content_uses_vault_only_openai_codex_source() { let server = MockServer::start_async().await; let response_mock = server .mock_async(|when, then| { @@ -1327,7 +1429,10 @@ mod tests { .header("authorization", "Bearer vault-openai-key"); then.status(200) .header("content-type", "application/json") - .json_body(openai_responses_payload("Narrative from vault source.")); + .json_body(openai_responses_payload(&pr_content_json( + "Vault title", + "Narrative from vault source.", + ))); }) .await; @@ -1355,17 +1460,19 @@ mod tests { let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); let run_store_handle: RunStoreHandle = run_store.into(); - let body = build_pr_body( + let PrContent { title, body } = build_pr_content( "diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n", "Implement feature", "gpt-5.4", &run_store_handle, llm_source.as_ref(), Some(&make_test_conclusion()), + None, ) .await .unwrap(); + assert_eq!(title, "Vault title"); assert!(body.contains("Narrative from vault source.")); response_mock.assert_async().await; } @@ -1462,7 +1569,7 @@ mod tests { fn pr_title_truncates_long_line() { let long = "x".repeat(300); let title = pr_title_from_goal(&long); - assert_eq!(title.chars().count(), 120); + assert_eq!(title.chars().count(), 72); assert!(title.ends_with('…')); } @@ -1485,6 +1592,42 @@ mod tests { assert_eq!(pr_title_from_goal("Fix bug"), "Fix bug"); } + #[test] + fn truncation_caps_scale_with_context_window_and_clamp() { + assert_eq!( + truncation_caps_for_context_window(100_000), + TruncationCaps { + diff: 40_000, + plan: 10_000, + } + ); + assert_eq!( + truncation_caps_for_context_window(200_000), + TruncationCaps { + diff: 80_000, + plan: 20_000, + } + ); + assert_eq!( + truncation_caps_for_context_window(1_000_000), + TruncationCaps { + diff: 400_000, + plan: 100_000, + } + ); + assert_eq!( + truncation_caps_for_context_window(10_000_000), + TruncationCaps { + diff: 500_000, + plan: 100_000, + } + ); + assert_eq!(truncation_caps("unknown-model"), TruncationCaps { + diff: 80_000, + plan: 20_000, + }); + } + #[tokio::test] async fn empty_diff_returns_none() { let store = test_store(); @@ -1575,4 +1718,373 @@ mod tests { assert!(diff.contains("from_store")); } + + // ── Structured-output PR content tests ────────────────────────────── + + /// MockProvider returns an over-long title; builder must cap it at 72 + /// chars and end with `…`. Exercises [`enforce_title_cap`] inside + /// [`build_pr_content_with_client`]. + #[tokio::test] + async fn build_pr_content_truncates_long_title() { + let store = test_store(); + let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); + let long_title = "x".repeat(200); + let payload = pr_content_json(&long_title, "Body content."); + let title = build_pr_content_with_client( + "diff --git a/src/lib.rs b/src/lib.rs\n+fn x() {}\n", + "Implement feature", + "mock-model", + &run_store.clone().into(), + Some(&make_test_conclusion()), + None, + explicit_client("mock", &payload), + ) + .await + .unwrap() + .title; + + assert_eq!(title.chars().count(), 72); + assert!(title.ends_with('\u{2026}')); + } + + #[tokio::test] + async fn build_pr_content_uses_default_title_when_generated_and_goal_titles_empty() { + let store = test_store(); + let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); + let payload = pr_content_json("", "Body content."); + let title = build_pr_content_with_client( + "diff --git a/src/lib.rs b/src/lib.rs\n+fn x() {}\n", + "## Plan:", + "mock-model", + &run_store.clone().into(), + Some(&make_test_conclusion()), + None, + explicit_client("mock", &payload), + ) + .await + .unwrap() + .title; + + assert_eq!(title, DEFAULT_PR_TITLE); + } + + /// Empty or whitespace-only bodies use the skeleton fallback instead of + /// aborting PR creation. + #[tokio::test] + async fn build_pr_content_uses_skeleton_when_body_empty() { + let store = test_store(); + let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); + + let run_spec = RunSpec { + run_id: fixtures::RUN_1, + settings: fabro_types::WorkflowSettings::default(), + graph: Graph::new("test"), + workflow_slug: Some("test".to_string()), + source_directory: Some("/tmp/project".to_string()), + git: None, + labels: HashMap::new(), + provenance: None, + manifest_blob: None, + definition_blob: None, + fork_source_ref: None, + in_place: false, + }; + append_event(&run_store, &fixtures::RUN_1, &Event::RunCreated { + run_id: fixtures::RUN_1, + settings: serde_json::to_value(&run_spec.settings).unwrap(), + graph: serde_json::to_value(&run_spec.graph).unwrap(), + workflow_source: Some("digraph test { plan -> code }".to_string()), + workflow_config: None, + labels: run_spec.labels.clone().into_iter().collect(), + run_dir: "/tmp/project".to_string(), + source_directory: run_spec.source_directory.clone(), + workflow_slug: run_spec.workflow_slug.clone(), + db_prefix: None, + provenance: None, + manifest_blob: None, + git: None, + fork_source_ref: None, + in_place: false, + web_url: None, + }) + .await + .unwrap(); + append_event(&run_store, &fixtures::RUN_1, &Event::StageCompleted { + node_id: "plan".to_string(), + name: "plan".to_string(), + index: 0, + duration_ms: 1, + status: "succeeded".to_string(), + preferred_label: None, + suggested_next_ids: vec![], + billing: None, + failure: None, + notes: None, + files_touched: vec![], + context_updates: None, + jump_to_node: None, + context_values: None, + node_visits: None, + loop_failure_signatures: None, + restart_failure_signatures: None, + response: Some("Plan from store".to_string()), + attempt: 1, + max_attempts: 1, + }) + .await + .unwrap(); + append_event(&run_store, &fixtures::RUN_1, &Event::RetroCompleted { + duration_ms: 1, + response: Some(String::new()), + retro: Some(serde_json::to_value(make_test_retro()).unwrap()), + }) + .await + .unwrap(); + + let payload = pr_content_json("Mock", " \n"); + let body = build_pr_content_with_client( + "diff --git a/src/lib.rs b/src/lib.rs\n+fn x() {}\n", + "Implement feature", + "mock-model", + &run_store.clone().into(), + Some(&make_test_conclusion()), + None, + explicit_client("mock", &payload), + ) + .await + .unwrap() + .body; + + assert!(body.contains("The LLM did not produce a description")); + assert!(body.contains("Full plan")); + assert!(body.contains("Plan from store")); + assert!(body.contains("### Retro")); + assert!(body.contains("### Fabro Details")); + assert!(body.contains("Generated with [Fabro](https://fabro.sh)")); + } + + // ── maybe_open_pull_request fallback tests ────────────────────────── + + /// Set of mock servers and credentials for the `maybe_open_pull_request` + /// fallback path. The builder's `Client::from_source` rebuilds the LLM + /// client from the credential source, so the in-process MockProvider + /// cannot intercept — we mock the OpenAI HTTP endpoint instead. + struct FallbackHarness { + _vault_dir: tempfile::TempDir, + // Held to keep the mock listener alive for the duration of the test; + // the test interacts with it via `Client::from_source` (which goes + // out via HTTP to the mock URL stored in `llm_source`). + openai_server: MockServer, + github_server: MockServer, + openai_mock_id: usize, + github_mock_id: usize, + llm_source: Arc, + creds: fabro_github::GitHubCredentials, + run_store: RunStoreHandle, + } + + impl FallbackHarness { + async fn assert_mocks_called_once(&self) { + httpmock::Mock::new(self.openai_mock_id, &self.openai_server) + .assert_async() + .await; + httpmock::Mock::new(self.github_mock_id, &self.github_server) + .assert_async() + .await; + } + } + + /// Stand up an OpenAI mock that returns the given structured-output + /// payload, a GitHub mock that accepts a PR creation, a vault-backed + /// credential source, and a run store seeded with a non-empty + /// `final_patch`. + async fn setup_fallback_test_harness(openai_payload_text: &str) -> FallbackHarness { + let openai_server = MockServer::start_async().await; + let openai_mock = openai_server + .mock_async(|when, then| { + when.method(POST) + .path("/v1/responses") + .header("authorization", "Bearer vault-openai-key"); + then.status(200) + .header("content-type", "application/json") + .json_body(openai_responses_payload(openai_payload_text)); + }) + .await; + + let github_server = MockServer::start_async().await; + let github_mock = github_server + .mock_async(|when, then| { + when.method(POST) + .path("/repos/owner/repo/pulls") + .header("authorization", "Bearer test-token"); + then.status(201) + .header("content-type", "application/json") + .json_body(serde_json::json!({ + "number": 1, + "html_url": "https://example.test/owner/repo/pull/1", + "node_id": "PR_kwTest1", + })); + }) + .await; + + let vault_dir = tempfile::tempdir().unwrap(); + let mut vault = Vault::load(vault_dir.path().join("secrets.json")).unwrap(); + vault + .set( + "openai_codex", + &serde_json::to_string(&openai_api_key_credential("vault-openai-key")).unwrap(), + SecretType::Credential, + None, + ) + .unwrap(); + let base_url = openai_server.url("/v1"); + let llm_source: Arc = + Arc::new(VaultCredentialSource::with_env_lookup( + Arc::new(AsyncRwLock::new(vault)), + move |name| match name { + "OPENAI_BASE_URL" => Some(base_url.clone()), + _ => None, + }, + )); + + let creds = fabro_github::GitHubCredentials::Token("test-token".to_string()); + + let store = test_store(); + let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); + // Seed a non-empty `final_patch` so `load_pull_request_diff` returns + // diff content and the early-return for empty diffs does not fire. + let run_spec = RunSpec { + run_id: fixtures::RUN_1, + settings: fabro_types::WorkflowSettings::default(), + graph: Graph::new("test"), + workflow_slug: None, + source_directory: None, + git: None, + labels: HashMap::new(), + provenance: None, + manifest_blob: None, + definition_blob: None, + fork_source_ref: None, + in_place: false, + }; + append_event(&run_store, &fixtures::RUN_1, &Event::RunCreated { + run_id: fixtures::RUN_1, + settings: serde_json::to_value(&run_spec.settings).unwrap(), + graph: serde_json::to_value(&run_spec.graph).unwrap(), + workflow_source: None, + workflow_config: None, + labels: run_spec.labels.clone().into_iter().collect(), + run_dir: "/tmp/x".to_string(), + source_directory: None, + workflow_slug: None, + db_prefix: None, + provenance: None, + manifest_blob: None, + git: None, + fork_source_ref: None, + in_place: false, + web_url: None, + }) + .await + .unwrap(); + append_event(&run_store, &fixtures::RUN_1, &Event::WorkflowRunCompleted { + duration_ms: 1, + artifact_count: 0, + status: "succeeded".to_string(), + reason: SuccessReason::Completed, + total_usd_micros: None, + final_git_commit_sha: None, + final_patch: Some( + "diff --git a/src/lib.rs b/src/lib.rs\n+fn from_store() {}\n".to_string(), + ), + billing: None, + }) + .await + .unwrap(); + + let openai_mock_id = openai_mock.id; + let github_mock_id = github_mock.id; + + FallbackHarness { + _vault_dir: vault_dir, + openai_server, + github_server, + openai_mock_id, + github_mock_id, + llm_source, + creds, + run_store: run_store.into(), + } + } + + /// LLM returns a usable body but an empty title; the content builder + /// falls back to `pr_title_from_goal` (first line, decoration stripped) + /// and PR creation succeeds with that title. + #[tokio::test] + async fn maybe_open_pull_request_falls_back_to_goal_title_when_llm_returns_empty_title() { + let payload = pr_content_json("", "Narrative."); + let harness = setup_fallback_test_harness(&payload).await; + + let github_base_url = harness.github_server.url(""); + let github = github_app::GitHubContext::new(&harness.creds, &github_base_url); + + let result = maybe_open_pull_request(OpenPullRequestRequest { + github, + origin_url: "https://github.com/owner/repo.git", + base_branch: "main", + head_branch: "fabro/run/123", + goal: "Fix telemetry leak\n\ndetails...", + diff: "diff --git a/src/lib.rs b/src/lib.rs\n+fn x() {}\n", + model: "gpt-5.4", + draft: false, + auto_merge: None, + run_store: &harness.run_store, + llm_source: harness.llm_source.as_ref(), + conclusion: None, + run_state: None, + }) + .await + .expect("PR creation should succeed"); + + let record = result.expect("PR record should be Some"); + assert_eq!(record.title, "Fix telemetry leak"); + harness.assert_mocks_called_once().await; + } + + /// LLM returns an empty title; the content builder fallback still caps + /// the deterministic goal title at 72 chars ending with `…`. + #[tokio::test] + async fn maybe_open_pull_request_caps_fallback_title_at_72_chars() { + let payload = pr_content_json("", "Narrative."); + let harness = setup_fallback_test_harness(&payload).await; + + let github_base_url = harness.github_server.url(""); + let github = github_app::GitHubContext::new(&harness.creds, &github_base_url); + + // Single ~200-char line, no `Plan:` / heading prefix, no newlines. + let goal = "x".repeat(200); + + let result = maybe_open_pull_request(OpenPullRequestRequest { + github, + origin_url: "https://github.com/owner/repo.git", + base_branch: "main", + head_branch: "fabro/run/123", + goal: &goal, + diff: "diff --git a/src/lib.rs b/src/lib.rs\n+fn x() {}\n", + model: "gpt-5.4", + draft: false, + auto_merge: None, + run_store: &harness.run_store, + llm_source: harness.llm_source.as_ref(), + conclusion: None, + run_state: None, + }) + .await + .expect("PR creation should succeed"); + + let record = result.expect("PR record should be Some"); + assert_eq!(record.title.chars().count(), 72); + assert!(record.title.ends_with('\u{2026}')); + harness.assert_mocks_called_once().await; + } } diff --git a/lib/crates/fabro-workflow/src/pipeline/retro.rs b/lib/crates/fabro-workflow/src/pipeline/retro.rs index 47e0070b0..2749cf023 100644 --- a/lib/crates/fabro-workflow/src/pipeline/retro.rs +++ b/lib/crates/fabro-workflow/src/pipeline/retro.rs @@ -53,7 +53,7 @@ pub async fn run_retro(options: &RetroOptions, dry_run: bool) -> Option { return None; } }; - let stage_durations = crate::extract_stage_durations_from_events(&events); + let stage_durations = crate::latest_stage_duration_by_node(&events); let mut retro = derive_retro( options.run_id, &options.workflow_name, diff --git a/lib/crates/fabro-workflow/src/pull_request.rs b/lib/crates/fabro-workflow/src/pull_request.rs index 3989bf63c..3b65dfcbc 100644 --- a/lib/crates/fabro-workflow/src/pull_request.rs +++ b/lib/crates/fabro-workflow/src/pull_request.rs @@ -1,4 +1,4 @@ pub use crate::pipeline::{ - AutoMergeOptions, OpenPullRequestRequest, PullRequestRecord, build_pr_body, + AutoMergeOptions, OpenPullRequestRequest, PrContent, PullRequestRecord, build_pr_content, maybe_open_pull_request, }; diff --git a/lib/crates/fabro-workflow/src/test_support.rs b/lib/crates/fabro-workflow/src/test_support.rs index 12aa40f10..0bf6377a6 100644 --- a/lib/crates/fabro-workflow/src/test_support.rs +++ b/lib/crates/fabro-workflow/src/test_support.rs @@ -17,7 +17,7 @@ use crate::handler::HandlerRegistry; use crate::outcome::Outcome; use crate::pipeline; use crate::pipeline::types::{Executed, Initialized}; -use crate::pipeline::{billing_from_checkpoint, build_terminal_event}; +use crate::pipeline::{billing_from_projection, build_terminal_event}; use crate::records::Checkpoint; use crate::run_metadata::RunMetadataRuntime; use crate::run_options::RunOptions; @@ -36,10 +36,7 @@ async fn execute_and_emit_terminal(initialized: InitializedState) -> Executed { let executed = Box::pin(pipeline::execute(initialized.initialized)).await; initialized.store_logger.flush().await; let state = executed.engine.run.run_store.state().await.ok(); - let billing = state - .as_ref() - .and_then(|s| s.checkpoint.as_ref()) - .and_then(billing_from_checkpoint); + let billing = state.as_ref().and_then(billing_from_projection); let event = build_terminal_event( &executed.outcome, executed.duration_ms, diff --git a/lib/crates/fabro-workflow/tests/it/integration.rs b/lib/crates/fabro-workflow/tests/it/integration.rs index 6ee580bc8..cdffc7837 100644 --- a/lib/crates/fabro-workflow/tests/it/integration.rs +++ b/lib/crates/fabro-workflow/tests/it/integration.rs @@ -6225,6 +6225,8 @@ mod real_llm { _node: &Node, prompt: &str, _system_prompt: Option<&str>, + _emitter: &Arc, + _stage_scope: &fabro_workflow::event::StageScope, ) -> Result { self.complete(prompt).await } @@ -6813,7 +6815,13 @@ async fn workflow_run_with_vault_only_openai_codex_builds_pr_body() { .header("authorization", "Bearer vault-openai-key"); then.status(200) .header("content-type", "application/json") - .json_body(openai_responses_payload("Narrative from vault source.")); + .json_body(openai_responses_payload( + &serde_json::to_string(&serde_json::json!({ + "title": "Vault title", + "body": "Narrative from vault source.", + })) + .unwrap(), + )); }) .await; @@ -6894,7 +6902,7 @@ async fn workflow_run_with_vault_only_openai_codex_builds_pr_body() { let run_store = store.open_run_reader(&run_options.run_id).await.unwrap(); let run_store_handle: fabro_workflow::runtime_store::RunStoreHandle = run_store.into(); - let body = fabro_workflow::pull_request::build_pr_body( + let content = fabro_workflow::pull_request::build_pr_content( "diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n", "Implement feature", "gpt-5.4", @@ -6910,11 +6918,13 @@ async fn workflow_run_with_vault_only_openai_codex_builds_pr_body() { billing: None, total_retries: 0, }), + None, ) .await .expect("PR body should build from vault-only credentials"); - assert!(body.contains("Narrative from vault source.")); + assert_eq!(content.title, "Vault title"); + assert!(content.body.contains("Narrative from vault source.")); response_mock.assert_async().await; } diff --git a/lib/packages/fabro-api-client/src/.openapi-generator/FILES b/lib/packages/fabro-api-client/src/.openapi-generator/FILES index 3113a0298..2dffdba54 100644 --- a/lib/packages/fabro-api-client/src/.openapi-generator/FILES +++ b/lib/packages/fabro-api-client/src/.openapi-generator/FILES @@ -30,7 +30,6 @@ models/artifact-batch-upload-manifest.ts models/artifact-entry.ts models/artifact-list-response.ts models/artifacts-settings.ts -models/assistant-stage-turn.ts models/auth-method.ts models/billed-token-counts.ts models/billing-by-model.ts @@ -168,7 +167,6 @@ models/paginated-run-file-list.ts models/paginated-run-list.ts models/paginated-run-stage-list.ts models/paginated-saved-query-list.ts -models/paginated-stage-turn-list.ts models/pagination-meta.ts models/pending-interview-record.ts models/pre-run-push-outcome-failed.ts @@ -300,7 +298,6 @@ models/stage-completion.ts models/stage-outcome.ts models/stage-projection.ts models/stage-state.ts -models/stage-turn.ts models/start-run-request.ts models/submit-answer-request.ts models/success-reason.ts @@ -308,13 +305,10 @@ models/system-actor-kind.ts models/system-features.ts models/system-info-response.ts models/system-run-counts.ts -models/system-stage-turn.ts models/teams-integration-settings.ts models/terminal-status.ts models/timeline-entry-response.ts models/tls-mode.ts -models/tool-stage-turn.ts -models/tool-use.ts models/user-response.ts models/validate-response.ts models/webhook-strategy.ts diff --git a/lib/packages/fabro-api-client/src/api/run-internals-api.ts b/lib/packages/fabro-api-client/src/api/run-internals-api.ts index 0852b16f6..6520b0fc6 100644 --- a/lib/packages/fabro-api-client/src/api/run-internals-api.ts +++ b/lib/packages/fabro-api-client/src/api/run-internals-api.ts @@ -36,8 +36,6 @@ import type { PaginatedEventList } from '../models'; // @ts-ignore import type { PaginatedRunStageList } from '../models'; // @ts-ignore -import type { PaginatedStageTurnList } from '../models'; -// @ts-ignore import type { RunArtifactListResponse } from '../models'; // @ts-ignore import type { RunCheckpoint } from '../models'; @@ -527,21 +525,21 @@ export const RunInternalsApiAxiosParamCreator = function (configuration?: Config }; }, /** - * Returns a paginated list of conversation turns within a specific stage, including system prompts, assistant responses, and tool invocations. - * @summary List Stage Turns + * Returns a paginated JSON list of stored run events scoped to a single stage visit. + * @summary List Stage Events * @param {string} id Unique run identifier (ULID). * @param {string} stageId Identifier of a stage within a run\'s workflow graph, serialized as `node_id@visit`. - * @param {number} [pageLimit] Maximum number of items to return per page. - * @param {number} [pageOffset] Number of items to skip before returning results. + * @param {number} [sinceSeq] First event sequence number to include. + * @param {number} [limit] Maximum number of events to return. * @param {*} [options] Override http request option. * @throws {RequiredError} */ - listStageTurns: async (id: string, stageId: string, pageLimit?: number, pageOffset?: number, options: RawAxiosRequestConfig = {}): Promise => { + listStageEvents: async (id: string, stageId: string, sinceSeq?: number, limit?: number, options: RawAxiosRequestConfig = {}): Promise => { // verify required parameter 'id' is not null or undefined - assertParamExists('listStageTurns', 'id', id) + assertParamExists('listStageEvents', 'id', id) // verify required parameter 'stageId' is not null or undefined - assertParamExists('listStageTurns', 'stageId', stageId) - const localVarPath = `/api/v1/runs/{id}/stages/{stageId}/turns` + assertParamExists('listStageEvents', 'stageId', stageId) + const localVarPath = `/api/v1/runs/{id}/stages/{stageId}/events` .replace(`{${"id"}}`, encodeURIComponent(String(id))) .replace(`{${"stageId"}}`, encodeURIComponent(String(stageId))); // use dummy base URL string because the URL constructor only accepts absolute URLs. @@ -561,12 +559,12 @@ export const RunInternalsApiAxiosParamCreator = function (configuration?: Config // http bearer authentication required await setBearerAuthToObject(localVarHeaderParameter, configuration) - if (pageLimit !== undefined) { - localVarQueryParameter['page[limit]'] = pageLimit; + if (sinceSeq !== undefined) { + localVarQueryParameter['since_seq'] = sinceSeq; } - if (pageOffset !== undefined) { - localVarQueryParameter['page[offset]'] = pageOffset; + if (limit !== undefined) { + localVarQueryParameter['limit'] = limit; } localVarHeaderParameter['Accept'] = 'application/json'; @@ -964,19 +962,19 @@ export const RunInternalsApiFp = function(configuration?: Configuration) { return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, localVarOperationServerBasePath || basePath); }, /** - * Returns a paginated list of conversation turns within a specific stage, including system prompts, assistant responses, and tool invocations. - * @summary List Stage Turns + * Returns a paginated JSON list of stored run events scoped to a single stage visit. + * @summary List Stage Events * @param {string} id Unique run identifier (ULID). * @param {string} stageId Identifier of a stage within a run\'s workflow graph, serialized as `node_id@visit`. - * @param {number} [pageLimit] Maximum number of items to return per page. - * @param {number} [pageOffset] Number of items to skip before returning results. + * @param {number} [sinceSeq] First event sequence number to include. + * @param {number} [limit] Maximum number of events to return. * @param {*} [options] Override http request option. * @throws {RequiredError} */ - async listStageTurns(id: string, stageId: string, pageLimit?: number, pageOffset?: number, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise> { - const localVarAxiosArgs = await localVarAxiosParamCreator.listStageTurns(id, stageId, pageLimit, pageOffset, options); + async listStageEvents(id: string, stageId: string, sinceSeq?: number, limit?: number, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise> { + const localVarAxiosArgs = await localVarAxiosParamCreator.listStageEvents(id, stageId, sinceSeq, limit, options); const localVarOperationServerIndex = configuration?.serverIndex ?? 0; - const localVarOperationServerBasePath = operationServerMap['RunInternalsApi.listStageTurns']?.[localVarOperationServerIndex]?.url; + const localVarOperationServerBasePath = operationServerMap['RunInternalsApi.listStageEvents']?.[localVarOperationServerIndex]?.url; return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, localVarOperationServerBasePath || basePath); }, /** @@ -1174,17 +1172,17 @@ export const RunInternalsApiFactory = function (configuration?: Configuration, b return localVarFp.listStageArtifacts(id, stageId, options).then((request) => request(axios, basePath)); }, /** - * Returns a paginated list of conversation turns within a specific stage, including system prompts, assistant responses, and tool invocations. - * @summary List Stage Turns + * Returns a paginated JSON list of stored run events scoped to a single stage visit. + * @summary List Stage Events * @param {string} id Unique run identifier (ULID). * @param {string} stageId Identifier of a stage within a run\'s workflow graph, serialized as `node_id@visit`. - * @param {number} [pageLimit] Maximum number of items to return per page. - * @param {number} [pageOffset] Number of items to skip before returning results. + * @param {number} [sinceSeq] First event sequence number to include. + * @param {number} [limit] Maximum number of events to return. * @param {*} [options] Override http request option. * @throws {RequiredError} */ - listStageTurns(id: string, stageId: string, pageLimit?: number, pageOffset?: number, options?: RawAxiosRequestConfig): AxiosPromise { - return localVarFp.listStageTurns(id, stageId, pageLimit, pageOffset, options).then((request) => request(axios, basePath)); + listStageEvents(id: string, stageId: string, sinceSeq?: number, limit?: number, options?: RawAxiosRequestConfig): AxiosPromise { + return localVarFp.listStageEvents(id, stageId, sinceSeq, limit, options).then((request) => request(axios, basePath)); }, /** * Uploads one or more artifacts for a stage. Intended for trusted internal callers. The server accepts both: - `application/octet-stream` for single-file uploads with the `filename` query parameter - strict manifest-first `multipart/form-data` uploads documented by `ArtifactBatchUploadManifest` The generated Rust client currently exposes the octet-stream variant because the OpenAPI code generator in this repo does not support multiple request media types on one operation. @@ -1374,17 +1372,17 @@ export class RunInternalsApi extends BaseAPI { } /** - * Returns a paginated list of conversation turns within a specific stage, including system prompts, assistant responses, and tool invocations. - * @summary List Stage Turns + * Returns a paginated JSON list of stored run events scoped to a single stage visit. + * @summary List Stage Events * @param {string} id Unique run identifier (ULID). * @param {string} stageId Identifier of a stage within a run\'s workflow graph, serialized as `node_id@visit`. - * @param {number} [pageLimit] Maximum number of items to return per page. - * @param {number} [pageOffset] Number of items to skip before returning results. + * @param {number} [sinceSeq] First event sequence number to include. + * @param {number} [limit] Maximum number of events to return. * @param {*} [options] Override http request option. * @throws {RequiredError} */ - public listStageTurns(id: string, stageId: string, pageLimit?: number, pageOffset?: number, options?: RawAxiosRequestConfig) { - return RunInternalsApiFp(this.configuration).listStageTurns(id, stageId, pageLimit, pageOffset, options).then((request) => request(this.axios, this.basePath)); + public listStageEvents(id: string, stageId: string, sinceSeq?: number, limit?: number, options?: RawAxiosRequestConfig) { + return RunInternalsApiFp(this.configuration).listStageEvents(id, stageId, sinceSeq, limit, options).then((request) => request(this.axios, this.basePath)); } /** diff --git a/lib/packages/fabro-api-client/src/models/assistant-stage-turn.ts b/lib/packages/fabro-api-client/src/models/assistant-stage-turn.ts deleted file mode 100644 index 8e0a8edcf..000000000 --- a/lib/packages/fabro-api-client/src/models/assistant-stage-turn.ts +++ /dev/null @@ -1,34 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ -/** - * Fabro Run API - * HTTP API for managing Fabro workflow run executions. - * - * The version of the OpenAPI document: 0.1.0 - * - * - * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). - * https://openapi-generator.tech - * Do not edit the class manually. - */ - - - -/** - * An assistant response turn within a stage. - */ -export interface AssistantStageTurn { - 'kind': AssistantStageTurnKindEnum; - /** - * Assistant response text. - */ - 'content': string; -} - -export const AssistantStageTurnKindEnum = { - ASSISTANT: 'assistant' -} as const; - -export type AssistantStageTurnKindEnum = typeof AssistantStageTurnKindEnum[keyof typeof AssistantStageTurnKindEnum]; - - diff --git a/lib/packages/fabro-api-client/src/models/billing-by-model.ts b/lib/packages/fabro-api-client/src/models/billing-by-model.ts index d8d802575..74f074401 100644 --- a/lib/packages/fabro-api-client/src/models/billing-by-model.ts +++ b/lib/packages/fabro-api-client/src/models/billing-by-model.ts @@ -26,7 +26,7 @@ import type { ModelReference } from './model-reference'; export interface BillingByModel { 'model': ModelReference; /** - * Number of stages that used this model. + * Number of usage-bearing stage visits that used this model. */ 'stages': number; 'billing': BilledTokenCounts; diff --git a/lib/packages/fabro-api-client/src/models/billing-stage-ref.ts b/lib/packages/fabro-api-client/src/models/billing-stage-ref.ts index bcc94a656..7b87ef631 100644 --- a/lib/packages/fabro-api-client/src/models/billing-stage-ref.ts +++ b/lib/packages/fabro-api-client/src/models/billing-stage-ref.ts @@ -15,7 +15,7 @@ /** - * Reference to a billing stage. + * Reference to a workflow node in a billing stage row. */ export interface BillingStageRef { /** diff --git a/lib/packages/fabro-api-client/src/models/board-column-definition.ts b/lib/packages/fabro-api-client/src/models/board-column-definition.ts index f1730a16c..20deba1f5 100644 --- a/lib/packages/fabro-api-client/src/models/board-column-definition.ts +++ b/lib/packages/fabro-api-client/src/models/board-column-definition.ts @@ -13,9 +13,11 @@ */ +// May contain unused imports in some cases +// @ts-ignore +import type { BoardColumn } from './board-column'; export interface BoardColumnDefinition { - 'id': string; + 'id': BoardColumn; 'name': string; } - diff --git a/lib/packages/fabro-api-client/src/models/board-column.ts b/lib/packages/fabro-api-client/src/models/board-column.ts index 9d97dc120..66c24c107 100644 --- a/lib/packages/fabro-api-client/src/models/board-column.ts +++ b/lib/packages/fabro-api-client/src/models/board-column.ts @@ -19,6 +19,7 @@ */ export const BoardColumn = { + QUEUED: 'queued', INITIALIZING: 'initializing', RUNNING: 'running', BLOCKED: 'blocked', diff --git a/lib/packages/fabro-api-client/src/models/index.ts b/lib/packages/fabro-api-client/src/models/index.ts index 0b72c59bd..fcb9e05ef 100644 --- a/lib/packages/fabro-api-client/src/models/index.ts +++ b/lib/packages/fabro-api-client/src/models/index.ts @@ -10,7 +10,6 @@ export * from './artifact-batch-upload-manifest'; export * from './artifact-entry'; export * from './artifact-list-response'; export * from './artifacts-settings'; -export * from './assistant-stage-turn'; export * from './auth-method'; export * from './billed-token-counts'; export * from './billing-by-model'; @@ -147,7 +146,6 @@ export * from './paginated-run-file-list'; export * from './paginated-run-list'; export * from './paginated-run-stage-list'; export * from './paginated-saved-query-list'; -export * from './paginated-stage-turn-list'; export * from './pagination-meta'; export * from './pending-interview-record'; export * from './pre-run-push-outcome'; @@ -279,7 +277,6 @@ export * from './stage-completion'; export * from './stage-outcome'; export * from './stage-projection'; export * from './stage-state'; -export * from './stage-turn'; export * from './start-run-request'; export * from './submit-answer-request'; export * from './success-reason'; @@ -287,13 +284,10 @@ export * from './system-actor-kind'; export * from './system-features'; export * from './system-info-response'; export * from './system-run-counts'; -export * from './system-stage-turn'; export * from './teams-integration-settings'; export * from './terminal-status'; export * from './timeline-entry-response'; export * from './tls-mode'; -export * from './tool-stage-turn'; -export * from './tool-use'; export * from './user-response'; export * from './validate-response'; export * from './webhook-strategy'; diff --git a/lib/packages/fabro-api-client/src/models/manifest-git.ts b/lib/packages/fabro-api-client/src/models/manifest-git.ts deleted file mode 100644 index bef882689..000000000 --- a/lib/packages/fabro-api-client/src/models/manifest-git.ts +++ /dev/null @@ -1,41 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ -/** - * Fabro Run API - * HTTP API for managing Fabro workflow run executions. - * - * The version of the OpenAPI document: 0.1.0 - * - * - * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). - * https://openapi-generator.tech - * Do not edit the class manually. - */ - - -// May contain unused imports in some cases -// @ts-ignore -import type { ManifestPreRunPushOutcome } from './manifest-pre-run-push-outcome'; - -/** - * Observable git state from the CLI working directory. - */ -export interface ManifestGit { - /** - * Remote origin URL with any embedded credentials removed. - */ - 'origin_url': string; - /** - * Current branch name. - */ - 'branch': string; - /** - * Current commit SHA. - */ - 'sha': string; - /** - * Whether the working tree has uncommitted changes. - */ - 'clean': boolean; - 'push_outcome': ManifestPreRunPushOutcome; -} diff --git a/lib/packages/fabro-api-client/src/models/manifest-pre-run-push-outcome.ts b/lib/packages/fabro-api-client/src/models/manifest-pre-run-push-outcome.ts deleted file mode 100644 index 93f6be234..000000000 --- a/lib/packages/fabro-api-client/src/models/manifest-pre-run-push-outcome.ts +++ /dev/null @@ -1,36 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ -/** - * Fabro Run API - * HTTP API for managing Fabro workflow run executions. - * - * The version of the OpenAPI document: 0.1.0 - * - * - * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). - * https://openapi-generator.tech - * Do not edit the class manually. - */ - - - -/** - * Outcome of the CLI\'s best-effort pre-run push. - */ -export interface ManifestPreRunPushOutcome { - 'type': ManifestPreRunPushOutcomeTypeEnum; - 'remote'?: string | null; - 'branch'?: string | null; - 'message'?: string | null; - 'repo_origin_url'?: string | null; -} - -export const ManifestPreRunPushOutcomeTypeEnum = { - NOT_ATTEMPTED: 'not_attempted', - SUCCEEDED: 'succeeded', - FAILED: 'failed', - SKIPPED_NO_REMOTE: 'skipped_no_remote', - SKIPPED_REMOTE_MISMATCH: 'skipped_remote_mismatch' -} as const; - -export type ManifestPreRunPushOutcomeTypeEnum = typeof ManifestPreRunPushOutcomeTypeEnum[keyof typeof ManifestPreRunPushOutcomeTypeEnum]; diff --git a/lib/packages/fabro-api-client/src/models/paginated-stage-turn-list.ts b/lib/packages/fabro-api-client/src/models/paginated-stage-turn-list.ts deleted file mode 100644 index 26f8d9cf2..000000000 --- a/lib/packages/fabro-api-client/src/models/paginated-stage-turn-list.ts +++ /dev/null @@ -1,30 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ -/** - * Fabro Run API - * HTTP API for managing Fabro workflow run executions. - * - * The version of the OpenAPI document: 0.1.0 - * - * - * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). - * https://openapi-generator.tech - * Do not edit the class manually. - */ - - -// May contain unused imports in some cases -// @ts-ignore -import type { PaginationMeta } from './pagination-meta'; -// May contain unused imports in some cases -// @ts-ignore -import type { StageTurn } from './stage-turn'; - -/** - * Paginated list of stage turns. - */ -export interface PaginatedStageTurnList { - 'data': Array; - 'meta': PaginationMeta; -} - diff --git a/lib/packages/fabro-api-client/src/models/pre-run-git-context.ts b/lib/packages/fabro-api-client/src/models/pre-run-git-context.ts deleted file mode 100644 index aaddc93e7..000000000 --- a/lib/packages/fabro-api-client/src/models/pre-run-git-context.ts +++ /dev/null @@ -1,30 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ -/** - * Fabro Run API - * HTTP API for managing Fabro workflow run executions. - * - * The version of the OpenAPI document: 0.1.0 - * - * - * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). - * https://openapi-generator.tech - * Do not edit the class manually. - */ - - -// May contain unused imports in some cases -// @ts-ignore -import type { DirtyStatus } from './dirty-status'; -// May contain unused imports in some cases -// @ts-ignore -import type { PreRunPushOutcome } from './pre-run-push-outcome'; - -/** - * Submitter-side git context captured before run creation. - */ -export interface PreRunGitContext { - 'display_base_sha'?: string | null; - 'local_dirty': DirtyStatus; - 'push_outcome': PreRunPushOutcome; -} diff --git a/lib/packages/fabro-api-client/src/models/run-billing-stage.ts b/lib/packages/fabro-api-client/src/models/run-billing-stage.ts index b6d807d8f..acdf57c97 100644 --- a/lib/packages/fabro-api-client/src/models/run-billing-stage.ts +++ b/lib/packages/fabro-api-client/src/models/run-billing-stage.ts @@ -22,17 +22,24 @@ import type { BillingStageRef } from './billing-stage-ref'; // May contain unused imports in some cases // @ts-ignore import type { ModelReference } from './model-reference'; +// May contain unused imports in some cases +// @ts-ignore +import type { StageState } from './stage-state'; /** - * Token counts and billed totals for a single stage within a run. + * Token counts and billed totals for one workflow node within a run. Rows are grouped by node; billing and runtime sum every visit of that node. */ export interface RunBillingStage { 'stage': BillingStageRef; 'model': ModelReference | null; 'billing': BilledTokenCounts; /** - * Wall-clock runtime in seconds. + * Wall-clock runtime in seconds, summed across every visit of this node. */ 'runtime_secs': number; + /** + * Wall-clock time the latest attempt of this stage started, if known. + */ + 'started_at'?: string | null; + 'state'?: StageState | null; } - diff --git a/lib/packages/fabro-api-client/src/models/run-billing.ts b/lib/packages/fabro-api-client/src/models/run-billing.ts index 88544b1cc..09cfed9e3 100644 --- a/lib/packages/fabro-api-client/src/models/run-billing.ts +++ b/lib/packages/fabro-api-client/src/models/run-billing.ts @@ -28,7 +28,7 @@ import type { RunBillingTotals } from './run-billing-totals'; */ export interface RunBilling { /** - * Per-stage billing breakdown. + * Per-node billing breakdown. Each row sums billing and runtime across all visits of that node. */ 'stages': Array; 'totals': RunBillingTotals; diff --git a/lib/packages/fabro-api-client/src/models/run-stage.ts b/lib/packages/fabro-api-client/src/models/run-stage.ts index c98ec9bee..060a6166e 100644 --- a/lib/packages/fabro-api-client/src/models/run-stage.ts +++ b/lib/packages/fabro-api-client/src/models/run-stage.ts @@ -22,7 +22,7 @@ import type { StageState } from './stage-state'; */ export interface RunStage { /** - * Unique stage identifier within the run. + * StageId in \"node_id@visit\" form, e.g. verify@2. */ 'id': string; /** @@ -35,10 +35,15 @@ export interface RunStage { */ 'duration_secs'?: number; /** - * Node identifier in the Graphviz graph source. + * Node id in the workflow graph; multiple stages with different visits share the same node_id. */ - 'dot_id'?: string; + 'node_id': string; + /** + * 1-based visit count; bumped each time the workflow re-enters this node. + */ + 'visit': number; + /** + * Wall-clock time the latest attempt of this stage started, if known. + */ + 'started_at'?: string | null; } - - - diff --git a/lib/packages/fabro-api-client/src/models/set-secret-request.ts b/lib/packages/fabro-api-client/src/models/set-secret-request.ts deleted file mode 100644 index b343832c1..000000000 --- a/lib/packages/fabro-api-client/src/models/set-secret-request.ts +++ /dev/null @@ -1,25 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ -/** - * Fabro Run API - * HTTP API for managing Fabro workflow run executions. - * - * The version of the OpenAPI document: 0.1.0 - * - * - * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). - * https://openapi-generator.tech - * Do not edit the class manually. - */ - - - -/** - * Request to store a secret value. - */ -export interface SetSecretRequest { - /** - * The secret value to store. - */ - 'value': string; -} diff --git a/lib/packages/fabro-api-client/src/models/stage-projection.ts b/lib/packages/fabro-api-client/src/models/stage-projection.ts index 41321ec78..139fdd437 100644 --- a/lib/packages/fabro-api-client/src/models/stage-projection.ts +++ b/lib/packages/fabro-api-client/src/models/stage-projection.ts @@ -19,6 +19,9 @@ import type { CommandTermination } from './command-termination'; // May contain unused imports in some cases // @ts-ignore import type { StageCompletion } from './stage-completion'; +// May contain unused imports in some cases +// @ts-ignore +import type { StageState } from './stage-state'; /** * Observable projection data for one workflow stage execution. @@ -52,7 +55,13 @@ export interface StageProjection { 'streams_separated'?: boolean | null; 'live_streaming'?: boolean | null; 'termination'?: CommandTermination | null; + /** + * Wall-clock time the latest attempt of this stage started, if known. + */ + 'started_at'?: string | null; + /** + * Wall-clock duration of the stage\'s latest terminal attempt, if known. + */ + 'duration_ms'?: number | null; + 'state'?: StageState | null; } - - - diff --git a/lib/packages/fabro-api-client/src/models/stage-turn.ts b/lib/packages/fabro-api-client/src/models/stage-turn.ts deleted file mode 100644 index 3c5ffdf3d..000000000 --- a/lib/packages/fabro-api-client/src/models/stage-turn.ts +++ /dev/null @@ -1,35 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ -/** - * Fabro Run API - * HTTP API for managing Fabro workflow run executions. - * - * The version of the OpenAPI document: 0.1.0 - * - * - * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). - * https://openapi-generator.tech - * Do not edit the class manually. - */ - - -// May contain unused imports in some cases -// @ts-ignore -import type { AssistantStageTurn } from './assistant-stage-turn'; -// May contain unused imports in some cases -// @ts-ignore -import type { SystemStageTurn } from './system-stage-turn'; -// May contain unused imports in some cases -// @ts-ignore -import type { ToolStageTurn } from './tool-stage-turn'; -// May contain unused imports in some cases -// @ts-ignore -import type { ToolUse } from './tool-use'; - -/** - * @type StageTurn - * A single turn in a stage conversation — a system prompt, assistant response, or tool invocation block. - */ -export type StageTurn = { kind: 'assistant' } & AssistantStageTurn | { kind: 'system' } & SystemStageTurn | { kind: 'tool' } & ToolStageTurn; - - diff --git a/lib/packages/fabro-api-client/src/models/system-stage-turn.ts b/lib/packages/fabro-api-client/src/models/system-stage-turn.ts deleted file mode 100644 index 68ad6386f..000000000 --- a/lib/packages/fabro-api-client/src/models/system-stage-turn.ts +++ /dev/null @@ -1,34 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ -/** - * Fabro Run API - * HTTP API for managing Fabro workflow run executions. - * - * The version of the OpenAPI document: 0.1.0 - * - * - * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). - * https://openapi-generator.tech - * Do not edit the class manually. - */ - - - -/** - * A system prompt turn that sets the stage\'s instructions. - */ -export interface SystemStageTurn { - 'kind': SystemStageTurnKindEnum; - /** - * System prompt text. - */ - 'content': string; -} - -export const SystemStageTurnKindEnum = { - SYSTEM: 'system' -} as const; - -export type SystemStageTurnKindEnum = typeof SystemStageTurnKindEnum[keyof typeof SystemStageTurnKindEnum]; - - diff --git a/lib/packages/fabro-api-client/src/models/tool-stage-turn.ts b/lib/packages/fabro-api-client/src/models/tool-stage-turn.ts deleted file mode 100644 index e637a0520..000000000 --- a/lib/packages/fabro-api-client/src/models/tool-stage-turn.ts +++ /dev/null @@ -1,41 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ -/** - * Fabro Run API - * HTTP API for managing Fabro workflow run executions. - * - * The version of the OpenAPI document: 0.1.0 - * - * - * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). - * https://openapi-generator.tech - * Do not edit the class manually. - */ - - -// May contain unused imports in some cases -// @ts-ignore -import type { ToolUse } from './tool-use'; - -/** - * A tool invocation turn containing one or more tool calls. - */ -export interface ToolStageTurn { - 'kind': ToolStageTurnKindEnum; - /** - * Text accompanying the tool invocations, or null when the turn contains only tool calls. - */ - 'content'?: string; - /** - * Tool invocations executed in this turn. - */ - 'tools': Array; -} - -export const ToolStageTurnKindEnum = { - TOOL: 'tool' -} as const; - -export type ToolStageTurnKindEnum = typeof ToolStageTurnKindEnum[keyof typeof ToolStageTurnKindEnum]; - - diff --git a/lib/packages/fabro-api-client/src/models/tool-use.ts b/lib/packages/fabro-api-client/src/models/tool-use.ts deleted file mode 100644 index c6f5e721e..000000000 --- a/lib/packages/fabro-api-client/src/models/tool-use.ts +++ /dev/null @@ -1,46 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ -/** - * Fabro Run API - * HTTP API for managing Fabro workflow run executions. - * - * The version of the OpenAPI document: 0.1.0 - * - * - * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). - * https://openapi-generator.tech - * Do not edit the class manually. - */ - - - -/** - * A single tool invocation with its input, result, and execution metadata. - */ -export interface ToolUse { - /** - * Unique identifier for this tool invocation. Enables correlation in parallel tool use. - */ - 'id': string; - /** - * Name of the tool that was invoked. - */ - 'tool_name': string; - /** - * JSON-encoded input passed to the tool. - */ - 'input': string; - /** - * Output returned by the tool. Contains the error message when is_error is true. - */ - 'result': string; - /** - * Whether the tool invocation failed. When true, the result field contains the error message. - */ - 'is_error': boolean; - /** - * Wall-clock execution time of the tool invocation in milliseconds. - */ - 'duration_ms'?: number; -} -