From 0a39ba9e0695b6cf26f5f2db57f012181be65e2f Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 24 Jul 2026 06:19:11 -0400 Subject: [PATCH 01/24] Shared-checkout parallel execution (recovered from run 01KY7YH7RYCJ1BDVTTP96ZA4HV) Cumulative implement + simplify_fable diff recovered from the run's meta branch (fabro/meta/01KY7YH7RYCJ1BDVTTP96ZA4HV, stage 006 diff.patch). The run validated this tree clean: cargo nextest (7,007 passed), clippy, fmt, TS client regen + typecheck, web tests (679 passed), docs check. Co-Authored-By: Claude Fable 5 --- .../stage-renderers/fan-in-results.test.tsx | 77 + .../stage-renderers/fan-in-results.tsx | 132 +- .../stage-renderers/helpers.test.ts | 109 +- .../app/components/stage-renderers/helpers.ts | 103 +- .../parallel-children.test.tsx | 85 ++ .../stage-renderers/parallel-children.tsx | 57 +- apps/fabro-web/app/lib/test-utils.tsx | 16 + apps/fabro-web/app/routes/run-stages.tsx | 11 +- docs/internal/demo/06-parallel.fabro | 2 +- docs/internal/demo/11-ensemble.fabro | 2 +- docs/internal/events.md | 75 +- docs/internal/parallel-strategy.md | 393 ++--- docs/public/api-reference/fabro-api.yaml | 20 +- docs/public/examples/clone-substack.mdx | 10 +- docs/public/execution/context.mdx | 9 +- docs/public/execution/outcomes.mdx | 2 +- docs/public/reference/dot-language.mdx | 3 +- docs/public/tutorials/ensemble.mdx | 8 +- docs/public/tutorials/parallel-review.mdx | 41 +- docs/public/workflows/stages-and-nodes.mdx | 22 +- lib/crates/fabro-agent/src/lib.rs | 3 +- lib/crates/fabro-agent/src/sandbox.rs | 3 +- lib/crates/fabro-api/build.rs | 5 + lib/crates/fabro-api/src/lib.rs | 4 +- .../fabro-api/tests/run_event_round_trip.rs | 59 + .../tests/stage_projection_round_trip.rs | 31 +- .../src/commands/run/run_progress/event.rs | 4 +- .../src/commands/run/run_progress/mod.rs | 11 +- .../run/run_progress/stage_display.rs | 4 +- .../tests/it/workflow/dry_run_examples.rs | 6 +- lib/crates/fabro-dump/src/lib.rs | 11 +- lib/crates/fabro-sandbox/src/daytona/mod.rs | 16 - lib/crates/fabro-sandbox/src/docker.rs | 16 - lib/crates/fabro-sandbox/src/lib.rs | 3 - lib/crates/fabro-sandbox/src/sandbox.rs | 27 - lib/crates/fabro-sandbox/src/worktree.rs | 908 ------------ lib/crates/fabro-store/src/run_state.rs | 25 +- .../tests/serializable_projection.rs | 18 +- lib/crates/fabro-types/src/lib.rs | 2 + lib/crates/fabro-types/src/parallel.rs | 14 + lib/crates/fabro-types/src/run_event/infra.rs | 1 - lib/crates/fabro-types/src/run_event/misc.rs | 27 +- lib/crates/fabro-types/src/run_event/mod.rs | 12 - lib/crates/fabro-types/src/run_projection.rs | 3 +- lib/crates/fabro-validate/src/lib.rs | 48 + .../src/rules/inert_attribute.rs | 9 +- .../src/rules/join_policy_removed.rs | 35 + lib/crates/fabro-validate/src/rules/mod.rs | 2 + lib/crates/fabro-workflow/README.md | 2 +- lib/crates/fabro-workflow/src/artifact.rs | 219 ++- lib/crates/fabro-workflow/src/context.rs | 29 +- .../fabro-workflow/src/event/convert.rs | 104 +- .../fabro-workflow/src/event/emitter.rs | 17 - lib/crates/fabro-workflow/src/event/events.rs | 48 +- lib/crates/fabro-workflow/src/event/names.rs | 3 - lib/crates/fabro-workflow/src/git.rs | 148 +- .../fabro-workflow/src/handler/agent.rs | 7 +- .../fabro-workflow/src/handler/fan_in.rs | 733 +++------ .../src/handler/manager_loop.rs | 32 +- .../fabro-workflow/src/handler/parallel.rs | 1312 +++++++---------- .../fabro-workflow/src/pipeline/execute.rs | 14 - .../fabro-workflow/src/pipeline/initialize.rs | 1 - lib/crates/fabro-workflow/src/sandbox_git.rs | 54 - lib/crates/fabro-workflow/src/services.rs | 41 +- lib/crates/fabro-workflow/src/stage_scope.rs | 3 +- lib/crates/fabro-workflow/src/test_support.rs | 1 - .../tests/it/daytona_integration.rs | 230 +-- .../tests/it/git_integration.rs | 23 +- .../fabro-workflow/tests/it/integration.rs | 249 ++-- .../src/.openapi-generator/FILES | 3 + .../fabro-api-client/src/api/models-api.ts | 86 ++ .../src/models/hook-definition.ts | 3 + .../fabro-api-client/src/models/index.ts | 3 + .../src/models/parallel-branch-result.ts | 27 + .../provider-credential-test-request.ts | 22 + .../provider-credential-test-response.ts | 22 + .../src/models/stage-projection.ts | 7 +- test/attractor/reference_template.dot | 16 +- .../clone-substack/clone-substack.fabro | 10 +- test/docs/tutorials/ensemble/ensemble.fabro | 2 +- .../tutorials/parallel-review/parallel.fabro | 2 +- .../stages-and-nodes/all-node-types.fabro | 2 +- test/parallel.fabro | 2 +- 83 files changed, 2072 insertions(+), 3889 deletions(-) create mode 100644 apps/fabro-web/app/components/stage-renderers/fan-in-results.test.tsx create mode 100644 apps/fabro-web/app/components/stage-renderers/parallel-children.test.tsx delete mode 100644 lib/crates/fabro-sandbox/src/worktree.rs create mode 100644 lib/crates/fabro-types/src/parallel.rs create mode 100644 lib/crates/fabro-validate/src/rules/join_policy_removed.rs create mode 100644 lib/packages/fabro-api-client/src/models/parallel-branch-result.ts create mode 100644 lib/packages/fabro-api-client/src/models/provider-credential-test-request.ts create mode 100644 lib/packages/fabro-api-client/src/models/provider-credential-test-response.ts diff --git a/apps/fabro-web/app/components/stage-renderers/fan-in-results.test.tsx b/apps/fabro-web/app/components/stage-renderers/fan-in-results.test.tsx new file mode 100644 index 000000000..26115eaee --- /dev/null +++ b/apps/fabro-web/app/components/stage-renderers/fan-in-results.test.tsx @@ -0,0 +1,77 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import type { EventEnvelope } from "@qltysh/fabro-api-client"; +import TestRenderer, { act } from "react-test-renderer"; + +import { makeEventEnvelope, setupReactTestEnv } from "../../lib/test-utils"; +import type { Stage } from "../stage-sidebar"; +import { FanInResults } from "./fan-in-results"; + +let teardown: () => void; +beforeEach(() => { + teardown = setupReactTestEnv(); +}); +afterEach(() => teardown()); + +const fanInStage: Stage = { + id: "join@1", + name: "join", + handler: "parallel.fan_in", + status: "succeeded", + duration: "1s", + nodeId: "join", + visit: 1, + startedAt: "2026-04-09T12:00:00Z", + providerUsed: null, +}; + +function event(seq: number, partial: Partial): EventEnvelope { + return makeEventEnvelope(seq, { stage_id: "join@1", ...partial }); +} + +function renderFanIn(events: EventEnvelope[]): string { + (globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true; + let renderer!: TestRenderer.ReactTestRenderer; + act(() => { + renderer = TestRenderer.create(); + }); + return JSON.stringify(renderer.toJSON()); +} + +describe("FanInResults", () => { + test("renders a neutral joined state without best-branch selection UI", () => { + const rendered = renderFanIn([]); + + expect(rendered).toContain("Joined"); + expect(rendered).not.toContain("Selected branch"); + expect(rendered).not.toContain("Selected by"); + expect(rendered).not.toContain("TrophyIcon"); + }); + + test("optionally renders the standard reducer transcript", () => { + const rendered = renderFanIn([ + event(1, { + event: "stage.prompt", + properties: { + mode: "prompt", + text: "Combine the useful findings.", + model: "claude-sonnet-4-6", + }, + }), + event(2, { + event: "prompt.completed", + properties: { + response: "All branch findings are now available.", + billing: { input_tokens: 1200, output_tokens: 340 }, + }, + }), + ]); + + expect(rendered).toContain("Reducer transcript"); + expect(rendered).toContain("Combine the useful findings."); + expect(rendered).toContain("All branch findings are now available."); + expect(rendered).toContain("claude-sonnet-4-6"); + expect(rendered).toContain("1k"); + expect(rendered).toContain("340"); + expect(rendered).toContain("tokens"); + }); +}); diff --git a/apps/fabro-web/app/components/stage-renderers/fan-in-results.tsx b/apps/fabro-web/app/components/stage-renderers/fan-in-results.tsx index f9126d2a4..0f977ce7b 100644 --- a/apps/fabro-web/app/components/stage-renderers/fan-in-results.tsx +++ b/apps/fabro-web/app/components/stage-renderers/fan-in-results.tsx @@ -1,143 +1,53 @@ import { useMemo } from "react"; import { + CheckCircleIcon, CpuChipIcon, - SparklesIcon, - TrophyIcon, } from "@heroicons/react/20/solid"; import type { EventEnvelope } from "@qltysh/fabro-api-client"; import type { Stage } from "../stage-sidebar"; import { formatTokenCount } from "../../lib/format"; -import { getString } from "../../lib/unknown"; import { Markdown } from "./primitives"; -import { prettyJson } from "./pretty-json"; import { StageMetaBar } from "./meta-bar"; -import { parseFanInOutcome } from "./helpers"; - -interface ReducerTurn { - prompt: string; - response: string; - model: string | null; - inputTokens: number; - outputTokens: number; -} - -function extractReducerTurn(events: EventEnvelope[]): ReducerTurn | null { - let prompt = ""; - let response = ""; - let model: string | null = null; - let inputTokens = 0; - let outputTokens = 0; - let hasReducer = false; - - for (const event of events) { - const props = event.properties ?? {}; - if (event.event === "stage.prompt" && getString(props, "mode") === "fan_in") { - prompt = getString(props, "text") ?? prompt; - hasReducer = true; - } else if (event.event === "prompt.completed") { - response = getString(props, "response") ?? response; - model = getString(props, "model") ?? model; - const billing = props.billing as Record | undefined; - if (billing) { - const it = billing.input_tokens; - const ot = billing.output_tokens; - if (typeof it === "number") inputTokens = it; - if (typeof ot === "number") outputTokens = ot; - } - hasReducer = true; - } - } - - return hasReducer ? { prompt, response, model, inputTokens, outputTokens } : null; -} - -/** - * The fan-in `stage.prompt.text` is built by the handler as - * "\n\n". Split it for nicer display so the JSON candidate set - * lands in a code block rather than fighting markdown rendering. - */ -function splitPromptAndCandidates(text: string): { prompt: string; candidatesJson: string } { - const trimmed = text.trim(); - // Find the start of the JSON envelope. The handler always uses array results - // so look for the first opening bracket that begins a balanced array/object. - const idx = (() => { - for (let i = 0; i < trimmed.length; i += 1) { - const ch = trimmed[i]; - if (ch === "[" || ch === "{") return i; - } - return -1; - })(); - if (idx < 0) return { prompt: trimmed, candidatesJson: "" }; - const promptPart = trimmed.slice(0, idx).trim(); - const jsonPart = trimmed.slice(idx).trim(); - const pretty = prettyJson(jsonPart); - return { - prompt: promptPart, - candidatesJson: pretty.isJson ? pretty.text : jsonPart, - }; -} +import { parseReducerTranscript } from "./helpers"; export function FanInResults({ stage, events, - notes, }: { stage: Stage; events: EventEnvelope[]; - notes: string | null; }) { - const outcome = useMemo(() => parseFanInOutcome(events, notes), [events, notes]); - const reducer = useMemo(() => extractReducerTurn(events), [events]); - const promptParts = useMemo( - () => (reducer ? splitPromptAndCandidates(reducer.prompt) : null), - [reducer], - ); + const reducer = useMemo(() => parseReducerTranscript(events), [events]); return (
- {outcome.reducerModel ? ( + {reducer?.model ? ( ) : null} -
+
-
-
- {reducer && promptParts && ( + {reducer && (

Reducer transcript @@ -149,18 +59,10 @@ export function FanInResults({ Prompt - {promptParts.prompt && ( - - )} - {promptParts.candidatesJson && ( -
- - Candidates JSON - - 
-                  {promptParts.candidatesJson}
-
-
+ {reducer.prompt ? ( + + ) : ( +

No prompt recorded.

)} diff --git a/apps/fabro-web/app/components/stage-renderers/helpers.test.ts b/apps/fabro-web/app/components/stage-renderers/helpers.test.ts index d5603d8bf..4fcf47e95 100644 --- a/apps/fabro-web/app/components/stage-renderers/helpers.test.ts +++ b/apps/fabro-web/app/components/stage-renderers/helpers.test.ts @@ -3,10 +3,9 @@ import type { EventEnvelope } from "@qltysh/fabro-api-client"; import { extractStageContext, - extractStageNotes, - parseFanInOutcome, parseHumanInterviewPairs, parseParallelOverview, + parseReducerTranscript, } from "./helpers"; function envelope(seq: number, partial: Partial): EventEnvelope { @@ -144,45 +143,70 @@ describe("parseHumanInterviewPairs", () => { }); describe("parseParallelOverview", () => { - test("rolls up branch_count from started and results from completed", () => { + test("rolls up branch_count and status-only results", () => { const events: EventEnvelope[] = [ envelope(1, { event: "parallel.started", - properties: { branch_count: 3, join_policy: "wait_all" }, + properties: { branch_count: 3 }, }), envelope(2, { event: "parallel.completed", properties: { + duration_ms: 12000, success_count: 2, failure_count: 1, results: [ - { id: "branch-a", status: "succeeded", head_sha: "abc1234567890" }, - { id: "branch-b", status: "succeeded" }, - { id: "branch-c", status: "failed" }, + { + id: "branch-a", + status: "succeeded", + context_updates: { "response.branch-a": "A" }, + }, + { + id: "branch-b", + status: "succeeded", + context_updates: { "command.output": { stdout: "B" } }, + }, + { + id: "branch-c", + status: "failed", + context_updates: { "response.branch-c": "C" }, + }, ], }, }), ]; const overview = parseParallelOverview(events); - expect(overview).toMatchObject({ + expect(overview).toEqual({ branchCount: 3, - joinPolicy: "wait_all", successCount: 2, failureCount: 1, + durationMs: 12000, + results: [ + { + id: "branch-a", + status: "succeeded", + context_updates: { "response.branch-a": "A" }, + }, + { + id: "branch-b", + status: "succeeded", + context_updates: { "command.output": { stdout: "B" } }, + }, + { + id: "branch-c", + status: "failed", + context_updates: { "response.branch-c": "C" }, + }, + ], isComplete: true, }); - expect(overview.results).toEqual([ - { id: "branch-a", status: "succeeded", headSha: "abc1234567890" }, - { id: "branch-b", status: "succeeded", headSha: null }, - { id: "branch-c", status: "failed", headSha: null }, - ]); }); test("reports in-flight when only the started event is present", () => { const events: EventEnvelope[] = [ envelope(1, { event: "parallel.started", - properties: { branch_count: 4, join_policy: "first_success" }, + properties: { branch_count: 4 }, }), ]; const overview = parseParallelOverview(events); @@ -192,49 +216,52 @@ describe("parseParallelOverview", () => { }); }); -describe("parseFanInOutcome", () => { - test("extracts the selected branch id from notes", () => { - const outcome = parseFanInOutcome([], "Selected best candidate: branch-42"); - expect(outcome.selectedId).toBe("branch-42"); - expect(outcome.hasReducerTranscript).toBe(false); +describe("parseReducerTranscript", () => { + test("returns null when fan-in joins without a reducer", () => { + expect(parseReducerTranscript([])).toBeNull(); }); - test("flags reducer presence when fan-in prompt events exist", () => { + test("parses the standard prompt transcript when a reducer ran", () => { const events: EventEnvelope[] = [ envelope(1, { event: "stage.prompt", - properties: { mode: "fan_in", text: "rank these", model: "claude-sonnet-4-6" }, + properties: { + mode: "prompt", + text: "Combine the branch results.", + model: "claude-sonnet-4-6", + }, }), envelope(2, { event: "prompt.completed", - properties: { response: "branch-a wins", model: "ignored-downstream-model" }, + properties: { + response: "The branch results are joined.", + billing: { input_tokens: 1200, output_tokens: 340 }, + }, }), ]; - const outcome = parseFanInOutcome(events, "Selected best candidate: branch-a"); - expect(outcome.hasReducerTranscript).toBe(true); - expect(outcome.reducerModel).toBe("claude-sonnet-4-6"); - expect(outcome.selectedId).toBe("branch-a"); + + expect(parseReducerTranscript(events)).toEqual({ + prompt: "Combine the branch results.", + response: "The branch results are joined.", + model: "claude-sonnet-4-6", + inputTokens: 1200, + outputTokens: 340, + }); }); - test("returns null selection when notes lack the selected line", () => { - const outcome = parseFanInOutcome([], "all candidates failed"); - expect(outcome.selectedId).toBeNull(); - }); -}); - -describe("extractStageNotes", () => { - test("returns notes from the stage.completed event", () => { + test("uses normal prompt mode for the reducer transcript", () => { const events: EventEnvelope[] = [ envelope(1, { - event: "stage.completed", - properties: { notes: "Stop condition satisfied at cycle 7" }, + event: "stage.prompt", + properties: { mode: "prompt", text: "Standard reducer" }, + }), + envelope(2, { + event: "prompt.completed", + properties: { response: "Standard response" }, }), ]; - expect(extractStageNotes(events)).toBe("Stop condition satisfied at cycle 7"); - }); - test("returns null when there is no stage.completed event", () => { - expect(extractStageNotes([])).toBeNull(); + expect(parseReducerTranscript(events)?.response).toBe("Standard response"); }); }); diff --git a/apps/fabro-web/app/components/stage-renderers/helpers.ts b/apps/fabro-web/app/components/stage-renderers/helpers.ts index f7f6fa10d..3c631a9e2 100644 --- a/apps/fabro-web/app/components/stage-renderers/helpers.ts +++ b/apps/fabro-web/app/components/stage-renderers/helpers.ts @@ -1,7 +1,16 @@ -import type { EventEnvelope } from "@qltysh/fabro-api-client"; +import { StageOutcome } from "@qltysh/fabro-api-client"; +import type { EventEnvelope, ParallelBranchResult } from "@qltysh/fabro-api-client"; + +export type { ParallelBranchResult }; import { getArray, getNumber, getObject, getString, type UnknownRecord } from "../../lib/unknown"; +const STAGE_OUTCOMES: ReadonlySet = new Set(Object.values(StageOutcome)); + +function asStageOutcome(value: string | undefined): StageOutcome | null { + return value !== undefined && STAGE_OUTCOMES.has(value) ? (value as StageOutcome) : null; +} + export interface InterviewOption { key: string; label: string; @@ -137,15 +146,8 @@ export function parseHumanInterviewPairs(events: EventEnvelope[]): HumanIntervie return Array.from(pairs.values()).sort((a, b) => a.question.ts.localeCompare(b.question.ts)); } -export interface ParallelBranchResult { - id: string; - status: string; - headSha: string | null; -} - export interface ParallelOverview { branchCount: number | null; - joinPolicy: string | null; successCount: number | null; failureCount: number | null; durationMs: number | null; @@ -160,7 +162,6 @@ export interface ParallelOverview { */ export function parseParallelOverview(events: EventEnvelope[]): ParallelOverview { let branchCount: number | null = null; - let joinPolicy: string | null = null; let successCount: number | null = null; let failureCount: number | null = null; let durationMs: number | null = null; @@ -171,7 +172,6 @@ export function parseParallelOverview(events: EventEnvelope[]): ParallelOverview const props: UnknownRecord = event.properties ?? {}; if (event.event === "parallel.started") { branchCount = getNumber(props, "branch_count") ?? branchCount; - joinPolicy = getString(props, "join_policy") ?? joinPolicy; } else if (event.event === "parallel.completed") { isComplete = true; successCount = getNumber(props, "success_count") ?? successCount; @@ -182,20 +182,23 @@ export function parseParallelOverview(events: EventEnvelope[]): ParallelOverview .map((entry) => { const record = entry && typeof entry === "object" ? (entry as UnknownRecord) : null; if (!record) return null; + const id = getString(record, "id"); + const status = asStageOutcome(getString(record, "status")); + const contextUpdates = getObject(record, "context_updates"); + if (!id || !status || !contextUpdates) return null; return { - id: getString(record, "id") ?? "", - status: getString(record, "status") ?? "unknown", - headSha: getString(record, "head_sha") ?? null, + id, + status, + context_updates: contextUpdates, } satisfies ParallelBranchResult; }) - .filter((r): r is ParallelBranchResult => r != null && r.id !== ""); + .filter((r): r is ParallelBranchResult => r != null); if (branchCount == null) branchCount = results.length; } } return { branchCount, - joinPolicy, successCount, failureCount, durationMs, @@ -204,59 +207,39 @@ export function parseParallelOverview(events: EventEnvelope[]): ParallelOverview }; } -export interface FanInOutcome { - selectedId: string | null; - hasReducerTranscript: boolean; - reducerModel: string | null; +export interface ReducerTranscript { + prompt: string; + response: string; + model: string | null; + inputTokens: number; + outputTokens: number; } -const FAN_IN_NOTES_RE = /Selected best candidate:\s*(.+?)\s*$/; +/** Extract the standard prompt/response transcript emitted by an optional fan-in reducer. */ +export function parseReducerTranscript(events: EventEnvelope[]): ReducerTranscript | null { + let prompt = ""; + let response = ""; + let model: string | null = null; + let inputTokens = 0; + let outputTokens = 0; + let hasReducer = false; -/** - * Derive the fan-in winner from the `parallel.completed`-style notes string, - * and report whether reducer LLM events were emitted (so the UI knows to show - * the embedded transcript). - */ -export function parseFanInOutcome(events: EventEnvelope[], notes: string | null): FanInOutcome { - const match = notes ? FAN_IN_NOTES_RE.exec(notes) : null; - let hasReducerTranscript = false; - let reducerModel: string | null = null; for (const event of events) { + const props: UnknownRecord = event.properties ?? {}; if (event.event === "stage.prompt") { - const mode = getString(event.properties ?? {}, "mode"); - if (mode === "fan_in") { - hasReducerTranscript = true; - const model = getString(event.properties ?? {}, "model"); - if (model) reducerModel = model; - } - } - if (event.event === "prompt.completed") { - hasReducerTranscript = true; + prompt = getString(props, "text") ?? prompt; + model = getString(props, "model") ?? model; + hasReducer = true; + } else if (event.event === "prompt.completed" && hasReducer) { + response = getString(props, "response") ?? response; + model = getString(props, "model") ?? model; + const billing = getObject(props, "billing") ?? {}; + inputTokens = getNumber(billing, "input_tokens") ?? inputTokens; + outputTokens = getNumber(billing, "output_tokens") ?? outputTokens; } } - return { - selectedId: match ? match[1].trim() : null, - hasReducerTranscript, - reducerModel, - }; -} -function asUnknownRecord(value: unknown): UnknownRecord | null { - if (!value || typeof value !== "object" || Array.isArray(value)) return null; - return value as UnknownRecord; -} - -/** - * Extract `notes` from the `stage.completed` event scoped to this stage. - * Returns null when the stage hasn't finished yet or when notes are absent. - */ -export function extractStageNotes(events: EventEnvelope[]): string | null { - for (const event of events) { - if (event.event !== "stage.completed") continue; - const notes = getString(event.properties ?? {}, "notes"); - if (notes) return notes; - } - return null; + return hasReducer ? { prompt, response, model, inputTokens, outputTokens } : null; } export interface StageContextData { diff --git a/apps/fabro-web/app/components/stage-renderers/parallel-children.test.tsx b/apps/fabro-web/app/components/stage-renderers/parallel-children.test.tsx new file mode 100644 index 000000000..f1313b780 --- /dev/null +++ b/apps/fabro-web/app/components/stage-renderers/parallel-children.test.tsx @@ -0,0 +1,85 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import type { EventEnvelope } from "@qltysh/fabro-api-client"; +import TestRenderer, { act } from "react-test-renderer"; +import { MemoryRouter } from "react-router"; + +import { makeEventEnvelope, setupReactTestEnv } from "../../lib/test-utils"; +import type { Stage } from "../stage-sidebar"; +import { ParallelChildren } from "./parallel-children"; + +let teardown: () => void; +beforeEach(() => { + teardown = setupReactTestEnv(); +}); +afterEach(() => teardown()); + +const parallelStage: Stage = { + id: "fork@1", + name: "fork", + handler: "parallel", + status: "succeeded", + duration: "12s", + nodeId: "fork", + visit: 1, + startedAt: "2026-04-09T12:00:00Z", + providerUsed: null, +}; + +function event(partial: Partial): EventEnvelope { + return makeEventEnvelope(partial.seq ?? 1, { event: "parallel.completed", ...partial }); +} + +function renderParallel(events: EventEnvelope[]): TestRenderer.ReactTestRenderer { + let renderer!: TestRenderer.ReactTestRenderer; + act(() => { + renderer = TestRenderer.create( + + + , + ); + }); + return renderer; +} + +describe("ParallelChildren", () => { + test("renders branch status and stage links without checkout metadata", () => { + const renderer = renderParallel([ + event({ + event: "parallel.started", + properties: { branch_count: 2 }, + }), + event({ + seq: 2, + event: "parallel.completed", + properties: { + duration_ms: 12000, + success_count: 1, + failure_count: 1, + results: [ + { id: "branch-a", status: "succeeded", context_updates: {} }, + { id: "branch-b", status: "failed", context_updates: {} }, + ], + }, + }), + ]); + + const rendered = JSON.stringify(renderer.toJSON()); + expect(rendered).toContain("branch-a"); + expect(rendered).toContain("Succeeded"); + expect(rendered).toContain("branch-b"); + expect(rendered).toContain("Failed"); + const hrefs = renderer.root.findAllByType("a").map((link) => link.props.href); + expect(hrefs).toEqual([ + "/runs/run-1/stages/branch-a@1", + "/runs/run-1/stages/branch-b@1", + ]); + }); +}); diff --git a/apps/fabro-web/app/components/stage-renderers/parallel-children.tsx b/apps/fabro-web/app/components/stage-renderers/parallel-children.tsx index 0b9e4b951..30cb6169c 100644 --- a/apps/fabro-web/app/components/stage-renderers/parallel-children.tsx +++ b/apps/fabro-web/app/components/stage-renderers/parallel-children.tsx @@ -1,34 +1,19 @@ import { useMemo } from "react"; import { Link } from "react-router"; import { ArrowTopRightOnSquareIcon } from "@heroicons/react/20/solid"; +import { StageState } from "@qltysh/fabro-api-client"; import type { EventEnvelope } from "@qltysh/fabro-api-client"; import type { Stage } from "../stage-sidebar"; -import { CopyButton } from "../ui"; +import { stageStatusLabel, stageStatusTone } from "../../lib/stage-sidebar"; import { formatDurationMs } from "../../lib/format"; import { StageMetaBar } from "./meta-bar"; -import { parseParallelOverview, type ParallelBranchResult } from "./helpers"; +import { parseParallelOverview } from "./helpers"; -const RESULT_STATUS_TONE: Record = { - succeeded: "bg-mint/15 text-mint", - partially_succeeded: "bg-amber/15 text-amber", - failed: "bg-coral/15 text-coral", - cancelled: "bg-overlay-strong text-fg-muted", - skipped: "bg-overlay-strong text-fg-muted", -}; - -function statusTone(status: string): string { - return RESULT_STATUS_TONE[status] ?? "bg-overlay-strong text-fg-muted"; -} - -function statusLabel(status: string): string { - if (!status) return "—"; - return status.charAt(0).toUpperCase() + status.slice(1).replace(/_/g, " "); -} - -function shortSha(sha: string | null): string | null { - if (!sha) return null; - return sha.length > 8 ? sha.slice(0, 8) : sha; +/** Branch row view state: completed outcomes plus a synthesized in-flight row. */ +interface BranchRow { + id: string; + status: StageState; } function StatItem({ @@ -56,27 +41,21 @@ function ChildRow({ result, stageHref, }: { - result: ParallelBranchResult; + result: BranchRow; stageHref: string | null; }) { - const sha = shortSha(result.headSha); - const tone = statusTone(result.status); + const tone = stageStatusTone(result.status); const inner = ( <> - {statusLabel(result.status)} + {stageStatusLabel(result.status)} {result.id} - {sha && ( - - {sha} - - )} {stageHref && ( {inner} )} - {result.headSha && ( - - )} ); } @@ -128,25 +104,18 @@ export function ParallelChildren({ return new Map(Array.from(latest.entries()).map(([nodeId, s]) => [nodeId, s.id])); }, [allStages]); - const items = overview.results.length > 0 + const items: BranchRow[] = overview.results.length > 0 ? overview.results : overview.branchCount && overview.branchCount > 0 ? Array.from({ length: overview.branchCount }, (_, i) => ({ id: `branch ${i + 1}`, - status: "running", - headSha: null, + status: StageState.RUNNING, })) : []; return (
- - {overview.joinPolicy ? ( - - {overview.joinPolicy.replace(/_/g, " ")} - - ) : null} - +
diff --git a/apps/fabro-web/app/lib/test-utils.tsx b/apps/fabro-web/app/lib/test-utils.tsx index f881952de..995245aa5 100644 --- a/apps/fabro-web/app/lib/test-utils.tsx +++ b/apps/fabro-web/app/lib/test-utils.tsx @@ -1,4 +1,5 @@ import { createElement, type ReactNode } from "react"; +import type { EventEnvelope } from "@qltysh/fabro-api-client"; import TestRenderer, { act } from "react-test-renderer"; const IS_REACT_ACT_ENV = "IS_REACT_ACT_ENVIRONMENT" as const; @@ -39,6 +40,21 @@ export function setupReactTestEnv(): () => void { }; } +/** Build an event envelope fixture; override any field via `partial`. */ +export function makeEventEnvelope( + seq: number, + partial: Partial, +): EventEnvelope { + return { + seq, + id: `evt-${seq}`, + ts: `2026-04-09T12:00:0${seq}Z`, + run_id: "run-1", + event: "stage.prompt", + ...partial, + } as EventEnvelope; +} + export function renderHook( hook: () => T, options: { wrapper: React.ComponentType<{ children: ReactNode }> }, diff --git a/apps/fabro-web/app/routes/run-stages.tsx b/apps/fabro-web/app/routes/run-stages.tsx index aa0ae43af..deaf26d44 100644 --- a/apps/fabro-web/app/routes/run-stages.tsx +++ b/apps/fabro-web/app/routes/run-stages.tsx @@ -42,10 +42,7 @@ import { } from "../components/ui"; import { ConditionalDecision } from "../components/stage-renderers/conditional-decision"; import { FanInResults } from "../components/stage-renderers/fan-in-results"; -import { - extractStageContext, - extractStageNotes, -} from "../components/stage-renderers/helpers"; +import { extractStageContext } from "../components/stage-renderers/helpers"; import { HumanQA } from "../components/stage-renderers/human-qa"; import { ParallelChildren } from "../components/stage-renderers/parallel-children"; import { @@ -1572,11 +1569,7 @@ function StageActivityBody({ allStages={stages} /> ) : renderer === "fan_in" ? ( - + ) : renderer === "wait" ? ( ) : ( diff --git a/docs/internal/demo/06-parallel.fabro b/docs/internal/demo/06-parallel.fabro index 7f7191a3c..f4714e600 100644 --- a/docs/internal/demo/06-parallel.fabro +++ b/docs/internal/demo/06-parallel.fabro @@ -5,7 +5,7 @@ digraph Parallel { start [shape=Mdiamond, label="Start"] exit [shape=Msquare, label="Exit"] - fork [label="Fork Analysis", shape=component, join_policy="wait_all"] + fork [label="Fork Analysis", shape=component] security [label="Security Audit", prompt="Examine the codebase for security concerns: hardcoded secrets, injection risks, unsafe dependencies. List findings as bullet points.", shape=tab, reasoning_effort="low"] architecture [label="Architecture Review", prompt="Assess the codebase architecture: separation of concerns, dependency structure, modularity. List findings as bullet points.", shape=tab, reasoning_effort="low"] diff --git a/docs/internal/demo/11-ensemble.fabro b/docs/internal/demo/11-ensemble.fabro index 3a8655cde..87aadf36c 100644 --- a/docs/internal/demo/11-ensemble.fabro +++ b/docs/internal/demo/11-ensemble.fabro @@ -14,7 +14,7 @@ digraph Ensemble { start [shape=Mdiamond, label="Start"] exit [shape=Msquare, label="Exit"] - fork [label="Fan Out", shape=component, join_policy="wait_all"] + fork [label="Fan Out", shape=component] opus [label="Opus", prompt="Analyze the goal. Provide your independent assessment, recommendations, and any code or prose needed. Be thorough.", shape=tab] gemini [label="Gemini", prompt="Analyze the goal. Provide your independent assessment, recommendations, and any code or prose needed. Be thorough.", shape=tab] diff --git a/docs/internal/events.md b/docs/internal/events.md index 4af5ab491..53d10e1e4 100644 --- a/docs/internal/events.md +++ b/docs/internal/events.md @@ -523,16 +523,16 @@ Emitted when a parallel node begins executing branches. "id": "...", "ts": "...", "run_id": "...", "event": "parallel.started", "properties": { - "branch_count": 3, - "join_policy": "all" + "visit": 1, + "branch_count": 3 } } ``` | Property | Type | Description | |----------|------|-------------| +| `visit` | number | Visit number for this parallel stage | | `branch_count` | number | Number of parallel branches | -| `join_policy` | string | Join policy | ### `parallel.branch.started` @@ -587,18 +587,33 @@ Emitted when all parallel branches have finished. "id": "...", "ts": "...", "run_id": "...", "event": "parallel.completed", "properties": { + "visit": 1, "duration_ms": 12000, "success_count": 2, - "failure_count": 1 + "failure_count": 1, + "results": [ + { + "id": "branch_a", + "status": "succeeded", + "context_updates": {"response.branch_a": "review complete"} + }, + { + "id": "branch_b", + "status": "failed", + "context_updates": {"command.output": "validation failed"} + } + ] } } ``` | Property | Type | Description | |----------|------|-------------| +| `visit` | number | Visit number for this parallel stage | | `duration_ms` | number | Total parallel duration | | `success_count` | number | Branches that succeeded | | `failure_count` | number | Branches that failed | +| `results` | array | Ordered typed branch results with `id`, `status`, and isolated `context_updates` | --- @@ -760,58 +775,6 @@ Note: `node_id` is optional — may be absent for non-stage commits. | `branch` | string | Branch name | | `success` | boolean | Whether push succeeded | -### `git.branch` - -```json -{ - "id": "...", "ts": "...", "run_id": "...", - "event": "git.branch", - "properties": { - "branch": "fabro/run-01JQXYZ", - "sha": "abc123..." - } -} -``` - -| Property | Type | Description | -|----------|------|-------------| -| `branch` | string | Branch name | -| `sha` | string | Branch HEAD SHA | - -### `git.worktree.added` - -```json -{ - "id": "...", "ts": "...", "run_id": "...", - "event": "git.worktree.added", - "properties": { - "path": "/tmp/fabro-worktrees/...", - "branch": "fabro/run-01JQXYZ" - } -} -``` - -| Property | Type | Description | -|----------|------|-------------| -| `path` | string | Worktree directory path | -| `branch` | string | Branch name | - -### `git.worktree.removed` - -```json -{ - "id": "...", "ts": "...", "run_id": "...", - "event": "git.worktree.removed", - "properties": { - "path": "/tmp/fabro-worktrees/..." - } -} -``` - -| Property | Type | Description | -|----------|------|-------------| -| `path` | string | Worktree directory path | - ### `git.fetch` ```json diff --git a/docs/internal/parallel-strategy.md b/docs/internal/parallel-strategy.md index 351764ade..61a019326 100644 --- a/docs/internal/parallel-strategy.md +++ b/docs/internal/parallel-strategy.md @@ -1,318 +1,153 @@ -# Parallel fan-out / fan-in strategy +# Shared-checkout parallel execution strategy -Status: proposed (not yet implemented). Line numbers reference the tree at the -time of writing and will drift. +Status: implemented. -This document specifies the intended product behavior of parallel fan-out -(`shape=component`) and fan-in (`shape=tripleoctagon`). Appendix A catalogs -pre-existing bugs this design resolves. Appendix B lists behavior changes -relative to today's implementation. +This document defines Fabro's parallel fan-out (`shape=component`) and fan-in +(`shape=tripleoctagon`) behavior. -## 1. Introduction: goals and current weaknesses +## 1. Execution model -The goal of this design is a parallel execution model that is **simple**, -**coherent**, and **correct**: +A parallel node dispatches one branch for each outgoing edge. A branch executes +the single target node on that edge; parallel branches are not subgraph walks. +Every branch: -- **Simple** — a user should be able to predict what a fan-out/fan-in does - from the graph alone. One mental model ("branches produce candidates; fan-in - picks a commit; downstream sees all responses"), no new syntax for - synthesis, no incantations (special fidelity settings, escape-hatch - attributes) to make the flagship patterns work. -- **Coherent** — the same rules apply regardless of node type or channel. - What holds for a sequential command node's output should hold for a branch - command node's output; the workspace (git) facet and the text (context) - facet should follow parallel logic; selection should live in exactly one - place. -- **Correct** — the engine must do what the graph, the docs, and the recorded - run state say it does. Nodes drawn in the graph must run; documented outputs - must exist; a judge asked to pick the best candidate must be shown the - candidates. +- receives an independent fork of the parent workflow context; +- receives the same `Arc` as the parent run; +- inherits the same sandbox working directory and `internal.work_dir`; +- runs through the normal handler dispatch path, including dry-run behavior; +- retains its branch identity, lifecycle events, and hook scope. -Today's implementation misses all three. Issue #490 is the visible symptom: -a synthesis node after fan-in never sees branch responses — only -`{id, status, head_sha}` metadata — while two tutorials promise the opposite. -Investigation showed a broader incoherence: +Branches execute concurrently. `max_parallel` limits the number that may run at +once and defaults to 4. The parallel node always waits for every branch task, +even when a branch fails or run cancellation begins. There is no early-success +join mode. -- **Inherited spec gap.** The Attractor spec (fabro's ancestor) deliberately - isolates branch context and never defines a channel for branch output text. - Its fan-in pseudocode judges candidates it cannot see (`llm_evaluate` - receives only statuses) and sorts by a `score` field nothing sets. Fabro - inherited this gap faithfully. -- **Missing compensation.** Kilroy (a sibling Attractor implementation) - compensates with a file/git handoff: the post-merge node's prompt is - injected with each branch's `worktree_dir`, `logs_root`, and `head_sha` plus - instructions to read/merge them. Fabro has no equivalent, but its tutorials - promise kilroy-like behavior ("the synth node receives all four perspectives - in its preamble"). -- **Accidental semantics.** Several adjacent behaviors are unprincipled - accidents rather than decisions: branches execute a single node and silently - skip chained nodes; two handlers fast-forward to two different definitions - of "winner"; branch nodes run with a stale preamble built for the fan-out - node; selection is vacuous in both modes; nested fan-outs silently lose git - isolation. Appendix A catalogs these. +The parent context is not used as shared mutable branch state. A branch can +change its context fork without exposing those changes as top-level values to +other branches or to the parent. -## 2. Core model: candidates +## 2. Shared checkout -A parallel branch is an isolated unit of execution that produces a -**candidate**. A candidate has exactly three facets: +All branches use the run's existing sandbox and checkout. Parallel execution +creates no branch-specific: -| Facet | Content | Carried by | -|---|---|---| -| Commit | Workspace state produced by the branch | Per-branch git commit (`head_sha`) | -| Response | Text output (LLM response, command output) | Branch stage records + context keys | -| Verdict | Terminal status plus optional numeric `score` | `parallel.results` entries | +- Git refs or branches; +- worktrees; +- base checkpoints; +- commits; +- cleanup operations; +- merges or fast-forwards. -Fan-out produces N candidates in isolation. Fan-in selects **one commit** to -continue on. Downstream nodes (synthesis) get **all responses**. Selection and -synthesis are distinct concerns: selection needs a node (`tripleoctagon`); -synthesis is any ordinary downstream node, because responses propagate. +Normal run-level checkpointing still occurs after the parallel node. Any files +left in the shared checkout by its branches are captured together by that +checkpoint. -The post-fan-in contract, in one sentence: **after fan-in, the run looks as if -every branch had run sequentially, and the winner ran last.** +Read-only parallel work is best effort: an agent or command can still write if +its configured capabilities permit it. Concurrent writes are allowed and are +entirely user-managed. Fabro does not lock files, enforce read-only access, +detect overlapping edits, or warn about races. Workflows that write in parallel +should coordinate externally or assign disjoint paths. -## 3. Topology: branches are subgraphs +## 3. Branch results -Each outgoing edge of a fan-out node starts a branch. A branch executes as a -subgraph walk: the engine traverses nodes and edges from the branch entry node -until it reaches the join node (the fan-in). Multi-node chains -(`fork -> plan_a; plan_a -> impl_a; impl_a -> merge`) run every node in the -chain. This matches the Attractor spec (`execute_subgraph`) and kilroy -(`runSubgraphUntil`); today fabro executes exactly one node per branch and -silently skips the rest (Appendix A.2). +The shared result type is: -Structural validation (lint): +```rust +ParallelBranchResult { + id: String, + status: String, + context_updates: BTreeMap, +} +``` -- Every path from a branch entry must converge on the run's join node. - A branch path that escapes the join (reaches exit, or a node outside the - fan-out region) is a validation error. -- A nested `component` node inside a branch is rejected until - worktree-from-worktree isolation is implemented (today it silently runs with - no git isolation; Appendix A.6). -- `fidelity="full"` on a fan-in's outgoing edge gets a lint warning: branches - run on different threads, so full fidelity can never carry branch outputs - (it drops the preamble entirely). +The parallel handler stores one result per outgoing edge in +`parallel.results`. Results preserve outgoing-edge order, independent of branch +completion order. `parallel.branch_count` stores the number of dispatched +branches. -## 4. Node types in branches +`context_updates` includes changes made in the branch context and updates +returned by the branch outcome. This applies to successful and failed branches, +including structured values, `response.`, and `command.output`. +Engine-internal context keys are omitted. A task failure or panic cannot provide +updates that were never returned, but its result still preserves the original +branch ID and index. -No type-based restrictions. Restriction is structural (§3), not by allowlist: +Branch updates remain nested in their result. Fabro never merges them into the +parent's top-level context, so branches cannot collide through context keys. -- **Agent / prompt nodes** — the primary case. -- **Command / script / tool nodes** — first-class. Deterministic fan-out - (test matrices, benchmark bake-offs across worktrees) is a supported pattern - with no LLM anywhere: branches emit `score` via status fields, heuristic - selection picks the winner by measurement. -- **Conditionals** — meaningful under subgraph branches: they route *within* - the branch. -- **Human gates** — allowed; each branch may pause independently. -- **Nested parallel** — rejected by lint until isolation composes (§3). +The parallel stage outcome is: -## 5. Git isolation +- `succeeded` when every branch succeeds; +- `failed` when every branch fails; +- `partially_succeeded` for mixed outcomes, partial outcomes, and zero branches. -Unchanged mechanics, with ownership fixed: +## 4. Artifacts and downstream context -1. Before fan-out, checkpoint the sandbox to produce `base_sha`. -2. Each branch gets a worktree on a branch ref - (`fabro/run/parallel///pass/`), rooted at `base_sha`. - The branch's `internal.work_dir` points at the worktree. -3. After a branch completes, `git add -A` + commit (`--allow-empty`) yields the - candidate's `head_sha`. -4. Worktrees are removed after the join. Loser branch refs are **kept** so - downstream nodes and humans can `git show`/`git diff` any candidate. -5. **Fan-in exclusively owns the fast-forward.** The parallel handler performs - no merge. After selection, fan-in fast-forwards the primary workspace to the - winner's `head_sha`. (Today both handlers fast-forward, to potentially - different winners; Appendix A.3.) +Large context values use the normal artifact store. Offloading recursively +replaces oversized leaf values while retaining the object and array structure +of `parallel.results`. -Degradation without git (no repo, or git isolation disabled): branches share -the primary sandbox with no workspace isolation, `head_sha` is absent from -candidates, and fan-in performs no merge. Response and verdict facets work -unchanged — prompt-only ensembles do not require git. +When Fabro constructs execution or prompt context, it resolves nested textual +blob references under `response.*` and `command.output`, including those keys +inside a branch result's `context_updates`. This lets a prompted fan-in inspect +complete branch text without flattening branch state into the parent context. -## 6. Execution and stage recording +`parallel.results` is runtime context. Fabro does not materialize a +`parallel_results.json` file in the workspace. Diagnostic run dumps may export +stage projection data, but that export is not a workflow handoff mechanism and +is not visible as a checkout file to downstream nodes. -Branch nodes execute as **real stages**, recorded through the normal -`ExecutionState::record` path and namespaced under the fan-out -(e.g. stage `a@1` within `fork@1`). Consequences (all fixes to current -behavior): +## 5. Fan-in -- Branch prompts/responses appear in events, `fabro dump`, and the web UI as - ordinary stages. -- Each branch node gets a **freshly built preamble** for its own position, via - the standard lifecycle, instead of reusing the fan-out node's stale preamble. -- Branch stages participate in the standard retry policy per node. +Fan-in is an explicit join node. -## 7. Context merge-back at fan-in +A fan-in node without a nonblank prompt validates that `parallel.results` +exists and deserializes as typed branch results. It then succeeds with a joined +branches note. It is a no-op barrier: it does not alter context or workspace +state. -When fan-in completes, branch context updates are applied to the parent -context with a collision rule: +A fan-in node with a prompt delegates to the standard prompt handler. It sees +the aggregated runtime context in the normal prompt preamble and records the +normal prompt-stage outputs: -- **Per-node keys** (`response.`, structured-output fields - namespaced by node) apply for **all** branches. Branch node IDs are unique, - so no collisions. -- **Singleton keys** (`last_stage`, `last_response`, `command.output`, - un-namespaced status fields) are taken from the **winner only**. -- Failed branches' `response.` values are still applied (a synthesis node - analyzing disagreement wants to see the failure text). Their singletons are - never applied. +- `response.`; +- `last_response`; +- model usage and timing; +- prompt and response events. -Fan-in additionally writes (as today): +A prompted fan-in synthesizes results. It does not rank branches, select a +winner, restore files, or choose workspace state. -- `parallel.results` — one entry per candidate: `{id, status, head_sha?, - score?}`. -- `parallel.branch_count`, `parallel.fan_in.best_id`, - `parallel.fan_in.best_outcome`, `parallel.fan_in.best_head_sha`. +## 6. Events and projections -No file is materialized into the run workspace. `parallel.results` reaches LLM -consumers through the preamble's context section, and agents can `git show` -any candidate via its `head_sha`. (The current docs claim -`parallel_results.json` is available to downstream nodes; that claim is false -today and should be corrected rather than implemented — see open question 4 -for the one consumer this leaves unserved.) +Parallel execution emits: -## 8. Selection +- `parallel.started` with `visit` and `branch_count`; +- `parallel.branch.started` with stable branch identity and index; +- `parallel.branch.completed` with index, duration, and status; +- `parallel.completed` with counts and the ordered typed result array. -Fan-in selects the winning candidate. Two modes, as today, but with real -signal: +Every branch task emits one terminal branch completion event, including handler +failure, cancellation before semaphore acquisition, panic, or join failure. +The final typed array is also projected into +`StageProjection.parallel_results`. -- **Heuristic** (no prompt on the fan-in node): rank by status - (succeeded < partially_succeeded < failed), then `score` descending, then - lexical id. `score` becomes settable: branches emit it via structured-output - / status fields, which now survive into `parallel.results` (§7). -- **LLM judge** (fan-in node has a `prompt` and a backend is configured): the - judge prompt includes, per candidate: id, status, score, a bounded response - excerpt, and `git diff --stat` vs. `base_sha` when git isolation is active. - Today the judge sees only `{id, status, head_sha}` and cannot possibly - discriminate (Appendix A.4). +## 7. Cancellation -Selection determines the commit facet only. It does not suppress loser -responses (§7) or loser refs (§5). +Semaphore acquisition observes the run cancellation token. Branches waiting for +a permit can terminate as cancelled rather than waiting indefinitely. Branches +already executing continue through their handler's cooperative cancellation +path. The parallel handler joins every task before returning cancellation to the +run executor. -## 9. Downstream visibility (preamble) +Cancellation does not trigger branch Git cleanup because no branch Git state is +created. -Prompt templates render once at manifest build time with `{goal, inputs}` -only; the preamble is the sole channel for runtime context into a -fresh-session node. Therefore: +## 8. Product constraints -- At **`compact`** (default) fidelity, branch stage summaries render their - responses **inline**, bounded, with a `See: ` reference when - truncated — the same treatment `command.output` already receives at compact. - Rationale: post-fan-in branch responses are unrecoverable through any - fidelity setting (different threads), exactly like command output. -- The per-branch response budget is larger than command output's 25-line tail - (ensemble analyses front-load their substance; a small tail amputates it). - Exact budget TBD at implementation; must remain bounded so N long branches - cannot blow the downstream context. Agent-type synthesis nodes can read the - full text from the blob/artifact reference. -- `summary:high` renders the same with its larger budget. `truncate` carries - goal only (explicit opt-out). `full` remains the degenerate case and lints - (§3). - -## 10. Join policies - -- `wait_all` (default): all branches run to completion; join proceeds when all - are terminal. Succeeds if no branch failed, else partially succeeds (fan-in - fails only when *all* candidates failed). -- `first_success`: join proceeds at the first successful branch. Remaining - branches are cancelled; cancelled branches record a terminal cancelled stage - (their partial responses are not merged back). The sole successful branch is - the winner. -- `k_of_n` / `quorum` (kilroy has them): deliberately **not** added now. The - surface stays minimal until a concrete need appears. - -## Open questions - -1. Implementation phasing: subgraph branches (§3) are the largest lift. - Response propagation (§6–§9) fixes #490 and both tutorials on its own and - can ship first. -2. `first_success` cancellation semantics for in-flight agent sessions - (graceful stop vs. abort; what the cancelled stage records). -3. Exact preamble budget per branch response (§9). -4. Context access for deterministic post-fan-in consumers. Command/script - nodes have no channel to context (no preamble, no template rendering in - `script`), so a deterministic aggregator after fan-in cannot learn - candidate `head_sha`s. Candidate mechanisms: a results file under a - checkpoint-excluded workspace path, or an env var (e.g. - `FABRO_PARALLEL_RESULTS`) pointing at a file outside the workspace. A bare - workspace file is ruled out: checkpoint commits `git add -A`, so it would - leak into run history and PRs. Design alongside the deterministic fan-out - pattern (§4). - ---- - -## Appendix A: pre-existing bugs - -Cataloged against the current tree; line numbers will drift. - -1. **Branch outputs dropped (#490).** The fan-out task reads only - `outcome.status` and `head_sha` from each branch; branch `context_updates` - (including `response.`) are discarded with the forked context - (`fabro-workflow/src/handler/parallel.rs:389-397,465-470`). No channel - carries branch text to downstream nodes. Confirmed by live repro on - fabro-testing (run `01KX148ZAMMJRAADHK1HBF7PC3`, server 0.287.0-nightly.0). -2. **Chained branch nodes silently skipped.** Branches execute exactly one - node; the engine then jumps to the join (`parallel.rs:388-397,612,628`; - `fabro-core/src/executor.rs:421`). In `fork -> a; a -> a2; a2 -> merge`, - `a2` never runs and nothing warns. -3. **Double fast-forward with two different winner definitions.** The parallel - handler fast-forwards the *lexically first* successful branch - (`parallel.rs:511-538`); fan-in then fast-forwards *its* selected winner - (`fan_in.rs:120-131`). If selection ever picks a non-lexical-first branch, - the second `--ff-only` merge cannot succeed (sibling commits diverge). - Masked today only because selection is vacuous (A.4). -4. **Selection is vacuous.** Heuristic tie-breaks on a `score` field nothing - can set (scores would arrive via branch context updates, which are dropped - per A.1). The LLM judge prompt is `serde_json::to_string_pretty` of - `parallel.results` — id/status/head_sha only (`fan_in.rs:247-250`). Both - modes reduce to "first successful branch, alphabetically." -5. **Branch preambles are stale.** Branches run via `dispatch_handler`, - bypassing the lifecycle's per-node preamble rebuild; each branch node - inherits `current.preamble` as computed for the fan-out node itself. -6. **Nested parallel silently loses git isolation.** Branch `EngineServices` - are built with `git_state: RwLock::new(None)` (`parallel.rs:380`), so a - `component` node inside a branch runs its own branches with no worktrees - and no warning. -7. **Docs contradict the engine.** `tutorials/ensemble.mdx:85` and - `tutorials/parallel-review.mdx:81` claim the post-merge node receives all - branch perspectives in its preamble (false, per A.1). - `workflows/stages-and-nodes.mdx:195` claims merged results are available to - downstream nodes as `parallel_results.json` (the file exists only under - `stages/@1/` in dumps, not in any node's working directory). -8. **`fidelity="full"` across a fan-in is a trap.** It drops the preamble - (metadata included) and cannot attach to any branch thread; raising - fidelity strictly reduces what the downstream node sees. No lint warns. - -## Appendix B: behavior changes vs. today - -Changes a user could observe if this spec is implemented as written. - -1. **Chained branch nodes execute.** Graphs that (unknowingly) relied on - single-node branch semantics will now run the full chain (fixes A.2; may - lengthen existing runs). -2. **New validation errors.** Branch paths that don't converge on the join, - and nested `component` nodes inside branches, become lint failures for - graphs that previously ran (with wrong or silently degraded semantics). -3. **Fan-in owns the fast-forward.** The workspace after fan-in may land on a - different commit than today whenever selection (scores, LLM judge) - disagrees with lexical-first order. The parallel handler no longer merges. -4. **Post-fan-in context is richer.** `response.` for every branch, - winner-sourced singletons (`last_stage`, `last_response`, - `command.output`), and `score` in `parallel.results`. Today those - singletons retain their pre-fork values; workflows with edge conditions - over them could route differently. -5. **Preambles after fan-in grow.** Branch responses render inline at - `compact` fidelity (bounded). Downstream nodes see more tokens per run; - snapshot tests over preambles will churn. -6. **Branch executions become visible stages.** Events, dumps, the web UI, and - the stage list gain per-branch stages (`a@1` under `fork@1`). Consumers of - `events.jsonl` / the API will see new stage records. -7. **`parallel_results.json` docs claim is corrected, not implemented.** - `workflows/stages-and-nodes.mdx:195` is updated to describe the real - channels (context key + preamble); no file appears in the workspace - (open question 4 covers deterministic consumers). -8. **Loser branch refs are documented as retained** and become part of the - product contract instead of an accident of not deleting them. -9. **`first_success` cancels losers explicitly** and records cancelled stages; - today's exact cancellation behavior is unspecified. -10. **LLM judge prompts change shape.** Fan-in nodes with prompts now send - candidate excerpts and diff stats to the judge — more tokens, different - (better) selections than today's id-only prompt. +- Branches remain single-node executions. +- `max_parallel` remains supported. +- Results are deterministic in outgoing-edge order. +- There is no branch-selection score, SHA, model-usage mode, notice, or UI. +- Server-owned independent checkout/worktree behavior is separate and remains + unchanged. diff --git a/docs/public/api-reference/fabro-api.yaml b/docs/public/api-reference/fabro-api.yaml index d6e384405..a39a4c414 100644 --- a/docs/public/api-reference/fabro-api.yaml +++ b/docs/public/api-reference/fabro-api.yaml @@ -10411,6 +10411,22 @@ components: items: $ref: "#/components/schemas/StageContextWindowWarning" + ParallelBranchResult: + description: The outcome and isolated context updates from one parallel branch. + type: object + required: + - id + - status + - context_updates + properties: + id: + type: string + status: + $ref: "#/components/schemas/StageOutcome" + context_updates: + type: object + additionalProperties: true + StageProjection: description: Observable projection data for one workflow stage execution. type: object @@ -10447,8 +10463,8 @@ components: parallel_results: type: ["array", "null"] items: - type: object - description: Per-branch result objects produced by a parallel stage. + $ref: "#/components/schemas/ParallelBranchResult" + description: Ordered per-branch results produced by a parallel stage. output: type: ["string", "null"] output_bytes: diff --git a/docs/public/examples/clone-substack.mdx b/docs/public/examples/clone-substack.mdx index cc11157f7..3529436a2 100644 --- a/docs/public/examples/clone-substack.mdx +++ b/docs/public/examples/clone-substack.mdx @@ -153,8 +153,8 @@ Write to .workflow/plan_b.md." label="Debate & Consolidate", prompt="Synthesize the two implementation plans into a single best-of-breed \ final plan.\n\n\ -Read branch outputs via parallel_results.json. If parallel_results.json is missing, \ -fall back to reading .workflow/plan_a.md and .workflow/plan_b.md.\n\n\ +Review every branch result in parallel.results from the prompt context, then read \ +.workflow/plan_a.md and .workflow/plan_b.md from the shared checkout.\n\n\ If .workflow/postmortem_latest.md exists, read it FIRST. The postmortem contains \ root-cause analysis and concrete fixes from the previous iteration. The final plan \ MUST be adjusted to address every issue identified in the postmortem — add new \ @@ -481,8 +481,8 @@ Write to .workflow/review_b.md." goal_gate=true, retry_target="postmortem", prompt="Synthesize the two reviews into a consensus verdict.\n\n\ -Read branch outputs via parallel_results.json. If parallel_results.json is \ -missing, fall back to reading .workflow/review_a.md and .workflow/review_b.md.\n\n\ +Review every branch result in parallel.results from the prompt context, then read \ +.workflow/review_a.md and .workflow/review_b.md from the shared checkout.\n\n\ Read .workflow/definition_of_done.md for acceptance criteria reference.\n\n\ Consensus rules:\n\ - Both APPROVED with no critical gaps: the implementation passes\n\ @@ -513,7 +513,7 @@ Read (if they exist):\n\ - .workflow/test-evidence/latest/manifest.json\n\ - Evidence files referenced by manifest entries for failed or suspicious IT \ scenarios\n\ -- Branch review outputs via parallel_results.json (if available)\n\n\ +- Parallel branch review status and context updates from parallel.results (if available)\n\n\ Output to .workflow/postmortem_latest.md (overwrite previous):\n\ - Root causes of failure\n\ - What works and must be preserved\n\ diff --git a/docs/public/execution/context.mdx b/docs/public/execution/context.mdx index c67012b3c..25f6e536d 100644 --- a/docs/public/execution/context.mdx +++ b/docs/public/execution/context.mdx @@ -57,13 +57,14 @@ Agents can also emit arbitrary context updates by including a JSON object with a | `human.gate..answer` | The answer text for a specific human gate node | | `human.gate..label` | The selected label for a specific human gate node, when applicable | -### Parallel merge (fan-in) +### Parallel fan-out and fan-in | Key | Value | |---|---| -| `parallel.fan_in.best_id` | Node ID of the best-performing branch | -| `parallel.fan_in.best_outcome` | Status of the best branch | -| `parallel.fan_in.best_head_sha` | Git SHA from the best branch (if applicable) | +| `parallel.results` | Ordered branch results. Each entry contains `id`, `status`, and the branch's isolated `context_updates`. | +| `parallel.branch_count` | Number of outgoing branches dispatched by the parallel node. | + +Branch updates remain nested inside `parallel.results`; they are not merged into top-level context. Prompted fan-in nodes can synthesize the complete result array, while promptless fan-in nodes act as barriers. ### Engine-managed keys diff --git a/docs/public/execution/outcomes.mdx b/docs/public/execution/outcomes.mdx index 5ddbdc774..4ebf04a96 100644 --- a/docs/public/execution/outcomes.mdx +++ b/docs/public/execution/outcomes.mdx @@ -26,7 +26,7 @@ Each node type has its own rules for which outcomes it can return: |---|---|---| | **Command** | `succeeded`, `failed` | `succeeded` when exit code is 0; `failed` otherwise | | **Agent / Prompt** | `succeeded`, `failed`, `partially_succeeded`, `skipped` | Defaults to `succeeded`. The LLM can set any outcome via a [routing directive](/agents/outputs#routing-directives) JSON object in its response. Backend errors request retry when retryable or finish as `failed`. | -| **Parallel** | `succeeded`, `partially_succeeded`, `failed` | Depends on the `join_policy`. `wait_all`: `succeeded` if no failures, `partially_succeeded` if some branches failed. `first_success`: `succeeded` if threshold met, else `failed`. | +| **Parallel** | `succeeded`, `partially_succeeded`, `failed` | Waits for every branch. `succeeded` when all branches succeed, `failed` when all branches fail, and `partially_succeeded` for mixed, partial, or zero-branch results. | | **Human** | `succeeded` | Always succeeds — the user's selection becomes a routing signal via `preferred_label` | | **Conditional** | `succeeded` | Always succeeds — routing is handled by the engine's edge selection | | **Start / Exit / Wait** | `succeeded` | Always succeed | diff --git a/docs/public/reference/dot-language.mdx b/docs/public/reference/dot-language.mdx index f01fbf2ed..678d5ac23 100644 --- a/docs/public/reference/dot-language.mdx +++ b/docs/public/reference/dot-language.mdx @@ -249,8 +249,7 @@ audit [ | Attribute | Type | Description | |---|---|---| -| `join_policy` | String | When the merge can proceed: `wait_all` (default), `first_success` | -| `max_parallel` | Integer | Maximum concurrent branches (default: 4) | +| `max_parallel` | Integer | Maximum concurrent branches (default: 4). The node always waits for every branch. | ### Wait nodes diff --git a/docs/public/tutorials/ensemble.mdx b/docs/public/tutorials/ensemble.mdx index e6ff5adb8..c134c8d26 100644 --- a/docs/public/tutorials/ensemble.mdx +++ b/docs/public/tutorials/ensemble.mdx @@ -1,6 +1,6 @@ --- title: "Ensemble" -description: "Multi-provider fan-out, error policies, and result synthesis" +description: "Multi-provider fan-out and shared-checkout result synthesis" --- This tutorial combines parallel execution with multi-model routing to get independent opinions from four different LLM providers, then synthesizes the results. This is the ensemble pattern — useful when you want diverse perspectives, consensus-based decisions, or protection against any single model's blind spots. @@ -28,15 +28,15 @@ digraph Ensemble { start [shape=Mdiamond, label="Start"] exit [shape=Msquare, label="Exit"] - fork [label="Fan Out", shape=component, join_policy="wait_all"] + fork [label="Fan Out", shape=component] opus [label="Opus", prompt="Analyze the goal. Provide your independent assessment, recommendations, and any code or prose needed. Be thorough.", shape=tab] gemini [label="Gemini", prompt="Analyze the goal. Provide your independent assessment, recommendations, and any code or prose needed. Be thorough.", shape=tab] codex [label="Codex", prompt="Analyze the goal. Provide your independent assessment, recommendations, and any code or prose needed. Be thorough.", shape=tab] mercury [label="Mercury", prompt="Analyze the goal. Provide your independent assessment, recommendations, and any code or prose needed. Be thorough.", shape=tab] - merge [label="Merge", shape=tripleoctagon] - synth [label="Synthesize", prompt="You have received independent analyses from four different models (Opus, Gemini, Codex, Mercury). Compare their perspectives: identify consensus, highlight disagreements, and synthesize the strongest ideas into a single coherent recommendation. Note where models agreed and where they diverged.", shape=tab] + merge [label="Synthesize", shape=tripleoctagon, prompt="Compare every branch result: identify consensus, highlight disagreements, and synthesize the strongest ideas into one recommendation. Note where models agreed and diverged."] + synth [label="Write Recommendation", prompt="Write the synthesized analysis as a single coherent recommendation.", shape=tab] start -> fork fork -> opus diff --git a/docs/public/tutorials/parallel-review.mdx b/docs/public/tutorials/parallel-review.mdx index 41f589bea..39f6fd7f8 100644 --- a/docs/public/tutorials/parallel-review.mdx +++ b/docs/public/tutorials/parallel-review.mdx @@ -1,6 +1,6 @@ --- title: "Parallel Review" -description: "Fan-out, fan-in, join policies, and merge nodes" +description: "Shared-checkout fan-out, fan-in, and result synthesis" --- This tutorial runs three code review perspectives in parallel — security, architecture, and quality — then merges the results into a single report. @@ -19,14 +19,14 @@ digraph Parallel { start [shape=Mdiamond, label="Start"] exit [shape=Msquare, label="Exit"] - fork [label="Fork Analysis", shape=component, join_policy="wait_all"] + fork [label="Fork Analysis", shape=component] security [label="Security Audit", prompt="Examine the codebase for security concerns: hardcoded secrets, injection risks, unsafe dependencies. List findings as bullet points.", shape=tab, reasoning_effort="low"] architecture [label="Architecture Review", prompt="Assess the codebase architecture: separation of concerns, dependency structure, modularity. List findings as bullet points.", shape=tab, reasoning_effort="low"] quality [label="Code Quality", prompt="Check code quality: naming conventions, dead code, test coverage gaps, error handling. List findings as bullet points.", shape=tab, reasoning_effort="low"] - merge [label="Merge Findings", shape=tripleoctagon] - report [label="Final Report", prompt="Synthesize the security, architecture, and code quality findings into a prioritized summary report with top 5 action items.", shape=tab] + merge [label="Synthesize Findings", shape=tripleoctagon, prompt="Synthesize every branch result into a prioritized summary report with the top 5 action items."] + report [label="Write Final Report", prompt="Write the synthesized review as a clear final report.", shape=tab] start -> fork fork -> security @@ -48,37 +48,38 @@ fabro run docs/internal/demo/06-parallel.fabro The `fork` node has `shape=component`, making it a **parallel fan-out node**. Every outgoing edge becomes a concurrent branch: ```dot -fork [label="Fork Analysis", shape=component, join_policy="wait_all"] +fork [label="Fork Analysis", shape=component] fork -> security fork -> architecture fork -> quality ``` -All three branches start at the same time. Each gets an isolated copy of the run context, so branches can't interfere with each other. +All three branches start concurrently and operate in the same sandbox checkout and working directory. This review is safe because the branches are prompt nodes with no workspace tools. -### Join policies + +Parallel branches are not isolated from one another. If branches edit files, they can observe, race with, or overwrite each other's changes. Fabro does not lock files, detect overlapping writes, or warn about races. Keep branches read-only or give each branch responsibility for disjoint paths. + -The `join_policy` controls when execution can proceed past the merge: - -| Policy | Behavior | -|---|---| -| `wait_all` | Wait for every branch to finish (default) | -| `first_success` | Proceed as soon as one branch succeeds | +The parallel node always waits for every branch to finish before continuing. ## Fan-in with the merge node -The `merge` node has `shape=tripleoctagon`, making it a **merge (fan-in) node**. It collects results from all branches into a single context: +The `merge` node has `shape=tripleoctagon`, making it a **merge (fan-in) node**. It receives every branch's status and context updates in `parallel.results` and synthesizes them with its prompt: ```dot -merge [label="Merge Findings", shape=tripleoctagon] +merge [ + label="Synthesize Findings", + shape=tripleoctagon, + prompt="Synthesize every branch result into a prioritized summary report with the top 5 action items." +] security -> merge architecture -> merge quality -> merge ``` -The merged branch results are available to downstream nodes. The `report` node receives all three perspectives in its preamble and synthesizes them. +`parallel.results` is runtime context, not a `parallel_results.json` file in the checkout. Fan-in never selects a branch's files or changes workspace state; every branch has already worked in the same checkout. The downstream `report` node writes the synthesis produced by fan-in as the final report. ## Concurrency control @@ -92,10 +93,10 @@ This is useful when branches are resource-intensive (e.g., each running a full a ## What you've learned -- **Fan-out nodes** (`shape=component`) spawn concurrent branches -- **Merge nodes** (`shape=tripleoctagon`) collect branch results -- **Join policies** control when execution can proceed past the merge -- Each branch gets an isolated copy of the context +- **Fan-out nodes** (`shape=component`) spawn concurrent branches and wait for all of them +- Parallel branches share one checkout, so workflows must prevent or tolerate file races +- **Fan-in nodes** (`shape=tripleoctagon`) can synthesize `parallel.results` with a prompt +- Fan-in never selects or restores workspace state, and no results file is created ## Next diff --git a/docs/public/workflows/stages-and-nodes.mdx b/docs/public/workflows/stages-and-nodes.mdx index 1dd0f71ac..9263d1ea7 100644 --- a/docs/public/workflows/stages-and-nodes.mdx +++ b/docs/public/workflows/stages-and-nodes.mdx @@ -155,10 +155,10 @@ Conditions support `=`, `!=`, `&&`, and context variable lookups (e.g. `context. **Shape:** `component` -Fans out to execute multiple branches concurrently. Each branch gets its own isolated context. +Fans out to execute multiple branches concurrently. Every branch runs in the same sandbox checkout and working directory, and the parallel node waits for every branch to finish. ```dot -fork [label="Fan Out", shape=component, join_policy="wait_all"] +fork [label="Fan Out", shape=component] fork -> security fork -> architecture @@ -167,24 +167,22 @@ fork -> quality | Attribute | Description | |---|---| -| `join_policy` | When the merge can proceed (see table below) | | `max_parallel` | Maximum concurrent branches (default: 4) | -**Join policies:** - -| Policy | Behavior | -|---|---| -| `wait_all` | Wait for every branch to finish (default) | -| `first_success` | Proceed as soon as one branch succeeds | +Because the checkout is shared, file changes from one branch are immediately visible to the others. Concurrent writes can race or overwrite each other. Fabro does not isolate branch files, lock paths, detect conflicts, or warn about overlapping writes. Design branches to be read-only or assign each branch disjoint files and directories when deterministic workspace changes matter. ### Merge (fan-in) **Shape:** `tripleoctagon` -Collects results from parallel branches into a single context. Typically paired with a parallel fan-out node: +Converges parallel branches after all of them finish. Branch status and context updates are collected in the runtime context at `parallel.results`; Fabro does not create a `parallel_results.json` file in the checkout. ```dot -merge [label="Merge Results", shape=tripleoctagon] +merge [ + label="Synthesize Results", + shape=tripleoctagon, + prompt="Synthesize every branch result into one report." +] security -> merge architecture -> merge @@ -192,7 +190,7 @@ quality -> merge merge -> report ``` -The merged results are available to downstream nodes as `parallel_results.json`. +A fan-in node with a `prompt` synthesizes the collected results. It never chooses, restores, or merges a branch's workspace state: all branches have already operated on the same checkout. Without a prompt, fan-in is only a convergence barrier. ## Common node attributes diff --git a/lib/crates/fabro-agent/src/lib.rs b/lib/crates/fabro-agent/src/lib.rs index 47aa8cd75..fd6f84db7 100644 --- a/lib/crates/fabro-agent/src/lib.rs +++ b/lib/crates/fabro-agent/src/lib.rs @@ -57,8 +57,7 @@ pub use read_before_write_sandbox::ReadBeforeWriteSandbox; pub use sandbox::{ CommandOutputCallback, DirEntry, ExecResult, ExecStreamingResult, GrepOptions, RefreshOutcome, Sandbox, SandboxEvent, SandboxEventCallback, StderrCollector, StdioProcess, StdioProcessHandle, - WorktreeEvent, WorktreeEventCallback, WorktreeOptions, WorktreeSandbox, format_lines_numbered, - shell_quote, + format_lines_numbered, shell_quote, }; pub use session::{ CompletionCoordinator, Session, SessionControlHandle, SessionInputTiming, StaticEnvProvider, diff --git a/lib/crates/fabro-agent/src/sandbox.rs b/lib/crates/fabro-agent/src/sandbox.rs index 5884876ab..3ba22c7a9 100644 --- a/lib/crates/fabro-agent/src/sandbox.rs +++ b/lib/crates/fabro-agent/src/sandbox.rs @@ -4,6 +4,5 @@ pub use fabro_sandbox::{ CommandOutputCallback, DirEntry, ExecResult, ExecStreamingResult, GrepOptions, RefreshOutcome, Sandbox, SandboxEvent, SandboxEventCallback, StderrCollector, StdioProcess, StdioProcessHandle, - StdioProcessTermination, WorktreeEvent, WorktreeEventCallback, WorktreeOptions, - WorktreeSandbox, delegate_sandbox, format_lines_numbered, shell_quote, + StdioProcessTermination, delegate_sandbox, format_lines_numbered, shell_quote, }; diff --git a/lib/crates/fabro-api/build.rs b/lib/crates/fabro-api/build.rs index 0a81b9845..3fa54d1a6 100644 --- a/lib/crates/fabro-api/build.rs +++ b/lib/crates/fabro-api/build.rs @@ -348,6 +348,11 @@ fn main() { ("StageState", "fabro_types::StageState", &[]), ("CommandTermination", "fabro_types::CommandTermination", &[]), ("StageModelUsage", "fabro_types::StageModelUsage", &[]), + ( + "ParallelBranchResult", + "fabro_types::ParallelBranchResult", + &[], + ), ("StageProjection", "fabro_types::StageProjection", &[]), ("PermissionLevel", "fabro_types::PermissionLevel", &[]), ( diff --git a/lib/crates/fabro-api/src/lib.rs b/lib/crates/fabro-api/src/lib.rs index da898f8c2..b8401950c 100644 --- a/lib/crates/fabro-api/src/lib.rs +++ b/lib/crates/fabro-api/src/lib.rs @@ -50,8 +50,8 @@ pub mod types { McpServerReplace as ReplaceMcpServerRequest, McpServerStatus, McpServerView as McpServer, McpTransportView, Message, PairId, PairMessageId, PairMessageRecord, PairMessageRequest, PairRecord, PairStartRequest, PairStatus, PairTarget, PairTranscriptEntry, - PairTranscriptResponse, PendingInterviewRecord, PermissionLevel, PreRunPushOutcome, - Principal, PullRequest, PullRequestDetails, PullRequestDetailsStatus, + PairTranscriptResponse, ParallelBranchResult, PendingInterviewRecord, PermissionLevel, + PreRunPushOutcome, Principal, PullRequest, PullRequestDetails, PullRequestDetailsStatus, PullRequestDetailsUnavailableReason, PullRequestLink, PullRequestMeta, PullRequestResponse, QuestionType, RepositoryRef, Role, Run, RunApproval, RunApprovalState, RunClientProvenance, RunEvent, RunEventDetailContentKind, RunEventDetailResponse, RunFailure, diff --git a/lib/crates/fabro-api/tests/run_event_round_trip.rs b/lib/crates/fabro-api/tests/run_event_round_trip.rs index c36c1e67f..4f4ea5e99 100644 --- a/lib/crates/fabro-api/tests/run_event_round_trip.rs +++ b/lib/crates/fabro-api/tests/run_event_round_trip.rs @@ -250,6 +250,65 @@ fn run_event_round_trips_stage_started() { assert_run_event_round_trip(value); } +#[test] +fn run_event_round_trips_parallel_public_contracts() { + assert_run_event_round_trip(json!({ + "id": "evt_parallel_started", + "ts": "2026-04-29T12:02:00Z", + "run_id": fixtures::RUN_1, + "event": "parallel.started", + "node_id": "fanout", + "node_label": "Fanout", + "parallel_group_id": "fanout@2", + "properties": { + "visit": 2, + "branch_count": 2 + } + })); + assert_run_event_round_trip(json!({ + "id": "evt_parallel_branch_completed", + "ts": "2026-04-29T12:02:01Z", + "run_id": fixtures::RUN_1, + "event": "parallel.branch.completed", + "node_id": "review_api", + "node_label": "Review API", + "parallel_group_id": "fanout@2", + "parallel_branch_id": "fanout@2:0", + "properties": { + "index": 0, + "duration_ms": 1000, + "status": "succeeded" + } + })); + assert_run_event_round_trip(json!({ + "id": "evt_parallel_completed", + "ts": "2026-04-29T12:02:02Z", + "run_id": fixtures::RUN_1, + "event": "parallel.completed", + "node_id": "fanout", + "node_label": "Fanout", + "parallel_group_id": "fanout@2", + "properties": { + "visit": 2, + "duration_ms": 2000, + "success_count": 1, + "failure_count": 1, + "results": [ + { + "id": "review_api", + "status": "succeeded", + "context_updates": {"response.review_api": "looks good"} + }, + { + "id": "review_ux", + "status": "failed", + "context_updates": {} + } + ] + } + })); +} + #[test] fn run_event_round_trips_agent_tool_started() { let value = json!({ diff --git a/lib/crates/fabro-api/tests/stage_projection_round_trip.rs b/lib/crates/fabro-api/tests/stage_projection_round_trip.rs index 2f26ad612..7a680b135 100644 --- a/lib/crates/fabro-api/tests/stage_projection_round_trip.rs +++ b/lib/crates/fabro-api/tests/stage_projection_round_trip.rs @@ -7,8 +7,8 @@ use fabro_api::types::{ AgentToolSource as ApiAgentToolSource, AgentToolSummary as ApiAgentToolSummary, AgentToolsAvailableProps as ApiAgentToolsAvailableProps, McpServerProjection as ApiMcpServerProjection, McpServerStatus as ApiMcpServerStatus, - PermissionLevel as ApiPermissionLevel, SkillsProjection as ApiSkillsProjection, - StageContextWindow as ApiStageContextWindow, + ParallelBranchResult as ApiParallelBranchResult, PermissionLevel as ApiPermissionLevel, + SkillsProjection as ApiSkillsProjection, StageContextWindow as ApiStageContextWindow, StageContextWindowBreakdownItem as ApiStageContextWindowBreakdownItem, StageContextWindowCategory as ApiStageContextWindowCategory, StageContextWindowCountMethod as ApiStageContextWindowCountMethod, @@ -22,11 +22,11 @@ use fabro_api::types::{ use fabro_types::{ ActivatedSkill, AgentMcpToolSummary, AgentSkillActivationSource, AgentSkillSummary, AgentToolCategory, AgentToolSource, AgentToolSummary, AgentToolsAvailableProps, - McpServerProjection, McpServerStatus, PermissionLevel, SkillsProjection, StageContextWindow, - StageContextWindowBreakdownItem, StageContextWindowCategory, StageContextWindowCountMethod, - StageContextWindowProjection, StageContextWindowStaleness, StageContextWindowUnavailableReason, - StageContextWindowWarning, StageProjection, SubAgentProjection, SubAgentStatus, TodoListKind, - TodoListProjection, + McpServerProjection, McpServerStatus, ParallelBranchResult, PermissionLevel, SkillsProjection, + StageContextWindow, StageContextWindowBreakdownItem, StageContextWindowCategory, + StageContextWindowCountMethod, StageContextWindowProjection, StageContextWindowStaleness, + StageContextWindowUnavailableReason, StageContextWindowWarning, StageProjection, + SubAgentProjection, SubAgentStatus, TodoListKind, TodoListProjection, }; use serde_json::json; @@ -37,6 +37,7 @@ fn stage_projection_reuses_canonical_type() { #[test] fn stage_projection_reuses_nested_agent_state_types() { + assert_same_type::(); assert_same_type::(); assert_same_type::(); assert_same_type::(); @@ -85,7 +86,21 @@ fn stage_projection_round_trips_representative_json() { "diff": "diff --git a/file b/file", "script_invocation": { "command": "cargo test" }, "script_timing": { "duration_ms": 42 }, - "parallel_results": [{ "branch": 0, "status": "succeeded" }], + "parallel_results": [ + { + "id": "review_api", + "status": "succeeded", + "context_updates": { + "response.review_api": "looks good", + "score": 0.95 + } + }, + { + "id": "review_ux", + "status": "failed", + "context_updates": {} + } + ], "output": "ok", "termination": "exited", "started_at": "2026-04-29T12:34:00Z", diff --git a/lib/crates/fabro-cli/src/commands/run/run_progress/event.rs b/lib/crates/fabro-cli/src/commands/run/run_progress/event.rs index a4a72f0bc..6f8498411 100644 --- a/lib/crates/fabro-cli/src/commands/run/run_progress/event.rs +++ b/lib/crates/fabro-cli/src/commands/run/run_progress/event.rs @@ -130,7 +130,7 @@ pub(super) enum ProgressEvent { ParallelBranchCompleted { branch: String, duration_ms: u64, - status: String, + status: fabro_types::StageOutcome, }, ParallelCompleted, AssistantMessage { @@ -318,7 +318,7 @@ pub(super) fn from_run_event(stored: &RunEvent) -> Option { EventBody::ParallelBranchCompleted(props) => Some(ProgressEvent::ParallelBranchCompleted { branch: node_id, duration_ms: props.duration_ms, - status: props.status.clone(), + status: props.status, }), EventBody::ParallelCompleted(_) => Some(ProgressEvent::ParallelCompleted), EventBody::AgentMessage(props) => Some(ProgressEvent::AssistantMessage { diff --git a/lib/crates/fabro-cli/src/commands/run/run_progress/mod.rs b/lib/crates/fabro-cli/src/commands/run/run_progress/mod.rs index 6e1a6227b..24570dac4 100644 --- a/lib/crates/fabro-cli/src/commands/run/run_progress/mod.rs +++ b/lib/crates/fabro-cli/src/commands/run/run_progress/mod.rs @@ -248,7 +248,7 @@ impl ProgressUI { status, } => { self.stage - .on_parallel_branch_completed(renderer, &branch, duration_ms, &status); + .on_parallel_branch_completed(renderer, &branch, duration_ms, status); } ProgressEvent::ParallelCompleted => { self.stage.on_parallel_completed(); @@ -587,7 +587,6 @@ mod tests { node_id: "fork1".into(), visit: 1, branch_count: 2, - join_policy: "wait_all".into(), }); assert_eq!(ui.stage.parallel_parent.as_deref(), Some("fork1")); @@ -611,8 +610,7 @@ mod tests { branch: "security".into(), index: 0, duration_ms: 2000, - status: "succeeded".into(), - head_sha: None, + status: fabro_workflow::outcome::StageOutcome::Succeeded, }); let stage = &ui.stage.active_stages["fork1"]; assert!(matches!( @@ -630,7 +628,6 @@ mod tests { node_id: "fork1".into(), visit: 1, branch_count: 1, - join_policy: "wait_all".into(), }); emit(&mut ui, Event::ParallelBranchStarted { parallel_group_id: StageId::new("fork1", 1), @@ -1246,7 +1243,6 @@ mod tests { node_id: "fork1".into(), visit: 1, branch_count: 1, - join_policy: "wait_all".into(), }); emit(&mut ui, Event::ParallelBranchStarted { parallel_group_id: StageId::new("fork1", 1), @@ -1260,8 +1256,7 @@ mod tests { branch: "security".into(), index: 0, duration_ms: 500, - status: "succeeded".into(), - head_sha: None, + status: fabro_workflow::outcome::StageOutcome::Succeeded, }); let stage = &ui.stage.active_stages["fork1"]; diff --git a/lib/crates/fabro-cli/src/commands/run/run_progress/stage_display.rs b/lib/crates/fabro-cli/src/commands/run/run_progress/stage_display.rs index 13014c2a0..b58568b48 100644 --- a/lib/crates/fabro-cli/src/commands/run/run_progress/stage_display.rs +++ b/lib/crates/fabro-cli/src/commands/run/run_progress/stage_display.rs @@ -244,7 +244,7 @@ impl StageDisplay { renderer: &ProgressRenderer, branch: &str, duration_ms: u64, - status: &str, + status: StageOutcome, ) { let Some(parent_id) = self.parallel_parent.clone() else { return; @@ -261,7 +261,7 @@ impl StageDisplay { return; }; - let succeeded = matches!(status, "succeeded" | "partially_succeeded"); + let succeeded = status.is_successful(); entry.status = if succeeded { ToolCallStatus::Succeeded } else { diff --git a/lib/crates/fabro-cli/tests/it/workflow/dry_run_examples.rs b/lib/crates/fabro-cli/tests/it/workflow/dry_run_examples.rs index bb8a50c85..658dd1fea 100644 --- a/lib/crates/fabro-cli/tests/it/workflow/dry_run_examples.rs +++ b/lib/crates/fabro-cli/tests/it/workflow/dry_run_examples.rs @@ -80,7 +80,9 @@ fn dry_run_parallel() { let mut cmd = context.run_cmd(); cmd.args(["--dry-run", "--auto-approve"]); cmd.arg(&workflow); - fabro_snapshot!(run_output_filters(&context), cmd, @" + let mut filters = run_output_filters(&context); + filters.push((r"\bbranch[12]\b".to_string(), "[BRANCH]".to_string())); + fabro_snapshot!(filters, cmd, @" success: true exit_code: 0 ----- stdout ----- @@ -93,6 +95,8 @@ fn dry_run_parallel() { Web UI: http://localhost:3000/runs/[ULID] Sandbox: local (ready in [TIME]) ✓ start [TIME] + ✓ [BRANCH] [TIME] + ✓ [BRANCH] [TIME] ✓ Fork Work [TIME] ✓ Merge Results [TIME] ✓ Review [TIME] diff --git a/lib/crates/fabro-dump/src/lib.rs b/lib/crates/fabro-dump/src/lib.rs index b4da2215c..d8378cbd5 100644 --- a/lib/crates/fabro-dump/src/lib.rs +++ b/lib/crates/fabro-dump/src/lib.rs @@ -127,7 +127,7 @@ impl RunDump { if let Some(parallel_results) = stage.parallel_results.as_ref() { entries.push(RunDumpEntry::json_path( &base.join("parallel_results.json"), - parallel_results.clone(), + serde_json::to_value(parallel_results)?, )); } if let Some(output) = stage.output.as_ref() { @@ -605,7 +605,14 @@ mod tests { stage.diff = Some("diff --git a/a b/a".to_string()); stage.script_invocation = Some(serde_json::json!({ "command": "cargo test" })); stage.script_timing = Some(serde_json::json!({ "duration_ms": 10 })); - stage.parallel_results = Some(serde_json::json!([{ "stage": "fanout@1" }])); + stage.parallel_results = Some(vec![fabro_types::ParallelBranchResult { + id: "review".to_string(), + status: fabro_types::StageOutcome::Succeeded, + context_updates: std::collections::BTreeMap::from([( + "response.review".to_string(), + serde_json::json!("looks good"), + )]), + }]); stage.output = Some("output".to_string()); let dump = RunDump::from_projection(&projection).unwrap(); diff --git a/lib/crates/fabro-sandbox/src/daytona/mod.rs b/lib/crates/fabro-sandbox/src/daytona/mod.rs index 181b7895a..fa227d2c8 100644 --- a/lib/crates/fabro-sandbox/src/daytona/mod.rs +++ b/lib/crates/fabro-sandbox/src/daytona/mod.rs @@ -1311,22 +1311,6 @@ impl Sandbox for DaytonaSandbox { crate::git_push_via_exec(self, refspec).await } - fn parallel_worktree_path( - &self, - _run_dir: &std::path::Path, - run_id: &str, - node_id: &str, - key: &str, - ) -> String { - format!( - "{}/.fabro/scratch/{}/parallel/{}/{}", - self.working_directory(), - run_id, - node_id, - key - ) - } - async fn ssh_access_command(&self) -> crate::Result> { self.create_ssh_access(Some(60.0)).await.map(Some) } diff --git a/lib/crates/fabro-sandbox/src/docker.rs b/lib/crates/fabro-sandbox/src/docker.rs index 08dc04c3d..51f52a41c 100644 --- a/lib/crates/fabro-sandbox/src/docker.rs +++ b/lib/crates/fabro-sandbox/src/docker.rs @@ -1929,22 +1929,6 @@ impl Sandbox for DockerSandbox { crate::git_push_via_exec(self, refspec).await } - fn parallel_worktree_path( - &self, - _run_dir: &std::path::Path, - run_id: &str, - node_id: &str, - key: &str, - ) -> String { - format!( - "{}/.fabro/scratch/{}/parallel/{}/{}", - self.working_directory(), - run_id, - node_id, - key - ) - } - fn origin_url(&self) -> Option<&str> { if !self.repo_cloned() { return None; diff --git a/lib/crates/fabro-sandbox/src/lib.rs b/lib/crates/fabro-sandbox/src/lib.rs index 54daf581a..c4d10a7a2 100644 --- a/lib/crates/fabro-sandbox/src/lib.rs +++ b/lib/crates/fabro-sandbox/src/lib.rs @@ -22,8 +22,6 @@ pub mod details; pub mod reconnect; -pub mod worktree; - pub mod terminal; pub mod local; @@ -62,4 +60,3 @@ pub use sandbox::{ }; pub use sandbox_spec::SandboxSpec; pub use terminal::{TerminalSession, TerminalSize, open_terminal_for_run}; -pub use worktree::{WorktreeEvent, WorktreeEventCallback, WorktreeOptions, WorktreeSandbox}; diff --git a/lib/crates/fabro-sandbox/src/sandbox.rs b/lib/crates/fabro-sandbox/src/sandbox.rs index 352a51be4..fa2e9bacb 100644 --- a/lib/crates/fabro-sandbox/src/sandbox.rs +++ b/lib/crates/fabro-sandbox/src/sandbox.rs @@ -217,16 +217,6 @@ macro_rules! delegate_sandbox { self.$field.git_push_ref(refspec).await } - fn parallel_worktree_path( - &self, - run_dir: &std::path::Path, - run_id: &str, - node_id: &str, - key: &str, - ) -> String { - self.$field.parallel_worktree_path(run_dir, run_id, node_id, key) - } - async fn ssh_access_command(&self) -> $crate::Result> { self.$field.ssh_access_command().await } @@ -1005,23 +995,6 @@ pub trait Sandbox: Send + Sync { )) } - /// Compute the filesystem path for a parallel branch worktree. - fn parallel_worktree_path( - &self, - run_dir: &std::path::Path, - _run_id: &str, - node_id: &str, - key: &str, - ) -> String { - run_dir - .join("parallel") - .join(node_id) - .join(key) - .join("worktree") - .to_string_lossy() - .into_owned() - } - /// Return an SSH command string for connecting to this sandbox, if /// supported. async fn ssh_access_command(&self) -> crate::Result> { diff --git a/lib/crates/fabro-sandbox/src/worktree.rs b/lib/crates/fabro-sandbox/src/worktree.rs deleted file mode 100644 index 45370f11a..000000000 --- a/lib/crates/fabro-sandbox/src/worktree.rs +++ /dev/null @@ -1,908 +0,0 @@ -use std::collections::HashMap; -use std::path::Path; -use std::sync::Arc; - -use async_trait::async_trait; -use tokio_util::sync::CancellationToken; - -use crate::sandbox::fetch_source_run_ref; -use crate::{ - CommandOutputCallback, DirEntry, ExecResult, ExecStreamingResult, GitRunInfo, GitSetupIntent, - GrepOptions, Sandbox, StdioProcess, shell_quote, -}; - -/// Git command prefix that disables background maintenance. -const GIT: &str = "git -c maintenance.auto=0 -c gc.auto=0"; - -// --------------------------------------------------------------------------- -// Public types -// --------------------------------------------------------------------------- - -/// Events emitted during worktree lifecycle operations. -pub enum WorktreeEvent { - BranchCreated { branch: String, sha: String }, - WorktreeAdded { path: String, branch: String }, - WorktreeRemoved { path: String }, -} - -/// Callback type for worktree lifecycle events. -pub type WorktreeEventCallback = Arc; - -/// Configuration for a `WorktreeSandbox`. -pub struct WorktreeOptions { - pub branch_name: String, - pub base_sha: String, - pub worktree_path: String, - /// Skip branch creation and hard reset (for resume, where branch already - /// exists). - pub skip_branch_creation: bool, - pub setup_intent: Option, -} - -/// Wraps any `Sandbox`, manages a git worktree lifecycle in -/// `initialize()`/`cleanup()`, and overrides `working_directory()` and -/// `exec_command()` to use the worktree path. -/// -/// `initialize()` and `cleanup()` do NOT call the inner sandbox's lifecycle -/// methods. The inner sandbox's lifecycle is managed separately by the caller. -pub struct WorktreeSandbox { - inner: Arc, - config: WorktreeOptions, - event_callback: Option, - initialized: std::sync::atomic::AtomicBool, -} - -impl WorktreeSandbox { - /// Create a new `WorktreeSandbox` wrapping `inner` with the given - /// configuration. - pub fn new(inner: Arc, config: WorktreeOptions) -> Self { - Self { - inner, - config, - event_callback: None, - initialized: std::sync::atomic::AtomicBool::new(false), - } - } - - /// Set the callback to receive worktree lifecycle events. - pub fn set_event_callback(&mut self, cb: WorktreeEventCallback) { - self.event_callback = Some(cb); - } - - /// The git branch name managed by this sandbox. - pub fn branch_name(&self) -> &str { - &self.config.branch_name - } - - /// The base commit SHA used when initializing the worktree. - pub fn base_sha(&self) -> &str { - &self.config.base_sha - } - - /// The filesystem path to the worktree directory. - pub fn worktree_path(&self) -> &str { - &self.config.worktree_path - } - - fn emit(&self, event: WorktreeEvent) { - if let Some(ref cb) = self.event_callback { - cb(event); - } - } - - fn resolve_path(&self, path: &str) -> String { - if std::path::Path::new(path).is_absolute() { - path.to_string() - } else { - format!("{}/{path}", self.config.worktree_path) - } - } - - async fn fetch_fork_source_if_needed(&self) -> crate::Result<()> { - if let Some(GitSetupIntent::ForkFromCheckpoint { - source_run_id, - checkpoint_sha, - .. - }) = self.config.setup_intent.as_ref() - { - fetch_source_run_ref(&*self.inner, source_run_id, checkpoint_sha).await?; - } - Ok(()) - } -} - -// --------------------------------------------------------------------------- -// Sandbox implementation -// --------------------------------------------------------------------------- - -#[async_trait] -impl Sandbox for WorktreeSandbox { - // --- Lifecycle --- - - /// Set up the git worktree: - /// 1. Best-effort remove any stale worktree at `path` (so the branch is - /// free to be updated). - /// 2. Unless `skip_branch_creation`: force-create the branch at `base_sha`, - /// emit `BranchCreated`. - /// 3. Add the worktree, emit `WorktreeAdded`. - /// - /// Does NOT call `inner.initialize()`. - async fn initialize(&self) -> crate::Result<()> { - if self - .initialized - .swap(true, std::sync::atomic::Ordering::Relaxed) - { - return Ok(()); - } - let path = shell_quote(&self.config.worktree_path); - let branch = shell_quote(&self.config.branch_name); - let sha = shell_quote(&self.config.base_sha); - - self.fetch_fork_source_if_needed().await?; - - // Best-effort remove any stale worktree registration + directory first, - // so that the branch is not "in use" when we try to force-update it. - let rm_cmd = format!("{GIT} worktree remove --force {path}"); - let _ = self - .inner - .exec_command(&rm_cmd, 30_000, None, None, None) - .await; - - // Prune all stale worktree references whose directories no longer exist. - // Without this, a branch may remain locked by a worktree in a deleted - // temp directory from a previous run. - let prune_cmd = format!("{GIT} worktree prune"); - let _ = self - .inner - .exec_command(&prune_cmd, 30_000, None, None, None) - .await; - - if !self.config.skip_branch_creation { - let cmd = format!("{GIT} branch --force {branch} {sha}"); - let result = self - .inner - .exec_command(&cmd, 30_000, None, None, None) - .await?; - if !result.is_success() { - return Err(crate::Error::message(format!( - "git branch --force failed (exit {}): {}", - result.display_exit_code(), - result.stderr.trim() - ))); - } - self.emit(WorktreeEvent::BranchCreated { - branch: self.config.branch_name.clone(), - sha: self.config.base_sha.clone(), - }); - } - - let add_cmd = format!("{GIT} worktree add {path} {branch}"); - let result = self - .inner - .exec_command(&add_cmd, 30_000, None, None, None) - .await?; - if !result.is_success() { - // Roll back the branch created above so we don't leak partial state. - if !self.config.skip_branch_creation { - let rollback_cmd = format!("{GIT} branch -D {branch}"); - let _ = self - .inner - .exec_command(&rollback_cmd, 30_000, None, None, None) - .await; - } - return Err(crate::Error::message(format!( - "git worktree add failed (exit {}): {}", - result.display_exit_code(), - result.stderr.trim() - ))); - } - self.emit(WorktreeEvent::WorktreeAdded { - path: self.config.worktree_path.clone(), - branch: self.config.branch_name.clone(), - }); - - Ok(()) - } - - /// No-op — the worktree must survive cleanup for `fabro cp` access. - /// Worktrees are pruned separately by `system prune`. - async fn cleanup(&self) -> crate::Result<()> { - Ok(()) - } - - async fn start(&self) -> crate::Result<()> { - self.inner.start().await - } - - async fn stop(&self) -> crate::Result<()> { - self.inner.stop().await - } - - async fn delete(&self) -> crate::Result<()> { - self.inner.delete().await - } - - fn working_directory(&self) -> &str { - &self.config.worktree_path - } - - /// Execute a command, defaulting `working_dir` to the worktree path when - /// `None`. - async fn exec_command( - &self, - command: &str, - timeout_ms: u64, - working_dir: Option<&str>, - env_vars: Option<&HashMap>, - cancel_token: Option, - ) -> crate::Result { - let wd = working_dir.unwrap_or(&self.config.worktree_path); - self.inner - .exec_command(command, timeout_ms, Some(wd), env_vars, cancel_token) - .await - } - - /// Stream a command's output, forwarding to the inner sandbox's streaming - /// implementation so live output and `streams_separated` / `live_streaming` - /// flags survive the worktree wrapping. - async fn exec_command_streaming( - &self, - command: &str, - timeout_ms: Option, - working_dir: Option<&str>, - env_vars: Option<&HashMap>, - cancel_token: Option, - output_callback: CommandOutputCallback, - ) -> crate::Result { - let wd = working_dir.unwrap_or(&self.config.worktree_path); - self.inner - .exec_command_streaming( - command, - timeout_ms, - Some(wd), - env_vars, - cancel_token, - output_callback, - ) - .await - } - - async fn spawn_stdio_process( - &self, - command: &str, - working_dir: Option<&str>, - env_vars: Option<&HashMap>, - cancel_token: Option, - ) -> crate::Result { - let wd = working_dir.unwrap_or(&self.config.worktree_path); - self.inner - .spawn_stdio_process(command, Some(wd), env_vars, cancel_token) - .await - } - - // --- Delegated methods --- - - async fn read_file_bytes(&self, path: &str) -> crate::Result> { - let resolved = self.resolve_path(path); - self.inner.read_file_bytes(&resolved).await - } - - async fn write_file(&self, path: &str, content: &str) -> crate::Result<()> { - let resolved = self.resolve_path(path); - self.inner.write_file(&resolved, content).await - } - - async fn delete_file(&self, path: &str) -> crate::Result<()> { - let resolved = self.resolve_path(path); - self.inner.delete_file(&resolved).await - } - - async fn file_exists(&self, path: &str) -> crate::Result { - let resolved = self.resolve_path(path); - self.inner.file_exists(&resolved).await - } - - async fn list_directory( - &self, - path: &str, - depth: Option, - ) -> crate::Result> { - let resolved = self.resolve_path(path); - self.inner.list_directory(&resolved, depth).await - } - - async fn grep( - &self, - pattern: &str, - path: &str, - options: &GrepOptions, - ) -> crate::Result> { - let resolved = self.resolve_path(path); - self.inner.grep(pattern, &resolved, options).await - } - - async fn glob(&self, pattern: &str, path: Option<&str>) -> crate::Result> { - let resolved = path.map(|p| self.resolve_path(p)); - let glob_path = resolved.as_deref().unwrap_or(&self.config.worktree_path); - self.inner.glob(pattern, Some(glob_path)).await - } - - async fn download_file_to_local( - &self, - remote_path: &str, - local_path: &Path, - ) -> crate::Result<()> { - let resolved = self.resolve_path(remote_path); - self.inner - .download_file_to_local(&resolved, local_path) - .await - } - - async fn upload_file_from_local( - &self, - local_path: &Path, - remote_path: &str, - ) -> crate::Result<()> { - let resolved = self.resolve_path(remote_path); - self.inner - .upload_file_from_local(local_path, &resolved) - .await - } - - fn platform(&self) -> &str { - self.inner.platform() - } - - fn os_version(&self) -> String { - self.inner.os_version() - } - - fn sandbox_info(&self) -> String { - self.inner.sandbox_info() - } - - async fn refresh_push_credentials(&self) -> crate::Result { - self.inner.refresh_push_credentials().await - } - - async fn set_autostop_interval(&self, minutes: i32) -> crate::Result<()> { - self.inner.set_autostop_interval(minutes).await - } - - async fn setup_git( - &self, - intent: &crate::GitSetupIntent, - ) -> crate::Result> { - if let GitSetupIntent::ForkFromCheckpoint { - source_run_id, - checkpoint_sha, - .. - } = intent - { - fetch_source_run_ref(&*self.inner, source_run_id, checkpoint_sha).await?; - } - Ok(Some(GitRunInfo { - base_sha: self.config.base_sha.clone(), - run_branch: self.config.branch_name.clone(), - base_branch: None, - })) - } - - fn resume_setup_commands(&self, run_branch: &str) -> Vec { - self.inner.resume_setup_commands(run_branch) - } - - async fn git_push_ref(&self, refspec: &str) -> crate::Result<()> { - let has_origin = match self - .exec_command("git remote get-url origin", 10_000, None, None, None) - .await - { - Ok(result) if result.is_success() => true, - Ok(_) => false, - Err(err) => return Err(crate::Error::context("git remote get-url origin", err)), - }; - if !has_origin { - return Ok(()); - } - - crate::git_push_via_exec(self, refspec).await - } - - fn parallel_worktree_path( - &self, - run_dir: &Path, - run_id: &str, - node_id: &str, - key: &str, - ) -> String { - self.inner - .parallel_worktree_path(run_dir, run_id, node_id, key) - } - - async fn ssh_access_command(&self) -> crate::Result> { - self.inner.ssh_access_command().await - } - - fn origin_url(&self) -> Option<&str> { - self.inner.origin_url() - } - - async fn get_preview_url( - &self, - port: u16, - ) -> crate::Result)>> { - self.inner.get_preview_url(port).await - } - - fn mark_agent_read(&self, path: &str) { - let resolved = self.resolve_path(path); - self.inner.mark_agent_read(&resolved); - } -} - -// --------------------------------------------------------------------------- -// Tests -// --------------------------------------------------------------------------- - -#[cfg(test)] -#[expect( - clippy::disallowed_methods, - reason = "worktree tests stage fixtures with sync std::fs writes in temp dirs" -)] -mod tests { - use std::sync::Mutex; - - use fabro_types::CommandTermination; - - use super::*; - use crate::local::LocalSandbox; - use crate::test_support::MockSandbox; - - fn make_config(wt_path: &str) -> WorktreeOptions { - WorktreeOptions { - branch_name: "fabro/run/test-branch".to_string(), - base_sha: "abc123def456".to_string(), - worktree_path: wt_path.to_string(), - skip_branch_creation: false, - setup_intent: None, - } - } - - fn make_config_skip(wt_path: &str) -> WorktreeOptions { - WorktreeOptions { - branch_name: "fabro/run/test-branch".to_string(), - base_sha: "abc123def456".to_string(), - worktree_path: wt_path.to_string(), - skip_branch_creation: true, - setup_intent: None, - } - } - - /// Create a shared mock and return both the `Arc` (passed to - /// WorktreeSandbox) and the `Arc` (used to assert captured - /// state). - fn make_mock() -> (Arc, Arc) { - let mock = Arc::new(MockSandbox::linux()); - let as_sandbox: Arc = mock.clone(); - (as_sandbox, mock) - } - - // ----------------------------------------------------------------------- - // initialize() — full setup (skip_branch_creation = false) - // ----------------------------------------------------------------------- - - #[tokio::test] - async fn initialize_issues_correct_git_commands() { - let (inner, mock) = make_mock(); - let wt = WorktreeSandbox::new(inner, make_config("/tmp/wt")); - - wt.initialize().await.unwrap(); - - let cmds = mock.captured_commands.lock().unwrap().clone(); - // worktree remove (best-effort), worktree prune, branch --force, worktree add - assert_eq!(cmds.len(), 4, "expected 4 git commands, got: {cmds:?}"); - assert!( - cmds[0].contains("worktree remove --force"), - "cmd[0]: {}", - cmds[0] - ); - assert!(cmds[1].contains("worktree prune"), "cmd[1]: {}", cmds[1]); - assert!(cmds[2].contains("branch --force"), "cmd[2]: {}", cmds[2]); - assert!(cmds[3].contains("worktree add"), "cmd[3]: {}", cmds[3]); - } - - #[tokio::test] - async fn initialize_emits_branch_and_worktree_events() { - let (inner, _mock) = make_mock(); - let mut wt = WorktreeSandbox::new(inner, make_config("/tmp/wt")); - - let events: Arc>> = Arc::new(Mutex::new(Vec::new())); - let events_clone = Arc::clone(&events); - wt.set_event_callback(Arc::new(move |event| { - let label = match &event { - WorktreeEvent::BranchCreated { .. } => "BranchCreated", - WorktreeEvent::WorktreeAdded { .. } => "WorktreeAdded", - WorktreeEvent::WorktreeRemoved { .. } => "WorktreeRemoved", - }; - events_clone.lock().unwrap().push(label.to_string()); - })); - - wt.initialize().await.unwrap(); - - let captured = events.lock().unwrap(); - assert_eq!(*captured, vec!["BranchCreated", "WorktreeAdded"]); - } - - #[tokio::test] - async fn initialize_uses_shell_quoted_values_in_commands() { - let (inner, mock) = make_mock(); - let config = WorktreeOptions { - branch_name: "fabro/run/my-branch".to_string(), - base_sha: "deadbeef".to_string(), - worktree_path: "/tmp/my worktree".to_string(), // path with space - skip_branch_creation: false, - setup_intent: None, - }; - let wt = WorktreeSandbox::new(inner, config); - - wt.initialize().await.unwrap(); - - let cmds = mock.captured_commands.lock().unwrap().clone(); - // The path "/tmp/my worktree" should be quoted in the worktree remove command - // (cmd[0]) - assert!( - cmds[0].contains("'/tmp/my worktree'") || cmds[0].contains("\"/tmp/my worktree\""), - "worktree path should be shell-quoted: {}", - cmds[0] - ); - } - - // ----------------------------------------------------------------------- - // initialize() — skip_branch_creation = true - // ----------------------------------------------------------------------- - - #[tokio::test] - async fn initialize_skip_branch_creation_issues_only_worktree_commands() { - let (inner, mock) = make_mock(); - let wt = WorktreeSandbox::new(inner, make_config_skip("/tmp/wt")); - - wt.initialize().await.unwrap(); - - let cmds = mock.captured_commands.lock().unwrap().clone(); - // worktree remove (best-effort), worktree prune, worktree add - assert_eq!(cmds.len(), 3, "expected 3 git commands, got: {cmds:?}"); - assert!( - cmds[0].contains("worktree remove --force"), - "cmd[0]: {}", - cmds[0] - ); - assert!(cmds[1].contains("worktree prune"), "cmd[1]: {}", cmds[1]); - assert!(cmds[2].contains("worktree add"), "cmd[2]: {}", cmds[2]); - } - - #[tokio::test] - async fn initialize_skip_branch_creation_emits_only_worktree_added() { - let (inner, _mock) = make_mock(); - let mut wt = WorktreeSandbox::new(inner, make_config_skip("/tmp/wt")); - - let events: Arc>> = Arc::new(Mutex::new(Vec::new())); - let events_clone = Arc::clone(&events); - wt.set_event_callback(Arc::new(move |event| { - let label = match &event { - WorktreeEvent::BranchCreated { .. } => "BranchCreated", - WorktreeEvent::WorktreeAdded { .. } => "WorktreeAdded", - WorktreeEvent::WorktreeRemoved { .. } => "WorktreeRemoved", - }; - events_clone.lock().unwrap().push(label.to_string()); - })); - - wt.initialize().await.unwrap(); - - let captured = events.lock().unwrap(); - assert_eq!(*captured, vec!["WorktreeAdded"]); - } - - // ----------------------------------------------------------------------- - // initialize() — error propagation - // ----------------------------------------------------------------------- - - #[tokio::test] - async fn initialize_propagates_error_on_nonzero_exit() { - let inner: Arc = Arc::new(MockSandbox { - exec_result: ExecResult { - stdout: String::new(), - stderr: "fatal: not a git repo".to_string(), - exit_code: Some(128), - termination: CommandTermination::Exited, - duration_ms: 5, - }, - ..MockSandbox::linux() - }); - let wt = WorktreeSandbox::new(inner, make_config("/tmp/wt")); - - let result = wt.initialize().await; - - assert!(result.is_err(), "should return Err on non-zero exit"); - let err = result.unwrap_err().to_string(); - assert!( - err.contains("branch --force failed") || err.contains("128"), - "error should mention the failure: {err}" - ); - } - - // ----------------------------------------------------------------------- - // cleanup() - // ----------------------------------------------------------------------- - - // ----------------------------------------------------------------------- - // working_directory() - // ----------------------------------------------------------------------- - - #[test] - fn working_directory_returns_worktree_path() { - let (inner, _mock) = make_mock(); - let wt = WorktreeSandbox::new(inner, make_config("/tmp/my_worktree")); - - assert_eq!(wt.working_directory(), "/tmp/my_worktree"); - } - - // ----------------------------------------------------------------------- - // exec_command() working_dir defaulting - // ----------------------------------------------------------------------- - - #[tokio::test] - async fn exec_command_none_working_dir_defaults_to_worktree_path() { - let (inner, mock) = make_mock(); - let wt = WorktreeSandbox::new(inner, make_config("/tmp/wt")); - - wt.exec_command("echo hello", 5000, None, None, None) - .await - .unwrap(); - - let wdirs = mock.captured_working_dirs.lock().unwrap().clone(); - assert_eq!( - wdirs.last(), - Some(&Some("/tmp/wt".to_string())), - "None working_dir should be replaced with worktree path" - ); - } - - #[tokio::test] - async fn exec_command_explicit_working_dir_passes_through() { - let (inner, mock) = make_mock(); - let wt = WorktreeSandbox::new(inner, make_config("/tmp/wt")); - - wt.exec_command("echo hello", 5000, Some("/explicit/path"), None, None) - .await - .unwrap(); - - let wdirs = mock.captured_working_dirs.lock().unwrap().clone(); - assert_eq!( - wdirs.last(), - Some(&Some("/explicit/path".to_string())), - "explicit working_dir should be passed through unchanged" - ); - } - - #[tokio::test] - async fn stdio_process_none_working_dir_defaults_to_worktree_path() { - let (inner, mock) = make_mock(); - let wt = WorktreeSandbox::new(inner, make_config("/tmp/wt")); - - wt.spawn_stdio_process("python fake_agent.py", None, None, None) - .await - .unwrap(); - - let wdirs = mock.captured_working_dirs.lock().unwrap().clone(); - assert_eq!( - wdirs.last(), - Some(&Some("/tmp/wt".to_string())), - "None working_dir should be replaced with worktree path" - ); - } - - // ----------------------------------------------------------------------- - // Accessors - // ----------------------------------------------------------------------- - - // ----------------------------------------------------------------------- - // Bug: cleanup() destroys worktree, breaking `fabro cp` - // ----------------------------------------------------------------------- - - #[tokio::test] - async fn cleanup_should_preserve_worktree_for_post_run_access() { - // The worktree directory must survive cleanup() so that `fabro cp` can - // access run artifacts afterward. It is pruned separately by `system prune`. - // LocalSandbox.cleanup() was a no-op; WorktreeSandbox should match. - let (inner, mock) = make_mock(); - let wt = WorktreeSandbox::new(inner, make_config("/tmp/wt")); - - wt.cleanup().await.unwrap(); - - let cmds = mock.captured_commands.lock().unwrap().clone(); - assert!( - cmds.is_empty(), - "cleanup should not issue destructive git commands \ - (worktree must be preserved for fabro cp), but got: {cmds:?}" - ); - } - - #[tokio::test] - async fn lifecycle_operations_forward_to_inner_sandbox() { - let (inner, mock) = make_mock(); - let wt = WorktreeSandbox::new(inner, make_config("/tmp/wt")); - - wt.start().await.unwrap(); - wt.stop().await.unwrap(); - wt.delete().await.unwrap(); - - assert_eq!(mock.start_count(), 1); - assert_eq!(mock.stop_count(), 1); - assert_eq!(mock.delete_count(), 1); - } - - // ----------------------------------------------------------------------- - // Bug: initialize() is not idempotent — double call destroys worktree - // ----------------------------------------------------------------------- - - #[tokio::test] - async fn initialize_is_idempotent_on_second_call() { - // engine.run_with_lifecycle() calls sandbox.initialize() unconditionally, - // even when run.rs already called it during sandbox construction. - // The second call must be a no-op; it must NOT re-run - // `git worktree remove --force` which would destroy the worktree. - let (inner, mock) = make_mock(); - let wt = WorktreeSandbox::new(inner, make_config("/tmp/wt")); - - wt.initialize().await.unwrap(); - let first_count = mock.captured_commands.lock().unwrap().len(); - - wt.initialize().await.unwrap(); - let second_count = mock.captured_commands.lock().unwrap().len(); - - assert_eq!( - first_count, - second_count, - "second initialize() should be a no-op, but it issued {} additional commands", - second_count - first_count - ); - } - - // ----------------------------------------------------------------------- - // Bug: file operations resolve against inner working_directory, not worktree - // ----------------------------------------------------------------------- - - #[tokio::test] - async fn grep_should_search_worktree_not_inner_working_directory() { - // WorktreeSandbox delegates grep() to the inner sandbox without path - // adjustment. When the inner LocalSandbox was created with original_cwd, - // grep("pattern", ".") searches the original repo instead of the worktree. - let original = - std::env::temp_dir().join(format!("fabro-test-original-{}", uuid::Uuid::new_v4())); - let worktree = - std::env::temp_dir().join(format!("fabro-test-worktree-{}", uuid::Uuid::new_v4())); - std::fs::create_dir_all(&original).unwrap(); - std::fs::create_dir_all(&worktree).unwrap(); - - // Put a marker file ONLY in the worktree directory - std::fs::write(worktree.join("marker.txt"), "UNIQUE_WORKTREE_MARKER").unwrap(); - - let inner: Arc = Arc::new(LocalSandbox::new(original.clone())); - let config = WorktreeOptions { - branch_name: "test-branch".into(), - base_sha: "abc123".into(), - worktree_path: worktree.to_string_lossy().to_string(), - skip_branch_creation: false, - setup_intent: None, - }; - let wt = WorktreeSandbox::new(inner, config); - - // working_directory() correctly returns the worktree path - assert_eq!(wt.working_directory(), worktree.to_string_lossy().as_ref()); - - // grep with "." should search the worktree, not the original repo - let results = wt - .grep("UNIQUE_WORKTREE_MARKER", ".", &GrepOptions::default()) - .await - .unwrap(); - assert!( - !results.is_empty(), - "grep(\".\") should search the worktree directory, not the inner sandbox's working directory" - ); - - std::fs::remove_dir_all(&original).ok(); - std::fs::remove_dir_all(&worktree).ok(); - } - - #[tokio::test] - async fn glob_should_search_worktree_when_path_is_none() { - // WorktreeSandbox delegates glob() to the inner sandbox without path - // adjustment. LocalSandbox::glob(pattern, None) defaults to - // self.working_directory, which is the original repo path. - let original = - std::env::temp_dir().join(format!("fabro-test-original-{}", uuid::Uuid::new_v4())); - let worktree = - std::env::temp_dir().join(format!("fabro-test-worktree-{}", uuid::Uuid::new_v4())); - std::fs::create_dir_all(&original).unwrap(); - std::fs::create_dir_all(&worktree).unwrap(); - - // Put a file ONLY in the worktree directory - std::fs::write(worktree.join("worktree_only.txt"), "content").unwrap(); - - let inner: Arc = Arc::new(LocalSandbox::new(original.clone())); - let config = WorktreeOptions { - branch_name: "test-branch".into(), - base_sha: "abc123".into(), - worktree_path: worktree.to_string_lossy().to_string(), - skip_branch_creation: false, - setup_intent: None, - }; - let wt = WorktreeSandbox::new(inner, config); - - let results = wt.glob("*.txt", None).await.unwrap(); - assert!( - results.iter().any(|r| r.contains("worktree_only.txt")), - "glob(pattern, None) should search the worktree directory, not the inner sandbox's working directory. Got: {results:?}" - ); - - std::fs::remove_dir_all(&original).ok(); - std::fs::remove_dir_all(&worktree).ok(); - } - - #[tokio::test] - async fn read_file_relative_should_resolve_against_worktree() { - // WorktreeSandbox delegates read_file() to the inner sandbox without - // path adjustment. Relative paths resolve against the inner - // LocalSandbox's working_directory (original repo), not the worktree. - let original = - std::env::temp_dir().join(format!("fabro-test-original-{}", uuid::Uuid::new_v4())); - let worktree = - std::env::temp_dir().join(format!("fabro-test-worktree-{}", uuid::Uuid::new_v4())); - std::fs::create_dir_all(&original).unwrap(); - std::fs::create_dir_all(&worktree).unwrap(); - - // Put the file ONLY in the worktree directory - std::fs::write(worktree.join("only_in_worktree.txt"), "worktree content").unwrap(); - - let inner: Arc = Arc::new(LocalSandbox::new(original.clone())); - let config = WorktreeOptions { - branch_name: "test-branch".into(), - base_sha: "abc123".into(), - worktree_path: worktree.to_string_lossy().to_string(), - skip_branch_creation: false, - setup_intent: None, - }; - let wt = WorktreeSandbox::new(inner, config); - - let result = wt.read_file("only_in_worktree.txt", None, None).await; - assert!( - result.is_ok(), - "read_file with relative path should resolve against worktree, not inner sandbox's working directory. Error: {}", - result.unwrap_err() - ); - - std::fs::remove_dir_all(&original).ok(); - std::fs::remove_dir_all(&worktree).ok(); - } - - // ----------------------------------------------------------------------- - // Accessors - // ----------------------------------------------------------------------- - - #[test] - fn accessors_return_config_values() { - let (inner, _mock) = make_mock(); - let config = WorktreeOptions { - branch_name: "my-branch".to_string(), - base_sha: "sha123".to_string(), - worktree_path: "/path/to/wt".to_string(), - skip_branch_creation: false, - setup_intent: None, - }; - let wt = WorktreeSandbox::new(inner, config); - - assert_eq!(wt.branch_name(), "my-branch"); - assert_eq!(wt.base_sha(), "sha123"); - assert_eq!(wt.worktree_path(), "/path/to/wt"); - } -} diff --git a/lib/crates/fabro-store/src/run_state.rs b/lib/crates/fabro-store/src/run_state.rs index f242acc52..44783d4fc 100644 --- a/lib/crates/fabro-store/src/run_state.rs +++ b/lib/crates/fabro-store/src/run_state.rs @@ -505,13 +505,10 @@ impl RunProjectionReducer for RunProjection { )?; } EventBody::ParallelCompleted(props) => { - let parallel_results = serde_json::to_value(&props.results).map_err(|err| { - Error::InvalidEvent(format!("invalid parallel.completed payload: {err}")) - })?; let Some(stage) = stage_at_stored_or_current_visit(self, stored, event.seq) else { return Ok(()); }; - stage.parallel_results = Some(parallel_results); + stage.parallel_results = Some(props.results.clone()); } EventBody::ParallelBranchStarted(_) => { // Branches bypass the engine's StageStarted/StageCompleted @@ -529,21 +526,17 @@ impl RunProjectionReducer for RunProjection { // A branch never emits its own StageCompleted, so finalize it // here; otherwise the stage spins Running forever after the run // (and the fan-in) is done. - let outcome = - StageOutcome::from_str(&props.status).unwrap_or(StageOutcome::Failed { - retry_requested: false, - }); let Some(stage) = stage_at_stored_or_current_visit(self, stored, event.seq) else { return Ok(()); }; stage.completion = Some(StageCompletion { - outcome, - notes: None, + outcome: props.status, + notes: None, failure_reason: None, - timestamp: ts, + timestamp: ts, }); stage.timing = Some(fabro_types::StageTiming::wall_only(props.duration_ms)); - stage.state = StageState::from(outcome); + stage.state = StageState::from(props.status); } EventBody::TodoCreated(props) => { let Some(stage) = stage_at_stored_or_current_visit(self, stored, event.seq) else { @@ -2054,8 +2047,7 @@ mod tests { EventBody::ParallelBranchCompleted(ParallelBranchCompletedProps { index: 0, duration_ms: 1234, - status: "succeeded".to_string(), - head_sha: None, + status: StageOutcome::Succeeded, }), branch.clone(), )) @@ -2088,8 +2080,9 @@ mod tests { EventBody::ParallelBranchCompleted(ParallelBranchCompletedProps { index: 0, duration_ms: 500, - status: "failed".to_string(), - head_sha: None, + status: StageOutcome::Failed { + retry_requested: false, + }, }), branch.clone(), )) diff --git a/lib/crates/fabro-store/tests/serializable_projection.rs b/lib/crates/fabro-store/tests/serializable_projection.rs index ffc7fbb03..66600c154 100644 --- a/lib/crates/fabro-store/tests/serializable_projection.rs +++ b/lib/crates/fabro-store/tests/serializable_projection.rs @@ -6,9 +6,9 @@ use fabro_types::graph::Graph; use fabro_types::run::RunSpec; use fabro_types::{ BilledModelUsage, BilledTokenCounts, Checkpoint, CheckpointRecord, InterviewQuestionRecord, - QuestionType, RunDiff, RunSandbox, RunSandboxInstance, RunSandboxPlan, RunSandboxRuntime, - RunStatus, SandboxProviderKind, StageCompletion, StageModelUsage, StageOutcome, StartRecord, - WorkflowSettings, first_event_seq, fixtures, test_support, + ParallelBranchResult, QuestionType, RunDiff, RunSandbox, RunSandboxInstance, RunSandboxPlan, + RunSandboxRuntime, RunStatus, SandboxProviderKind, StageCompletion, StageModelUsage, + StageOutcome, StartRecord, WorkflowSettings, first_event_seq, fixtures, test_support, }; use serde_json::json; @@ -143,7 +143,12 @@ fn serializable_projection_round_trips_and_trims_bulky_node_fields() { stage.diff = Some("diff --git a/a b/a".to_string()); stage.script_invocation = Some(json!({ "command": "cargo test" })); stage.script_timing = Some(json!({ "duration_ms": 10 })); - stage.parallel_results = Some(json!([{ "stage": "fanout@1" }])); + let parallel_results = vec![ParallelBranchResult { + id: "review".to_string(), + status: StageOutcome::Succeeded, + context_updates: BTreeMap::from([("response.review".to_string(), json!("looks good"))]), + }]; + stage.parallel_results = Some(parallel_results.clone()); stage.timing = Some(fabro_types::StageTiming::wall_only(1234)); let usage = sample_usage(); let usage_counts = BilledTokenCounts::from_billed_usage(std::slice::from_ref(&usage)); @@ -203,10 +208,7 @@ fn serializable_projection_round_trips_and_trims_bulky_node_fields() { Some(json!({ "command": "cargo test" })) ); assert_eq!(node.script_timing, Some(json!({ "duration_ms": 10 }))); - assert_eq!( - node.parallel_results, - Some(json!([{ "stage": "fanout@1" }])) - ); + assert_eq!(node.parallel_results, Some(parallel_results)); assert_eq!(node.timing.map(|t| t.wall_time_ms), Some(1234)); assert_eq!(node.usage, usage_counts); assert_eq!(node.model.as_ref(), Some(usage.model())); diff --git a/lib/crates/fabro-types/src/lib.rs b/lib/crates/fabro-types/src/lib.rs index e339f285d..752d67dde 100644 --- a/lib/crates/fabro-types/src/lib.rs +++ b/lib/crates/fabro-types/src/lib.rs @@ -19,6 +19,7 @@ pub mod manifest_path; pub mod mcp_store; pub mod outcome; pub mod pair; +pub mod parallel; pub mod principal; pub mod pull_request; pub mod repository; @@ -93,6 +94,7 @@ pub use pair::{ PairTranscriptWarning, RunEventDetailContent, RunEventDetailContentKind, RunEventDetailEnvelope, RunEventDetailResponse, RunPairStatusResponse, }; +pub use parallel::ParallelBranchResult; pub use principal::{AuthMethod, Principal, SystemActorKind, UserPrincipal}; pub use pull_request::{ CheckRun, CheckRunStatus, PullRequest, PullRequestDetails, PullRequestDetailsStatus, diff --git a/lib/crates/fabro-types/src/parallel.rs b/lib/crates/fabro-types/src/parallel.rs new file mode 100644 index 000000000..fc4a2563a --- /dev/null +++ b/lib/crates/fabro-types/src/parallel.rs @@ -0,0 +1,14 @@ +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::StageOutcome; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ParallelBranchResult { + pub id: String, + pub status: StageOutcome, + #[serde(default)] + pub context_updates: BTreeMap, +} diff --git a/lib/crates/fabro-types/src/run_event/infra.rs b/lib/crates/fabro-types/src/run_event/infra.rs index 3abd80c12..8af295ffa 100644 --- a/lib/crates/fabro-types/src/run_event/infra.rs +++ b/lib/crates/fabro-types/src/run_event/infra.rs @@ -30,7 +30,6 @@ pub enum RunNoticeCode { GitPushFailed, GithubTokenFailed, GithubTokenRefreshLimited, - ParallelBaseCheckpointFailed, PullRequestFailed, SandboxCleanupFailed, SandboxGitUnavailable, diff --git a/lib/crates/fabro-types/src/run_event/misc.rs b/lib/crates/fabro-types/src/run_event/misc.rs index 3f88cdbd3..84276a3bd 100644 --- a/lib/crates/fabro-types/src/run_event/misc.rs +++ b/lib/crates/fabro-types/src/run_event/misc.rs @@ -1,8 +1,7 @@ use serde::{Deserialize, Serialize}; -use serde_json::Value; use super::ExecOutputTail; -use crate::{CommandTermination, PullRequestLink}; +use crate::{CommandTermination, ParallelBranchResult, PullRequestLink, StageOutcome}; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] pub struct InterviewOption { @@ -18,7 +17,6 @@ pub struct InterviewOption { pub struct ParallelStartedProps { pub visit: u32, pub branch_count: usize, - pub join_policy: String, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -30,9 +28,7 @@ pub struct ParallelBranchStartedProps { pub struct ParallelBranchCompletedProps { pub index: usize, pub duration_ms: u64, - pub status: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub head_sha: Option, + pub status: StageOutcome, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -42,7 +38,7 @@ pub struct ParallelCompletedProps { pub success_count: usize, pub failure_count: usize, #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub results: Vec, + pub results: Vec, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -106,23 +102,6 @@ pub struct GitPushProps { pub exec_output_tail: Option, } -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct GitBranchProps { - pub branch: String, - pub sha: String, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct GitWorktreeAddProps { - pub path: String, - pub branch: String, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct GitWorktreeRemoveProps { - pub path: String, -} - #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct GitFetchProps { pub branch: String, diff --git a/lib/crates/fabro-types/src/run_event/mod.rs b/lib/crates/fabro-types/src/run_event/mod.rs index cb1c38b81..205b17d74 100644 --- a/lib/crates/fabro-types/src/run_event/mod.rs +++ b/lib/crates/fabro-types/src/run_event/mod.rs @@ -176,12 +176,6 @@ pub enum EventBody { GitCommit(GitCommitProps), #[serde(rename = "git.push")] GitPush(GitPushProps), - #[serde(rename = "git.branch")] - GitBranch(GitBranchProps), - #[serde(rename = "git.worktree.added")] - GitWorktreeAdd(GitWorktreeAddProps), - #[serde(rename = "git.worktree.removed")] - GitWorktreeRemove(GitWorktreeRemoveProps), #[serde(rename = "git.fetch")] GitFetch(GitFetchProps), #[serde(rename = "git.reset")] @@ -477,9 +471,6 @@ impl EventBody { Self::CheckpointFailed(_) => "checkpoint.failed", Self::GitCommit(_) => "git.commit", Self::GitPush(_) => "git.push", - Self::GitBranch(_) => "git.branch", - Self::GitWorktreeAdd(_) => "git.worktree.added", - Self::GitWorktreeRemove(_) => "git.worktree.removed", Self::GitFetch(_) => "git.fetch", Self::GitReset(_) => "git.reset", Self::EdgeSelected(_) => "edge.selected", @@ -649,9 +640,6 @@ fn is_known_event_name(event: &str) -> bool { | "checkpoint.failed" | "git.commit" | "git.push" - | "git.branch" - | "git.worktree.added" - | "git.worktree.removed" | "git.fetch" | "git.reset" | "edge.selected" diff --git a/lib/crates/fabro-types/src/run_projection.rs b/lib/crates/fabro-types/src/run_projection.rs index 6b67282ee..f251d6182 100644 --- a/lib/crates/fabro-types/src/run_projection.rs +++ b/lib/crates/fabro-types/src/run_projection.rs @@ -75,7 +75,6 @@ impl StageModelUsage { pub const MODE_PROMPT: &'static str = "prompt"; pub const MODE_AGENT: &'static str = "agent"; pub const MODE_ACP: &'static str = "acp"; - pub const MODE_FAN_IN: &'static str = "fan_in"; /// Build the usage record from a `stage.prompt` event, returning `None` /// when the event carried no model metadata. @@ -328,7 +327,7 @@ pub struct StageProjection { pub diff: Option, pub script_invocation: Option, pub script_timing: Option, - pub parallel_results: Option, + pub parallel_results: Option>, pub output: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub output_bytes: Option, diff --git a/lib/crates/fabro-validate/src/lib.rs b/lib/crates/fabro-validate/src/lib.rs index e5430c481..3747b677c 100644 --- a/lib/crates/fabro-validate/src/lib.rs +++ b/lib/crates/fabro-validate/src/lib.rs @@ -257,6 +257,54 @@ reasoning = false assert!(result.is_ok()); } + #[test] + fn validate_rejects_join_policy_on_any_node() { + let mut g = minimal_valid_graph(); + + let mut fork = Node::new("fork"); + fork.attrs.insert( + "shape".to_string(), + AttrValue::String("component".to_string()), + ); + fork.attrs.insert( + "join_policy".to_string(), + AttrValue::String("wait_all".to_string()), + ); + g.nodes.insert("fork".to_string(), fork); + + let mut custom = Node::new("custom"); + custom.attrs.insert( + "type".to_string(), + AttrValue::String("custom.handler".to_string()), + ); + custom.attrs.insert( + "join_policy".to_string(), + AttrValue::String("first_success".to_string()), + ); + g.nodes.insert("custom".to_string(), custom); + + let diagnostics = validate(&g, &[]); + let removed = diagnostics + .iter() + .filter(|d| d.rule == "join_policy_removed") + .collect::>(); + + assert_eq!(removed.len(), 2, "diagnostics: {diagnostics:?}"); + assert!(removed.iter().all(|d| d.severity == Severity::Error)); + assert!( + removed + .iter() + .all(|d| d.message.contains("Remove 'join_policy'")) + ); + assert_eq!( + removed + .iter() + .filter_map(|d| d.node_id.as_deref()) + .collect::>(), + std::collections::BTreeSet::from(["custom", "fork"]), + ); + } + #[test] fn validate_or_raise_fails_for_missing_start() { let mut g = Graph::new("test"); diff --git a/lib/crates/fabro-validate/src/rules/inert_attribute.rs b/lib/crates/fabro-validate/src/rules/inert_attribute.rs index fef9c5454..a740d97b8 100644 --- a/lib/crates/fabro-validate/src/rules/inert_attribute.rs +++ b/lib/crates/fabro-validate/src/rules/inert_attribute.rs @@ -18,7 +18,6 @@ const HANDLER_SPECIFIC_ATTRS: &[(&str, &[&str])] = &[ ("script", &["command"]), ("language", &["command"]), ("duration", &["wait"]), - ("join_policy", &["parallel"]), ("max_parallel", &["parallel"]), ("output_schema", &["agent", "prompt"]), ("prompt", &["agent", "prompt", "parallel.fan_in"]), @@ -140,15 +139,11 @@ mod tests { fn warns_on_parallel_attrs_on_agent_node() { let mut g = minimal_graph(); let mut node = Node::new("work"); - node.attrs.insert( - "join_policy".to_string(), - AttrValue::String("wait_all".to_string()), - ); node.attrs .insert("max_parallel".to_string(), AttrValue::Integer(4)); g.nodes.insert("work".to_string(), node); let d = Rule.apply(&g); - assert_eq!(d.len(), 2); + assert_eq!(d.len(), 1); } #[test] @@ -180,7 +175,7 @@ mod tests { ); g.nodes.insert( "fork".to_string(), - node_with_attr("fork", "component", "join_policy", "wait_all"), + node_with_attr("fork", "component", "max_parallel", "4"), ); g.nodes.insert( "spec".to_string(), diff --git a/lib/crates/fabro-validate/src/rules/join_policy_removed.rs b/lib/crates/fabro-validate/src/rules/join_policy_removed.rs new file mode 100644 index 000000000..b96f8b446 --- /dev/null +++ b/lib/crates/fabro-validate/src/rules/join_policy_removed.rs @@ -0,0 +1,35 @@ +use fabro_graphviz::graph::Graph; + +use crate::{Diagnostic, LintRule, Severity}; + +pub(super) fn rule() -> Box { + Box::new(Rule) +} + +struct Rule; + +impl LintRule for Rule { + fn name(&self) -> &'static str { + "join_policy_removed" + } + + fn apply(&self, graph: &Graph) -> Vec { + graph + .nodes + .values() + .filter(|node| node.attrs.contains_key("join_policy")) + .map(|node| Diagnostic { + rule: self.name().to_string(), + severity: Severity::Error, + message: format!( + "Node '{}' sets the removed 'join_policy' attribute. Remove 'join_policy'; parallel nodes always wait for every branch to finish", + node.id, + ), + node_id: Some(node.id.clone()), + edge: None, + fix: Some("Remove 'join_policy' from this node".to_string()), + ..Diagnostic::default() + }) + .collect() + } +} diff --git a/lib/crates/fabro-validate/src/rules/mod.rs b/lib/crates/fabro-validate/src/rules/mod.rs index 428004240..cb4a5a18a 100644 --- a/lib/crates/fabro-validate/src/rules/mod.rs +++ b/lib/crates/fabro-validate/src/rules/mod.rs @@ -9,6 +9,7 @@ mod freeform_edge_count; mod goal_gate_has_retry; mod import_error; mod inert_attribute; +mod join_policy_removed; mod model_support; mod node_model_known; mod orphan_custom_outcome; @@ -58,6 +59,7 @@ pub fn built_in_rules() -> Vec> { orphan_custom_outcome::rule(), script_absolute_cd::rule(), import_error::rule(), + join_policy_removed::rule(), unresolved_file_ref::rule(), thread_id_requires_fidelity_full::rule(), selection_valid::rule(), diff --git a/lib/crates/fabro-workflow/README.md b/lib/crates/fabro-workflow/README.md index 655bca79a..21deba55b 100644 --- a/lib/crates/fabro-workflow/README.md +++ b/lib/crates/fabro-workflow/README.md @@ -140,7 +140,7 @@ Nodes with `shape=hexagon` or `type="human"` pause execution for human input. Ou ### Parallel Execution -Nodes with `shape=component` fan out to branches concurrently. Configurable join policies: `wait_all` (default), `first_success`. +Nodes with `shape=component` fan out to branches concurrently. Branches receive isolated context forks, share the same sandbox checkout, and always finish before the workflow continues. Use `max_parallel` to limit concurrency; concurrent workspace writes are user-managed. ### Checkpoints and Resume diff --git a/lib/crates/fabro-workflow/src/artifact.rs b/lib/crates/fabro-workflow/src/artifact.rs index 3f9f9eb7d..26de9edd8 100644 --- a/lib/crates/fabro-workflow/src/artifact.rs +++ b/lib/crates/fabro-workflow/src/artifact.rs @@ -3,7 +3,9 @@ use std::path::{Path, PathBuf}; use fabro_agent::Sandbox; use fabro_config::RunScratch; -use fabro_types::{RunBlobId, format_blob_ref, parse_blob_ref, parse_managed_blob_file_ref}; +use fabro_types::{ + ParallelBranchResult, RunBlobId, format_blob_ref, parse_blob_ref, parse_managed_blob_file_ref, +}; use futures::future::BoxFuture; use serde_json::Value; use tokio::fs; @@ -27,6 +29,10 @@ const ARTIFACT_POINTER_PREFIX: &str = "file://"; /// and replaced with a `"blob://sha256/{blob_id}"` reference. /// Small values are left untouched. /// +/// `parallel.results` is offloaded leaf-wise instead of as one value so it +/// stays a structured array that fan-in prompts, projections, and the UI can +/// read without hydrating the whole payload. +/// /// # Errors /// /// Returns an error if blob persistence fails. @@ -34,21 +40,79 @@ pub async fn offload_large_values( updates: &mut HashMap, run_store: &RunStoreHandle, ) -> Result<()> { - for value in updates.values_mut() { - let bytes = serde_json::to_vec(&*value) - .map_err(|e| Error::engine_with_source("artifact serialize failed", e))?; - - if bytes.len() > BLOB_OFFLOAD_THRESHOLD { - let blob_id = run_store - .write_blob(&bytes) - .await - .map_err(|e| Error::engine_with_anyhow("artifact blob write failed", e))?; - *value = Value::String(format_blob_ref(&blob_id)); + for (key, value) in updates { + if key == context::keys::PARALLEL_RESULTS { + offload_large_leaves(value, run_store).await?; + } else { + offload_value(value, run_store).await?; } } Ok(()) } +/// Offload large leaves of typed parallel branch results before they are +/// emitted through `parallel.completed` and stored in projections. +/// +/// # Errors +/// +/// Returns an error if blob persistence fails. +pub async fn offload_parallel_branch_updates( + results: &mut [ParallelBranchResult], + run_store: &RunStoreHandle, +) -> Result<()> { + for result in results.iter_mut() { + for value in result.context_updates.values_mut() { + offload_large_leaves(value, run_store).await?; + } + } + Ok(()) +} + +fn offload_large_leaves<'a>( + value: &'a mut Value, + run_store: &'a RunStoreHandle, +) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + match value { + Value::Array(items) => { + for item in items { + offload_large_leaves(item, run_store).await?; + } + } + Value::Object(map) => { + for item in map.values_mut() { + offload_large_leaves(item, run_store).await?; + } + } + Value::String(_) | Value::Null | Value::Bool(_) | Value::Number(_) => { + offload_value(value, run_store).await?; + } + } + Ok(()) + }) +} + +async fn offload_value(value: &mut Value, run_store: &RunStoreHandle) -> Result<()> { + // JSON escaping expands a string to at most 6 bytes per char plus quotes, + // so short strings can never cross the threshold — skip serializing them. + if let Value::String(text) = &*value { + if text.len().saturating_mul(6) + 2 <= BLOB_OFFLOAD_THRESHOLD { + return Ok(()); + } + } + let bytes = serde_json::to_vec(&*value) + .map_err(|e| Error::engine_with_source("artifact serialize failed", e))?; + + if bytes.len() > BLOB_OFFLOAD_THRESHOLD { + let blob_id = run_store + .write_blob(&bytes) + .await + .map_err(|e| Error::engine_with_anyhow("artifact blob write failed", e))?; + *value = Value::String(format_blob_ref(&blob_id)); + } + Ok(()) +} + /// Extract the file path from an artifact pointer value. /// /// Returns `Some(path)` if the value is a string starting with `"file://"`, @@ -264,6 +328,10 @@ fn resolve_execution_values<'a>( }) } +fn is_text_context_key(key: &str) -> bool { + key == context::keys::COMMAND_OUTPUT || key.starts_with(context::keys::RESPONSE_PREFIX) +} + fn resolve_execution_value<'a>( key: Option<&'a str>, value: &'a mut Value, @@ -274,7 +342,7 @@ fn resolve_execution_value<'a>( Box::pin(async move { match value { Value::String(current) => { - if matches!(key, Some(context::keys::COMMAND_OUTPUT)) { + if key.is_some_and(is_text_context_key) { *current = resolve_text_or_blob_ref_str(current, run_store).await?; } else if let Some(blob_id) = parse_blob_ref(current) { *current = materialize_blob_ref(&blob_id, run_store, env, run_dir).await?; @@ -286,12 +354,19 @@ fn resolve_execution_value<'a>( } Value::Array(items) => { for item in items { - resolve_execution_value(None, item, run_store, env, run_dir).await?; + resolve_execution_value(key, item, run_store, env, run_dir).await?; } } Value::Object(map) => { - for item in map.values_mut() { - resolve_execution_value(None, item, run_store, env, run_dir).await?; + for (child_key, item) in map.iter_mut() { + resolve_execution_value( + Some(child_key.as_str()), + item, + run_store, + env, + run_dir, + ) + .await?; } } Value::Null | Value::Bool(_) | Value::Number(_) => {} @@ -306,15 +381,12 @@ async fn materialize_blob_ref( env: &dyn Sandbox, run_dir: &Path, ) -> Result { - let bytes = run_store - .read_blob(blob_id) - .await - .map_err(|e| Error::engine_with_anyhow("artifact blob read failed", e))? - .ok_or_else(|| Error::engine(format!("artifact blob missing: {blob_id}")))?; - + // Blobs are content-addressed, so an existing materialized file is always + // current — check before paying for the store read. if is_local_execution(env, run_dir).await? { let path = local_materialized_blob_path(run_dir, blob_id); if !path.exists() { + let bytes = read_required_blob(blob_id, run_store).await?; if let Some(parent) = path.parent() { fs::create_dir_all(parent).await.map_err(|err| { Error::Io(format!( @@ -336,6 +408,7 @@ async fn materialize_blob_ref( .await .map_err(|e| Error::engine_with_source("failed to check blob existence", e))? { + let bytes = read_required_blob(blob_id, run_store).await?; let content = String::from_utf8(bytes.to_vec()) .map_err(|e| Error::engine_with_source("artifact blob was not valid UTF-8 JSON", e))?; env.write_file(&remote_path, &content).await.map_err(|e| { @@ -346,6 +419,17 @@ async fn materialize_blob_ref( Ok(format!("{ARTIFACT_POINTER_PREFIX}{remote_path}")) } +async fn read_required_blob( + blob_id: &RunBlobId, + run_store: &RunStoreHandle, +) -> Result { + run_store + .read_blob(blob_id) + .await + .map_err(|e| Error::engine_with_anyhow("artifact blob read failed", e))? + .ok_or_else(|| Error::engine(format!("artifact blob missing: {blob_id}"))) +} + async fn resolve_explicit_file_ref(value: &str, env: &dyn Sandbox) -> Result { let local_path = value .strip_prefix(ARTIFACT_POINTER_PREFIX) @@ -466,6 +550,47 @@ mod tests { assert_eq!(updates.get("small_key").unwrap(), &small_value); } + #[tokio::test] + async fn offload_preserves_parallel_results_and_replaces_only_large_leaves() { + let run_store = make_run_store("parallel-result-artifact-offload").await; + let large_response = "r".repeat(BLOB_OFFLOAD_THRESHOLD + 1); + let large_output = "o".repeat(BLOB_OFFLOAD_THRESHOLD + 1); + let mut updates = HashMap::from([( + context::keys::PARALLEL_RESULTS.to_string(), + serde_json::json!([{ + "id": "branch_a", + "status": "failed", + "context_updates": { + "response.branch_a": large_response, + "command.output": large_output, + "small": "kept inline", + } + }]), + )]); + + offload_large_values(&mut updates, &run_store.clone().into()) + .await + .unwrap(); + + let results = updates[context::keys::PARALLEL_RESULTS] + .as_array() + .expect("parallel.results must remain a structured array"); + let branch_updates = results[0]["context_updates"] + .as_object() + .expect("context_updates must remain a structured object"); + assert!( + branch_updates["response.branch_a"] + .as_str() + .is_some_and(|value| fabro_types::parse_blob_ref(value).is_some()) + ); + assert!( + branch_updates[context::keys::COMMAND_OUTPUT] + .as_str() + .is_some_and(|value| fabro_types::parse_blob_ref(value).is_some()) + ); + assert_eq!(branch_updates["small"], serde_json::json!("kept inline")); + } + #[test] fn artifact_path_extracts_path_from_pointer() { let value = serde_json::json!("file:///tmp/logs/runtime/blobs/response.plan.json"); @@ -487,6 +612,58 @@ mod tests { assert_eq!(artifact_path(&value), None); } + #[tokio::test] + async fn resolve_context_hydrates_nested_parallel_text_blob_references() { + let run_store = make_run_store("parallel-result-text-resolution").await; + let response = "full branch response"; + let output = "full command output"; + let response_blob = run_store + .write_blob(&serde_json::to_vec(response).unwrap()) + .await + .unwrap(); + let output_blob = run_store + .write_blob(&serde_json::to_vec(output).unwrap()) + .await + .unwrap(); + let unrelated_blob = run_store + .write_blob(&serde_json::to_vec("unrelated artifact").unwrap()) + .await + .unwrap(); + let context = Context::new(); + context.set( + context::keys::PARALLEL_RESULTS, + serde_json::json!([{ + "id": "branch_a", + "status": "succeeded", + "context_updates": { + "response.branch_a": fabro_types::format_blob_ref(&response_blob), + "command.output": fabro_types::format_blob_ref(&output_blob), + "report": fabro_types::format_blob_ref(&unrelated_blob), + } + }]), + ); + let env = TestSyncEnv::new(true, "/workspace"); + let run_dir = tempfile::tempdir().unwrap(); + + let resolved = + resolved_context_snapshot(&context, &run_store.clone().into(), &env, run_dir.path()) + .await + .unwrap(); + + let updates = &resolved[context::keys::PARALLEL_RESULTS][0]["context_updates"]; + assert_eq!(updates["response.branch_a"], serde_json::json!(response)); + assert_eq!( + updates[context::keys::COMMAND_OUTPUT], + serde_json::json!(output) + ); + assert!( + updates["report"] + .as_str() + .is_some_and(|value| value.starts_with("file://")), + "non-textual nested values should retain artifact semantics" + ); + } + #[test] fn normalize_durable_updates_rewrites_managed_blob_file_refs_recursively() { let blob_id = fabro_types::RunBlobId::new(b"hello"); diff --git a/lib/crates/fabro-workflow/src/context.rs b/lib/crates/fabro-workflow/src/context.rs index af1233586..1926c42db 100644 --- a/lib/crates/fabro-workflow/src/context.rs +++ b/lib/crates/fabro-workflow/src/context.rs @@ -40,9 +40,6 @@ pub mod keys { // --- parallel.* keys --- pub const PARALLEL_RESULTS: &str = "parallel.results"; pub const PARALLEL_BRANCH_COUNT: &str = "parallel.branch_count"; - pub const PARALLEL_FAN_IN_BEST_ID: &str = "parallel.fan_in.best_id"; - pub const PARALLEL_FAN_IN_BEST_OUTCOME: &str = "parallel.fan_in.best_outcome"; - pub const PARALLEL_FAN_IN_BEST_HEAD_SHA: &str = "parallel.fan_in.best_head_sha"; // --- Prefix constants (for filtering and dynamic keys) --- pub const GRAPH_PREFIX: &str = "graph."; @@ -132,18 +129,36 @@ pub mod keys { } } +use std::collections::HashMap; + pub use fabro_core::Context; use fabro_graphviz::Fidelity; -use fabro_types::{ParallelBranchId, StageId}; +use fabro_types::{ParallelBranchId, RunId, StageId}; +use crate::error::Error; use crate::event::StageScope; +/// Keys whose values changed or were added in `after` relative to `before`. +/// Takes `after` by value so changed entries move instead of clone. +pub(crate) fn context_diff( + before: &HashMap, + after: HashMap, +) -> HashMap { + after + .into_iter() + .filter(|(key, value)| before.get(key) != Some(value)) + .collect() +} + /// Domain-specific typed accessors for workflow context values. pub trait WorkflowContext { fn fidelity(&self) -> Fidelity; fn thread_id(&self) -> Option; fn preamble(&self) -> String; fn run_id(&self) -> String; + /// Parse `internal.run_id`, failing when the engine did not seed a + /// valid run ID. + fn parsed_run_id(&self) -> Result; fn parallel_group_id(&self) -> Option; fn parallel_branch_id(&self) -> Option; /// Build the stage-level emit scope from the currently-executing node and @@ -172,6 +187,12 @@ impl WorkflowContext for Context { self.get_string(keys::INTERNAL_RUN_ID, "unknown") } + fn parsed_run_id(&self) -> Result { + self.run_id() + .parse() + .map_err(|err| Error::handler_with_source("invalid internal run_id", err)) + } + fn parallel_group_id(&self) -> Option { self.get(keys::INTERNAL_PARALLEL_GROUP_ID) .and_then(|value| serde_json::from_value(value).ok()) diff --git a/lib/crates/fabro-workflow/src/event/convert.rs b/lib/crates/fabro-workflow/src/event/convert.rs index 3238e153d..5114bdac7 100644 --- a/lib/crates/fabro-workflow/src/event/convert.rs +++ b/lib/crates/fabro-workflow/src/event/convert.rs @@ -371,12 +371,10 @@ fn event_body_from_event(event: &Event) -> EventBody { Event::ParallelStarted { visit, branch_count, - join_policy, .. } => EventBody::ParallelStarted(fabro_types::ParallelStartedProps { visit: *visit, branch_count: *branch_count, - join_policy: join_policy.clone(), }), Event::ParallelBranchStarted { index, .. } => { EventBody::ParallelBranchStarted(fabro_types::ParallelBranchStartedProps { @@ -387,13 +385,11 @@ fn event_body_from_event(event: &Event) -> EventBody { index, duration_ms, status, - head_sha, .. } => EventBody::ParallelBranchCompleted(fabro_types::ParallelBranchCompletedProps { index: *index, duration_ms: *duration_ms, - status: status.clone(), - head_sha: head_sha.clone(), + status: *status, }), Event::ParallelCompleted { visit, @@ -516,19 +512,6 @@ fn event_body_from_event(event: &Event) -> EventBody { success: *success, exec_output_tail: exec_output_tail.clone(), }), - Event::GitBranch { branch, sha } => EventBody::GitBranch(fabro_types::GitBranchProps { - branch: branch.clone(), - sha: sha.clone(), - }), - Event::GitWorktreeAdd { path, branch } => { - EventBody::GitWorktreeAdd(fabro_types::GitWorktreeAddProps { - path: path.clone(), - branch: branch.clone(), - }) - } - Event::GitWorktreeRemove { path } => { - EventBody::GitWorktreeRemove(fabro_types::GitWorktreeRemoveProps { path: path.clone() }) - } Event::GitFetch { branch, success } => EventBody::GitFetch(fabro_types::GitFetchProps { branch: branch.clone(), success: *success, @@ -1716,15 +1699,96 @@ mod tests { } #[test] - fn parallel_started_populates_parallel_group_id() { + fn parallel_started_populates_group_id_and_public_properties() { let stored = to_run_event(&fixtures::RUN_1, &Event::ParallelStarted { node_id: "fanout".to_string(), visit: 2, branch_count: 3, - join_policy: "wait_all".to_string(), }); assert_eq!(stored.parallel_group_id, Some(StageId::new("fanout", 2))); assert!(stored.parallel_branch_id.is_none()); + assert_eq!( + stored.properties().unwrap(), + serde_json::json!({ + "visit": 2, + "branch_count": 3, + }) + ); + } + + #[test] + fn parallel_branch_completed_public_properties() { + let group_id = StageId::new("fanout", 2); + let stored = to_run_event(&fixtures::RUN_1, &Event::ParallelBranchCompleted { + parallel_group_id: group_id.clone(), + parallel_branch_id: ParallelBranchId::new(group_id, 1), + branch: "review".to_string(), + index: 1, + duration_ms: 42, + status: StageOutcome::Succeeded, + }); + + assert_eq!( + stored.properties().unwrap(), + serde_json::json!({ + "index": 1, + "duration_ms": 42, + "status": "succeeded", + }) + ); + } + + #[test] + fn parallel_completed_exposes_typed_results_in_input_order() { + let stored = to_run_event(&fixtures::RUN_1, &Event::ParallelCompleted { + node_id: "fanout".to_string(), + visit: 2, + duration_ms: 84, + success_count: 1, + failure_count: 1, + results: vec![ + ::fabro_types::ParallelBranchResult { + id: "review_api".to_string(), + status: StageOutcome::Succeeded, + context_updates: BTreeMap::from([( + "response.review_api".to_string(), + serde_json::json!("looks good"), + )]), + }, + ::fabro_types::ParallelBranchResult { + id: "review_ux".to_string(), + status: StageOutcome::Failed { + retry_requested: false, + }, + context_updates: BTreeMap::from([( + "response.review_ux".to_string(), + serde_json::json!("needs work"), + )]), + }, + ], + }); + + assert_eq!( + stored.properties().unwrap(), + serde_json::json!({ + "visit": 2, + "duration_ms": 84, + "success_count": 1, + "failure_count": 1, + "results": [ + { + "id": "review_api", + "status": "succeeded", + "context_updates": {"response.review_api": "looks good"}, + }, + { + "id": "review_ux", + "status": "failed", + "context_updates": {"response.review_ux": "needs work"}, + }, + ], + }) + ); } #[test] diff --git a/lib/crates/fabro-workflow/src/event/emitter.rs b/lib/crates/fabro-workflow/src/event/emitter.rs index dc50a56dd..339919b18 100644 --- a/lib/crates/fabro-workflow/src/event/emitter.rs +++ b/lib/crates/fabro-workflow/src/event/emitter.rs @@ -3,7 +3,6 @@ use std::sync::atomic::{AtomicI64, Ordering}; use ::fabro_types::{ExecOutputTail, RunEvent, RunId, RunNoticeCode, RunNoticeLevel}; use chrono::Utc; -use fabro_agent::{WorktreeEvent, WorktreeEventCallback}; use super::Event; use super::convert::to_run_event_at; @@ -139,22 +138,6 @@ impl Emitter { pub fn touch(&self) { self.last_event_at.store(epoch_millis(), Ordering::Relaxed); } - - /// Build a [`WorktreeEventCallback`] that forwards worktree lifecycle - /// events as [`Event`]s on this emitter. - pub fn worktree_callback(self: Arc) -> WorktreeEventCallback { - Arc::new(move |event| match event { - WorktreeEvent::BranchCreated { branch, sha } => { - self.emit(&Event::GitBranch { branch, sha }); - } - WorktreeEvent::WorktreeAdded { path, branch } => { - self.emit(&Event::GitWorktreeAdd { path, branch }); - } - WorktreeEvent::WorktreeRemoved { path } => { - self.emit(&Event::GitWorktreeRemove { path }); - } - }) - } } #[cfg(test)] diff --git a/lib/crates/fabro-workflow/src/event/events.rs b/lib/crates/fabro-workflow/src/event/events.rs index 0b5afb1c7..ed940c5b1 100644 --- a/lib/crates/fabro-workflow/src/event/events.rs +++ b/lib/crates/fabro-workflow/src/event/events.rs @@ -3,10 +3,10 @@ use std::collections::BTreeMap; use ::fabro_types::{ AutomationRef, BilledTokenCounts, BlockedReason, CommandTermination, DiffSummary, FailureReason, ForkSourceRef, GitContext, PairId, PairMessageId, PairSystemMessageKind, - PairTarget, ParallelBranchId, PendingReason, PermissionLevel, Principal, PullRequestLink, - RunBlobId, RunFailure, RunId, RunNoticeLevel, RunPairEndedReason, RunPairFailedReason, - RunProvenance, RunRunnableSource, RunTiming, SandboxProviderKind, StageId, StageTiming, - SuccessReason, run_event as fabro_types, + PairTarget, ParallelBranchId, ParallelBranchResult, PendingReason, PermissionLevel, Principal, + PullRequestLink, RunBlobId, RunFailure, RunId, RunNoticeLevel, RunPairEndedReason, + RunPairFailedReason, RunProvenance, RunRunnableSource, RunTiming, SandboxProviderKind, StageId, + StageOutcome, StageTiming, SuccessReason, run_event as fabro_types, }; use fabro_agent::{AgentEvent, SandboxEvent}; use fabro_model::{ReasoningEffort, Speed}; @@ -304,7 +304,6 @@ pub enum Event { node_id: String, visit: u32, branch_count: usize, - join_policy: String, }, ParallelBranchStarted { parallel_group_id: StageId, @@ -318,9 +317,7 @@ pub enum Event { branch: String, index: usize, duration_ms: u64, - status: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - head_sha: Option, + status: StageOutcome, }, ParallelCompleted { node_id: String, @@ -329,7 +326,7 @@ pub enum Event { success_count: usize, failure_count: usize, #[serde(default, skip_serializing_if = "Vec::is_empty")] - results: Vec, + results: Vec, }, InterviewStarted { question_id: String, @@ -414,17 +411,6 @@ pub enum Event { #[serde(default, skip_serializing_if = "Option::is_none")] exec_output_tail: Option, }, - GitBranch { - branch: String, - sha: String, - }, - GitWorktreeAdd { - path: String, - branch: String, - }, - GitWorktreeRemove { - path: String, - }, GitFetch { branch: String, success: bool, @@ -1116,12 +1102,8 @@ impl Event { "Stage retrying" ); } - Self::ParallelStarted { - branch_count, - join_policy, - .. - } => { - debug!(branch_count, join_policy, "Parallel execution started"); + Self::ParallelStarted { branch_count, .. } => { + debug!(branch_count, "Parallel execution started"); } Self::ParallelBranchStarted { branch, index, .. } => { debug!(branch, index, "Parallel branch started"); @@ -1135,7 +1117,10 @@ impl Event { } => { debug!( branch, - index, duration_ms, status, "Parallel branch completed" + index, + duration_ms, + status = %status, + "Parallel branch completed" ); } Self::ParallelCompleted { @@ -1233,15 +1218,6 @@ impl Event { ); } } - Self::GitBranch { branch, sha } => { - debug!(branch, sha, "Git branch created"); - } - Self::GitWorktreeAdd { path, branch } => { - debug!(path, branch, "Git worktree added"); - } - Self::GitWorktreeRemove { path } => { - debug!(path, "Git worktree removed"); - } Self::GitFetch { branch, success } => { if *success { debug!(branch, "Git fetch succeeded"); diff --git a/lib/crates/fabro-workflow/src/event/names.rs b/lib/crates/fabro-workflow/src/event/names.rs index acfee89e6..58abe28e7 100644 --- a/lib/crates/fabro-workflow/src/event/names.rs +++ b/lib/crates/fabro-workflow/src/event/names.rs @@ -56,9 +56,6 @@ pub fn event_name(event: &Event) -> &'static str { Event::CheckpointFailed { .. } => "checkpoint.failed", Event::GitCommit { .. } => "git.commit", Event::GitPush { .. } => "git.push", - Event::GitBranch { .. } => "git.branch", - Event::GitWorktreeAdd { .. } => "git.worktree.added", - Event::GitWorktreeRemove { .. } => "git.worktree.removed", Event::GitFetch { .. } => "git.fetch", Event::GitReset { .. } => "git.reset", Event::EdgeSelected { .. } => "edge.selected", diff --git a/lib/crates/fabro-workflow/src/git.rs b/lib/crates/fabro-workflow/src/git.rs index 8fa23a208..a1f0adf99 100644 --- a/lib/crates/fabro-workflow/src/git.rs +++ b/lib/crates/fabro-workflow/src/git.rs @@ -74,60 +74,6 @@ pub fn head_sha(repo: &Path) -> Result { Ok(String::from_utf8_lossy(&output.stdout).trim().to_string()) } -/// Create a new branch at HEAD without checking it out. -pub fn create_branch(repo: &Path, name: &str) -> Result<()> { - let output = git_cmd(repo) - .args(["branch", "--force", name, "HEAD"]) - .output() - .map_err(|e| Error::engine_with_source("git branch failed", e))?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(git_error(format!("git branch failed: {stderr}"))); - } - - Ok(()) -} - -/// Add a git worktree for the given branch at `path`. -pub fn add_worktree(repo: &Path, path: &Path, branch: &str) -> Result<()> { - let output = git_cmd(repo) - .args(["worktree", "add"]) - .arg(path) - .arg(branch) - .output() - .map_err(|e| Error::engine_with_source("git worktree add failed", e))?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(git_error(format!("git worktree add failed: {stderr}"))); - } - - Ok(()) -} - -/// Remove a git worktree. -pub fn remove_worktree(repo: &Path, path: &Path) -> Result<()> { - let output = git_cmd(repo) - .args(["worktree", "remove", "--force"]) - .arg(path) - .output() - .map_err(|e| Error::engine_with_source("git worktree remove failed", e))?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(git_error(format!("git worktree remove failed: {stderr}"))); - } - - Ok(()) -} - -/// Remove any stale worktree at `path` (best-effort), then add a fresh one. -pub fn replace_worktree(repo: &Path, path: &Path, branch: &str) -> Result<()> { - let _ = remove_worktree(repo, path); - add_worktree(repo, path, branch) -} - /// Run a `git push` command and check for success. fn run_git_push(cmd: &mut Command) -> Result<()> { let output = cmd @@ -313,23 +259,6 @@ pub fn sync_status(repo: &Path, remote: &str, branch: Option<&str>) -> GitSyncSt } } -/// Sanitize a string for use as a git ref component. -/// Lowercases, replaces non-alphanumeric chars with dashes, collapses runs. -pub fn sanitize_ref_component(s: &str) -> String { - let mut result = String::with_capacity(s.len()); - let mut prev_dash = false; - for c in s.chars() { - if c.is_ascii_alphanumeric() { - result.push(c.to_ascii_lowercase()); - prev_dash = false; - } else if !prev_dash { - result.push('-'); - prev_dash = true; - } - } - result.trim_matches('-').to_string() -} - /// Filenames allowed in per-node directories on the shadow branch. #[cfg(test)] #[expect( @@ -416,39 +345,6 @@ mod tests { assert!(sha.chars().all(|c| c.is_ascii_hexdigit())); } - #[test] - #[expect( - clippy::disallowed_methods, - reason = "This synchronous test verifies git branch listing against the real git CLI." - )] - fn create_branch_and_list() { - let dir = tempfile::tempdir().unwrap(); - init_repo(dir.path()); - create_branch(dir.path(), "test-branch").unwrap(); - - let output = Command::new("git") - .args(["branch", "--list", "test-branch"]) - .current_dir(dir.path()) - .output() - .unwrap(); - let stdout = String::from_utf8_lossy(&output.stdout); - assert!(stdout.contains("test-branch")); - } - - #[test] - fn add_and_remove_worktree() { - let dir = tempfile::tempdir().unwrap(); - init_repo(dir.path()); - create_branch(dir.path(), "wt-branch").unwrap(); - - let wt_path = dir.path().join("my-worktree"); - add_worktree(dir.path(), &wt_path, "wt-branch").unwrap(); - assert!(wt_path.join(".git").exists()); - - remove_worktree(dir.path(), &wt_path).unwrap(); - assert!(!wt_path.exists()); - } - #[tokio::test] async fn scan_node_files_from_state_reconstructs_allowlisted_entries() { use crate::event::{Event, append_event}; @@ -550,7 +446,11 @@ mod tests { duration_ms: 100, success_count: 1, failure_count: 0, - results: vec![serde_json::json!({"id": "a"})], + results: vec![fabro_types::ParallelBranchResult { + id: "a".to_string(), + status: fabro_types::StageOutcome::Succeeded, + context_updates: std::collections::BTreeMap::new(), + }], }) .await .unwrap(); @@ -588,44 +488,6 @@ mod tests { assert!(paths.contains(&"stages/001-work@2/parallel_results.json")); } - #[test] - fn sanitize_ref_component_lowercases() { - assert_eq!(sanitize_ref_component("Hello"), "hello"); - } - - #[test] - fn sanitize_ref_component_replaces_special_chars() { - assert_eq!(sanitize_ref_component("a/b:c d"), "a-b-c-d"); - } - - #[test] - fn sanitize_ref_component_collapses_consecutive_dashes() { - assert_eq!(sanitize_ref_component("a///b"), "a-b"); - } - - #[test] - fn sanitize_ref_component_trims_leading_trailing_dashes() { - assert_eq!(sanitize_ref_component("--abc--"), "abc"); - } - - #[test] - fn sanitize_ref_component_mixed() { - assert_eq!(sanitize_ref_component("My Node!@#123"), "my-node-123"); - } - - #[test] - fn replace_worktree_on_clean_path() { - let dir = tempfile::tempdir().unwrap(); - init_repo(dir.path()); - create_branch(dir.path(), "rw-branch").unwrap(); - - let wt_path = dir.path().join("rw-worktree"); - replace_worktree(dir.path(), &wt_path, "rw-branch").unwrap(); - assert!(wt_path.join(".git").exists()); - - remove_worktree(dir.path(), &wt_path).unwrap(); - } - #[test] fn push_branch_fails_for_nonexistent_remote() { let dir = tempfile::tempdir().unwrap(); diff --git a/lib/crates/fabro-workflow/src/handler/agent.rs b/lib/crates/fabro-workflow/src/handler/agent.rs index 16dbe0230..5a063e68e 100644 --- a/lib/crates/fabro-workflow/src/handler/agent.rs +++ b/lib/crates/fabro-workflow/src/handler/agent.rs @@ -4,7 +4,7 @@ use std::sync::Arc; use async_trait::async_trait; use fabro_agent::Sandbox; use fabro_graphviz::graph::{Graph, Node}; -use fabro_types::{RunId, StageModelUsage, StageTiming}; +use fabro_types::{StageModelUsage, StageTiming}; pub(crate) use structured_output::extract_status_fields; use tokio_util::sync::CancellationToken; @@ -268,10 +268,7 @@ impl Handler for AgentHandler { // 3. Call LLM backend (agent loop) let thread_id = context.thread_id(); - let run_id = context - .run_id() - .parse::() - .map_err(|err| Error::handler_with_source("invalid internal run_id", err))?; + let run_id = context.parsed_run_id()?; let tool_hooks: Option> = services.run.hook_runner.as_ref().map(|hr| { Arc::new(fabro_hooks::WorkflowToolHookCallback { diff --git a/lib/crates/fabro-workflow/src/handler/fan_in.rs b/lib/crates/fabro-workflow/src/handler/fan_in.rs index aa03d8f74..b6b73db05 100644 --- a/lib/crates/fabro-workflow/src/handler/fan_in.rs +++ b/lib/crates/fabro-workflow/src/handler/fan_in.rs @@ -2,665 +2,262 @@ use std::path::Path; use std::sync::Arc; use async_trait::async_trait; -use fabro_agent::Sandbox; use fabro_graphviz::graph::{Graph, Node}; -use fabro_types::{StageModelUsage, StageTiming}; -use tokio_util::sync::CancellationToken; -use super::agent::{CodergenBackend, CodergenResult, CodergenRunRequest}; +use super::agent::CodergenBackend; +use super::prompt::PromptHandler; use super::{EngineServices, Handler}; use crate::context::{Context, keys}; use crate::error::Error; -use crate::event::{Emitter, Event, StageScope}; -use crate::outcome::{Outcome, OutcomeExt}; -use crate::sandbox_git::git_merge_ff_only; +use crate::event::Emitter; +use crate::outcome::Outcome; -/// Consolidates results from a preceding parallel node and selects the best -/// candidate. +/// Joins results from a preceding parallel node. +/// +/// Promptless fan-in nodes are barriers. Prompted fan-in nodes use the same +/// execution path as standard prompt stages and synthesize the full ordered +/// branch result set without selecting workspace state. pub struct FanInHandler { - backend: Option>, + prompt_handler: PromptHandler, } impl FanInHandler { #[must_use] pub fn new(backend: Option>) -> Self { - Self { backend } + Self { + prompt_handler: PromptHandler::new(backend), + } + } +} + +impl FanInHandler { + async fn run_join( + &self, + node: &Node, + context: &Context, + graph: &Graph, + run_dir: &Path, + services: &EngineServices, + simulated: bool, + ) -> Result { + let branch_count = validated_branch_count(context)?; + if node + .prompt() + .is_some_and(|prompt| !prompt.trim().is_empty()) + { + return if simulated { + self.prompt_handler + .simulate(node, context, graph, run_dir, services) + .await + } else { + self.prompt_handler + .execute(node, context, graph, run_dir, services) + .await + }; + } + Ok(joined_outcome(branch_count, simulated)) } } #[async_trait] impl Handler for FanInHandler { async fn shutdown(&self, emitter: &Arc) { - if let Some(backend) = self.backend.as_ref() { - backend.shutdown(emitter).await; - } + self.prompt_handler.shutdown(emitter).await; } async fn simulate( &self, node: &Node, context: &Context, - _graph: &Graph, - _run_dir: &Path, - _services: &EngineServices, + graph: &Graph, + run_dir: &Path, + services: &EngineServices, ) -> Result { - let results = context.get(keys::PARALLEL_RESULTS); - let Some(results) = results else { - return Ok(Outcome::fail_deterministic( - "No parallel results to evaluate", - )); - }; - - let best = heuristic_select(&results); - - let mut outcome = Outcome::simulated(&node.id); - outcome.context_updates.insert( - keys::PARALLEL_FAN_IN_BEST_ID.to_string(), - serde_json::json!(best.id), - ); - outcome.context_updates.insert( - keys::PARALLEL_FAN_IN_BEST_OUTCOME.to_string(), - serde_json::json!(best.status), - ); - // Override the generic simulated notes with handler-specific detail. - outcome.notes = Some(format!("[Simulated] Selected best candidate: {}", best.id)); - Ok(outcome) + self.run_join(node, context, graph, run_dir, services, true) + .await } async fn execute( &self, node: &Node, context: &Context, - _graph: &Graph, + graph: &Graph, run_dir: &Path, services: &EngineServices, ) -> Result { - let results = context.get(keys::PARALLEL_RESULTS); - let Some(results) = results else { - return Ok(Outcome::fail_deterministic( - "No parallel results to evaluate", - )); - }; + self.run_join(node, context, graph, run_dir, services, false) + .await + } +} - let prompt = node.prompt().filter(|p| !p.is_empty()); +/// Validate that `parallel.results` exists and has the typed shape without +/// cloning the (potentially hydrated) branch payloads into a full +/// [`ParallelBranchResult`] vec that would go unused. +fn validated_branch_count(context: &Context) -> Result { + #[derive(serde::Deserialize)] + struct BranchShape { + #[expect(dead_code, reason = "deserialized only to validate the shape")] + id: String, + #[expect(dead_code, reason = "deserialized only to validate the shape")] + status: fabro_types::StageOutcome, + } - let best = if let (Some(prompt_text), Some(backend)) = (prompt, &self.backend) { - llm_evaluate( - backend.as_ref(), - prompt_text, - &results, - context, - run_dir, - &node.id, - &services.run.emitter, - &services.run.sandbox, - services.run.cancel_token(), - ) - .await? + let value = context + .get(keys::PARALLEL_RESULTS) + .ok_or_else(|| Error::handler("No parallel results to join"))?; + let results: Vec = serde_json::from_value(value) + .map_err(|err| Error::handler_with_source("Invalid parallel results", err))?; + Ok(results.len()) +} + +fn joined_outcome(branch_count: usize, simulated: bool) -> Outcome { + let mut outcome = Outcome::success(); + let prefix = if simulated { "[Simulated] " } else { "" }; + outcome.notes = Some(format!( + "{prefix}Joined {branch_count} parallel {}", + if branch_count == 1 { + "branch" } else { - heuristic_select(&results) - }; - - // Check if all candidates failed — if so, return fail - let all_failed = if best.status == "failed" { - let empty_vec = vec![]; - let arr = results.as_array().unwrap_or(&empty_vec); - arr.iter() - .all(|v| v.get("status").and_then(|v| v.as_str()).unwrap_or("failed") == "failed") - } else { - false - }; - - if all_failed { - let mut outcome = Outcome::fail_deterministic("all candidates failed"); - outcome.timing = Some(best.timing); - return Ok(outcome); + "branches" } - - // --- Fast-forward to winner's HEAD when git isolation is active --- - let best_head_sha = { - let empty_vec = vec![]; - let arr = results.as_array().unwrap_or(&empty_vec); - arr.iter() - .find(|v| v.get("id").and_then(|v| v.as_str()) == Some(&best.id)) - .and_then(|v| v.get("head_sha").and_then(|v| v.as_str()).map(String::from)) - }; - - if let (Some(ref sha), Some(_)) = (&best_head_sha, services.git_state()) { - git_merge_ff_only(&*services.run.sandbox, sha).await; - } - - let mut outcome = Outcome::success(); - outcome.context_updates.insert( - keys::PARALLEL_FAN_IN_BEST_ID.to_string(), - serde_json::json!(best.id), - ); - outcome.context_updates.insert( - keys::PARALLEL_FAN_IN_BEST_OUTCOME.to_string(), - serde_json::json!(best.status), - ); - if let Some(ref sha) = best_head_sha { - outcome.context_updates.insert( - keys::PARALLEL_FAN_IN_BEST_HEAD_SHA.to_string(), - serde_json::json!(sha), - ); - } - outcome.notes = Some(format!("Selected best candidate: {}", best.id)); - outcome.timing = Some(best.timing); - - Ok(outcome) - } -} - -struct Candidate { - id: String, - status: String, - score: f64, - timing: StageTiming, -} - -fn status_rank(status: &str) -> u32 { - match status { - "succeeded" => 0, - "partially_succeeded" => 1, - "failed" => 2, - _ => 4, - } -} - -fn heuristic_select(results: &serde_json::Value) -> Candidate { - let empty_vec = vec![]; - let arr = results.as_array().unwrap_or(&empty_vec); - if arr.is_empty() { - return Candidate { - id: "unknown".to_string(), - status: "failed".to_string(), - score: 0.0, - timing: StageTiming::default(), - }; - } - - let mut candidates: Vec = arr - .iter() - .map(|v| Candidate { - id: v - .get("id") - .and_then(|v| v.as_str()) - .unwrap_or("unknown") - .to_string(), - status: v - .get("status") - .and_then(|v| v.as_str()) - .unwrap_or("failed") - .to_string(), - score: v - .get("score") - .and_then(serde_json::Value::as_f64) - .unwrap_or(0.0), - timing: StageTiming::default(), - }) - .collect(); - - candidates.sort_by(|a, b| { - let rank_cmp = status_rank(&a.status).cmp(&status_rank(&b.status)); - if rank_cmp != std::cmp::Ordering::Equal { - return rank_cmp; - } - // Higher score is better, so reverse the comparison - let score_cmp = b - .score - .partial_cmp(&a.score) - .unwrap_or(std::cmp::Ordering::Equal); - if score_cmp != std::cmp::Ordering::Equal { - return score_cmp; - } - a.id.cmp(&b.id) - }); - - candidates.into_iter().next().unwrap_or_else(|| Candidate { - id: "unknown".to_string(), - status: "failed".to_string(), - score: 0.0, - timing: StageTiming::default(), - }) -} - -/// Use an LLM backend to evaluate and rank parallel branch results. -#[allow( - clippy::too_many_arguments, - reason = "Fan-in evaluation passes prompt, results, context, and runtime handles separately." -)] -async fn llm_evaluate( - backend: &dyn CodergenBackend, - prompt: &str, - results: &serde_json::Value, - context: &Context, - _run_dir: &Path, - node_id: &str, - emitter: &Arc, - sandbox: &Arc, - cancel_token: CancellationToken, -) -> Result { - let results_text = - serde_json::to_string_pretty(results).unwrap_or_else(|_| results.to_string()); - - let full_prompt = format!( - "{prompt}\n\nParallel branch results:\n{results_text}\n\n\ - Respond with the ID of the best candidate." - ); - - let stage_scope = StageScope::for_handler(context, node_id); - - emitter.emit_scoped( - &Event::Prompt { - stage: node_id.to_string(), - visit: stage_scope.visit, - text: full_prompt.clone(), - mode: Some(StageModelUsage::MODE_FAN_IN.to_string()), - provider: None, - model: None, - reasoning_effort: None, - speed: None, - }, - &stage_scope, - ); - - // Build a synthetic node for the backend call - let eval_node = Node::new("fan_in_eval"); - - // Fan-in evaluation runs outside a thread context, so pass None - match backend - .run(CodergenRunRequest { - node: &eval_node, - prompt: &full_prompt, - context, - thread_id: None, - emitter, - sandbox, - tool_hooks: None, - cancel_token, - agent_tool_runtime: fabro_agent::AgentToolRuntime::default(), - }) - .await - { - Ok(CodergenResult::Full(outcome)) => { - let timing = outcome.timing.unwrap_or_default(); - // If the backend returned a full Outcome, extract best_id from context_updates - let best_id = outcome - .context_updates - .get(keys::PARALLEL_FAN_IN_BEST_ID) - .and_then(|v| v.as_str()) - .map(String::from) - .or_else(|| outcome.notes.clone()) - .unwrap_or_else(|| "unknown".to_string()); - let response_text = - serde_json::to_string_pretty(&outcome).unwrap_or_else(|_| "{}".to_string()); - emitter.emit_scoped( - &Event::PromptCompleted { - node_id: node_id.to_string(), - response: response_text.clone(), - model: String::new(), - provider: String::new(), - billing: None, - }, - &stage_scope, - ); - Ok(Candidate { - id: best_id, - status: outcome.status.to_string(), - score: 0.0, - timing, - }) - } - Ok(CodergenResult::Text { text, timing, .. }) => { - emitter.emit_scoped( - &Event::PromptCompleted { - node_id: node_id.to_string(), - response: text.clone(), - model: String::new(), - provider: String::new(), - billing: None, - }, - &stage_scope, - ); - - // The LLM responded with text; try to find a matching candidate ID - let text = text.trim().to_string(); - let empty_vec = vec![]; - let arr = results.as_array().unwrap_or(&empty_vec); - - // Check if the response text matches any candidate ID - for v in arr { - if let Some(id) = v.get("id").and_then(|v| v.as_str()) { - if text.contains(id) { - let status = v - .get("status") - .and_then(|v| v.as_str()) - .unwrap_or("succeeded") - .to_string(); - let score = v - .get("score") - .and_then(serde_json::Value::as_f64) - .unwrap_or(0.0); - return Ok(Candidate { - id: id.to_string(), - status, - score, - timing, - }); - } - } - } - - // No match found; fall back to heuristic - let mut fallback = heuristic_select(results); - fallback.timing = timing; - Ok(fallback) - } - Err(_) => { - // LLM call failed; fall back to heuristic - Ok(heuristic_select(results)) - } - } + )); + outcome } #[cfg(test)] mod tests { + use fabro_graphviz::graph::AttrValue; + use fabro_types::StageTiming; + use tempfile::TempDir; + use super::*; + use crate::handler::agent::{CodergenResult, CodergenRunRequest, OneShotRequest}; use crate::outcome::StageOutcome; fn make_services() -> EngineServices { EngineServices::test_default() } - #[tokio::test] - async fn fan_in_no_results() { - let handler = FanInHandler::new(None); - let node = Node::new("fan_in"); - let context = Context::new(); - let graph = Graph::new("test"); - let run_dir = Path::new("/tmp/test"); - - let outcome = handler - .execute(&node, &context, &graph, run_dir, &make_services()) - .await - .unwrap(); - assert_eq!(outcome.status, StageOutcome::Failed { - retry_requested: false, - }); - } - - #[tokio::test] - async fn fan_in_selects_best() { - let handler = FanInHandler::new(None); - let node = Node::new("fan_in"); + fn context_with_results() -> Context { let context = Context::new(); context.set( keys::PARALLEL_RESULTS, serde_json::json!([ - {"id": "branch_a", "status": "failed"}, - {"id": "branch_b", "status": "succeeded"}, + { + "id": "branch_a", + "status": "failed", + "context_updates": {"command.output": "failure details"} + }, + { + "id": "branch_b", + "status": "succeeded", + "context_updates": {"response.branch_b": "complete response"} + } ]), ); - let graph = Graph::new("test"); - let run_dir = Path::new("/tmp/test"); + context + } - let outcome = handler - .execute(&node, &context, &graph, run_dir, &make_services()) + #[tokio::test] + async fn promptless_fan_in_is_a_noop_barrier() { + let outcome = FanInHandler::new(None) + .execute( + &Node::new("fan_in"), + &context_with_results(), + &Graph::new("test"), + Path::new("/tmp/test"), + &make_services(), + ) .await .unwrap(); + assert_eq!(outcome.status, StageOutcome::Succeeded); - assert_eq!( - outcome.context_updates.get(keys::PARALLEL_FAN_IN_BEST_ID), - Some(&serde_json::json!("branch_b")) - ); + assert_eq!(outcome.notes.as_deref(), Some("Joined 2 parallel branches")); + assert!(outcome.context_updates.is_empty()); } #[tokio::test] - async fn fan_in_lexical_tiebreak() { - let handler = FanInHandler::new(None); - let node = Node::new("fan_in"); + async fn fan_in_requires_typed_parallel_results() { let context = Context::new(); - context.set( - keys::PARALLEL_RESULTS, - serde_json::json!([ - {"id": "c", "status": "succeeded"}, - {"id": "a", "status": "succeeded"}, - {"id": "b", "status": "succeeded"}, - ]), - ); - let graph = Graph::new("test"); - let run_dir = Path::new("/tmp/test"); + let missing = FanInHandler::new(None) + .execute( + &Node::new("fan_in"), + &context, + &Graph::new("test"), + Path::new("/tmp/test"), + &make_services(), + ) + .await; + assert!(missing.is_err()); - let outcome = handler - .execute(&node, &context, &graph, run_dir, &make_services()) - .await - .unwrap(); - assert_eq!( - outcome.context_updates.get(keys::PARALLEL_FAN_IN_BEST_ID), - Some(&serde_json::json!("a")) - ); - } - - #[test] - fn status_rank_ordering() { - assert!(status_rank("succeeded") < status_rank("partially_succeeded")); - assert!(status_rank("partially_succeeded") < status_rank("failed")); - assert!(status_rank("failed") < status_rank("unknown")); + context.set(keys::PARALLEL_RESULTS, serde_json::json!([{"id": "a"}])); + let invalid = FanInHandler::new(None) + .execute( + &Node::new("fan_in"), + &context, + &Graph::new("test"), + Path::new("/tmp/test"), + &make_services(), + ) + .await; + assert!(invalid.is_err()); } #[tokio::test] - async fn fan_in_no_backend_ignores_prompt() { - // When there's a prompt but no backend, it should fall back to heuristic - let handler = FanInHandler::new(None); - let mut node = Node::new("fan_in"); - node.attrs.insert( - "prompt".to_string(), - fabro_graphviz::graph::AttrValue::String("Pick the best branch".to_string()), - ); - let context = Context::new(); - context.set( - keys::PARALLEL_RESULTS, - serde_json::json!([ - {"id": "branch_a", "status": "succeeded"}, - {"id": "branch_b", "status": "failed"}, - ]), - ); - let graph = Graph::new("test"); - let run_dir = Path::new("/tmp/test"); - - let outcome = handler - .execute(&node, &context, &graph, run_dir, &make_services()) - .await - .unwrap(); - assert_eq!(outcome.status, StageOutcome::Succeeded); - // Should still pick branch_a via heuristic (success beats fail) - assert_eq!( - outcome.context_updates.get(keys::PARALLEL_FAN_IN_BEST_ID), - Some(&serde_json::json!("branch_a")) - ); - } - - #[tokio::test] - async fn fan_in_with_backend_llm_eval() { - use tempfile::TempDir; - - use crate::handler::agent::{CodergenBackend, CodergenRunRequest}; - - struct MockBackend; + async fn prompted_fan_in_uses_standard_prompt_response_fields() { + struct ReducerBackend; #[async_trait] - impl CodergenBackend for MockBackend { + impl CodergenBackend for ReducerBackend { async fn run(&self, _request: CodergenRunRequest<'_>) -> Result { - // Return text that contains the ID "branch_b" + panic!("prompted fan-in must use one_shot like a standard prompt") + } + + async fn one_shot(&self, request: OneShotRequest<'_>) -> Result { + assert!(request.prompt.contains("Synthesize every result")); Ok(CodergenResult::Text { - text: "The best candidate is branch_b".to_string(), + text: "combined result".to_string(), usage: None, files_touched: Vec::new(), last_file_touched: None, - timing: StageTiming::default(), + timing: StageTiming::new(0, 20, 30), }) } } - let handler = FanInHandler::new(Some(Box::new(MockBackend))); + let handler = FanInHandler::new(Some(Box::new(ReducerBackend))); let mut node = Node::new("fan_in"); node.attrs.insert( "prompt".to_string(), - fabro_graphviz::graph::AttrValue::String("Pick the best branch".to_string()), + AttrValue::String("Synthesize every result".to_string()), ); - let context = Context::new(); - context.set( - keys::PARALLEL_RESULTS, - serde_json::json!([ - {"id": "branch_a", "status": "succeeded"}, - {"id": "branch_b", "status": "succeeded"}, - ]), - ); - let graph = Graph::new("test"); - let tmp = TempDir::new().unwrap(); - + let run_dir = TempDir::new().unwrap(); let outcome = handler - .execute(&node, &context, &graph, tmp.path(), &make_services()) + .execute( + &node, + &context_with_results(), + &Graph::new("test"), + run_dir.path(), + &make_services(), + ) .await .unwrap(); + assert_eq!(outcome.status, StageOutcome::Succeeded); - // LLM chose branch_b assert_eq!( - outcome.context_updates.get(keys::PARALLEL_FAN_IN_BEST_ID), - Some(&serde_json::json!("branch_b")) + outcome.context_updates.get(&keys::response_key("fan_in")), + Some(&serde_json::json!("combined result")) ); - } - - #[tokio::test] - async fn fan_in_with_backend_copies_llm_timing_to_outcome() { - use tempfile::TempDir; - - use crate::handler::agent::{CodergenBackend, CodergenRunRequest}; - - struct TimingBackend; - - #[async_trait] - impl CodergenBackend for TimingBackend { - async fn run(&self, _request: CodergenRunRequest<'_>) -> Result { - Ok(CodergenResult::Text { - text: "branch_b".to_string(), - usage: None, - files_touched: Vec::new(), - last_file_touched: None, - timing: StageTiming::new(0, 200, 300), - }) - } - } - - let handler = FanInHandler::new(Some(Box::new(TimingBackend))); - let mut node = Node::new("fan_in"); - node.attrs.insert( - "prompt".to_string(), - fabro_graphviz::graph::AttrValue::String("Pick the best branch".to_string()), + assert_eq!( + outcome.context_updates.get(keys::LAST_RESPONSE), + Some(&serde_json::json!("combined result")) ); - let context = Context::new(); - context.set( - keys::PARALLEL_RESULTS, - serde_json::json!([ - {"id": "branch_a", "status": "succeeded"}, - {"id": "branch_b", "status": "succeeded"}, - ]), - ); - let graph = Graph::new("test"); - let tmp = TempDir::new().unwrap(); - - let outcome = handler - .execute(&node, &context, &graph, tmp.path(), &make_services()) - .await - .unwrap(); - - assert_eq!(outcome.timing, Some(StageTiming::new(0, 200, 300))); - } - - #[tokio::test] - async fn fan_in_all_fail_returns_fail() { - let handler = FanInHandler::new(None); - let node = Node::new("fan_in"); - let context = Context::new(); - context.set( - keys::PARALLEL_RESULTS, - serde_json::json!([ - {"id": "branch_a", "status": "failed"}, - {"id": "branch_b", "status": "failed"}, - {"id": "branch_c", "status": "failed"}, - ]), - ); - let graph = Graph::new("test"); - let run_dir = Path::new("/tmp/test"); - - let outcome = handler - .execute(&node, &context, &graph, run_dir, &make_services()) - .await - .unwrap(); - assert_eq!(outcome.status, StageOutcome::Failed { - retry_requested: false, - }); + assert_eq!(outcome.timing, Some(StageTiming::new(0, 20, 30))); assert!( outcome - .failure_reason() - .unwrap() - .contains("all candidates failed") - ); - } - - #[tokio::test] - async fn fan_in_score_tiebreak() { - let handler = FanInHandler::new(None); - let node = Node::new("fan_in"); - let context = Context::new(); - context.set( - keys::PARALLEL_RESULTS, - serde_json::json!([ - {"id": "branch_a", "status": "succeeded", "score": 0.5}, - {"id": "branch_b", "status": "succeeded", "score": 0.9}, - {"id": "branch_c", "status": "succeeded", "score": 0.7}, - ]), - ); - let graph = Graph::new("test"); - let run_dir = Path::new("/tmp/test"); - - let outcome = handler - .execute(&node, &context, &graph, run_dir, &make_services()) - .await - .unwrap(); - assert_eq!(outcome.status, StageOutcome::Succeeded); - // branch_b has highest score - assert_eq!( - outcome.context_updates.get(keys::PARALLEL_FAN_IN_BEST_ID), - Some(&serde_json::json!("branch_b")) - ); - } - - #[tokio::test] - async fn fan_in_simulate_uses_heuristic() { - let handler = FanInHandler::new(None); - let node = Node::new("fan_in"); - let context = Context::new(); - context.set( - keys::PARALLEL_RESULTS, - serde_json::json!([ - {"id": "branch_a", "status": "failed"}, - {"id": "branch_b", "status": "succeeded"}, - ]), - ); - let graph = Graph::new("test"); - let run_dir = Path::new("/tmp/test"); - - let outcome = handler - .simulate(&node, &context, &graph, run_dir, &make_services()) - .await - .unwrap(); - assert_eq!(outcome.status, StageOutcome::Succeeded); - assert!(outcome.notes.as_deref().unwrap().contains("[Simulated]")); - assert_eq!( - outcome.context_updates.get(keys::PARALLEL_FAN_IN_BEST_ID), - Some(&serde_json::json!("branch_b")) + .context_updates + .keys() + .all(|key| !key.starts_with("parallel.fan_in.best_")) ); } } diff --git a/lib/crates/fabro-workflow/src/handler/manager_loop.rs b/lib/crates/fabro-workflow/src/handler/manager_loop.rs index d16291b68..431ed5d76 100644 --- a/lib/crates/fabro-workflow/src/handler/manager_loop.rs +++ b/lib/crates/fabro-workflow/src/handler/manager_loop.rs @@ -14,7 +14,7 @@ use tokio::time::{sleep, timeout}; use super::{EngineServices, Handler}; use crate::artifact_upload::ArtifactSink; use crate::condition::evaluate_condition; -use crate::context::{Context, WorkflowContext, keys}; +use crate::context::{Context, WorkflowContext, context_diff, keys}; use crate::error::Error; use crate::operations::{ValidateInput, WorkflowInput, validate}; use crate::outcome::{Outcome, OutcomeExt, StageOutcome}; @@ -133,21 +133,6 @@ fn parse_child_graph(node: &Node, services: &EngineServices) -> Result, - after: &HashMap, -) -> HashMap { - let mut diff = HashMap::new(); - for (key, value) in after { - if before.get(key) != Some(value) { - diff.insert(key.clone(), value.clone()); - } - } - diff -} - #[async_trait] impl Handler for SubWorkflowHandler { async fn execute( @@ -273,7 +258,6 @@ impl Handler for SubWorkflowHandler { run: child_run, registry, interviewer, - git_state: std::sync::RwLock::new(None), base_env, github_token, inputs, @@ -299,8 +283,8 @@ impl Handler for SubWorkflowHandler { }; // Compute context diff, filtering engine-internal keys - let after_snapshot = child_final_context.snapshot(); - let raw_diff = context_diff(&before_snapshot, &after_snapshot); + let raw_diff = + context_diff(&before_snapshot, child_final_context.snapshot()); let diff: HashMap = raw_diff .into_iter() .filter(|(key, _)| !keys::is_engine_internal_key(key)) @@ -824,7 +808,7 @@ mod tests { let before = HashMap::new(); let mut after = HashMap::new(); after.insert("key".to_string(), serde_json::json!("value")); - let diff = context_diff(&before, &after); + let diff = context_diff(&before, after); assert_eq!(diff.len(), 1); assert_eq!(diff.get("key"), Some(&serde_json::json!("value"))); } @@ -835,7 +819,7 @@ mod tests { before.insert("key".to_string(), serde_json::json!("old")); let mut after = HashMap::new(); after.insert("key".to_string(), serde_json::json!("new")); - let diff = context_diff(&before, &after); + let diff = context_diff(&before, after); assert_eq!(diff.len(), 1); assert_eq!(diff.get("key"), Some(&serde_json::json!("new"))); } @@ -846,7 +830,7 @@ mod tests { before.insert("key".to_string(), serde_json::json!("same")); let mut after = HashMap::new(); after.insert("key".to_string(), serde_json::json!("same")); - let diff = context_diff(&before, &after); + let diff = context_diff(&before, after); assert!(diff.is_empty()); } @@ -855,7 +839,7 @@ mod tests { let mut before = HashMap::new(); before.insert("removed".to_string(), serde_json::json!("gone")); let after = HashMap::new(); - let diff = context_diff(&before, &after); + let diff = context_diff(&before, after); assert!(diff.is_empty()); } @@ -876,7 +860,7 @@ mod tests { after.insert("response.plan".to_string(), serde_json::json!("the plan")); after.insert("review.result".to_string(), serde_json::json!("approved")); - let raw_diff = context_diff(&before, &after); + let raw_diff = context_diff(&before, after); let filtered: HashMap = raw_diff .into_iter() .filter(|(key, _)| !keys::is_engine_internal_key(key)) diff --git a/lib/crates/fabro-workflow/src/handler/parallel.rs b/lib/crates/fabro-workflow/src/handler/parallel.rs index 22aa2aa9c..307647193 100644 --- a/lib/crates/fabro-workflow/src/handler/parallel.rs +++ b/lib/crates/fabro-workflow/src/handler/parallel.rs @@ -1,59 +1,39 @@ -use std::path::{Path, PathBuf}; +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::path::Path; use std::sync::Arc; use std::time::Instant; use async_trait::async_trait; -use fabro_agent::{Sandbox, WorktreeOptions, WorktreeSandbox}; use fabro_graphviz::graph::{AttrValue, Graph, Node}; use fabro_hooks::{HookContext, HookEvent}; -use fabro_types::{ParallelBranchId, RunId, StageId}; +use fabro_types::{ParallelBranchId, ParallelBranchResult, StageId, StageOutcome}; +use futures::FutureExt; use tokio::sync::Semaphore; +use tokio::task::JoinHandle; use super::{EngineServices, Handler}; -use crate::context::{Context, WorkflowContext, keys}; +use crate::context::{Context, WorkflowContext, context_diff, keys}; use crate::error::Error; use crate::event::{Event, RunNoticeCode, RunNoticeLevel, StageScope}; -use crate::git::sanitize_ref_component; use crate::hook_context::set_hook_node; -use crate::millis_u64; -use crate::outcome::{FailureCategory, FailureDetail, Outcome, OutcomeExt, StageOutcome}; -use crate::run_dir::visit_from_context; -use crate::sandbox_git::{ - GIT_REMOTE, checked_git_checkpoint, git_merge_ff_only, git_remove_worktree, -}; +use crate::outcome::{FailureCategory, FailureDetail, Outcome, OutcomeExt}; +use crate::{artifact, millis_u64}; /// Fans out execution to multiple branches concurrently. -/// Each branch gets an isolated context clone and runs independently. +/// Each branch gets an isolated context fork and shares the run sandbox. pub struct ParallelHandler; -/// Parse join policy from node attributes. -#[derive(Debug, Clone)] -enum JoinPolicy { - WaitAll, - FirstSuccess, -} - -impl std::fmt::Display for JoinPolicy { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::WaitAll => write!(f, "wait_all"), - Self::FirstSuccess => write!(f, "first_success"), - } - } -} - -fn parse_join_policy(raw: &str) -> JoinPolicy { - if raw == "first_success" { - return JoinPolicy::FirstSuccess; - } - JoinPolicy::WaitAll -} - struct BranchResult { - id: String, - outcome: Outcome, - head_sha: Option, - worktree_path: Option, + result: ParallelBranchResult, + outcome: Outcome, +} + +struct BranchDispatch { + index: usize, + target_id: String, + branch_id: ParallelBranchId, + scope: StageScope, + handle: JoinHandle>, } #[async_trait] @@ -66,58 +46,7 @@ impl Handler for ParallelHandler { run_dir: &Path, services: &EngineServices, ) -> Result { - let branches = graph.outgoing_edges(&node.id); - if branches.is_empty() { - return Ok(Outcome::fail_classify("No branches for parallel node")); - } - - // Dispatch each branch child via dispatch_handler (which will call simulate) - let mut branch_results: Vec = Vec::new(); - for edge in &branches { - let target_id = &edge.to; - if let Some(target_node) = graph.nodes.get(target_id) { - let handler = services.registry.resolve(target_node); - let branch_context = context.fork(); - let outcome = super::dispatch_handler( - handler, - target_node, - &branch_context, - graph, - run_dir, - services, - ) - .await?; - branch_results.push(BranchResult { - id: target_id.clone(), - outcome, - head_sha: None, - worktree_path: None, - }); - } - } - - let total = branch_results.len(); - context.set(keys::PARALLEL_BRANCH_COUNT, serde_json::json!(total)); - - let results_json: Vec = branch_results - .iter() - .map(|r| { - serde_json::json!({ - "id": r.id, - "status": r.outcome.status.to_string(), - }) - }) - .collect(); - context.set(keys::PARALLEL_RESULTS, serde_json::json!(results_json)); - - let join_node = find_join_node(&branch_results, graph); - - let mut outcome = Outcome::simulated(&node.id); - outcome.notes = Some(format!( - "[Simulated] Parallel node dispatched {total} branches" - )); - outcome.jump_to_node = join_node; - Ok(outcome) + run_branches(node, context, graph, run_dir, services, true).await } async fn execute( @@ -128,567 +57,380 @@ impl Handler for ParallelHandler { run_dir: &Path, services: &EngineServices, ) -> Result { - // Build per-branch sandboxes (sequentially for git setup) - struct BranchSetup { - target_id: String, - branch_index: usize, - parallel_branch_id: ParallelBranchId, - branch_context: Context, - sandbox: Arc, - worktree_path: Option, - } + run_branches(node, context, graph, run_dir, services, false).await + } +} - let parallel_start = Instant::now(); - let branches = graph.outgoing_edges(&node.id); - if branches.is_empty() { - return Ok(Outcome::fail_classify("No branches for parallel node")); - } +async fn run_branches( + node: &Node, + context: &Context, + graph: &Graph, + run_dir: &Path, + services: &EngineServices, + simulated: bool, +) -> Result { + let parallel_start = Instant::now(); + let branches = graph.outgoing_edges(&node.id); - let join_policy = parse_join_policy( - node.attrs - .get("join_policy") - .and_then(|v| v.as_str()) - .unwrap_or("wait_all"), + let parallel_stage_scope = StageScope::for_handler(context, &node.id); + let parallel_group_id = StageId::new(node.id.clone(), parallel_stage_scope.visit); + services.run.emitter.emit_scoped( + &Event::ParallelStarted { + node_id: node.id.clone(), + visit: parallel_stage_scope.visit, + branch_count: branches.len(), + }, + ¶llel_stage_scope, + ); + emit_parallel_hook(services, context, graph, node, HookEvent::ParallelStart).await?; + + let max_parallel = node + .attrs + .get("max_parallel") + .and_then(AttrValue::as_i64) + .unwrap_or(4); + let max_parallel = usize::try_from(max_parallel).unwrap_or(4).max(1); + let semaphore = Arc::new(Semaphore::new(max_parallel)); + let shared_graph = Arc::new(graph.clone()); + let parent_snapshot = Arc::new(context.snapshot()); + + let mut dispatches = Vec::with_capacity(branches.len()); + for (branch_index, edge) in branches.iter().enumerate() { + let target_id = edge.to.clone(); + let parallel_branch_id = ParallelBranchId::new( + parallel_group_id.clone(), + u32::try_from(branch_index).unwrap_or(u32::MAX), + ); + let branch_context = Context::from_values(parent_snapshot.as_ref().clone()); + branch_context.set( + keys::INTERNAL_PARALLEL_GROUP_ID, + serde_json::Value::String(parallel_group_id.to_string()), + ); + branch_context.set( + keys::INTERNAL_PARALLEL_BRANCH_ID, + serde_json::Value::String(parallel_branch_id.to_string()), ); - let parallel_stage_scope = StageScope::for_handler(context, &node.id); - let parallel_group_id = StageId::new(node.id.clone(), parallel_stage_scope.visit); - - services.run.emitter.emit_scoped( - &Event::ParallelStarted { - node_id: node.id.clone(), - visit: parallel_stage_scope.visit, - branch_count: branches.len(), - join_policy: join_policy.to_string(), - }, - ¶llel_stage_scope, + let mut branch_services = services.clone(); + branch_services.dry_run = simulated || services.dry_run; + let parent_snapshot = Arc::clone(&parent_snapshot); + let graph = Arc::clone(&shared_graph); + let run_dir = run_dir.to_path_buf(); + let semaphore = Arc::clone(&semaphore); + let group_id = parallel_group_id.clone(); + let branch_scope = StageScope::for_parallel_branch( + target_id.clone(), + 1, + group_id.clone(), + parallel_branch_id.clone(), ); - { - let run_id = context - .run_id() - .parse::() - .map_err(|err| Error::handler_with_source("invalid internal run_id", err))?; - let mut hook_ctx = - HookContext::new(HookEvent::ParallelStart, run_id, graph.name.clone()); - set_hook_node(&mut hook_ctx, node); - let _ = services.run.run_hooks(&hook_ctx).await; - } - let max_parallel = node - .attrs - .get("max_parallel") - .and_then(AttrValue::as_i64) - .unwrap_or(4); - let max_parallel = usize::try_from(max_parallel).unwrap_or(4).max(1); - let semaphore = Arc::new(Semaphore::new(max_parallel)); - let git_state = services.git_state(); - - // --- Git isolation: checkpoint "parallel base" before fan-out --- - let base_sha: Option = if let Some(ref gs) = git_state { - let result = checked_git_checkpoint( - &services.run.sandbox_git, - &*services.run.sandbox, - &gs.run_id.to_string(), - &node.id, - "parallel_base", - 0, - None, - &gs.checkpoint, - &gs.git_author, - ) - .await; - match result { - Ok(sha) => Some(sha), - Err(e) if e.to_string() == "sandbox git unavailable" => { - return Err(Error::handler_with_source("sandbox git unavailable", e)); - } - Err(e) => { - tracing::warn!( - error = %fabro_sandbox::display_for_log(&e), - "parallel base checkpoint failed" - ); - services.run.emitter.notice_with_tail( - RunNoticeLevel::Warn, - RunNoticeCode::ParallelBaseCheckpointFailed, - format!("Could not checkpoint base state before parallel branches: {e}"), - fabro_sandbox::default_redacted_output_tail(&e), - ); - None - } - } - } else { - None - }; - - let mut branch_setups: Vec = Vec::new(); - for (branch_index, edge) in branches.iter().enumerate() { - let target_id = edge.to.clone(); - let branch_context = context.fork(); - let parallel_branch_id = ParallelBranchId::new( - parallel_group_id.clone(), - u32::try_from(branch_index).unwrap_or(u32::MAX), - ); - branch_context.set( - keys::INTERNAL_PARALLEL_GROUP_ID, - serde_json::Value::String(parallel_group_id.to_string()), - ); - branch_context.set( - keys::INTERNAL_PARALLEL_BRANCH_ID, - serde_json::Value::String(parallel_branch_id.to_string()), - ); - - let (branch_sandbox, worktree_path): (Arc, Option) = if let ( - Some(ref gs), - Some(ref bsha), - ) = - (&git_state, &base_sha) - { - let branch_key = &target_id; - let visit = visit_from_context(&branch_context); - let branch_name = format!( - "fabro/run/parallel/{}/{}/pass{}/{}", - gs.run_id, - sanitize_ref_component(&node.id), - visit, - sanitize_ref_component(branch_key), - ); - - // Compute worktree path (each sandbox type knows its own path scheme) - let wt_path_str = services.run.sandbox.parallel_worktree_path( - run_dir, - &gs.run_id.to_string(), - &node.id, - branch_key, - ); - tracing::debug!(branch = %branch_name, path = %wt_path_str, "Creating worktree for parallel branch"); - - // Set up worktree via WorktreeSandbox - let wt_config = WorktreeOptions { - branch_name: branch_name.clone(), - base_sha: bsha.clone(), - worktree_path: wt_path_str.clone(), - skip_branch_creation: false, - setup_intent: None, - }; - let mut wt_sandbox = - WorktreeSandbox::new(Arc::clone(&services.run.sandbox), wt_config); - wt_sandbox - .set_event_callback(Arc::clone(&services.run.emitter).worktree_callback()); - wt_sandbox - .initialize() - .await - .map_err(|e| Error::handler_with_source("worktree setup failed", e))?; - - branch_context.set(keys::INTERNAL_WORK_DIR, serde_json::json!(&wt_path_str)); - - let wt_path = PathBuf::from(&wt_path_str); - let env: Arc = Arc::new(wt_sandbox); - (env, Some(wt_path)) - } else { - (Arc::clone(&services.run.sandbox), None) - }; - - branch_setups.push(BranchSetup { - target_id, - branch_index, - parallel_branch_id, - branch_context, - sandbox: branch_sandbox, - worktree_path, - }); - } - - // --- Fan out: concurrent execution --- - let mut handles = Vec::new(); - for setup in branch_setups { - let parent_run = Arc::clone(&services.run); - let registry = Arc::clone(&services.registry); - let interviewer = Arc::clone(&services.interviewer); - let base_env = services.base_env.clone(); - let github_token = services.github_token.clone(); - let inputs = services.inputs.clone(); - let dry_run = services.dry_run; - let workflow_path = services.workflow_path.clone(); - let workflow_bundle = services.workflow_bundle.clone(); - let graph = graph.clone(); - let run_dir = run_dir.to_path_buf(); - let sem = Arc::clone(&semaphore); - let has_git = git_state.is_some(); - let run_id = git_state.as_ref().map(|gs| gs.run_id); - let git_author = git_state - .as_ref() - .map(|gs| gs.git_author.clone()) - .unwrap_or_default(); - let checkpoint = git_state - .as_ref() - .map(|gs| gs.checkpoint.clone()) - .unwrap_or_default(); - let group_id = parallel_group_id.clone(); - let branch_scope = StageScope::for_parallel_branch( - setup.target_id.clone(), - 1, - group_id.clone(), - setup.parallel_branch_id.clone(), - ); - - let handle = tokio::spawn(async move { - let _permit = sem - .acquire() - .await - .map_err(|e| Error::handler_with_source("semaphore error", e))?; - - parent_run.emitter.emit_scoped( - &Event::ParallelBranchStarted { - parallel_group_id: group_id.clone(), - parallel_branch_id: setup.parallel_branch_id.clone(), - branch: setup.target_id.clone(), - index: setup.branch_index, - }, - &branch_scope, - ); + dispatches.push(BranchDispatch { + index: branch_index, + target_id: target_id.clone(), + branch_id: parallel_branch_id.clone(), + scope: branch_scope.clone(), + handle: tokio::spawn(async move { let branch_start = Instant::now(); - - let Some(target_node) = graph.nodes.get(&setup.target_id) else { - let outcome = Outcome::fail_classify(format!( - "branch target node not found: {}", - setup.target_id - )); - parent_run.emitter.emit_scoped( - &Event::ParallelBranchCompleted { + let task = async { + let permit = semaphore.acquire(); + tokio::pin!(permit); + let cancel_token = branch_services.run.cancel_token(); + let _permit = tokio::select! { + biased; + () = cancel_token.cancelled() => { + return Err(Error::Cancelled); + } + permit = &mut permit => permit + .map_err(|err| Error::handler_with_source("semaphore error", err))?, + }; + branch_services.run.emitter.emit_scoped( + &Event::ParallelBranchStarted { parallel_group_id: group_id.clone(), - parallel_branch_id: setup.parallel_branch_id.clone(), - branch: setup.target_id.clone(), - index: setup.branch_index, - duration_ms: millis_u64(branch_start.elapsed()), - status: "failed".to_string(), - head_sha: None, + parallel_branch_id: parallel_branch_id.clone(), + branch: target_id.clone(), + index: branch_index, }, &branch_scope, ); - return Ok(BranchResult { - id: setup.target_id.clone(), - outcome, - head_sha: None, - worktree_path: setup.worktree_path, - }); - }; - let branch_services = EngineServices { - run: parent_run.with_sandbox(Arc::clone(&setup.sandbox)), - registry: Arc::clone(®istry), - interviewer, - git_state: std::sync::RwLock::new(None), - base_env: base_env.clone(), - github_token: github_token.clone(), - inputs: inputs.clone(), - dry_run, - workflow_path, - workflow_bundle, - }; - let handler = registry.resolve(target_node); - let outcome = super::dispatch_handler( - handler, - target_node, - &setup.branch_context, - &graph, - &run_dir, - &branch_services, - ) - .await?; - - // Checkpoint commit after branch execution (capture head_sha) - let head_sha = if has_git { - let rid = - run_id.map_or_else(|| "unknown".to_string(), |run_id| run_id.to_string()); - let nid = &setup.target_id; - let status_str = outcome.status.to_string(); - // Use exec_command to commit and capture HEAD in the branch worktree - let git_r = GIT_REMOTE; - let add_cmd = format!("{git_r} add -A"); - let add_result = setup - .sandbox - .exec_command(&add_cmd, checkpoint.commit_timeout_ms, None, None, None) - .await; - if add_result - .as_ref() - .is_ok_and(fabro_sandbox::ExecResult::is_success) - { - let msg = format!("fabro({rid}): {nid} ({status_str})"); - let commit_cmd = parallel_branch_commit_cmd( - git_r, - &git_author.name, - &git_author.email, - &msg, - checkpoint.skip_git_hooks, - ); - let _ = setup - .sandbox - .exec_command( - &commit_cmd, - checkpoint.commit_timeout_ms, - None, - None, - None, + let outcome = match graph.nodes.get(&target_id) { + Some(target_node) => { + let handler = branch_services.registry.resolve(target_node); + match super::dispatch_handler( + handler, + target_node, + &branch_context, + &graph, + &run_dir, + &branch_services, ) - .await; - } - let sha_cmd = format!("{git_r} rev-parse HEAD"); - let sha_result = setup - .sandbox - .exec_command(&sha_cmd, 10_000, None, None, None) - .await; - match sha_result { - Ok(r) if r.is_success() => { - let sha = r.stdout.trim().to_string(); - parent_run.emitter.emit_scoped( - &Event::GitCommit { - node_id: Some(setup.target_id.clone()), - sha: sha.clone(), - }, - &branch_scope, - ); - Some(sha) + .await + { + Ok(outcome) => outcome, + Err(Error::Cancelled) => return Err(Error::Cancelled), + Err(err) => err.to_fail_outcome(), + } } - _ => None, - } - } else { - None + None => Outcome::fail_classify(format!( + "branch target node not found: {target_id}" + )), + }; + + let context_updates = branch_context_updates( + &parent_snapshot, + branch_context.snapshot(), + &outcome.context_updates, + ); + let result = ParallelBranchResult { + id: target_id.clone(), + status: outcome.status, + context_updates, + }; + branch_services.run.emitter.emit_scoped( + &Event::ParallelBranchCompleted { + parallel_group_id: group_id.clone(), + parallel_branch_id: parallel_branch_id.clone(), + branch: target_id.clone(), + index: branch_index, + duration_ms: millis_u64(branch_start.elapsed()), + status: result.status, + }, + &branch_scope, + ); + Ok::(BranchResult { result, outcome }) }; - parent_run.emitter.emit_scoped( - &Event::ParallelBranchCompleted { - parallel_group_id: group_id.clone(), - parallel_branch_id: setup.parallel_branch_id.clone(), - branch: setup.target_id.clone(), - index: setup.branch_index, - duration_ms: millis_u64(branch_start.elapsed()), - status: outcome.status.to_string(), - head_sha: head_sha.clone(), - }, - &branch_scope, - ); - - Ok::(BranchResult { - id: setup.target_id, - outcome, - head_sha, - worktree_path: setup.worktree_path, - }) - }); - handles.push(handle); - } - - // Collect results - let mut results: Vec = Vec::new(); - let mut handles = handles.into_iter(); - while let Some(handle) = handles.next() { - match handle.await { - Ok(Ok(result)) => { - results.push(result); - } - Ok(Err(Error::Cancelled)) => { - for handle in handles { - handle.abort(); - } - return Err(Error::Cancelled); - } - Ok(Err(e)) => { - results.push(BranchResult { - id: String::new(), - outcome: e.to_fail_outcome(), - head_sha: None, - worktree_path: None, - }); - } - Err(join_err) => { - results.push(BranchResult { - id: String::new(), - outcome: Outcome::fail_classify(format!( - "task join error: {join_err}" - )), - head_sha: None, - worktree_path: None, - }); - } - } - } - - // --- Git isolation: clean up worktrees, then ff-merge winner --- - if git_state.is_some() { - // Clean up worktrees first - for result in &results { - if let Some(ref wt_path) = result.worktree_path { - let wt_str = wt_path.to_string_lossy().into_owned(); - git_remove_worktree(&*services.run.sandbox, &wt_str).await; - services - .run - .emitter - .emit(&Event::GitWorktreeRemove { path: wt_str }); - } - } - - // Fast-forward main branch to first successful branch (lexically sorted). - // This must happen here — before the engine creates its own checkpoint commit - // on the main branch — so that subsequent commits are descendants of the - // winner. - let mut successful: Vec<_> = results - .iter() - .filter(|r| r.outcome.status == StageOutcome::Succeeded && r.head_sha.is_some()) - .collect(); - successful.sort_by(|a, b| a.id.cmp(&b.id)); - if let Some(winner) = successful.first() { - if let Some(sha) = winner.head_sha.as_ref() { - git_merge_ff_only(&*services.run.sandbox, sha).await; - } - } - } - - // Count successes and failures - let success_count = results - .iter() - .filter(|r| r.outcome.status == StageOutcome::Succeeded) - .count(); - let fail_count = results - .iter() - .filter(|r| r.outcome.status.is_failure()) - .count(); - let total = results.len(); - - // Store results as JSON in context for downstream fan-in - let results_json: Vec = results - .iter() - .map(|r| { - let mut entry = serde_json::json!({ - "id": r.id, - "status": r.outcome.status.to_string(), - }); - if let Some(ref sha) = r.head_sha { - entry["head_sha"] = serde_json::json!(sha); - } - entry - }) - .collect(); - context.set(keys::PARALLEL_RESULTS, serde_json::json!(results_json)); - context.set(keys::PARALLEL_BRANCH_COUNT, serde_json::json!(total)); - - services.run.emitter.emit_scoped( - &Event::ParallelCompleted { - node_id: node.id.clone(), - visit: parallel_stage_scope.visit, - duration_ms: millis_u64(parallel_start.elapsed()), - success_count, - failure_count: fail_count, - results: results_json.clone(), - }, - ¶llel_stage_scope, - ); - { - let run_id = context - .run_id() - .parse::() - .map_err(|err| Error::handler_with_source("invalid internal run_id", err))?; - let mut hook_ctx = - HookContext::new(HookEvent::ParallelComplete, run_id, graph.name.clone()); - set_hook_node(&mut hook_ctx, node); - let _ = services.run.run_hooks(&hook_ctx).await; - } - - // Evaluate join policy - let status = match join_policy { - JoinPolicy::WaitAll => { - if fail_count == 0 { - StageOutcome::Succeeded - } else { - StageOutcome::PartiallySucceeded - } - } - JoinPolicy::FirstSuccess => { - if success_count > 0 { - StageOutcome::Succeeded - } else { - StageOutcome::Failed { - retry_requested: false, + match std::panic::AssertUnwindSafe(task).catch_unwind().await { + Ok(result) => result, + Err(payload) => { + let result = + failed_branch_result(&target_id, super::format_panic_message(&payload)); + branch_services.run.emitter.emit_scoped( + &Event::ParallelBranchCompleted { + parallel_group_id: group_id, + parallel_branch_id, + branch: target_id, + index: branch_index, + duration_ms: millis_u64(branch_start.elapsed()), + status: result.result.status, + }, + &branch_scope, + ); + Ok(result) } } + }), + }); + } + + // Awaiting in dispatch order keeps `results` aligned with the node's + // outgoing-edge order regardless of branch completion order. + let mut results = Vec::with_capacity(dispatches.len()); + let mut cancelled = false; + for dispatch in dispatches { + let (result, emit_completion) = match dispatch.handle.await { + Ok(Ok(result)) => (result, false), + Ok(Err(Error::Cancelled)) => { + cancelled = true; + ( + failed_branch_result(&dispatch.target_id, "branch cancelled"), + true, + ) } + Ok(Err(err)) => ( + failed_branch_result(&dispatch.target_id, err.to_string()), + true, + ), + Err(join_err) => ( + failed_branch_result(&dispatch.target_id, format!("task join error: {join_err}")), + true, + ), }; - - // Find the join/convergence node: follow each branch's outgoing edges - // and find the common downstream target (typically the fan-in node). - let join_node = find_join_node(&results, graph); - - let is_fail = status.is_failure(); - let mut outcome = Outcome { - status, - notes: Some(format!( - "Parallel node dispatched {total} branches ({success_count} succeeded, {fail_count} failed)" - )), - failure: if is_fail { - Some(FailureDetail::new( - format!("Join policy not satisfied: {success_count}/{total} succeeded"), - FailureCategory::Deterministic, - )) - } else { - None - }, - jump_to_node: if is_fail { None } else { join_node }, - ..Outcome::success() - }; - - if is_fail { - outcome.suggested_next_ids.clear(); + if emit_completion { + services.run.emitter.emit_scoped( + &Event::ParallelBranchCompleted { + parallel_group_id: parallel_group_id.clone(), + parallel_branch_id: dispatch.branch_id, + branch: dispatch.target_id, + index: dispatch.index, + duration_ms: 0, + status: result.result.status, + }, + &dispatch.scope, + ); } - - Ok(outcome) + if result.outcome.failure_category() == Some(FailureCategory::Canceled) { + cancelled = true; + } + results.push(result); } -} - -/// Find the convergence (join/fan-in) node by following each branch's outgoing -/// edges and finding the first node reachable from all branches. -fn find_join_node(results: &[BranchResult], graph: &Graph) -> Option { - if results.is_empty() { - return None; + if cancelled { + return Err(Error::Cancelled); } - // Collect outgoing targets for each branch - let mut target_sets: Vec> = Vec::new(); - for result in results { - let targets: std::collections::HashSet = graph - .outgoing_edges(&result.id) - .into_iter() - .map(|e| e.to.clone()) - .collect(); - target_sets.push(targets); - } - - // Find the intersection — nodes reachable from ALL branches - let first = target_sets.first()?; - let common: std::collections::HashSet<&String> = first + let success_count = results .iter() - .filter(|id| target_sets.iter().all(|set| set.contains(*id))) - .collect(); + .filter(|branch| branch.outcome.status == StageOutcome::Succeeded) + .count(); + let failure_count = results + .iter() + .filter(|branch| branch.outcome.status.is_failure()) + .count(); + let total = results.len(); + let status = aggregate_status(&results); + let is_failure = status.is_failure(); + let jump_to_node = if is_failure { + None + } else { + find_join_node(&results, graph) + }; - // Return the first common target (lexically sorted for determinism) - let mut common_sorted: Vec<&String> = common.into_iter().collect(); - common_sorted.sort(); - common_sorted.first().map(|id| (*id).clone()) + let mut typed_results = results + .into_iter() + .map(|branch| branch.result) + .collect::>(); + // Offload large leaves before the results reach the event log and + // projection: the artifact lifecycle's offload pass runs only after the + // handler returns, too late for the `parallel.completed` payload. + if let Err(err) = + artifact::offload_parallel_branch_updates(&mut typed_results, &services.run.run_store).await + { + services.run.emitter.notice( + RunNoticeLevel::Warn, + RunNoticeCode::ArtifactOffloadFailed, + format!("[node: {}] parallel result offload failed: {err}", node.id), + ); + } + let results_value = serde_json::to_value(&typed_results) + .map_err(|err| Error::handler_with_source("parallel result serialization failed", err))?; + let context_updates = HashMap::from([ + (keys::PARALLEL_RESULTS.to_string(), results_value), + ( + keys::PARALLEL_BRANCH_COUNT.to_string(), + serde_json::json!(total), + ), + ]); + + services.run.emitter.emit_scoped( + &Event::ParallelCompleted { + node_id: node.id.clone(), + visit: parallel_stage_scope.visit, + duration_ms: millis_u64(parallel_start.elapsed()), + success_count, + failure_count, + results: typed_results, + }, + ¶llel_stage_scope, + ); + emit_parallel_hook(services, context, graph, node, HookEvent::ParallelComplete).await?; + + let prefix = if simulated { "[Simulated] " } else { "" }; + let mut outcome = Outcome { + status, + notes: Some(format!( + "{prefix}Parallel node dispatched {total} branches ({success_count} succeeded, {failure_count} failed)" + )), + failure: is_failure.then(|| { + FailureDetail::new( + "All parallel branches failed", + FailureCategory::Deterministic, + ) + }), + jump_to_node, + context_updates, + ..Outcome::success() + }; + if is_failure { + outcome.suggested_next_ids.clear(); + } + Ok(outcome) } -/// Build the parallel-branch checkpoint commit command. Appends -/// `--no-verify` when `skip_git_hooks` is true so the commit bypasses the -/// repository's local Git commit hooks (e.g. `pre-commit`, `commit-msg`). -fn parallel_branch_commit_cmd( - git_remote: &str, - author_name: &str, - author_email: &str, - message: &str, - skip_git_hooks: bool, -) -> String { - let no_verify = if skip_git_hooks { " --no-verify" } else { "" }; - let name = fabro_sandbox::shell_quote(&format!("user.name={author_name}")); - let email = fabro_sandbox::shell_quote(&format!("user.email={author_email}")); - let msg = fabro_sandbox::shell_quote(message); - format!("{git_remote} -c {name} -c {email} commit --allow-empty{no_verify} -m {msg}") +async fn emit_parallel_hook( + services: &EngineServices, + context: &Context, + graph: &Graph, + node: &Node, + hook_event: HookEvent, +) -> Result<(), Error> { + let run_id = context.parsed_run_id()?; + let mut hook_context = HookContext::new(hook_event, run_id, graph.name.clone()); + set_hook_node(&mut hook_context, node); + let _ = services.run.run_hooks(&hook_context).await; + Ok(()) +} + +fn branch_context_updates( + before: &HashMap, + after: HashMap, + outcome_updates: &HashMap, +) -> BTreeMap { + let mut updates = outcome_updates + .iter() + .map(|(key, value)| (key.clone(), value.clone())) + .collect::>(); + updates.extend( + context_diff(before, after) + .into_iter() + .filter(|(key, _)| !keys::is_engine_internal_key(key)), + ); + updates +} + +fn failed_branch_result(id: &str, reason: impl Into) -> BranchResult { + let outcome = Outcome::fail_classify(reason); + BranchResult { + result: ParallelBranchResult { + id: id.to_string(), + status: outcome.status, + context_updates: BTreeMap::new(), + }, + outcome, + } +} + +fn aggregate_status(results: &[BranchResult]) -> StageOutcome { + if results.is_empty() { + StageOutcome::PartiallySucceeded + } else if results + .iter() + .all(|result| result.outcome.status == StageOutcome::Succeeded) + { + StageOutcome::Succeeded + } else if results + .iter() + .all(|result| result.outcome.status.is_failure()) + { + StageOutcome::Failed { + retry_requested: false, + } + } else { + StageOutcome::PartiallySucceeded + } +} + +/// Find the convergence node by finding a common direct target of every branch. +fn find_join_node(results: &[BranchResult], graph: &Graph) -> Option { + let first_result = results.first()?; + let first_targets = graph + .outgoing_edges(&first_result.result.id) + .into_iter() + .map(|edge| edge.to.clone()) + .collect::>(); + let mut common = first_targets + .into_iter() + .filter(|target| { + results.iter().skip(1).all(|result| { + graph + .outgoing_edges(&result.result.id) + .into_iter() + .any(|edge| &edge.to == target) + }) + }) + .collect::>(); + common.sort(); + common.into_iter().next() } #[cfg(test)] @@ -728,7 +470,7 @@ mod tests { graph: serde_json::to_value(fabro_types::Graph::new("test")).unwrap(), workflow_source: None, workflow_config: None, - labels: std::collections::BTreeMap::default(), + labels: BTreeMap::default(), run_dir: "/tmp".to_string(), source_directory: None, workflow_slug: None, @@ -750,240 +492,194 @@ mod tests { fn test_context() -> Context { let context = Context::new(); context.set( - crate::context::keys::INTERNAL_RUN_ID, + keys::INTERNAL_RUN_ID, serde_json::json!(fixtures::RUN_1.to_string()), ); context } + fn parallel_graph() -> (Node, Graph) { + let mut node = Node::new("par"); + node.attrs.insert( + "shape".to_string(), + AttrValue::String("component".to_string()), + ); + let mut graph = Graph::new("test"); + graph.nodes.insert("par".to_string(), node.clone()); + graph + .nodes + .insert("branch_a".to_string(), Node::new("branch_a")); + graph + .nodes + .insert("branch_b".to_string(), Node::new("branch_b")); + graph.edges.push(Edge::new("par", "branch_a")); + graph.edges.push(Edge::new("par", "branch_b")); + (node, graph) + } + #[tokio::test] async fn parallel_handler_no_branches() { - let services = make_services(); - let node = Node::new("par"); - let context = test_context(); - let graph = Graph::new("test"); - let run_dir = Path::new("/tmp/test"); - let outcome = ParallelHandler - .execute(&node, &context, &graph, run_dir, &services) + .execute( + &Node::new("par"), + &test_context(), + &Graph::new("test"), + Path::new("/tmp/test"), + &make_services(), + ) .await .unwrap(); - assert_eq!(outcome.status, StageOutcome::Failed { - retry_requested: false, - }); + assert_eq!(outcome.status, StageOutcome::PartiallySucceeded); + assert_eq!( + outcome.context_updates[keys::PARALLEL_RESULTS], + serde_json::json!([]) + ); + assert_eq!( + outcome.context_updates[keys::PARALLEL_BRANCH_COUNT], + serde_json::json!(0) + ); } #[tokio::test] - async fn parallel_handler_with_branches() { + async fn parallel_handler_returns_typed_ordered_results() { let store = test_store(); let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); seed_created(&run_store).await; - let mut services = EngineServices::test_default(); + let mut services = make_services(); services.run = services .run .with_emitter(Arc::new(crate::event::Emitter::new(fixtures::RUN_1))) .with_run_store(run_store.clone().into()); let logger = crate::event::StoreProgressLogger::new(run_store.clone()); logger.register(services.run.emitter.as_ref()); - let mut node = Node::new("par"); - node.attrs.insert( - "shape".to_string(), - AttrValue::String("component".to_string()), - ); + let (node, graph) = parallel_graph(); let context = test_context(); - let mut graph = Graph::new("test"); - graph.nodes.insert("par".to_string(), node.clone()); - graph - .nodes - .insert("branch_a".to_string(), Node::new("branch_a")); - graph - .nodes - .insert("branch_b".to_string(), Node::new("branch_b")); - graph.edges.push(Edge::new("par", "branch_a")); - graph.edges.push(Edge::new("par", "branch_b")); - let tmp = tempfile::tempdir().unwrap(); let outcome = ParallelHandler - .execute(&node, &context, &graph, tmp.path(), &services) + .execute(&node, &context, &graph, Path::new("/tmp/test"), &services) .await .unwrap(); logger.flush().await; assert_eq!(outcome.status, StageOutcome::Succeeded); - assert!(outcome.notes.as_deref().unwrap().contains("2 branches")); - - // Check context was set - let results = context.get(keys::PARALLEL_RESULTS); - assert!(results.is_some()); - - let state = run_store.state().await.unwrap(); - let node_state = state.stage(&StageId::new("par", 1)).unwrap(); - let parsed = node_state.parallel_results.as_ref().unwrap(); + let results: Vec = + serde_json::from_value(outcome.context_updates[keys::PARALLEL_RESULTS].clone()) + .unwrap(); + assert_eq!( + results + .iter() + .map(|result| result.id.as_str()) + .collect::>(), + ["branch_a", "branch_b"] + ); assert!( - parsed.is_array(), - "parallel_results.json should be a JSON array" + results + .iter() + .all(|result| result.status == StageOutcome::Succeeded) ); - assert_eq!(parsed.as_array().unwrap().len(), 2); - } - - #[tokio::test] - async fn parallel_handler_stores_results_in_run_store() { - let store = test_store(); - let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); - seed_created(&run_store).await; - let mut services = EngineServices::test_default(); - services.run = services - .run - .with_emitter(Arc::new(crate::event::Emitter::new(fixtures::RUN_1))) - .with_run_store(run_store.clone().into()); - let logger = crate::event::StoreProgressLogger::new(run_store.clone()); - logger.register(services.run.emitter.as_ref()); - let mut node = Node::new("par"); - node.attrs.insert( - "shape".to_string(), - AttrValue::String("component".to_string()), - ); - let context = test_context(); - let mut graph = Graph::new("test"); - graph.nodes.insert("par".to_string(), node.clone()); - graph - .nodes - .insert("branch_a".to_string(), Node::new("branch_a")); - graph - .nodes - .insert("branch_b".to_string(), Node::new("branch_b")); - graph.edges.push(Edge::new("par", "branch_a")); - graph.edges.push(Edge::new("par", "branch_b")); - - let tmp = tempfile::tempdir().unwrap(); - ParallelHandler - .execute(&node, &context, &graph, tmp.path(), &services) - .await - .unwrap(); - logger.flush().await; - let state = run_store.state().await.unwrap(); - let node_state = state.stage(&fabro_store::StageId::new("par", 1)).unwrap(); - let results = node_state.parallel_results.as_ref().unwrap(); - assert!(results.is_array()); - assert_eq!(results.as_array().unwrap().len(), 2); + assert_eq!( + state + .stage(&StageId::new("par", 1)) + .unwrap() + .parallel_results + .as_ref() + .unwrap() + .len(), + 2 + ); } #[tokio::test] - async fn parallel_handler_first_success_policy() { - let services = make_services(); - let mut node = Node::new("par"); - node.attrs.insert( - "join_policy".to_string(), - AttrValue::String("first_success".to_string()), - ); + async fn parallel_handler_simulate_returns_results_as_outcome_updates() { + let (node, graph) = parallel_graph(); let context = test_context(); - let mut graph = Graph::new("test"); - graph.nodes.insert("par".to_string(), node.clone()); - graph - .nodes - .insert("branch_a".to_string(), Node::new("branch_a")); - graph.edges.push(Edge::new("par", "branch_a")); - - let run_dir = Path::new("/tmp/test"); let outcome = ParallelHandler - .execute(&node, &context, &graph, run_dir, &services) - .await - .unwrap(); - - assert_eq!(outcome.status, StageOutcome::Succeeded); - } - - #[test] - fn join_policy_display() { - assert_eq!(JoinPolicy::WaitAll.to_string(), "wait_all"); - assert_eq!(JoinPolicy::FirstSuccess.to_string(), "first_success"); - } - - #[test] - fn parse_join_policy_variants() { - assert!(matches!(parse_join_policy("wait_all"), JoinPolicy::WaitAll)); - assert!(matches!( - parse_join_policy("first_success"), - JoinPolicy::FirstSuccess - )); - // Invalid falls back to WaitAll - assert!(matches!(parse_join_policy("invalid"), JoinPolicy::WaitAll)); - } - - #[tokio::test] - async fn parallel_handler_simulate() { - let services = make_services(); - let mut node = Node::new("par"); - node.attrs.insert( - "shape".to_string(), - AttrValue::String("component".to_string()), - ); - let context = test_context(); - let mut graph = Graph::new("test"); - graph.nodes.insert("par".to_string(), node.clone()); - graph - .nodes - .insert("branch_a".to_string(), Node::new("branch_a")); - graph - .nodes - .insert("branch_b".to_string(), Node::new("branch_b")); - // Add a fan_in node reachable from both branches - graph - .nodes - .insert("fan_in".to_string(), Node::new("fan_in")); - graph.edges.push(Edge::new("par", "branch_a")); - graph.edges.push(Edge::new("par", "branch_b")); - graph.edges.push(Edge::new("branch_a", "fan_in")); - graph.edges.push(Edge::new("branch_b", "fan_in")); - - let run_dir = Path::new("/tmp/test"); - let mut dry_services = services; - dry_services.dry_run = true; - - let outcome = ParallelHandler - .simulate(&node, &context, &graph, run_dir, &dry_services) + .simulate( + &node, + &context, + &graph, + Path::new("/tmp/test"), + &make_services(), + ) .await .unwrap(); assert_eq!(outcome.status, StageOutcome::Succeeded); assert!(outcome.notes.as_deref().unwrap().contains("[Simulated]")); - assert!(outcome.notes.as_deref().unwrap().contains("2 branches")); - assert_eq!(outcome.jump_to_node, Some("fan_in".to_string())); - - let branch_count = context.get(keys::PARALLEL_BRANCH_COUNT); - assert_eq!(branch_count, Some(serde_json::json!(2))); + assert_eq!( + outcome.context_updates[keys::PARALLEL_BRANCH_COUNT], + serde_json::json!(2) + ); } #[test] - fn parallel_branch_commit_cmd_includes_no_verify_when_skip_hooks_enabled() { - let cmd = super::parallel_branch_commit_cmd( - super::GIT_REMOTE, - "Fabro", - "fabro@example.com", - "fabro(r1): branch_a (succeeded)", - true, + fn aggregate_status_follows_parallel_truth_table() { + let success = |index: usize| BranchResult { + result: ParallelBranchResult { + id: format!("branch_{index}"), + status: StageOutcome::Succeeded, + context_updates: BTreeMap::new(), + }, + outcome: Outcome::success(), + }; + let failure = |index: usize| failed_branch_result(&format!("branch_{index}"), "failed"); + let partial = |index: usize| BranchResult { + result: ParallelBranchResult { + id: format!("branch_{index}"), + status: StageOutcome::PartiallySucceeded, + context_updates: BTreeMap::new(), + }, + outcome: Outcome { + status: StageOutcome::PartiallySucceeded, + ..Outcome::success() + }, + }; + + assert_eq!(aggregate_status(&[]), StageOutcome::PartiallySucceeded); + assert_eq!( + aggregate_status(&[success(0), success(1)]), + StageOutcome::Succeeded ); - assert!( - cmd.contains("--no-verify"), - "expected --no-verify when skip_git_hooks=true; got {cmd:?}" + assert!(aggregate_status(&[failure(0), failure(1)]).is_failure()); + assert_eq!( + aggregate_status(&[success(0), failure(1)]), + StageOutcome::PartiallySucceeded + ); + assert_eq!( + aggregate_status(&[success(0), partial(1)]), + StageOutcome::PartiallySucceeded + ); + assert_eq!( + aggregate_status(&[failure(0), partial(1)]), + StageOutcome::PartiallySucceeded ); - assert!(cmd.contains("commit --allow-empty")); } #[test] - fn parallel_branch_commit_cmd_omits_no_verify_when_skip_hooks_disabled() { - let cmd = super::parallel_branch_commit_cmd( - super::GIT_REMOTE, - "Fabro", - "fabro@example.com", - "fabro(r1): branch_a (succeeded)", - false, + fn branch_context_updates_include_failed_outcome_updates_without_internal_keys() { + let before = HashMap::from([("shared".to_string(), serde_json::json!("parent"))]); + let after = HashMap::from([ + ("shared".to_string(), serde_json::json!("branch")), + ( + keys::INTERNAL_WORK_DIR.to_string(), + serde_json::json!("/workspace"), + ), + ]); + let outcome = HashMap::from([( + keys::COMMAND_OUTPUT.to_string(), + serde_json::json!({"stdout": "failure output"}), + )]); + + assert_eq!( + branch_context_updates(&before, after, &outcome), + BTreeMap::from([ + ( + keys::COMMAND_OUTPUT.to_string(), + serde_json::json!({"stdout": "failure output"}) + ), + ("shared".to_string(), serde_json::json!("branch")), + ]) ); - assert!( - !cmd.contains("--no-verify"), - "expected no --no-verify when skip_git_hooks=false; got {cmd:?}" - ); - assert!(cmd.contains("commit --allow-empty")); } } diff --git a/lib/crates/fabro-workflow/src/pipeline/execute.rs b/lib/crates/fabro-workflow/src/pipeline/execute.rs index 0be068e88..cf02a736d 100644 --- a/lib/crates/fabro-workflow/src/pipeline/execute.rs +++ b/lib/crates/fabro-workflow/src/pipeline/execute.rs @@ -17,7 +17,6 @@ use crate::lifecycle::WorkflowLifecycle; use crate::node_handler::WorkflowNodeHandler; use crate::outcome::Outcome; use crate::records::Checkpoint; -use crate::sandbox_git::GitState; fn seed_context_from_checkpoint(checkpoint: Option<&Checkpoint>) -> Context { let context = Context::new(); @@ -55,19 +54,6 @@ pub async fn execute(init: Initialized) -> Executed { let graph_arc = Arc::new(graph.clone()); let wf_graph = WorkflowGraph(Arc::clone(&graph_arc)); - let git_state = run_options.git.as_ref().and_then(|git| { - let base_sha = git.base_sha.clone()?; - Some(Arc::new(GitState { - run_id: run_options.run_id, - base_sha, - run_branch: git.run_branch.clone(), - meta_branch: git.meta_branch.clone(), - checkpoint: run_options.checkpoint().clone(), - git_author: run_options.git_author(), - })) - }); - engine.set_git_state(git_state); - let handler = Arc::new(WorkflowNodeHandler { services: Arc::clone(&engine), run_dir: run_options.run_dir.clone(), diff --git a/lib/crates/fabro-workflow/src/pipeline/initialize.rs b/lib/crates/fabro-workflow/src/pipeline/initialize.rs index 54a6f1f45..8f1e56ee2 100644 --- a/lib/crates/fabro-workflow/src/pipeline/initialize.rs +++ b/lib/crates/fabro-workflow/src/pipeline/initialize.rs @@ -611,7 +611,6 @@ pub async fn initialize( run: Arc::clone(&run_services), registry, interviewer: Arc::clone(&options.interviewer), - git_state: std::sync::RwLock::new(None), base_env, github_token, inputs: options.run_options.settings.run.inputs.clone(), diff --git a/lib/crates/fabro-workflow/src/sandbox_git.rs b/lib/crates/fabro-workflow/src/sandbox_git.rs index 68b7e1bed..914153955 100644 --- a/lib/crates/fabro-workflow/src/sandbox_git.rs +++ b/lib/crates/fabro-workflow/src/sandbox_git.rs @@ -4,7 +4,6 @@ use fabro_agent::Sandbox; use fabro_checkpoint::trailer as trailerlink; use fabro_checkpoint::trailer::Trailer; use fabro_sandbox::shell_quote; -use fabro_types::RunId; use fabro_types::settings::run::RunCheckpointSettings; use fabro_util::error::SharedError; @@ -20,17 +19,6 @@ pub struct GitCommandError { pub source: fabro_sandbox::Error, } -/// Captured git state for a workflow run, shared with handlers. -#[derive(Debug, Clone)] -pub struct GitState { - pub run_id: RunId, - pub base_sha: String, - pub run_branch: Option, - pub meta_branch: Option, - pub checkpoint: RunCheckpointSettings, - pub git_author: GitAuthor, -} - pub const GIT_REMOTE: &str = "git -c maintenance.auto=0 -c gc.auto=0 -c commit.gpgsign=false -c tag.gpgsign=false"; @@ -237,48 +225,6 @@ pub(crate) async fn git_diff_with_timeout( } } -/// Create a branch at a specific SHA via the sandbox. -pub async fn git_create_branch_at(sandbox: &dyn Sandbox, name: &str, sha: &str) -> bool { - let cmd = format!("{GIT_REMOTE} branch --force {name} {sha}"); - matches!( - sandbox.exec_command(&cmd, 30_000, None, None, None).await, - Ok(r) if r.is_success() - ) -} - -/// Add a git worktree via the sandbox. -pub async fn git_add_worktree(sandbox: &dyn Sandbox, path: &str, branch: &str) -> bool { - let cmd = format!("{GIT_REMOTE} worktree add {path} {branch}"); - matches!( - sandbox.exec_command(&cmd, 30_000, None, None, None).await, - Ok(r) if r.is_success() - ) -} - -/// Remove a git worktree via the sandbox. -pub async fn git_remove_worktree(sandbox: &dyn Sandbox, path: &str) -> bool { - let cmd = format!("{GIT_REMOTE} worktree remove --force {path}"); - matches!( - sandbox.exec_command(&cmd, 30_000, None, None, None).await, - Ok(r) if r.is_success() - ) -} - -/// Fast-forward merge to a given SHA via the sandbox. -pub async fn git_merge_ff_only(sandbox: &dyn Sandbox, sha: &str) -> bool { - let cmd = format!("{GIT_REMOTE} merge --ff-only {sha}"); - matches!( - sandbox.exec_command(&cmd, 30_000, None, None, None).await, - Ok(r) if r.is_success() - ) -} - -/// Remove any stale worktree at `path` (best-effort), then add a fresh one. -pub async fn git_replace_worktree(sandbox: &dyn Sandbox, path: &str, branch: &str) -> bool { - let _ = git_remove_worktree(sandbox, path).await; - git_add_worktree(sandbox, path, branch).await -} - // ── Machine-readable diff enumeration (Run Files endpoint) ───────────────── /// Hardened git-command prefix for the Run Files endpoint. diff --git a/lib/crates/fabro-workflow/src/services.rs b/lib/crates/fabro-workflow/src/services.rs index 44e0eca21..506206f45 100644 --- a/lib/crates/fabro-workflow/src/services.rs +++ b/lib/crates/fabro-workflow/src/services.rs @@ -20,7 +20,6 @@ use crate::handler::HandlerRegistry; use crate::interview_runtime::RunInterviewBlocker; use crate::run_metadata::{RunMetadataRuntime, RunMetadataWriterHandle}; use crate::runtime_store::RunStoreHandle; -use crate::sandbox_git::GitState; use crate::sandbox_git_runtime::SandboxGitRuntime; use crate::workflow_bundle::WorkflowBundle; @@ -225,26 +224,24 @@ impl RunServices { } /// Services available only while executing workflow nodes. +#[derive(Clone)] pub struct EngineServices { - pub run: Arc, - pub registry: Arc, - pub interviewer: Arc, - /// Git state for the current run. Set via `set_git_state` at the start of - /// `execute` and read by parallel/fan-in handlers. - pub(crate) git_state: std::sync::RwLock>>, + pub run: Arc, + pub registry: Arc, + pub interviewer: Arc, /// Environment variables from `[sandbox.env]` config. - pub base_env: HashMap, + pub base_env: HashMap, /// GitHub token source used to inject `GITHUB_TOKEN` at the point of use. - pub github_token: Option>, + pub github_token: Option>, /// Typed values from `[run.inputs]`, available to prompt templates. - pub inputs: HashMap, + pub inputs: HashMap, /// When true, handlers should skip real execution and return simulated /// results. - pub dry_run: bool, + pub dry_run: bool, /// Manifest path of the current workflow when running from a bundle. - pub workflow_path: Option, + pub workflow_path: Option, /// Bundled workflows available for child-workflow resolution. - pub workflow_bundle: Option>, + pub workflow_bundle: Option>, } impl EngineServices { @@ -252,23 +249,6 @@ impl EngineServices { resolve_workflow_env(&self.base_env, self.github_token.as_ref()).await } - /// Read the current git state (if any). - pub fn git_state(&self) -> Option> { - self.git_state - .read() - .expect("git_state lock is never poisoned: no code panics while holding this lock") - .clone() - } - - /// Set the git state for the current run. - pub fn set_git_state(&self, state: Option>) { - *self - .git_state - .write() - .expect("git_state lock is never poisoned: no code panics while holding this lock") = - state; - } - /// Test-only default: empty registry and cross-phase services. #[cfg(test)] #[expect( @@ -343,7 +323,6 @@ impl EngineServices { ), registry: Arc::new(HandlerRegistry::new(Box::new(start::StartHandler))), interviewer: Arc::new(fabro_interview::AutoApproveInterviewer::engine()), - git_state: std::sync::RwLock::new(None), base_env: HashMap::new(), github_token: None, inputs: HashMap::new(), diff --git a/lib/crates/fabro-workflow/src/stage_scope.rs b/lib/crates/fabro-workflow/src/stage_scope.rs index fa309044b..396e84156 100644 --- a/lib/crates/fabro-workflow/src/stage_scope.rs +++ b/lib/crates/fabro-workflow/src/stage_scope.rs @@ -37,8 +37,7 @@ impl StageScope { } /// Build scope for the branch-lifecycle events emitted by the parallel - /// handler (`ParallelBranchStarted`, `ParallelBranchCompleted`, and the - /// pre-dispatch `GitCommit` for the branch worktree). + /// handler (`ParallelBranchStarted` and `ParallelBranchCompleted`). /// /// `target_visit` is the visit count of `target_node_id` for this /// particular branch dispatch. The parallel handler currently passes diff --git a/lib/crates/fabro-workflow/src/test_support.rs b/lib/crates/fabro-workflow/src/test_support.rs index 1acf410a4..35c341a83 100644 --- a/lib/crates/fabro-workflow/src/test_support.rs +++ b/lib/crates/fabro-workflow/src/test_support.rs @@ -241,7 +241,6 @@ async fn initialized( ), registry: Arc::new(registry), interviewer: Arc::new(AutoApproveInterviewer::engine()), - git_state: std::sync::RwLock::new(None), base_env: options.env, github_token: None, inputs: run_options.settings.run.inputs.clone(), diff --git a/lib/crates/fabro-workflow/tests/it/daytona_integration.rs b/lib/crates/fabro-workflow/tests/it/daytona_integration.rs index 692911491..ba72bd221 100644 --- a/lib/crates/fabro-workflow/tests/it/daytona_integration.rs +++ b/lib/crates/fabro-workflow/tests/it/daytona_integration.rs @@ -35,7 +35,7 @@ use fabro_workflow::event::Emitter; use fabro_workflow::handler::exit::ExitHandler; use fabro_workflow::handler::start::StartHandler; use fabro_workflow::handler::{Handler, HandlerRegistry}; -use fabro_workflow::outcome::{Outcome, OutcomeExt, StageOutcome}; +use fabro_workflow::outcome::{Outcome, StageOutcome}; use fabro_workflow::records::Checkpoint; use fabro_workflow::run_options::{GitCheckpointOptions, RunOptions}; use fabro_workflow::test_support::{WorkflowRunner, test_store_dir}; @@ -767,234 +767,6 @@ async fn daytona_git_checkpoint_remote_emits_events() { env.cleanup().await.unwrap(); } -// --------------------------------------------------------------------------- -// Parallel git branching on Daytona -// --------------------------------------------------------------------------- - -use fabro_workflow::handler::fan_in::FanInHandler; -use fabro_workflow::handler::parallel::ParallelHandler; - -/// End-to-end: parallel branches get isolated worktrees in Daytona sandbox, -/// fan-in fast-forwards to winner. -#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))] -async fn daytona_parallel_git_branching_e2e() { - let env = create_env().await; - env.initialize().await.unwrap(); - let env: Arc = Arc::new(env); - - // Install git if not available - let git_check = env - .exec_command("git --version", 10_000, None, None, None) - .await; - if git_check.as_ref().map_or(true, |r| !r.is_success()) { - let install = env - .exec_command( - "apt-get update -qq && apt-get install -y -qq git >/dev/null 2>&1", - 120_000, - None, - None, - None, - ) - .await - .expect("apt-get install git should not error"); - assert_eq!( - install.exit_code, - Some(0), - "git install failed: {}", - install.stderr - ); - } - - // Set up git in the sandbox (uses existing repo from Daytona project clone) - let (run_id, base_sha, branch_name) = setup_daytona_git(&*env).await; - - // Pipeline: start -> fan_out -> {branch_a, branch_b} -> fan_in -> exit - let mut graph = Graph::new("DaytonaParallelGitBranching"); - graph.attrs.insert( - "goal".to_string(), - AttrValue::String("Test parallel git branching on Daytona".to_string()), - ); - - let mut start = Node::new("start"); - start.attrs.insert( - "shape".to_string(), - AttrValue::String("Mdiamond".to_string()), - ); - graph.nodes.insert("start".to_string(), start); - - let mut fan_out = Node::new("fan_out"); - fan_out.attrs.insert( - "shape".to_string(), - AttrValue::String("component".to_string()), - ); - graph.nodes.insert("fan_out".to_string(), fan_out); - - let branch_a = Node::new("branch_a"); - graph.nodes.insert("branch_a".to_string(), branch_a); - - let branch_b = Node::new("branch_b"); - graph.nodes.insert("branch_b".to_string(), branch_b); - - let mut fan_in = Node::new("fan_in"); - fan_in.attrs.insert( - "shape".to_string(), - AttrValue::String("tripleoctagon".to_string()), - ); - graph.nodes.insert("fan_in".to_string(), fan_in); - - let mut exit_node = Node::new("exit"); - exit_node.attrs.insert( - "shape".to_string(), - AttrValue::String("Msquare".to_string()), - ); - graph.nodes.insert("exit".to_string(), exit_node); - - graph.edges.push(Edge::new("start", "fan_out")); - graph.edges.push(Edge::new("fan_out", "branch_a")); - graph.edges.push(Edge::new("fan_out", "branch_b")); - graph.edges.push(Edge::new("branch_a", "fan_in")); - graph.edges.push(Edge::new("branch_b", "fan_in")); - graph.edges.push(Edge::new("fan_in", "exit")); - - let run_tmp = tempfile::tempdir().unwrap(); - let emitter = Emitter::default(); - let events = Arc::new(std::sync::Mutex::new(Vec::new())); - { - let events_clone = Arc::clone(&events); - emitter.on_event(move |event| { - events_clone.lock().unwrap().push(event.clone()); - }); - } - - let mut registry = HandlerRegistry::new(Box::new(FileWriterHandler)); - registry.register("start", Box::new(StartHandler)); - registry.register("exit", Box::new(ExitHandler)); - registry.register("parallel", Box::new(ParallelHandler)); - registry.register("parallel.fan_in", Box::new(FanInHandler::new(None))); - - let engine = WorkflowRunner::new(registry, Arc::new(emitter), Arc::clone(&env)); - - let run_options = RunOptions { - settings: WorkflowSettings::default(), - run_dir: run_tmp.path().to_path_buf(), - cancel_token: CancellationToken::new(), - run_id, - labels: std::collections::HashMap::new(), - workflow_slug: None, - github_app: None, - base_branch: None, - display_base_sha: None, - pre_run_git: None, - fork_source_ref: None, - git: Some(GitCheckpointOptions { - base_sha: Some(base_sha), - run_branch: Some(branch_name), - meta_branch: None, - }), - }; - let outcome = engine - .run(&graph, &run_options) - .await - .expect("daytona parallel pipeline should succeed"); - assert_eq!( - outcome.status, - StageOutcome::Succeeded, - "pipeline failed: {:?}", - outcome.failure_reason() - ); - - // Verify parallel.results has head_sha for each branch - let checkpoint = load_run_checkpoint(run_tmp.path()).expect("checkpoint should load"); - let parallel_results = checkpoint - .context_values - .get("parallel.results") - .expect("parallel.results should be in context"); - let results_arr = parallel_results.as_array().expect("should be an array"); - assert_eq!(results_arr.len(), 2, "should have 2 branch results"); - - // Both branches should have head_sha (40-char hex) - let has_sha = results_arr.iter().all(|v| { - v.get("head_sha") - .and_then(|v| v.as_str()) - .is_some_and(|s| s.len() == 40 && s.chars().all(|c| c.is_ascii_hexdigit())) - }); - assert!(has_sha, "all branches should have 40-char hex head_sha"); - - // Branch SHAs should differ (each branch made unique changes) - let sha_a = results_arr - .iter() - .find(|v| v.get("id").and_then(|v| v.as_str()) == Some("branch_a")) - .and_then(|v| v.get("head_sha").and_then(|v| v.as_str())) - .unwrap(); - let sha_b = results_arr - .iter() - .find(|v| v.get("id").and_then(|v| v.as_str()) == Some("branch_b")) - .and_then(|v| v.get("head_sha").and_then(|v| v.as_str())) - .unwrap(); - assert_ne!(sha_a, sha_b, "branch SHAs should differ"); - - // Verify fan_in selected a winner and set best_head_sha - let best_id = checkpoint - .context_values - .get("parallel.fan_in.best_id") - .and_then(|v| v.as_str().map(String::from)) - .expect("fan_in should have selected a best_id"); - assert_eq!( - best_id, "branch_a", - "heuristic should pick branch_a (lexical)" - ); - - let best_head_sha = checkpoint - .context_values - .get("parallel.fan_in.best_head_sha") - .and_then(|v| v.as_str().map(String::from)); - assert!( - best_head_sha.is_some(), - "fan_in should have set best_head_sha" - ); - - // Verify winner's file exists in sandbox - let winner_check = env - .exec_command("cat branch_a.txt", 10_000, None, None, None) - .await - .expect("cat should succeed"); - assert_eq!( - winner_check.exit_code, - Some(0), - "winner's file should exist" - ); - assert!( - winner_check.stdout.contains("branch_a"), - "winner's file should have correct content, got: {}", - winner_check.stdout - ); - - // Verify events - { - let events = events.lock().unwrap(); - let parallel_started: Vec<_> = events - .iter() - .filter(|e| e.event_name() == "parallel.started") - .collect(); - assert_eq!( - parallel_started.len(), - 1, - "should have exactly one ParallelStarted event" - ); - let parallel_completed: Vec<_> = events - .iter() - .filter(|e| e.event_name() == "parallel.completed") - .collect(); - assert_eq!( - parallel_completed.len(), - 1, - "should have exactly one ParallelCompleted event" - ); - } - - env.cleanup().await.expect("Daytona cleanup should succeed"); -} - // --------------------------------------------------------------------------- // Daytona shadow commit E2E with sandbox-native metadata // --------------------------------------------------------------------------- diff --git a/lib/crates/fabro-workflow/tests/it/git_integration.rs b/lib/crates/fabro-workflow/tests/it/git_integration.rs index 8ff8f342d..bcdf8f9e0 100644 --- a/lib/crates/fabro-workflow/tests/it/git_integration.rs +++ b/lib/crates/fabro-workflow/tests/it/git_integration.rs @@ -12,10 +12,7 @@ use fabro_agent::Sandbox; use fabro_graphviz::graph::{AttrValue, Edge, Graph, Node}; use fabro_types::{RunEvent, WorkflowSettings, fixtures}; use fabro_workflow::event::Emitter; -use fabro_workflow::git::{ - add_worktree, branch_needs_push, create_branch, push_branch, push_ref, remove_worktree, - replace_worktree, -}; +use fabro_workflow::git::{branch_needs_push, push_branch, push_ref}; use fabro_workflow::handler::HandlerRegistry; use fabro_workflow::handler::exit::ExitHandler; use fabro_workflow::handler::start::StartHandler; @@ -169,22 +166,6 @@ fn test_run_options(run_dir: &Path) -> RunOptions { } } -#[test] -fn replace_worktree_replaces_stale() { - let dir = tempfile::tempdir().unwrap(); - init_repo(dir.path()); - create_branch(dir.path(), "stale-branch").unwrap(); - - let wt_path = dir.path().join("stale-wt"); - add_worktree(dir.path(), &wt_path, "stale-branch").unwrap(); - assert!(wt_path.join(".git").exists()); - - replace_worktree(dir.path(), &wt_path, "stale-branch").unwrap(); - assert!(wt_path.join(".git").exists()); - - remove_worktree(dir.path(), &wt_path).unwrap(); -} - #[test] fn push_ref_to_bare_remote() { let dir = tempfile::tempdir().unwrap(); @@ -195,7 +176,7 @@ fn push_ref_to_bare_remote() { init_repo(&repo_dir); add_origin(&repo_dir, &remote_dir); - create_branch(&repo_dir, "test-push").unwrap(); + rename_branch(&repo_dir, "test-push"); let url = format!("file://{}", remote_dir.display()); push_ref(&repo_dir, &url, "refs/heads/test-push").unwrap(); diff --git a/lib/crates/fabro-workflow/tests/it/integration.rs b/lib/crates/fabro-workflow/tests/it/integration.rs index 87ebeee4d..3f26b21f9 100644 --- a/lib/crates/fabro-workflow/tests/it/integration.rs +++ b/lib/crates/fabro-workflow/tests/it/integration.rs @@ -9755,7 +9755,7 @@ async fn artifact_pointers_rewritten_for_remote_sandbox() { } #[tokio::test] -async fn downstream_local_execution_materializes_blob_refs_to_runtime_files() { +async fn downstream_local_execution_resolves_response_blob_refs_as_text() { let mut graph = make_graph_with_start_exit("ArtifactMaterializeLocal"); graph.attrs.insert( "goal".to_string(), @@ -9818,31 +9818,19 @@ async fn downstream_local_execution_materializes_blob_refs_to_runtime_files() { .expect("pipeline should succeed"); assert_eq!(outcome.status, StageOutcome::Succeeded); - let expected_blob_id = fabro_types::RunBlobId::new( - &serde_json::to_vec(&serde_json::json!("x".repeat(150 * 1024))) - .expect("large value should serialize"), - ); let captured_value = captured.lock().unwrap().first().cloned().unwrap(); - let expected_path = RunScratch::new(dir.path()) - .runtime_dir() - .join("blobs") - .join(format!("{expected_blob_id}.json")); - assert_eq!( - captured_value, - format!("file://{}", expected_path.display()), - "downstream handlers should receive a local file ref" + assert_eq!(captured_value, "x".repeat(150 * 1024)); + assert!( + !RunScratch::new(dir.path()) + .runtime_dir() + .join("blobs") + .exists(), + "textual response values should resolve without file materialization" ); - let artifact_content = std::fs::read_to_string(&expected_path).expect("should read artifact"); - let artifact_value: serde_json::Value = - serde_json::from_str(&artifact_content).expect("should parse artifact JSON"); - let artifact_str = artifact_value - .as_str() - .expect("artifact should be a string"); - assert_eq!(artifact_str.len(), 150 * 1024); } #[tokio::test] -async fn downstream_remote_execution_materializes_blob_refs_to_sandbox_files() { +async fn downstream_remote_execution_resolves_response_blob_refs_as_text() { let mut graph = make_graph_with_start_exit("ArtifactMaterializeRemote"); graph.attrs.insert( "goal".to_string(), @@ -9906,27 +9894,11 @@ async fn downstream_remote_execution_materializes_blob_refs_to_sandbox_files() { .expect("pipeline should succeed"); assert_eq!(outcome.status, StageOutcome::Succeeded); - let expected_blob_id = fabro_types::RunBlobId::new( - &serde_json::to_vec(&serde_json::json!("x".repeat(150 * 1024))) - .expect("large value should serialize"), - ); let captured_value = captured.lock().unwrap().first().cloned().unwrap(); - assert_eq!( - captured_value, - format!("file:///sandbox/.fabro/blobs/{expected_blob_id}.json"), - "downstream handlers should receive a sandbox-local file ref" - ); - - let written = remote_env.written.lock().unwrap(); - assert_eq!(written.len(), 1, "should materialize the blob once"); - assert_eq!( - written[0].0, - format!("/sandbox/.fabro/blobs/{expected_blob_id}.json") - ); + assert_eq!(captured_value, "x".repeat(150 * 1024)); assert!( - written[0].1.len() > 100 * 1024, - "written content should be >100KB, got {} bytes", - written[0].1.len() + remote_env.written.lock().unwrap().is_empty(), + "textual response values should resolve without sandbox file materialization" ); } @@ -10065,7 +10037,7 @@ use fabro_workflow::handler::fan_in::FanInHandler; use fabro_workflow::handler::parallel::ParallelHandler; /// A handler that writes a file named `{node_id}.txt` into the sandbox's -/// working directory. Used to verify git worktree isolation in parallel +/// working directory. Used to verify shared-checkout writes from parallel /// branches. struct FileWriterHandler; @@ -10413,18 +10385,13 @@ async fn git_checkpoint_host_skips_metadata_branch_without_writer_prereqs() { } // --------------------------------------------------------------------------- -// Host e2e: parallel git branching with worktree isolation +// Host e2e: shared-checkout parallel execution // --------------------------------------------------------------------------- -/// End-to-end: parallel branches get isolated worktrees, fan-in fast-forwards -/// to winner. -/// -/// Pipeline: start -> fan_out -> {branch_a, branch_b} -> fan_in -> exit -/// -/// Each branch writes a unique file. After fan-in, only the winner's file -/// should be present in the main worktree. +/// End-to-end: parallel branches write to one shared checkout and normal +/// run-level checkpointing captures all branch changes after the parallel node. #[tokio::test] -async fn parallel_git_branching_host_e2e() { +async fn parallel_shared_checkout_host_e2e() { // 1. Create a temporary git repo with an initial commit let repo = tempfile::tempdir().unwrap(); std::process::Command::new("git") @@ -10532,10 +10499,7 @@ async fn parallel_git_branching_host_e2e() { registry.register("start", Box::new(StartHandler)); registry.register("exit", Box::new(ExitHandler)); registry.register("parallel", Box::new(ParallelHandler)); - registry.register( - "parallel.fan_in", - Box::new(FanInHandler::new(None)), // heuristic select — picks branch_a (lexical tiebreak) - ); + registry.register("parallel.fan_in", Box::new(FanInHandler::new(None))); let engine = WorkflowRunner::new(registry, Arc::new(emitter), env); @@ -10569,113 +10533,107 @@ async fn parallel_git_branching_host_e2e() { outcome.failure_reason() ); - // 6. Verify parallel.results has head_sha for each branch + // 6. Verify ordered typed results and that no fan-in selection state exists. let checkpoint = load_run_checkpoint(run_dir.path()).expect("checkpoint should load"); let parallel_results = checkpoint .context_values .get("parallel.results") .expect("parallel.results should be in context"); - let results_arr = parallel_results.as_array().expect("should be an array"); - assert_eq!(results_arr.len(), 2, "should have 2 branch results"); - - // Both branches should have head_sha - let branch_a_result = results_arr - .iter() - .find(|v| v.get("id").and_then(|v| v.as_str()) == Some("branch_a")) - .expect("branch_a result should exist"); - let branch_b_result = results_arr - .iter() - .find(|v| v.get("id").and_then(|v| v.as_str()) == Some("branch_b")) - .expect("branch_b result should exist"); - - let sha_a = branch_a_result - .get("head_sha") - .and_then(|v| v.as_str()) - .expect("branch_a should have head_sha"); - let sha_b = branch_b_result - .get("head_sha") - .and_then(|v| v.as_str()) - .expect("branch_b should have head_sha"); - - assert_eq!(sha_a.len(), 40, "SHA should be 40 hex chars"); - assert_eq!(sha_b.len(), 40, "SHA should be 40 hex chars"); - assert_ne!(sha_a, sha_b, "branch SHAs should differ"); - - // 7. Verify fan_in selected a winner and set best_head_sha - let best_id = checkpoint - .context_values - .get("parallel.fan_in.best_id") - .and_then(|v| v.as_str().map(String::from)) - .expect("fan_in should have selected a best_id"); - let best_head_sha = checkpoint - .context_values - .get("parallel.fan_in.best_head_sha") - .and_then(|v| v.as_str().map(String::from)) - .expect("fan_in should have set best_head_sha"); - - // Heuristic select with both success: lexical tiebreak picks "branch_a" + let results: Vec = + serde_json::from_value(parallel_results.clone()).expect("results should be typed"); assert_eq!( - best_id, "branch_a", - "heuristic should pick branch_a (lexical)" + results + .iter() + .map(|result| (result.id.as_str(), result.status)) + .collect::>(), + [ + ("branch_a", fabro_types::StageOutcome::Succeeded), + ("branch_b", fabro_types::StageOutcome::Succeeded), + ] + ); + assert!( + results + .iter() + .all(|result| result.context_updates.is_empty()) + ); + assert!( + checkpoint + .context_values + .keys() + .all(|key| !key.starts_with("parallel.fan_in.")) ); - // 8. Verify winner's file is in the main worktree, loser's is NOT - let winner_file = worktree_path.join(format!("{best_id}.txt")); - assert!( - winner_file.exists(), - "winner's file ({best_id}.txt) should exist in main worktree after ff-merge" - ); - let winner_content = std::fs::read_to_string(&winner_file).unwrap(); - assert!( - winner_content.contains(&format!("written by {best_id}")), - "winner's file should have correct content" - ); + // 7. Both branches wrote into the one shared checkout. + for branch in ["branch_a", "branch_b"] { + let file = worktree_path.join(format!("{branch}.txt")); + assert!( + file.exists(), + "{branch} output should remain in the checkout" + ); + assert_eq!( + std::fs::read_to_string(file).unwrap(), + format!("written by {branch}") + ); + } - let loser_id = if best_id == "branch_a" { - "branch_b" - } else { - "branch_a" - }; - let loser_file = worktree_path.join(format!("{loser_id}.txt")); - assert!( - !loser_file.exists(), - "loser's file ({loser_id}.txt) should NOT exist in main worktree" - ); - - // 9. Verify the main worktree HEAD matches the winner's head_sha - let main_head = { - let out = std::process::Command::new("git") - .args(["rev-parse", "HEAD"]) - .current_dir(&worktree_path) - .output() - .unwrap(); - String::from_utf8_lossy(&out.stdout).trim().to_string() - }; - // After fan-in ff-only + engine's own checkpoint commits, HEAD should be a - // descendant of best_head_sha. - let is_ancestor = std::process::Command::new("git") - .args(["merge-base", "--is-ancestor", &best_head_sha, &main_head]) + // 8. Normal run-level checkpointing captured both files together. + let committed_files = std::process::Command::new("git") + .args(["ls-tree", "-r", "--name-only", "HEAD"]) .current_dir(&worktree_path) .output() .unwrap(); - assert!( - is_ancestor.status.success(), - "best_head_sha ({best_head_sha}) should be an ancestor of current HEAD ({main_head})" - ); + assert!(committed_files.status.success()); + let committed_files = String::from_utf8_lossy(&committed_files.stdout); + assert!(committed_files.lines().any(|path| path == "branch_a.txt")); + assert!(committed_files.lines().any(|path| path == "branch_b.txt")); - // 10. Verify parallel branch refs still exist (for debugging) - let branch_ref_a = format!("fabro/run/parallel/{run_id}/fan-out/pass1/branch-a"); - let ref_check = std::process::Command::new("git") - .args(["rev-parse", "--verify", &branch_ref_a]) + // 9. Fabro created no branch-specific refs, commits, or worktrees. + let parallel_refs = std::process::Command::new("git") + .args([ + "for-each-ref", + "--format=%(refname)", + "refs/heads/fabro/run/parallel/", + ]) .current_dir(repo.path()) .output() .unwrap(); + assert!(parallel_refs.status.success()); assert!( - ref_check.status.success(), - "parallel branch ref should still exist for debugging" + parallel_refs.stdout.is_empty(), + "parallel refs must not exist" ); - // 11. Verify events + let worktrees = std::process::Command::new("git") + .args(["worktree", "list", "--porcelain"]) + .current_dir(repo.path()) + .output() + .unwrap(); + assert!(worktrees.status.success()); + let worktree_count = String::from_utf8_lossy(&worktrees.stdout) + .lines() + .filter(|line| line.starts_with("worktree ")) + .count(); + assert_eq!( + worktree_count, 2, + "parallel branches must not add worktrees" + ); + + let commit_count = std::process::Command::new("git") + .args(["rev-list", "--count", &format!("{base_sha}..HEAD")]) + .current_dir(&worktree_path) + .output() + .unwrap(); + assert!(commit_count.status.success()); + let commit_count: usize = String::from_utf8_lossy(&commit_count.stdout) + .trim() + .parse() + .unwrap(); + assert!( + (1..=2).contains(&commit_count), + "only run-level parallel/fan-in checkpoints should be committed, got {commit_count}" + ); + + // 10. Verify lifecycle events without parallel Git/worktree events. let events = events.lock().unwrap(); let parallel_started: Vec<_> = events .iter() @@ -10696,6 +10654,13 @@ async fn parallel_git_branching_host_e2e() { 1, "should have exactly one ParallelCompleted event" ); + assert!( + events.iter().all(|event| !matches!( + event.event_name(), + "git.branch" | "git.worktree.added" | "git.worktree.removed" + )), + "parallel execution must not emit Git branch or worktree lifecycle events" + ); // Cleanup let _ = std::process::Command::new("git") diff --git a/lib/packages/fabro-api-client/src/.openapi-generator/FILES b/lib/packages/fabro-api-client/src/.openapi-generator/FILES index 6c4bc5e4e..e7d9f3584 100644 --- a/lib/packages/fabro-api-client/src/.openapi-generator/FILES +++ b/lib/packages/fabro-api-client/src/.openapi-generator/FILES @@ -265,6 +265,7 @@ models/pair-transcript-system-message.ts models/pair-transcript-tool-call.ts models/pair-transcript-user-message.ts models/pair-transcript-warning.ts +models/parallel-branch-result.ts models/pending-interview-record.ts models/pending-reason.ts models/permission-level.ts @@ -293,6 +294,8 @@ models/principal-webhook.ts models/principal-worker.ts models/principal.ts models/project-namespace.ts +models/provider-credential-test-request.ts +models/provider-credential-test-response.ts models/provider-list.ts models/provider-test-list.ts models/provider-test-result.ts diff --git a/lib/packages/fabro-api-client/src/api/models-api.ts b/lib/packages/fabro-api-client/src/api/models-api.ts index 6b19c88c3..783f597ad 100644 --- a/lib/packages/fabro-api-client/src/api/models-api.ts +++ b/lib/packages/fabro-api-client/src/api/models-api.ts @@ -30,6 +30,10 @@ import type { ModelTestResult } from '../models'; // @ts-ignore import type { PaginatedModelList } from '../models'; // @ts-ignore +import type { ProviderCredentialTestRequest } from '../models'; +// @ts-ignore +import type { ProviderCredentialTestResponse } from '../models'; +// @ts-ignore import type { ProviderList } from '../models'; // @ts-ignore import type { ProviderTestList } from '../models'; @@ -175,6 +179,51 @@ export const ModelsApiAxiosParamCreator = function (configuration?: Configuratio options: localVarRequestOptions, }; }, + /** + * Validates an LLM provider API key against the server\'s effective catalog without persisting it. + * @summary Test Provider Credentials + * @param {string} provider The provider identifier. + * @param {ProviderCredentialTestRequest} providerCredentialTestRequest + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + testProviderCredentials: async (provider: string, providerCredentialTestRequest: ProviderCredentialTestRequest, options: RawAxiosRequestConfig = {}): Promise => { + // verify required parameter 'provider' is not null or undefined + assertParamExists('testProviderCredentials', 'provider', provider) + // verify required parameter 'providerCredentialTestRequest' is not null or undefined + assertParamExists('testProviderCredentials', 'providerCredentialTestRequest', providerCredentialTestRequest) + const localVarPath = `/api/v1/providers/{provider}/credentials/test` + .replace(`{${"provider"}}`, encodeURIComponent(String(provider))); + // use dummy base URL string because the URL constructor only accepts absolute URLs. + const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL); + let baseOptions; + if (configuration) { + baseOptions = configuration.baseOptions; + } + + const localVarRequestOptions = { method: 'POST', ...baseOptions, ...options}; + const localVarHeaderParameter = {} as any; + const localVarQueryParameter = {} as any; + + // authentication SessionCookie required + + // authentication BearerAuth required + // http bearer authentication required + await setBearerAuthToObject(localVarHeaderParameter, configuration) + + localVarHeaderParameter['Content-Type'] = 'application/json'; + localVarHeaderParameter['Accept'] = 'application/json'; + + setSearchParams(localVarUrlObj, localVarQueryParameter); + let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {}; + localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers}; + localVarRequestOptions.data = serializeDataIfNeeded(providerCredentialTestRequest, localVarRequestOptions, configuration) + + return { + url: toPathString(localVarUrlObj), + options: localVarRequestOptions, + }; + }, /** * Tests every configured LLM provider once using the catalog probe model. Provider-level failures are returned in the response body with HTTP 200. * @summary Test Providers @@ -262,6 +311,20 @@ export const ModelsApiFp = function(configuration?: Configuration) { const localVarOperationServerBasePath = operationServerMap['ModelsApi.testModel']?.[localVarOperationServerIndex]?.url; return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, localVarOperationServerBasePath || basePath); }, + /** + * Validates an LLM provider API key against the server\'s effective catalog without persisting it. + * @summary Test Provider Credentials + * @param {string} provider The provider identifier. + * @param {ProviderCredentialTestRequest} providerCredentialTestRequest + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + async testProviderCredentials(provider: string, providerCredentialTestRequest: ProviderCredentialTestRequest, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise> { + const localVarAxiosArgs = await localVarAxiosParamCreator.testProviderCredentials(provider, providerCredentialTestRequest, options); + const localVarOperationServerIndex = configuration?.serverIndex ?? 0; + const localVarOperationServerBasePath = operationServerMap['ModelsApi.testProviderCredentials']?.[localVarOperationServerIndex]?.url; + return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, localVarOperationServerBasePath || basePath); + }, /** * Tests every configured LLM provider once using the catalog probe model. Provider-level failures are returned in the response body with HTTP 200. * @summary Test Providers @@ -316,6 +379,17 @@ export const ModelsApiFactory = function (configuration?: Configuration, basePat testModel(id: string, mode?: ModelTestMode, options?: RawAxiosRequestConfig): AxiosPromise { return localVarFp.testModel(id, mode, options).then((request) => request(axios, basePath)); }, + /** + * Validates an LLM provider API key against the server\'s effective catalog without persisting it. + * @summary Test Provider Credentials + * @param {string} provider The provider identifier. + * @param {ProviderCredentialTestRequest} providerCredentialTestRequest + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + testProviderCredentials(provider: string, providerCredentialTestRequest: ProviderCredentialTestRequest, options?: RawAxiosRequestConfig): AxiosPromise { + return localVarFp.testProviderCredentials(provider, providerCredentialTestRequest, options).then((request) => request(axios, basePath)); + }, /** * Tests every configured LLM provider once using the catalog probe model. Provider-level failures are returned in the response body with HTTP 200. * @summary Test Providers @@ -368,6 +442,18 @@ export class ModelsApi extends BaseAPI { return ModelsApiFp(this.configuration).testModel(id, mode, options).then((request) => request(this.axios, this.basePath)); } + /** + * Validates an LLM provider API key against the server\'s effective catalog without persisting it. + * @summary Test Provider Credentials + * @param {string} provider The provider identifier. + * @param {ProviderCredentialTestRequest} providerCredentialTestRequest + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + public testProviderCredentials(provider: string, providerCredentialTestRequest: ProviderCredentialTestRequest, options?: RawAxiosRequestConfig) { + return ModelsApiFp(this.configuration).testProviderCredentials(provider, providerCredentialTestRequest, options).then((request) => request(this.axios, this.basePath)); + } + /** * Tests every configured LLM provider once using the catalog probe model. Provider-level failures are returned in the response body with HTTP 200. * @summary Test Providers diff --git a/lib/packages/fabro-api-client/src/models/hook-definition.ts b/lib/packages/fabro-api-client/src/models/hook-definition.ts index a7d2f7cf1..85092d14f 100644 --- a/lib/packages/fabro-api-client/src/models/hook-definition.ts +++ b/lib/packages/fabro-api-client/src/models/hook-definition.ts @@ -27,6 +27,9 @@ export interface HookDefinition { 'type'?: HookDefinitionTypeEnum | null; 'url'?: string | null; 'headers'?: { [key: string]: string; } | null; + /** + * Allowlist of environment variable names that an http hook header may read via `{{ env.NAME }}`. An empty list (the default) permits no env vars in headers. + */ 'allowed_env_vars'?: Array; 'tls'?: TlsMode; 'prompt'?: string | null; diff --git a/lib/packages/fabro-api-client/src/models/index.ts b/lib/packages/fabro-api-client/src/models/index.ts index 7b51ee78c..937ec7900 100644 --- a/lib/packages/fabro-api-client/src/models/index.ts +++ b/lib/packages/fabro-api-client/src/models/index.ts @@ -235,6 +235,7 @@ export * from './pair-transcript-system-message'; export * from './pair-transcript-tool-call'; export * from './pair-transcript-user-message'; export * from './pair-transcript-warning'; +export * from './parallel-branch-result'; export * from './pending-interview-record'; export * from './pending-reason'; export * from './permission-level'; @@ -264,6 +265,8 @@ export * from './principal-webhook'; export * from './principal-worker'; export * from './project-namespace'; export * from './provider'; +export * from './provider-credential-test-request'; +export * from './provider-credential-test-response'; export * from './provider-list'; export * from './provider-test-list'; export * from './provider-test-result'; diff --git a/lib/packages/fabro-api-client/src/models/parallel-branch-result.ts b/lib/packages/fabro-api-client/src/models/parallel-branch-result.ts new file mode 100644 index 000000000..994117dec --- /dev/null +++ b/lib/packages/fabro-api-client/src/models/parallel-branch-result.ts @@ -0,0 +1,27 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * Fabro Run API + * HTTP API for managing Fabro workflow run executions. + * + * The version of the OpenAPI document: 0.1.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + +// May contain unused imports in some cases +// @ts-ignore +import type { StageOutcome } from './stage-outcome'; + +/** + * The outcome and isolated context updates from one parallel branch. + */ +export interface ParallelBranchResult { + 'id': string; + 'status': StageOutcome; + 'context_updates': { [key: string]: any; }; +} diff --git a/lib/packages/fabro-api-client/src/models/provider-credential-test-request.ts b/lib/packages/fabro-api-client/src/models/provider-credential-test-request.ts new file mode 100644 index 000000000..390b00848 --- /dev/null +++ b/lib/packages/fabro-api-client/src/models/provider-credential-test-request.ts @@ -0,0 +1,22 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * Fabro Run API + * HTTP API for managing Fabro workflow run executions. + * + * The version of the OpenAPI document: 0.1.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + + +/** + * API key to validate against an LLM provider without persisting it. + */ +export interface ProviderCredentialTestRequest { + 'api_key': string; +} diff --git a/lib/packages/fabro-api-client/src/models/provider-credential-test-response.ts b/lib/packages/fabro-api-client/src/models/provider-credential-test-response.ts new file mode 100644 index 000000000..b74a688eb --- /dev/null +++ b/lib/packages/fabro-api-client/src/models/provider-credential-test-response.ts @@ -0,0 +1,22 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * Fabro Run API + * HTTP API for managing Fabro workflow run executions. + * + * The version of the OpenAPI document: 0.1.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + + +/** + * Successful response from provider credential validation. + */ +export interface ProviderCredentialTestResponse { + 'ok': boolean; +} diff --git a/lib/packages/fabro-api-client/src/models/stage-projection.ts b/lib/packages/fabro-api-client/src/models/stage-projection.ts index b49e864ca..6e9f8c446 100644 --- a/lib/packages/fabro-api-client/src/models/stage-projection.ts +++ b/lib/packages/fabro-api-client/src/models/stage-projection.ts @@ -30,6 +30,9 @@ import type { CommandTermination } from './command-termination'; import type { McpServerProjection } from './mcp-server-projection'; // May contain unused imports in some cases // @ts-ignore +import type { ParallelBranchResult } from './parallel-branch-result'; +// May contain unused imports in some cases +// @ts-ignore import type { PermissionLevel } from './permission-level'; // May contain unused imports in some cases // @ts-ignore @@ -75,9 +78,9 @@ export interface StageProjection { */ 'script_timing'?: object | null; /** - * Per-branch result objects produced by a parallel stage. + * Ordered per-branch results produced by a parallel stage. */ - 'parallel_results'?: Array | null; + 'parallel_results'?: Array | null; 'output'?: string | null; 'output_bytes'?: number | null; 'live_streaming'?: boolean | null; diff --git a/test/attractor/reference_template.dot b/test/attractor/reference_template.dot index 92767f3b3..35c2fac31 100644 --- a/test/attractor/reference_template.dot +++ b/test/attractor/reference_template.dot @@ -106,8 +106,8 @@ digraph reference_template { // PROMPT: consolidate_dod // Role: synthesize dod_a/b/c into consensus DoD // Must address: - // - Read branch outputs via parallel_results.json + worktree_dir - // - Fall back to current worktree if parallel_results.json missing + // - Review every branch result in parallel.results from prompt context + // - Read branch files from the shared checkout // - Read .ai/spec.md for context // - Resolve contradictions, apply DoD rubric + coverage checklist // Writes: .ai/definition_of_done.md @@ -138,8 +138,8 @@ digraph reference_template { // PROMPT: debate_consolidate // Role: synthesize plan_a/b/c into best-of-breed final plan // Must address: - // - Read branch outputs via parallel_results.json + worktree_dir - // - Fall back to current worktree if parallel_results.json missing + // - Review every branch result in parallel.results from prompt context + // - Read branch files from the shared checkout // - If .ai/postmortem_latest.md exists, verify plan addresses // every identified issue // - Resolve conflicts, ensure dependency order @@ -269,8 +269,8 @@ digraph reference_template { // PROMPT: review_consensus // Role: synthesize reviews into consensus verdict // Must address: - // - Read branch outputs via parallel_results.json + worktree_dir - // - Fall back to current worktree if parallel_results.json missing + // - Review every branch result in parallel.results from prompt context + // - Read branch files from the shared checkout // - Read .ai/definition_of_done.md for criteria // - Consensus: 2+ APPROVED with no critical gaps -> success; // otherwise -> retry with specific issues @@ -287,8 +287,8 @@ digraph reference_template { // Must address: // - Read .ai/review_consensus.md (if review stage reached) // - Read .ai/verify_fidelity.md (if semantic verify ran) - // - Read branch review outputs via parallel_results.json + - // worktree_dir if available + // - Review branch status and context updates in parallel.results + // from prompt context if available // - Read .ai/implementation_log.md // - Output: root causes, what worked (preserve), what failed // (fix), concrete next changes diff --git a/test/docs/examples/clone-substack/clone-substack.fabro b/test/docs/examples/clone-substack/clone-substack.fabro index 7c06e36a7..f3404383c 100644 --- a/test/docs/examples/clone-substack/clone-substack.fabro +++ b/test/docs/examples/clone-substack/clone-substack.fabro @@ -130,8 +130,8 @@ Write to .workflow/plan_b.md." label="Debate & Consolidate", prompt="Synthesize the two implementation plans into a single best-of-breed \ final plan.\n\n\ -Read branch outputs via parallel_results.json. If parallel_results.json is missing, \ -fall back to reading .workflow/plan_a.md and .workflow/plan_b.md.\n\n\ +Review every branch result in parallel.results from the prompt context, then read \ +.workflow/plan_a.md and .workflow/plan_b.md from the shared checkout.\n\n\ If .workflow/postmortem_latest.md exists, read it FIRST. The postmortem contains \ root-cause analysis and concrete fixes from the previous iteration. The final plan \ MUST be adjusted to address every issue identified in the postmortem — add new \ @@ -458,8 +458,8 @@ Write to .workflow/review_b.md." goal_gate=true, retry_target="postmortem", prompt="Synthesize the two reviews into a consensus verdict.\n\n\ -Read branch outputs via parallel_results.json. If parallel_results.json is \ -missing, fall back to reading .workflow/review_a.md and .workflow/review_b.md.\n\n\ +Review every branch result in parallel.results from the prompt context, then read \ +.workflow/review_a.md and .workflow/review_b.md from the shared checkout.\n\n\ Read .workflow/definition_of_done.md for acceptance criteria reference.\n\n\ Consensus rules:\n\ - Both APPROVED with no critical gaps: the implementation passes\n\ @@ -490,7 +490,7 @@ Read (if they exist):\n\ - .workflow/test-evidence/latest/manifest.json\n\ - Evidence files referenced by manifest entries for failed or suspicious IT \ scenarios\n\ -- Branch review outputs via parallel_results.json (if available)\n\n\ +- Parallel branch review status and context updates from parallel.results (if available)\n\n\ Output to .workflow/postmortem_latest.md (overwrite previous):\n\ - Root causes of failure\n\ - What works and must be preserved\n\ diff --git a/test/docs/tutorials/ensemble/ensemble.fabro b/test/docs/tutorials/ensemble/ensemble.fabro index 3a8655cde..87aadf36c 100644 --- a/test/docs/tutorials/ensemble/ensemble.fabro +++ b/test/docs/tutorials/ensemble/ensemble.fabro @@ -14,7 +14,7 @@ digraph Ensemble { start [shape=Mdiamond, label="Start"] exit [shape=Msquare, label="Exit"] - fork [label="Fan Out", shape=component, join_policy="wait_all"] + fork [label="Fan Out", shape=component] opus [label="Opus", prompt="Analyze the goal. Provide your independent assessment, recommendations, and any code or prose needed. Be thorough.", shape=tab] gemini [label="Gemini", prompt="Analyze the goal. Provide your independent assessment, recommendations, and any code or prose needed. Be thorough.", shape=tab] diff --git a/test/docs/tutorials/parallel-review/parallel.fabro b/test/docs/tutorials/parallel-review/parallel.fabro index a4cab30e9..c699750bc 100644 --- a/test/docs/tutorials/parallel-review/parallel.fabro +++ b/test/docs/tutorials/parallel-review/parallel.fabro @@ -5,7 +5,7 @@ digraph Parallel { start [shape=Mdiamond, label="Start"] exit [shape=Msquare, label="Exit"] - fork [label="Fork Analysis", shape=component, join_policy="wait_all"] + fork [label="Fork Analysis", shape=component] security [label="Security Audit", prompt="Examine the codebase for security concerns: hardcoded secrets, injection risks, unsafe dependencies. List findings as bullet points.", shape=tab, reasoning_effort="low"] architecture [label="Architecture Review", prompt="Assess the codebase architecture: separation of concerns, dependency structure, modularity. List findings as bullet points.", shape=tab, reasoning_effort="low"] diff --git a/test/docs/workflows/stages-and-nodes/all-node-types.fabro b/test/docs/workflows/stages-and-nodes/all-node-types.fabro index 75d4f3fc5..1970c147e 100644 --- a/test/docs/workflows/stages-and-nodes/all-node-types.fabro +++ b/test/docs/workflows/stages-and-nodes/all-node-types.fabro @@ -7,7 +7,7 @@ digraph AllNodeTypes { test [label="Run Tests", shape=parallelogram, script="cargo test 2>&1 || true"] gate [shape=diamond, label="Tests passing?"] cooldown [label="Wait 30s", shape=insulator, duration="30s"] - fork [label="Fan Out", shape=component, join_policy="wait_all"] + fork [label="Fan Out", shape=component] security [label="Security Review"] architecture [label="Architecture Review"] quality [label="Quality Review"] diff --git a/test/parallel.fabro b/test/parallel.fabro index 96cd323fa..381a9f2c5 100644 --- a/test/parallel.fabro +++ b/test/parallel.fabro @@ -4,7 +4,7 @@ digraph Parallel { start [shape=Mdiamond] exit [shape=Msquare] - fork [label="Fork Work", shape=component, join_policy="wait_all"] + fork [label="Fork Work", shape=component] branch1 [label="Branch 1"] branch2 [label="Branch 2"] merge [label="Merge Results", shape=tripleoctagon] From 27cf30ff8b4ccf68032a2be86e78b268c6ad8ce3 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 24 Jul 2026 06:33:22 -0400 Subject: [PATCH 02/24] feat: add GPT-5.6 name aliases --- lib/foundation/fabro-model/src/catalog.rs | 3 +++ .../fabro-model/src/catalog/providers/openai.toml | 6 +++--- .../fabro-model/src/catalog/providers/openrouter.toml | 6 +++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/lib/foundation/fabro-model/src/catalog.rs b/lib/foundation/fabro-model/src/catalog.rs index cac9f878f..f7c42f5ba 100644 --- a/lib/foundation/fabro-model/src/catalog.rs +++ b/lib/foundation/fabro-model/src/catalog.rs @@ -3118,8 +3118,11 @@ enabled = true for provider in [ProviderId::openai(), ProviderId::new("openrouter")] { for (alias, canonical_id) in [ ("sol", "gpt-5.6-sol"), + ("gpt-sol", "gpt-5.6-sol"), ("terra", "gpt-5.6-terra"), + ("gpt-terra", "gpt-5.6-terra"), ("luna", "gpt-5.6-luna"), + ("gpt-luna", "gpt-5.6-luna"), ] { let model = catalog .resolve_on_provider(&provider, alias) diff --git a/lib/foundation/fabro-model/src/catalog/providers/openai.toml b/lib/foundation/fabro-model/src/catalog/providers/openai.toml index 10b7b2193..6e44cd97e 100644 --- a/lib/foundation/fabro-model/src/catalog/providers/openai.toml +++ b/lib/foundation/fabro-model/src/catalog/providers/openai.toml @@ -14,7 +14,7 @@ family = "gpt-5" training = "2026-02-16" knowledge_cutoff = "February 16, 2026" default = true -aliases = ["sol", "gpt56-sol", "gpt-56-sol", "gpt-5.6", "gpt56", "gpt-56"] +aliases = ["sol", "gpt-sol", "gpt56-sol", "gpt-56-sol", "gpt-5.6", "gpt56", "gpt-56"] [providers.openai.models."gpt-5.6-sol".limits] context_window = 272000 @@ -37,7 +37,7 @@ display_name = "GPT-5.6 Terra" family = "gpt-5" training = "2026-02-16" knowledge_cutoff = "February 16, 2026" -aliases = ["terra", "gpt56-terra", "gpt-56-terra"] +aliases = ["terra", "gpt-terra", "gpt56-terra", "gpt-56-terra"] [providers.openai.models."gpt-5.6-terra".limits] context_window = 272000 @@ -60,7 +60,7 @@ display_name = "GPT-5.6 Luna" family = "gpt-5" training = "2026-02-16" knowledge_cutoff = "February 16, 2026" -aliases = ["luna", "gpt56-luna", "gpt-56-luna"] +aliases = ["luna", "gpt-luna", "gpt56-luna", "gpt-56-luna"] [providers.openai.models."gpt-5.6-luna".limits] context_window = 272000 diff --git a/lib/foundation/fabro-model/src/catalog/providers/openrouter.toml b/lib/foundation/fabro-model/src/catalog/providers/openrouter.toml index b83aa7662..201608654 100644 --- a/lib/foundation/fabro-model/src/catalog/providers/openrouter.toml +++ b/lib/foundation/fabro-model/src/catalog/providers/openrouter.toml @@ -167,7 +167,7 @@ display_name = "GPT-5.6 Sol (via OpenRouter)" family = "gpt-5" training = "2026-02-16" knowledge_cutoff = "February 16, 2026" -aliases = ["sol", "gpt56-sol", "gpt-56-sol", "gpt-5.6", "gpt56", "gpt-56"] +aliases = ["sol", "gpt-sol", "gpt56-sol", "gpt-56-sol", "gpt-5.6", "gpt56", "gpt-56"] [providers.openrouter.models."gpt-5.6-sol".limits] context_window = 1050000 @@ -192,7 +192,7 @@ display_name = "GPT-5.6 Terra (via OpenRouter)" family = "gpt-5" training = "2026-02-16" knowledge_cutoff = "February 16, 2026" -aliases = ["terra", "gpt56-terra", "gpt-56-terra"] +aliases = ["terra", "gpt-terra", "gpt56-terra", "gpt-56-terra"] [providers.openrouter.models."gpt-5.6-terra".limits] context_window = 1050000 @@ -217,7 +217,7 @@ display_name = "GPT-5.6 Luna (via OpenRouter)" family = "gpt-5" training = "2026-02-16" knowledge_cutoff = "February 16, 2026" -aliases = ["luna", "gpt56-luna", "gpt-56-luna"] +aliases = ["luna", "gpt-luna", "gpt56-luna", "gpt-56-luna"] [providers.openrouter.models."gpt-5.6-luna".limits] context_window = 1050000 From 558c1010f8758ea8a1bf4d29dc9adaaa87571edb Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 24 Jul 2026 06:44:56 -0400 Subject: [PATCH 03/24] Regenerate TS client to drop duplicated method from merge The merge of main kept two copies of testProviderCredentials in the generated models-api.ts; regeneration is authoritative. Co-Authored-By: Claude Fable 5 --- .../fabro-api-client/src/api/models-api.ts | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/lib/packages/fabro-api-client/src/api/models-api.ts b/lib/packages/fabro-api-client/src/api/models-api.ts index 1f34aeb76..03a90fac4 100644 --- a/lib/packages/fabro-api-client/src/api/models-api.ts +++ b/lib/packages/fabro-api-client/src/api/models-api.ts @@ -397,17 +397,6 @@ export const ModelsApiFactory = function (configuration?: Configuration, basePat testProviderCredentials(provider: string, providerCredentialTestRequest: ProviderCredentialTestRequest, options?: RawAxiosRequestConfig): AxiosPromise { return localVarFp.testProviderCredentials(provider, providerCredentialTestRequest, options).then((request) => request(axios, basePath)); }, - /** - * Validates an LLM provider API key against the server\'s effective catalog without persisting it. - * @summary Test Provider Credentials - * @param {string} provider The provider identifier. - * @param {ProviderCredentialTestRequest} providerCredentialTestRequest - * @param {*} [options] Override http request option. - * @throws {RequiredError} - */ - testProviderCredentials(provider: string, providerCredentialTestRequest: ProviderCredentialTestRequest, options?: RawAxiosRequestConfig): AxiosPromise { - return localVarFp.testProviderCredentials(provider, providerCredentialTestRequest, options).then((request) => request(axios, basePath)); - }, /** * Tests every configured LLM provider once using the catalog probe model. Provider-level failures are returned in the response body with HTTP 200. * @summary Test Providers @@ -473,18 +462,6 @@ export class ModelsApi extends BaseAPI { return ModelsApiFp(this.configuration).testProviderCredentials(provider, providerCredentialTestRequest, options).then((request) => request(this.axios, this.basePath)); } - /** - * Validates an LLM provider API key against the server\'s effective catalog without persisting it. - * @summary Test Provider Credentials - * @param {string} provider The provider identifier. - * @param {ProviderCredentialTestRequest} providerCredentialTestRequest - * @param {*} [options] Override http request option. - * @throws {RequiredError} - */ - public testProviderCredentials(provider: string, providerCredentialTestRequest: ProviderCredentialTestRequest, options?: RawAxiosRequestConfig) { - return ModelsApiFp(this.configuration).testProviderCredentials(provider, providerCredentialTestRequest, options).then((request) => request(this.axios, this.basePath)); - } - /** * Tests every configured LLM provider once using the catalog probe model. Provider-level failures are returned in the response body with HTTP 200. * @summary Test Providers From 7216c49e44b88dc440113b3178b228f7c755c888 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 24 Jul 2026 07:00:44 -0400 Subject: [PATCH 04/24] docs: align parallel strategy with typed StageOutcome results MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-ups: the ParallelBranchResult snippet showed status as String (it is StageOutcome), and the cancellation section implied a cancelled branch status that the type does not have — cancelled-while- waiting branches record a failed outcome (reason "branch cancelled") and the handler returns Error::Cancelled to the run executor. Co-Authored-By: Claude Fable 5 --- docs/internal/parallel-strategy.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/internal/parallel-strategy.md b/docs/internal/parallel-strategy.md index 61a019326..a7f9ed77e 100644 --- a/docs/internal/parallel-strategy.md +++ b/docs/internal/parallel-strategy.md @@ -55,7 +55,7 @@ The shared result type is: ```rust ParallelBranchResult { id: String, - status: String, + status: StageOutcome, context_updates: BTreeMap, } ``` @@ -134,11 +134,12 @@ The final typed array is also projected into ## 7. Cancellation -Semaphore acquisition observes the run cancellation token. Branches waiting for -a permit can terminate as cancelled rather than waiting indefinitely. Branches -already executing continue through their handler's cooperative cancellation -path. The parallel handler joins every task before returning cancellation to the -run executor. +Semaphore acquisition observes the run cancellation token. Branches still +waiting for a permit when cancellation fires stop without executing; because +`StageOutcome` has no cancelled variant, their results record a failed outcome +(reason `branch cancelled`). Branches already executing continue through their +handler's cooperative cancellation path. The parallel handler joins every task +before returning `Error::Cancelled` to the run executor. Cancellation does not trigger branch Git cleanup because no branch Git state is created. From 673a7064fe2ad3e1d72c448691cc57f4bbf70a77 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 24 Jul 2026 07:04:31 -0400 Subject: [PATCH 05/24] Validate completion reasoning effort --- docs/public/api-reference/fabro-api.yaml | 2 +- .../src/server/handler/completions.rs | 2 +- lib/apps/fabro-server/src/server/tests.rs | 21 ++++++++++++++++ .../create_completion_request_round_trip.rs | 25 +++++++++++++++++++ .../src/models/create-completion-request.ts | 5 +++- 5 files changed, 52 insertions(+), 3 deletions(-) create mode 100644 lib/foundation/fabro-api/tests/create_completion_request_round_trip.rs diff --git a/docs/public/api-reference/fabro-api.yaml b/docs/public/api-reference/fabro-api.yaml index 519ee4188..e13097f1b 100644 --- a/docs/public/api-reference/fabro-api.yaml +++ b/docs/public/api-reference/fabro-api.yaml @@ -8498,7 +8498,7 @@ components: type: string description: Stop sequences. reasoning_effort: - type: string + $ref: "#/components/schemas/ReasoningEffort" description: Reasoning effort level. provider: type: string diff --git a/lib/apps/fabro-server/src/server/handler/completions.rs b/lib/apps/fabro-server/src/server/handler/completions.rs index b53e56d31..7afe6e943 100644 --- a/lib/apps/fabro-server/src/server/handler/completions.rs +++ b/lib/apps/fabro-server/src/server/handler/completions.rs @@ -109,7 +109,7 @@ async fn create_completion( } else { Some(req.stop_sequences) }, - reasoning_effort: req.reasoning_effort.as_deref().and_then(|s| s.parse().ok()), + reasoning_effort: req.reasoning_effort, speed: None, metadata: None, provider_options: req.provider_options, diff --git a/lib/apps/fabro-server/src/server/tests.rs b/lib/apps/fabro-server/src/server/tests.rs index 454455be0..641ea0b7b 100644 --- a/lib/apps/fabro-server/src/server/tests.rs +++ b/lib/apps/fabro-server/src/server/tests.rs @@ -15240,6 +15240,27 @@ async fn create_completion_missing_messages_returns_422() { assert_status!(response, StatusCode::UNPROCESSABLE_ENTITY).await; } +#[tokio::test] +async fn create_completion_invalid_reasoning_effort_returns_422() { + let app = test_app_with(); + + let req = Request::builder() + .method("POST") + .uri(api("/completions")) + .header("content-type", "application/json") + .body(Body::from( + serde_json::json!({ + "messages": [], + "reasoning_effort": "bogus" + }) + .to_string(), + )) + .unwrap(); + + let response = app.oneshot(req).await.unwrap(); + assert_status!(response, StatusCode::UNPROCESSABLE_ENTITY).await; +} + #[tokio::test] async fn create_completion_unknown_provider_returns_clear_error() { let app = test_app_with(); diff --git a/lib/foundation/fabro-api/tests/create_completion_request_round_trip.rs b/lib/foundation/fabro-api/tests/create_completion_request_round_trip.rs new file mode 100644 index 000000000..913947453 --- /dev/null +++ b/lib/foundation/fabro-api/tests/create_completion_request_round_trip.rs @@ -0,0 +1,25 @@ +use fabro_api::types::CreateCompletionRequest; +use fabro_model::ReasoningEffort; +use serde_json::json; + +#[test] +fn create_completion_request_reuses_canonical_reasoning_effort() { + let request: CreateCompletionRequest = serde_json::from_value(json!({ + "messages": [], + "reasoning_effort": "high" + })) + .unwrap(); + + let reasoning_effort: Option = request.reasoning_effort; + assert_eq!(reasoning_effort, Some(ReasoningEffort::High)); +} + +#[test] +fn create_completion_request_rejects_unknown_reasoning_effort() { + let result = serde_json::from_value::(json!({ + "messages": [], + "reasoning_effort": "bogus" + })); + + assert!(result.is_err()); +} diff --git a/lib/packages/fabro-api-client/src/models/create-completion-request.ts b/lib/packages/fabro-api-client/src/models/create-completion-request.ts index 923431e9b..836e20c0d 100644 --- a/lib/packages/fabro-api-client/src/models/create-completion-request.ts +++ b/lib/packages/fabro-api-client/src/models/create-completion-request.ts @@ -22,6 +22,9 @@ import type { CompletionToolChoice } from './completion-tool-choice'; // May contain unused imports in some cases // @ts-ignore import type { CompletionToolDefinition } from './completion-tool-definition'; +// May contain unused imports in some cases +// @ts-ignore +import type { ReasoningEffort } from './reasoning-effort'; export interface CreateCompletionRequest { /** @@ -56,7 +59,7 @@ export interface CreateCompletionRequest { /** * Reasoning effort level. */ - 'reasoning_effort'?: string; + 'reasoning_effort'?: ReasoningEffort; /** * Optional provider pin. */ From 05fa48563792d1fb53af4f226c5cae8330e11502 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 24 Jul 2026 07:07:56 -0400 Subject: [PATCH 06/24] feat: add portable GLM and DeepSeek aliases --- docs/public/core-concepts/models.mdx | 4 +- docs/public/integrations/openrouter.mdx | 5 +- lib/foundation/fabro-model/src/catalog.rs | 62 ++++++++++++++++++- .../src/catalog/providers/openrouter.toml | 3 + .../src/catalog/providers/zai.toml | 2 +- 5 files changed, 70 insertions(+), 6 deletions(-) diff --git a/docs/public/core-concepts/models.mdx b/docs/public/core-concepts/models.mdx index ab88318dd..e5eb837b3 100644 --- a/docs/public/core-concepts/models.mdx +++ b/docs/public/core-concepts/models.mdx @@ -59,7 +59,7 @@ Fabro performs this selection once when creating a run and persists the chosen p | `kimi-k2.5` | kimi | `kimi` | 262K | $0.60 / $3.00 | 50 tok/s | | `laguna-s-2.1` | poolside | `laguna`, `laguna-s` | 1M | $0.10 / $0.20 | n/a | | `laguna-xs-2.1` | poolside | `laguna-xs` | 262K | $0.10 / $0.20 | n/a | -| `glm-4.7` | zai | `glm`, `glm4` | 203K | $0.60 / $2.20 | 100 tok/s | +| `glm-5.2` | zai | `glm`, `glm5`, `glm52`, `glm5.2` | 1M | $1.40 / $4.40 | n/a | | `minimax-m2.5` | minimax | `minimax` | 197K | $0.30 / $1.20 | 45 tok/s | | `mercury-2` | inception | `mercury` | 131K | $0.25 / $0.75 | 1000 tok/s | @@ -206,7 +206,7 @@ When no model or provider is specified, Fabro chooses the default offering on th | `gemini` | `gemini-3.5-flash` | | `kimi` | `kimi-k2.5` | | `poolside` | `laguna-s-2.1` | -| `zai` | `glm-4.7` | +| `zai` | `glm-5.2` | | `minimax` | `minimax-m2.5` | | `inception` | `mercury-2` | diff --git a/docs/public/integrations/openrouter.mdx b/docs/public/integrations/openrouter.mdx index 6f1f046b5..c725ddca6 100644 --- a/docs/public/integrations/openrouter.mdx +++ b/docs/public/integrations/openrouter.mdx @@ -53,10 +53,11 @@ The built-in catalog gives OpenRouter offerings the same human-facing model slug | `claude-haiku-4-5` | `anthropic/claude-haiku-4.5`; provider small default | | `gpt-5.4`, `gpt-5.5` | `openai/gpt-5.4`, `openai/gpt-5.5` | | `gemini-3.1-pro-preview`, `gemini-3.5-flash` | `google/...` API IDs | -| `deepseek-v4-pro`, `deepseek-v4-flash` | `deepseek/...` API IDs | +| `deepseek-v4-pro` (`deepseek`, `deepseek-v4`), `deepseek-v4-flash` (`deepseek-flash`) | `deepseek/...` API IDs | | `kimi-k2.6`, `qwen3-coder`, `qwen3.6-flash` | Vendor-prefixed API IDs | | `laguna-s-2.1`, `laguna-xs-2.1` | `poolside/...`; native reasoning and tool use | -| `glm-4.6`, `minimax-m2.7`, `mimo-v2.5-pro` | Vendor-prefixed API IDs | +| `glm-5.2` (`glm`, `glm5`, `glm52`, `glm5.2`), `glm-4.6` | `z-ai/...` API IDs | +| `minimax-m2.7`, `mimo-v2.5-pro` | Vendor-prefixed API IDs | | `nemotron-3-super-120b-a12b`, `devstral-2512` | Vendor-prefixed API IDs | Any other OpenRouter model can be added under the provider. Choose a stable Fabro model slug as the table key and put OpenRouter's exact vendor/model string in `api_id`: diff --git a/lib/foundation/fabro-model/src/catalog.rs b/lib/foundation/fabro-model/src/catalog.rs index f7c42f5ba..c58035f8d 100644 --- a/lib/foundation/fabro-model/src/catalog.rs +++ b/lib/foundation/fabro-model/src/catalog.rs @@ -3135,6 +3135,57 @@ enabled = true } } + #[test] + fn builtin_glm_5_2_aliases_are_portable() { + let catalog = Catalog::from_builtin_with_overrides(&minimal_settings( + r" +[providers.openrouter] +enabled = true +", + )) + .expect("enabled OpenRouter override should build from the built-in provider settings"); + + for provider in [ProviderId::new("zai"), ProviderId::new("openrouter")] { + for alias in ["glm", "glm5", "glm52", "glm5.2"] { + let model = catalog + .resolve_on_provider(&provider, alias) + .unwrap_or_else(|error| { + panic!("{alias} should resolve on {provider}: {error}") + }); + assert_eq!(model.provider, provider, "{alias}"); + assert_eq!(model.id, "glm-5.2", "{alias}"); + } + } + } + + #[test] + fn builtin_deepseek_v4_selectors_resolve_on_openrouter() { + let catalog = Catalog::from_builtin_with_overrides(&minimal_settings( + r" +[providers.openrouter] +enabled = true +", + )) + .expect("enabled OpenRouter override should build from the built-in provider settings"); + let openrouter = ProviderId::new("openrouter"); + + for (selector, canonical_id) in [ + ("deepseek-v4-pro", "deepseek-v4-pro"), + ("deepseek-v4", "deepseek-v4-pro"), + ("deepseek", "deepseek-v4-pro"), + ("deepseek-v4-flash", "deepseek-v4-flash"), + ("deepseek-flash", "deepseek-v4-flash"), + ] { + let model = catalog + .resolve_on_provider(&openrouter, selector) + .unwrap_or_else(|error| { + panic!("{selector} should resolve on {openrouter}: {error}") + }); + assert_eq!(model.provider, openrouter, "{selector}"); + assert_eq!(model.id, canonical_id, "{selector}"); + } + } + #[test] fn builtin_legacy_vendor_ids_normalize_for_pinned_and_unpinned_selection() { let catalog = Catalog::from_builtin_with_overrides(&minimal_settings( @@ -3260,7 +3311,12 @@ enabled = true ), }, estimated_output_tps: None, - aliases: [], + aliases: [ + "glm", + "glm5", + "glm52", + "glm5.2", + ], default: false, small_default: false, configured: false, @@ -6107,6 +6163,8 @@ sampling_params = false aliases: [ "glm", "glm5", + "glm52", + "glm5.2", ], default: true, small_default: false, @@ -6124,6 +6182,8 @@ sampling_params = false ]); assert_eq!(catalog.get("glm").unwrap().id, "glm-5.2"); assert_eq!(catalog.get("glm5").unwrap().id, "glm-5.2"); + assert_eq!(catalog.get("glm52").unwrap().id, "glm-5.2"); + assert_eq!(catalog.get("glm5.2").unwrap().id, "glm-5.2"); } #[test] diff --git a/lib/foundation/fabro-model/src/catalog/providers/openrouter.toml b/lib/foundation/fabro-model/src/catalog/providers/openrouter.toml index 201608654..175d76cad 100644 --- a/lib/foundation/fabro-model/src/catalog/providers/openrouter.toml +++ b/lib/foundation/fabro-model/src/catalog/providers/openrouter.toml @@ -354,6 +354,7 @@ output_cost_per_mtok = 1.20 api_id = "deepseek/deepseek-v4-pro" display_name = "DeepSeek V4 Pro" family = "deepseek-v4" +aliases = ["deepseek-v4", "deepseek"] [providers.openrouter.models."deepseek-v4-pro".limits] context_window = 1050000 @@ -372,6 +373,7 @@ output_cost_per_mtok = 0.87 api_id = "deepseek/deepseek-v4-flash" display_name = "DeepSeek V4 Flash" family = "deepseek-v4" +aliases = ["deepseek-flash"] [providers.openrouter.models."deepseek-v4-flash".limits] context_window = 1050000 @@ -513,6 +515,7 @@ output_cost_per_mtok = 1.125 api_id = "z-ai/glm-5.2" display_name = "GLM 5.2 (via OpenRouter)" family = "glm-5" +aliases = ["glm", "glm5", "glm52", "glm5.2"] [providers.openrouter.models."glm-5.2".limits] context_window = 1048576 diff --git a/lib/foundation/fabro-model/src/catalog/providers/zai.toml b/lib/foundation/fabro-model/src/catalog/providers/zai.toml index c6d70cd64..58fb8c38a 100644 --- a/lib/foundation/fabro-model/src/catalog/providers/zai.toml +++ b/lib/foundation/fabro-model/src/catalog/providers/zai.toml @@ -12,7 +12,7 @@ credentials = ["env:ZAI_API_KEY", "vault:ZAI_API_KEY"] display_name = "GLM 5.2" family = "glm-5" default = true -aliases = ["glm", "glm5"] +aliases = ["glm", "glm5", "glm52", "glm5.2"] [providers.zai.models."glm-5.2".limits] context_window = 1048576 From 3cfac20343b1b73a79d31e64b830d532dad604a5 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 24 Jul 2026 07:12:01 -0400 Subject: [PATCH 07/24] test: capture preflight provider routing gap --- lib/apps/fabro-server/src/run_manifest.rs | 149 ++++++++++++++++++++++ 1 file changed, 149 insertions(+) diff --git a/lib/apps/fabro-server/src/run_manifest.rs b/lib/apps/fabro-server/src/run_manifest.rs index 259e3d2b5..375ef57e5 100644 --- a/lib/apps/fabro-server/src/run_manifest.rs +++ b/lib/apps/fabro-server/src/run_manifest.rs @@ -1470,6 +1470,81 @@ mod tests { Arc::new(Catalog::from_builtin().unwrap()) } + fn openai_compatible_completion(model: &str) -> serde_json::Value { + serde_json::json!({ + "id": "chatcmpl_preflight", + "object": "chat.completion", + "created": 1_700_000_000, + "model": model, + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": "OK"}, + "finish_reason": "stop" + }], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2} + }) + } + + fn ready_kimi_and_openrouter_state( + server: &httpmock::MockServer, + ) -> Arc { + let kimi_url = server.url("/kimi/v1"); + let openrouter_url = server.url("/openrouter/v1"); + let llm_catalog_settings: LlmCatalogSettings = toml::from_str(&format!( + r#" +[providers.kimi] +base_url = "{kimi_url}" + +[providers.openrouter] +base_url = "{openrouter_url}" +enabled = true +"# + )) + .expect("catalog overrides should parse"); + + crate::test_support::TestAppStateBuilder::new() + .llm_catalog_settings(llm_catalog_settings) + .vault_entries([ + (EnvVars::KIMI_API_KEY, "test-kimi-key"), + (EnvVars::OPENROUTER_API_KEY, "test-openrouter-key"), + ]) + .build() + } + + async fn preflight_for_model( + state: &Arc, + model: &str, + ) -> (types::PreflightResponse, bool) { + let mut ready_providers = state.ready_llm_provider_ids().await; + ready_providers.sort(); + assert_eq!(ready_providers, vec![ + ProviderId::new("kimi"), + ProviderId::new("openrouter") + ]); + + let mut manifest = minimal_manifest(); + manifest.workflows.get_mut("workflow.fabro").unwrap().source = format!( + r#" +digraph Demo {{ + start [shape=Mdiamond] + exit [shape=Msquare] + work [prompt="Do work", model="{model}"] + start -> work -> exit +}} +"# + ); + let prepared = prepare_manifest( + &manifest_run_defaults(Some(&default_settings_fixture())), + &manifest, + ) + .unwrap(); + let validated = validate_prepared_manifest(&prepared, state.catalog()).unwrap(); + + run_preflight(state.as_ref(), &prepared, &validated) + .await + .unwrap() + } + fn manifest_workflow() -> types::ManifestWorkflow { types::ManifestWorkflow { config: None, @@ -2390,6 +2465,80 @@ digraph Demo { assert!(response_mock.calls_async().await >= 1); } + #[tokio::test] + async fn preflight_uses_ready_providers_for_known_shared_alias() { + let server = httpmock::MockServer::start_async().await; + let openrouter_probe = server + .mock_async(|when, then| { + when.method(httpmock::Method::POST) + .path("/openrouter/v1/chat/completions") + .header("authorization", "Bearer test-openrouter-key") + .json_body_includes(r#"{"model":"anthropic/claude-fable-5"}"#); + then.status(200) + .header("content-type", "application/json") + .json_body(openai_compatible_completion("anthropic/claude-fable-5")); + }) + .await; + let state = ready_kimi_and_openrouter_state(&server); + + let (response, _ok) = preflight_for_model(&state, "claude-fable").await; + + let llm_check = response.checks.sections[0] + .checks + .iter() + .find(|check| check.name == "LLM" && check.summary == "claude-fable-5") + .expect("preflight should include Claude Fable"); + assert_eq!( + llm_check + .details + .iter() + .map(|detail| detail.text.as_str()) + .find(|detail| detail.starts_with("Provider: ")), + Some("Provider: openrouter") + ); + assert_eq!(llm_check.status, types::PreflightCheckResultStatus::Pass); + openrouter_probe.assert_async().await; + } + + #[tokio::test] + async fn preflight_uses_ready_providers_for_unknown_unqualified_model() { + let server = httpmock::MockServer::start_async().await; + let kimi_probe = server + .mock_async(|when, then| { + when.method(httpmock::Method::POST) + .path("/kimi/v1/chat/completions") + .header("authorization", "Bearer test-kimi-key") + .json_body_includes(r#"{"model":"provider-private-preview"}"#); + then.status(200) + .header("content-type", "application/json") + .json_body(openai_compatible_completion("provider-private-preview")); + }) + .await; + let state = ready_kimi_and_openrouter_state(&server); + + let (response, _ok) = preflight_for_model(&state, "provider-private-preview").await; + + assert!(response.workflow.diagnostics.iter().any(|diagnostic| { + diagnostic.rule == "node_model_known" + && diagnostic.message.contains("provider-private-preview") + })); + let llm_check = response.checks.sections[0] + .checks + .iter() + .find(|check| check.name == "LLM" && check.summary == "provider-private-preview") + .expect("preflight should include the unknown passthrough model"); + assert_eq!( + llm_check + .details + .iter() + .map(|detail| detail.text.as_str()) + .find(|detail| detail.starts_with("Provider: ")), + Some("Provider: kimi") + ); + assert_eq!(llm_check.status, types::PreflightCheckResultStatus::Pass); + kimi_probe.assert_async().await; + } + #[test] fn static_validation_rejects_unknown_llm_provider() { let mut manifest = minimal_manifest(); From 5aebb17fa229c9d1796386d1b68779da818a6217 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 24 Jul 2026 07:22:47 -0400 Subject: [PATCH 08/24] Return bad request for unsupported reasoning effort --- .../src/server/handler/completions.rs | 22 +++++--- lib/apps/fabro-server/src/server/tests.rs | 50 +++++++++++++++++++ lib/components/fabro-llm/src/client.rs | 12 ++--- lib/components/fabro-llm/src/error.rs | 29 +++++++++++ lib/components/fabro-workflow/src/error.rs | 1 + 5 files changed, 100 insertions(+), 14 deletions(-) diff --git a/lib/apps/fabro-server/src/server/handler/completions.rs b/lib/apps/fabro-server/src/server/handler/completions.rs index b53e56d31..726847d5a 100644 --- a/lib/apps/fabro-server/src/server/handler/completions.rs +++ b/lib/apps/fabro-server/src/server/handler/completions.rs @@ -26,6 +26,17 @@ fn finish_reason_to_api_stop_reason(reason: &FinishReason) -> String { } } +fn llm_error_response(error: fabro_llm::Error) -> Response { + match error { + fabro_llm::Error::InvalidRequest { message } => { + ApiError::bad_request(message).into_response() + } + error => { + ApiError::new(StatusCode::BAD_GATEWAY, format!("LLM error: {error}")).into_response() + } + } +} + async fn create_completion( _auth: RequiredUser, State(state): State>, @@ -127,10 +138,7 @@ async fn create_completion( // Streaming path: forward all StreamEvents as SSE let stream_result = match client.stream(&request).await { Ok(s) => s, - Err(e) => { - return ApiError::new(StatusCode::BAD_GATEWAY, format!("LLM error: {e}")) - .into_response(); - } + Err(error) => return llm_error_response(error), }; llm_sse::stream_response(stream_result, state.shutdown_token()) @@ -179,8 +187,7 @@ async fn create_completion( }) .into_response() } - Err(e) => ApiError::new(StatusCode::BAD_GATEWAY, format!("LLM error: {e}")) - .into_response(), + Err(error) => llm_error_response(error), } } else { match client.complete(&request).await { @@ -202,8 +209,7 @@ async fn create_completion( }) .into_response() } - Err(e) => ApiError::new(StatusCode::BAD_GATEWAY, format!("LLM error: {e}")) - .into_response(), + Err(error) => llm_error_response(error), } } } diff --git a/lib/apps/fabro-server/src/server/tests.rs b/lib/apps/fabro-server/src/server/tests.rs index 454455be0..372cc1e6c 100644 --- a/lib/apps/fabro-server/src/server/tests.rs +++ b/lib/apps/fabro-server/src/server/tests.rs @@ -15272,6 +15272,56 @@ async fn create_completion_unknown_provider_returns_clear_error() { ); } +#[tokio::test] +async fn create_completion_unsupported_reasoning_efforts_return_bad_request() { + let upstream = MockServer::start(); + let completion = upstream.mock(|when, then| { + when.method(POST); + then.status(500); + }); + let state = TestAppStateBuilder::new() + .provider_base_url("kimi", upstream.url("/v1")) + .vault_entries([(EnvVars::KIMI_API_KEY, "test-kimi-api-key")]) + .build(); + let app = crate::test_support::build_test_router(state); + + for stream in [false, true] { + for effort in ["medium", "xhigh"] { + let req = Request::builder() + .method("POST") + .uri(api("/completions")) + .header("content-type", "application/json") + .body(Body::from( + serde_json::json!({ + "provider": "kimi", + "model": "kimi-k3", + "reasoning_effort": effort, + "stream": stream, + "messages": [ + { + "role": "user", + "content": [{"kind": "text", "data": "hi"}] + } + ] + }) + .to_string(), + )) + .unwrap(); + + let response = app.clone().oneshot(req).await.unwrap(); + let body = response_json!(response, StatusCode::BAD_REQUEST).await; + assert_eq!( + body["errors"][0]["detail"], + format!( + "model 'kimi-k3' does not support reasoning_effort '{effort}'; allowed values: low, high, max" + ) + ); + } + } + + completion.assert_calls(0); +} + #[tokio::test] async fn create_completion_default_model_uses_app_state_catalog() { let upstream = MockServer::start(); diff --git a/lib/components/fabro-llm/src/client.rs b/lib/components/fabro-llm/src/client.rs index 446b0b77a..8cfaadf1b 100644 --- a/lib/components/fabro-llm/src/client.rs +++ b/lib/components/fabro-llm/src/client.rs @@ -410,12 +410,11 @@ impl Client { if let Some(effort) = request.reasoning_effort { if !settings.controls.reasoning_effort.contains(&effort) { - return Err(Error::Configuration { + return Err(Error::InvalidRequest { message: format!( "model '{model_id}' does not support reasoning_effort '{effort}'; allowed values: {}", format_control_values(&settings.controls.reasoning_effort), ), - source: None, }); } } @@ -439,7 +438,8 @@ impl Client { /// /// # Errors /// - /// Returns `Error::Configuration` if no provider is specified or + /// Returns `Error::InvalidRequest` when a catalog-declared request control + /// is unsupported, `Error::Configuration` if no provider is specified or /// registered, or any provider/middleware error encountered during the /// request. pub async fn complete(&self, request: &Request) -> Result { @@ -475,7 +475,8 @@ impl Client { /// /// # Errors /// - /// Returns `Error::Configuration` if no provider is specified or + /// Returns `Error::InvalidRequest` when a catalog-declared request control + /// is unsupported, `Error::Configuration` if no provider is specified or /// registered, or any provider/middleware error encountered during the /// request. pub async fn stream(&self, request: &Request) -> Result { @@ -1481,9 +1482,8 @@ output_cost_per_mtok = 20.0 assert!(matches!( err, - Error::Configuration { + Error::InvalidRequest { ref message, - .. } if message.contains("model 'kimi-k2.5' does not support reasoning_effort 'high'") )); } diff --git a/lib/components/fabro-llm/src/error.rs b/lib/components/fabro-llm/src/error.rs index fd2987047..000f36943 100644 --- a/lib/components/fabro-llm/src/error.rs +++ b/lib/components/fabro-llm/src/error.rs @@ -95,6 +95,9 @@ pub enum Error { #[error("No object generated: {message}")] NoObjectGenerated { message: String }, + #[error("Invalid request: {message}")] + InvalidRequest { message: String }, + #[error("Configuration error: {message}")] Configuration { message: String, @@ -164,6 +167,7 @@ impl Error { Self::InvalidToolCall { .. } | Self::NoObjectGenerated { .. } | Self::Interrupt { .. } + | Self::InvalidRequest { .. } | Self::Configuration { .. } | Self::UnsupportedToolChoice { .. } | Self::RequestTimeout { .. } => false, @@ -266,6 +270,9 @@ impl Error { Self::NoObjectGenerated { .. } => { format!("api_deterministic|{provider}|no_object") } + Self::InvalidRequest { .. } => { + format!("api_deterministic|{provider}|invalid_request") + } Self::UnsupportedToolChoice { .. } => { format!("api_deterministic|{provider}|unsupported_tool_choice") } @@ -805,6 +812,14 @@ mod tests { source: None, }; assert_eq!(err.to_string(), "Configuration error: no provider"); + + let err = Error::InvalidRequest { + message: "unsupported reasoning effort".into(), + }; + assert_eq!( + err.to_string(), + "Invalid request: unsupported reasoning effort" + ); } #[test] @@ -996,6 +1011,13 @@ mod tests { .failover_eligible() ); + assert!( + !Error::InvalidRequest { + message: "bad".into(), + } + .failover_eligible() + ); + assert!( !Error::UnsupportedToolChoice { message: "nope".into(), @@ -1146,6 +1168,13 @@ mod tests { .failure_signature_hint(), "api_deterministic|unknown|no_object" ); + assert_eq!( + Error::InvalidRequest { + message: "bad".into(), + } + .failure_signature_hint(), + "api_deterministic|unknown|invalid_request" + ); assert_eq!( Error::UnsupportedToolChoice { message: "nope".into(), diff --git a/lib/components/fabro-workflow/src/error.rs b/lib/components/fabro-workflow/src/error.rs index 75b6d317f..128134480 100644 --- a/lib/components/fabro-workflow/src/error.rs +++ b/lib/components/fabro-workflow/src/error.rs @@ -38,6 +38,7 @@ pub fn classify_sdk_error(err: &LlmError) -> FailureCategory { LlmError::Interrupt { .. } => FailureCategory::Canceled, LlmError::InvalidToolCall { .. } | LlmError::NoObjectGenerated { .. } + | LlmError::InvalidRequest { .. } | LlmError::Configuration { .. } | LlmError::UnsupportedToolChoice { .. } => FailureCategory::Deterministic, } From 377eb961ec805dfba15317a06fe06d0d2f4534ea Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 24 Jul 2026 07:26:22 -0400 Subject: [PATCH 09/24] feat: add Poolside provider logo Adapted from poolside's official favicon mark: monochrome fill="currentColor" at 24x24 to match the other provider logos, with the brand's gradient-fade tail preserved via the original alpha mask. Co-Authored-By: Claude Fable 5 --- .../public/images/providers/poolside.svg | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 apps/fabro-web/public/images/providers/poolside.svg diff --git a/apps/fabro-web/public/images/providers/poolside.svg b/apps/fabro-web/public/images/providers/poolside.svg new file mode 100644 index 000000000..8c160c5ad --- /dev/null +++ b/apps/fabro-web/public/images/providers/poolside.svg @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + From 0cc4a018824f1c4bae06333ed5e5a659eaf7e307 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 24 Jul 2026 07:28:06 -0400 Subject: [PATCH 10/24] Forward reasoning effort for structured completions --- .../src/server/handler/completions.rs | 3 + lib/apps/fabro-server/src/server/tests.rs | 66 +++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/lib/apps/fabro-server/src/server/handler/completions.rs b/lib/apps/fabro-server/src/server/handler/completions.rs index b53e56d31..78a973087 100644 --- a/lib/apps/fabro-server/src/server/handler/completions.rs +++ b/lib/apps/fabro-server/src/server/handler/completions.rs @@ -155,6 +155,9 @@ async fn create_completion( if let Some(top_p) = request.top_p { params = params.top_p(top_p); } + if let Some(reasoning_effort) = request.reasoning_effort { + params = params.reasoning_effort(reasoning_effort); + } match generate_object(params, schema).await { Ok(result) => { // `result.finish_reason` / `result.usage` resolve through diff --git a/lib/apps/fabro-server/src/server/tests.rs b/lib/apps/fabro-server/src/server/tests.rs index 454455be0..c4d5cac28 100644 --- a/lib/apps/fabro-server/src/server/tests.rs +++ b/lib/apps/fabro-server/src/server/tests.rs @@ -15347,6 +15347,72 @@ reasoning = false completion.assert(); } +#[tokio::test] +async fn create_completion_structured_output_forwards_reasoning_effort() { + let upstream = MockServer::start(); + let completion = upstream.mock(|when, then| { + when.method(POST) + .path("/chat/completions") + .json_body_includes(r#"{"model":"kimi-k3","reasoning_effort":"high"}"#); + then.status(200) + .header("content-type", "application/json") + .json_body(json!({ + "id": "chatcmpl-kimi-structured", + "model": "kimi-k3", + "choices": [{ + "message": { + "role": "assistant", + "content": "{\"answer\":42}" + }, + "finish_reason": "stop" + }], + "usage": { + "prompt_tokens": 10, + "completion_tokens": 4, + "total_tokens": 14 + } + })); + }); + let state = TestAppStateBuilder::new() + .provider_base_url("kimi", upstream.base_url()) + .vault_entries([(EnvVars::KIMI_API_KEY, "test-kimi-api-key")]) + .build(); + let app = crate::test_support::build_test_router(state); + + let req = Request::builder() + .method("POST") + .uri(api("/completions")) + .header("content-type", "application/json") + .body(Body::from( + serde_json::json!({ + "provider": "kimi", + "model": "kimi-k3", + "reasoning_effort": "high", + "stream": false, + "schema": { + "type": "object", + "properties": { + "answer": {"type": "integer"} + }, + "required": ["answer"] + }, + "messages": [ + { + "role": "user", + "content": [{"kind": "text", "data": "Return the answer."}] + } + ] + }) + .to_string(), + )) + .unwrap(); + + let response = app.oneshot(req).await.unwrap(); + let body = response_json!(response, StatusCode::OK).await; + assert_eq!(body["output"], json!({"answer": 42})); + completion.assert_calls(1); +} + #[tokio::test] async fn demo_list_runs_returns_run_list_items() { let state = test_app_state(); From 1c1ea53093cced949bf9c49c1d547b9bd60b132d Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 24 Jul 2026 07:28:51 -0400 Subject: [PATCH 11/24] fix: prefer ready providers during preflight --- lib/apps/fabro-server/src/run_manifest.rs | 192 ++++++++++++------ .../fabro-server/src/server/handler/runs.rs | 24 ++- .../fabro-workflow/src/operations/create.rs | 6 + .../fabro-workflow/src/operations/mod.rs | 2 +- .../fabro-workflow/src/operations/validate.rs | 33 ++- .../fabro-workflow/src/pipeline/transform.rs | 5 + .../fabro-workflow/src/pipeline/types.rs | 1 + .../fabro-workflow/src/pipeline/validate.rs | 1 + .../fabro-workflow/src/run_materialization.rs | 47 ++++- .../src/transforms/model_resolution.rs | 74 ++++++- .../fabro-workflow/tests/it/integration.rs | 1 + lib/foundation/fabro-model/src/catalog.rs | 61 ++++++ 12 files changed, 365 insertions(+), 82 deletions(-) diff --git a/lib/apps/fabro-server/src/run_manifest.rs b/lib/apps/fabro-server/src/run_manifest.rs index 375ef57e5..aff59b0c5 100644 --- a/lib/apps/fabro-server/src/run_manifest.rs +++ b/lib/apps/fabro-server/src/run_manifest.rs @@ -34,14 +34,17 @@ use fabro_types::{ use fabro_util::check_report::{CheckDetail, CheckReport, CheckResult, CheckSection, CheckStatus}; use fabro_validate::Severity; use fabro_workflow::Error as WorkflowError; -use fabro_workflow::operations::{CreateRunInput, ValidateInput, WorkflowInput, validate}; +use fabro_workflow::operations::{ + CreateRunInput, ValidateInput, WorkflowInput, validate, validate_with_provider_fallback, +}; use fabro_workflow::pipeline::Validated; +#[cfg(test)] use fabro_workflow::run_materialization::materialize_run; +use fabro_workflow::run_materialization::materialize_run_with_provider_fallback; use fabro_workflow::workflow_bundle::{BundledWorkflow, ParsedWorkflowConfig, WorkflowBundle}; use futures_util::stream::{self, StreamExt}; use tokio::process::Command; use tokio::time; -use tracing::warn; use crate::interp::process_env_var; use crate::server::AppState; @@ -206,6 +209,27 @@ pub(crate) fn validate_prepared_manifest_with_vars( }) } +pub(crate) fn validate_prepared_manifest_for_preflight( + prepared: &PreparedManifest, + catalog: Arc, + vars: HashMap, + ready_providers: &[ProviderId], +) -> Result { + let fallback_providers = catalog.all_provider_ids().into_iter().collect::>(); + validate_with_provider_fallback( + ValidateInput { + workflow: WorkflowInput::Bundled(prepared.workflow_input.clone()), + settings: prepared.settings.clone(), + vars, + cwd: prepared.cwd.clone(), + custom_transforms: Vec::new(), + catalog, + }, + ready_providers, + &fallback_providers, + ) +} + pub(crate) fn create_run_input( prepared: PreparedManifest, configured_providers: Vec, @@ -238,8 +262,11 @@ pub(crate) async fn run_preflight( state: &AppState, prepared: &PreparedManifest, validated: &Validated, + preferred_providers: &[ProviderId], + llm_result: Result, ) -> Result<(types::PreflightResponse, bool)> { - let (report, checks_ok) = build_preflight_report(state, prepared, validated).await?; + let (report, checks_ok) = + build_preflight_report(state, prepared, validated, preferred_providers, llm_result).await?; let preflight_ok = !validated.has_errors() && checks_ok; Ok(( preflight_response( @@ -458,6 +485,8 @@ async fn build_preflight_report( state: &AppState, prepared: &PreparedManifest, validated: &Validated, + preferred_providers: &[ProviderId], + llm_result: Result, ) -> Result<(CheckReport, bool)> { let graph = validated.graph(); let mut checks = base_preflight_checks(prepared, graph); @@ -475,20 +504,13 @@ async fn build_preflight_report( } let catalog = state.catalog(); - let llm_result = state.resolve_llm_client().await; - if let Err(err) = &llm_result { - warn!(error = ?err, "Failed to resolve LLM client while checking ready providers"); - } - // Preflight is credential-independent static validation. Materialize - // against every enabled catalog provider so aliases and defaults can be - // inspected even when the corresponding adapter is not currently ready; - // `run_llm_check` below reports actual credential/registration readiness. - let enabled_providers = catalog.all_provider_ids().into_iter().collect::>(); - let materialized = materialize_run( + let fallback_providers = catalog.all_provider_ids().into_iter().collect::>(); + let materialized = materialize_run_with_provider_fallback( prepared.settings.clone(), graph, catalog.as_ref(), - &enabled_providers, + preferred_providers, + &fallback_providers, )?; let resolved_run = materialized.run; let server_settings = state.server_settings(); @@ -1033,13 +1055,21 @@ async fn run_llm_check( catalog: &Catalog, llm_result: Result, ) -> bool { - let model = settings - .model - .name - .as_deref() - .unwrap_or_else(|| catalog.default_for_configured_ids(&[]).id.as_str()); - let provider = settings.model.provider.as_deref(); - let default_provider = provider.unwrap_or("anthropic"); + let (Some(model), Some(default_provider)) = ( + settings.model.name.as_deref(), + settings.model.provider.as_deref(), + ) else { + checks.push(CheckResult { + name: "LLM".into(), + status: CheckStatus::Error, + summary: "model resolution failed".into(), + details: Vec::new(), + remediation: Some( + "Preflight did not produce a resolved run model and provider".to_string(), + ), + }); + return false; + }; let mut model_providers = std::collections::BTreeSet::new(); let mut has_llm_nodes = false; @@ -1050,24 +1080,7 @@ async fn run_llm_check( has_llm_nodes = true; let node_model = node.model().unwrap_or(model); let node_provider = node.provider().unwrap_or(default_provider); - let resolved = if node.provider().is_some() { - catalog.get_on_provider(&ProviderId::new(node_provider), node_model) - } else { - catalog - .select(node_model, None, &catalog.all_provider_ids()) - .ok() - }; - let (resolved_model, resolved_provider) = if let Some(info) = resolved { - (info.id.to_string(), info.provider.to_string()) - } else { - (node_model.to_string(), node_provider.to_string()) - }; - let final_provider = if node.provider().is_some() { - node_provider.to_string() - } else { - resolved_provider - }; - model_providers.insert((resolved_model, final_provider)); + model_providers.insert((node_model.to_string(), node_provider.to_string())); } if !has_llm_nodes { @@ -1515,7 +1528,11 @@ enabled = true state: &Arc, model: &str, ) -> (types::PreflightResponse, bool) { - let mut ready_providers = state.ready_llm_provider_ids().await; + let llm_result = state.resolve_llm_client().await; + let mut ready_providers = llm_result + .as_ref() + .map(LlmClientResult::provider_ids) + .unwrap_or_default(); ready_providers.sort(); assert_eq!(ready_providers, vec![ ProviderId::new("kimi"), @@ -1538,11 +1555,37 @@ digraph Demo {{ &manifest, ) .unwrap(); - let validated = validate_prepared_manifest(&prepared, state.catalog()).unwrap(); + let validated = validate_prepared_manifest_for_preflight( + &prepared, + state.catalog(), + HashMap::new(), + &ready_providers, + ) + .unwrap(); - run_preflight(state.as_ref(), &prepared, &validated) - .await - .unwrap() + run_preflight( + state.as_ref(), + &prepared, + &validated, + &ready_providers, + llm_result, + ) + .await + .unwrap() + } + + async fn run_preflight_with_catalog_routes( + state: &AppState, + prepared: &PreparedManifest, + validated: &Validated, + ) -> Result<(types::PreflightResponse, bool)> { + let llm_result = state.resolve_llm_client().await; + let preferred_providers = state + .catalog() + .all_provider_ids() + .into_iter() + .collect::>(); + run_preflight(state, prepared, validated, &preferred_providers, llm_result).await } fn manifest_workflow() -> types::ManifestWorkflow { @@ -2174,9 +2217,10 @@ name = "Control Plane" assert!(validated.has_errors()); - let (response, ok) = run_preflight(state.as_ref(), &prepared, &validated) - .await - .unwrap(); + let (response, ok) = + run_preflight_with_catalog_routes(state.as_ref(), &prepared, &validated) + .await + .unwrap(); assert!(!ok); assert_eq!(response.workflow.name, "Invalid"); @@ -2218,9 +2262,10 @@ issues = "read" let validated = validate_prepared_manifest(&prepared, test_catalog()).unwrap(); assert!(!validated.has_errors()); - let (response, _ok) = run_preflight(state.as_ref(), &prepared, &validated) - .await - .unwrap(); + let (response, _ok) = + run_preflight_with_catalog_routes(state.as_ref(), &prepared, &validated) + .await + .unwrap(); assert!( response.checks.sections[0] @@ -2269,9 +2314,10 @@ id = "local" assert!(!validated.has_errors()); - let (response, ok) = run_preflight(state.as_ref(), &prepared, &validated) - .await - .unwrap(); + let (response, ok) = + run_preflight_with_catalog_routes(state.as_ref(), &prepared, &validated) + .await + .unwrap(); assert!(ok); assert!(response.workflow.diagnostics.is_empty()); @@ -2376,9 +2422,10 @@ id = "daytona" .unwrap(); let validated = validate_prepared_manifest(&prepared, test_catalog()).unwrap(); - let (response, _ok) = run_preflight(state.as_ref(), &prepared, &validated) - .await - .unwrap(); + let (response, _ok) = + run_preflight_with_catalog_routes(state.as_ref(), &prepared, &validated) + .await + .unwrap(); assert!(response.workflow.diagnostics.is_empty()); assert!( @@ -2444,9 +2491,10 @@ digraph Demo { .unwrap(); let validated = validate_prepared_manifest(&prepared, test_catalog()).unwrap(); - let (response, ok) = run_preflight(state.as_ref(), &prepared, &validated) - .await - .unwrap(); + let (response, ok) = + run_preflight_with_catalog_routes(state.as_ref(), &prepared, &validated) + .await + .unwrap(); assert!(!ok); let llm_check = response.checks.sections[0] @@ -2615,11 +2663,29 @@ digraph Demo { &manifest, ) .unwrap(); - let validated = validate_prepared_manifest(&prepared, state.catalog()).unwrap(); + let llm_result = state.resolve_llm_client().await; + let ready_providers = llm_result + .as_ref() + .map(LlmClientResult::provider_ids) + .unwrap_or_default(); + assert!(ready_providers.is_empty()); + let validated = validate_prepared_manifest_for_preflight( + &prepared, + state.catalog(), + HashMap::new(), + &ready_providers, + ) + .unwrap(); - let (response, ok) = run_preflight(state.as_ref(), &prepared, &validated) - .await - .unwrap(); + let (response, ok) = run_preflight( + state.as_ref(), + &prepared, + &validated, + &ready_providers, + llm_result, + ) + .await + .unwrap(); assert!(!ok); let llm_check = response.checks.sections[0] diff --git a/lib/apps/fabro-server/src/server/handler/runs.rs b/lib/apps/fabro-server/src/server/handler/runs.rs index 614f2d719..20a4b3f9d 100644 --- a/lib/apps/fabro-server/src/server/handler/runs.rs +++ b/lib/apps/fabro-server/src/server/handler/runs.rs @@ -835,10 +835,22 @@ async fn run_preflight( return ApiError::bad_request(format!("Run config variable interpolation failed: {err}")) .into_response(); } - let mut validated = match run_manifest::validate_prepared_manifest_with_vars( + let llm_result = state.resolve_llm_client().await; + if let Err(error) = &llm_result { + tracing::warn!( + error = ?error, + "Failed to resolve LLM client while checking ready providers" + ); + } + let ready_providers = llm_result + .as_ref() + .map(LlmClientResult::provider_ids) + .unwrap_or_default(); + let mut validated = match run_manifest::validate_prepared_manifest_for_preflight( &prepared, state.catalog(), vars, + &ready_providers, ) { Ok(validated) => validated, Err(WorkflowError::Parse(_)) => { @@ -847,7 +859,15 @@ async fn run_preflight( Err(err) => return ApiError::bad_request(err.to_string()).into_response(), }; validated.promote_template_undefined_variables_to_errors(); - let response = match run_manifest::run_preflight(&state, &prepared, &validated).await { + let response = match run_manifest::run_preflight( + &state, + &prepared, + &validated, + &ready_providers, + llm_result, + ) + .await + { Ok((response, _ok)) => response, Err(err) => { return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) diff --git a/lib/components/fabro-workflow/src/operations/create.rs b/lib/components/fabro-workflow/src/operations/create.rs index 144cf5976..39bd097bf 100644 --- a/lib/components/fabro-workflow/src/operations/create.rs +++ b/lib/components/fabro-workflow/src/operations/create.rs @@ -312,6 +312,7 @@ fn create_from_source( .filter(|provider| !provider.is_empty()) .map(ProviderId::new), &options.configured_providers, + None, &options.catalog, )?; @@ -336,6 +337,7 @@ pub(super) fn preprocess_and_validate( render_mode: RenderMode, default_provider: Option, eligible_providers: &[ProviderId], + fallback_providers: Option<&[ProviderId]>, catalog: &Arc, ) -> Result { let mut parsed = pipeline::parse(dot_source)?; @@ -351,6 +353,7 @@ pub(super) fn preprocess_and_validate( catalog: Arc::clone(catalog), default_provider, eligible_providers: eligible_providers.iter().cloned().collect(), + fallback_providers: fallback_providers.map(|providers| providers.iter().cloned().collect()), })?; Ok(pipeline::validate(transformed, catalog.as_ref(), &[])) } @@ -579,6 +582,7 @@ reasoning = false RenderMode::Structural, None, &test_provider_ids(), + None, &test_catalog(), ) .unwrap() @@ -749,6 +753,7 @@ reasoning = false RenderMode::Strict, None, &test_provider_ids(), + None, &test_catalog(), ); let Err(err) = result else { @@ -787,6 +792,7 @@ reasoning = false RenderMode::Strict, None, &test_provider_ids(), + None, &test_catalog(), ); let Err(err) = result else { diff --git a/lib/components/fabro-workflow/src/operations/mod.rs b/lib/components/fabro-workflow/src/operations/mod.rs index 4710d97a9..9e5726515 100644 --- a/lib/components/fabro-workflow/src/operations/mod.rs +++ b/lib/components/fabro-workflow/src/operations/mod.rs @@ -22,7 +22,7 @@ pub use rewind::{RewindInput, RewindOutcome, rewind}; pub use source::WorkflowInput; pub use start::{StartServices, Started, start}; pub use timeline::{ForkTarget, RunTimeline, TimelineEntry, build_timeline, timeline}; -pub use validate::{ValidateInput, validate}; +pub use validate::{ValidateInput, validate, validate_with_provider_fallback}; pub use crate::pipeline::{LlmSpec, SandboxEnvSpec}; pub use crate::transforms::RenderMode; diff --git a/lib/components/fabro-workflow/src/operations/validate.rs b/lib/components/fabro-workflow/src/operations/validate.rs index a95b13920..2fb7235b0 100644 --- a/lib/components/fabro-workflow/src/operations/validate.rs +++ b/lib/components/fabro-workflow/src/operations/validate.rs @@ -2,7 +2,7 @@ use std::collections::HashMap; use std::path::PathBuf; use std::sync::Arc; -use fabro_model::Catalog; +use fabro_model::{Catalog, ProviderId}; use fabro_types::WorkflowSettings; use super::create::{preprocess_and_validate, template_context}; @@ -28,17 +28,35 @@ pub struct ValidateInput { /// Returns `Validated` even when validation produced errors. Call /// `validated.raise_on_errors()` if the caller wants to fail fast. pub fn validate(input: ValidateInput) -> Result { + let eligible_providers = input + .catalog + .all_provider_ids() + .into_iter() + .collect::>(); + validate_with_provider_sets(input, &eligible_providers, None) +} + +/// Parse, transform, and validate while preferring one provider snapshot and +/// falling back to another only for provider-readiness selection failures. +pub fn validate_with_provider_fallback( + input: ValidateInput, + preferred_providers: &[ProviderId], + fallback_providers: &[ProviderId], +) -> Result { + validate_with_provider_sets(input, preferred_providers, Some(fallback_providers)) +} + +fn validate_with_provider_sets( + input: ValidateInput, + eligible_providers: &[ProviderId], + fallback_providers: Option<&[ProviderId]>, +) -> Result { let resolved = resolve_workflow(ResolveWorkflowInput { workflow: input.workflow, settings: input.settings, cwd: input.cwd, }) .map_err(|err| Error::Parse(err.to_string()))?; - let eligible_providers = input - .catalog - .all_provider_ids() - .into_iter() - .collect::>(); preprocess_and_validate( &resolved.raw_source, @@ -60,7 +78,8 @@ pub fn validate(input: ValidateInput) -> Result { .as_deref() .filter(|provider| !provider.is_empty()) .map(fabro_model::ProviderId::new), - &eligible_providers, + eligible_providers, + fallback_providers, &input.catalog, ) } diff --git a/lib/components/fabro-workflow/src/pipeline/transform.rs b/lib/components/fabro-workflow/src/pipeline/transform.rs index 6b76380d0..cc3fe089d 100644 --- a/lib/components/fabro-workflow/src/pipeline/transform.rs +++ b/lib/components/fabro-workflow/src/pipeline/transform.rs @@ -68,6 +68,7 @@ pub fn transform(parsed: Parsed, options: &TransformOptions) -> Result, pub default_provider: Option, pub eligible_providers: HashSet, + pub fallback_providers: Option>, } /// Options for the FINALIZE phase. diff --git a/lib/components/fabro-workflow/src/pipeline/validate.rs b/lib/components/fabro-workflow/src/pipeline/validate.rs index 06120e8ce..8e19bb5a6 100644 --- a/lib/components/fabro-workflow/src/pipeline/validate.rs +++ b/lib/components/fabro-workflow/src/pipeline/validate.rs @@ -51,6 +51,7 @@ mod tests { catalog: std::sync::Arc::clone(&catalog), default_provider: None, eligible_providers: catalog.all_provider_ids(), + fallback_providers: None, }) .unwrap(); validate(transformed, catalog.as_ref(), &[]) diff --git a/lib/components/fabro-workflow/src/run_materialization.rs b/lib/components/fabro-workflow/src/run_materialization.rs index 7e82964ac..27183bcf5 100644 --- a/lib/components/fabro-workflow/src/run_materialization.rs +++ b/lib/components/fabro-workflow/src/run_materialization.rs @@ -9,10 +9,36 @@ use fabro_types::settings::run::RunGoal; use crate::error::Error; pub fn materialize_run( + settings: WorkflowSettings, + graph: &Graph, + catalog: &Catalog, + configured_providers: &[ProviderId], +) -> Result { + materialize_run_with_provider_sets(settings, graph, catalog, configured_providers, None) +} + +pub fn materialize_run_with_provider_fallback( + settings: WorkflowSettings, + graph: &Graph, + catalog: &Catalog, + preferred_providers: &[ProviderId], + fallback_providers: &[ProviderId], +) -> Result { + materialize_run_with_provider_sets( + settings, + graph, + catalog, + preferred_providers, + Some(fallback_providers), + ) +} + +fn materialize_run_with_provider_sets( mut settings: WorkflowSettings, graph: &Graph, catalog: &Catalog, configured_providers: &[ProviderId], + fallback_providers: Option<&[ProviderId]>, ) -> Result { let configured_model = settings.run.model.name.take(); let configured_provider = settings.run.model.provider.take(); @@ -30,11 +56,24 @@ pub fn materialize_run( let provider = configured_provider.or(graph_provider); let model = configured_model.or(graph_model); let eligible = configured_providers.iter().cloned().collect::>(); - let (resolved_model, resolved_provider) = - resolve_run_model(catalog, &eligible, model.as_deref(), provider.as_deref())?; + let fallback = + fallback_providers.map(|providers| providers.iter().cloned().collect::>()); + let provider = provider + .as_deref() + .filter(|provider| !provider.is_empty()) + .map(ProviderId::new); + let selected = match fallback { + Some(fallback) => catalog.resolve_selection_with_fallback( + model.as_deref(), + provider.as_ref(), + &eligible, + &fallback, + ), + None => catalog.resolve_selection(model.as_deref(), provider.as_ref(), &eligible), + }?; - settings.run.model.name = Some(resolved_model); - settings.run.model.provider = Some(resolved_provider.into_inner()); + settings.run.model.name = Some(selected.model); + settings.run.model.provider = Some(selected.provider.into_inner()); let goal = graph.goal().to_string(); settings.run.goal = if goal.is_empty() { diff --git a/lib/components/fabro-workflow/src/transforms/model_resolution.rs b/lib/components/fabro-workflow/src/transforms/model_resolution.rs index 7eecbcfd1..00e71ba74 100644 --- a/lib/components/fabro-workflow/src/transforms/model_resolution.rs +++ b/lib/components/fabro-workflow/src/transforms/model_resolution.rs @@ -13,6 +13,7 @@ pub struct ModelResolutionTransform { catalog: Arc, default_provider: Option, eligible_providers: HashSet, + fallback_providers: Option>, } impl ModelResolutionTransform { @@ -23,6 +24,7 @@ impl ModelResolutionTransform { catalog, default_provider: None, eligible_providers, + fallback_providers: None, } } @@ -32,6 +34,7 @@ impl ModelResolutionTransform { catalog, default_provider: None, eligible_providers, + fallback_providers: None, } } @@ -41,16 +44,33 @@ impl ModelResolutionTransform { self } + #[must_use] + pub fn with_fallback_providers( + mut self, + fallback_providers: Option>, + ) -> Self { + self.fallback_providers = fallback_providers; + self + } + fn resolve_model( &self, model: &str, explicit_provider: Option<&ProviderId>, ) -> Result<(String, ProviderId), Error> { - let selected = self.catalog.resolve_selection( - Some(model), - explicit_provider, - &self.eligible_providers, - )?; + let selected = match &self.fallback_providers { + Some(fallback_providers) => self.catalog.resolve_selection_with_fallback( + Some(model), + explicit_provider, + &self.eligible_providers, + fallback_providers, + ), + None => self.catalog.resolve_selection( + Some(model), + explicit_provider, + &self.eligible_providers, + ), + }?; Ok((selected.model, selected.provider)) } } @@ -326,6 +346,50 @@ reasoning = false ); } + #[test] + fn fallback_resolution_keeps_ready_preference_for_unpinned_nodes() { + let overrides: LlmCatalogSettings = toml::from_str( + r" +[providers.openrouter] +enabled = true +", + ) + .unwrap(); + let catalog = Arc::new(Catalog::from_builtin_with_overrides(&overrides).unwrap()); + let mut graph = Graph::new("test"); + let mut portable = Node::new("portable"); + portable.attrs.insert( + "model".to_string(), + AttrValue::String("claude-fable".to_string()), + ); + graph.nodes.insert("portable".to_string(), portable); + let mut pinned = Node::new("pinned"); + pinned.attrs.insert( + "model".to_string(), + AttrValue::String("claude-fable".to_string()), + ); + pinned.attrs.insert( + "provider".to_string(), + AttrValue::String("anthropic".to_string()), + ); + graph.nodes.insert("pinned".to_string(), pinned); + + let graph = ModelResolutionTransform::for_eligible( + Arc::clone(&catalog), + HashSet::from([ProviderId::new("openrouter")]), + ) + .with_fallback_providers(Some(catalog.all_provider_ids())) + .apply(graph) + .unwrap(); + + assert_eq!( + graph.nodes["portable"].provider(), + Some("openrouter"), + "the unrelated unavailable pin must not force catalog-wide routing" + ); + assert_eq!(graph.nodes["pinned"].provider(), Some("anthropic")); + } + #[test] fn graph_default_alias_materializes_to_canonical_offering() { let mut graph = Graph::new("test"); diff --git a/lib/components/fabro-workflow/tests/it/integration.rs b/lib/components/fabro-workflow/tests/it/integration.rs index 5f3f75a55..44307ca81 100644 --- a/lib/components/fabro-workflow/tests/it/integration.rs +++ b/lib/components/fabro-workflow/tests/it/integration.rs @@ -4908,6 +4908,7 @@ async fn import_e2e_through_engine() { catalog: std::sync::Arc::clone(&catalog), default_provider: None, eligible_providers: catalog.all_provider_ids(), + fallback_providers: None, }) .unwrap(); let validated = validate(transformed, catalog.as_ref(), &[]); diff --git a/lib/foundation/fabro-model/src/catalog.rs b/lib/foundation/fabro-model/src/catalog.rs index cac9f878f..da9ab96fd 100644 --- a/lib/foundation/fabro-model/src/catalog.rs +++ b/lib/foundation/fabro-model/src/catalog.rs @@ -1101,6 +1101,32 @@ impl Catalog { } } + /// Resolve a selection against a preferred provider snapshot, falling back + /// to a broader eligible set only when the preferred set cannot supply the + /// requested provider or model. + /// + /// This is useful for readiness checks: ready providers remain preferred, + /// while a catalog-only offering can still be selected so the caller can + /// report why its provider is unavailable. Semantic failures such as an + /// unknown provider do not fall back. + pub fn resolve_selection_with_fallback( + &self, + selector: Option<&str>, + explicit_provider: Option<&ProviderId>, + preferred_providers: &HashSet, + fallback_providers: &HashSet, + ) -> Result { + match self.resolve_selection(selector, explicit_provider, preferred_providers) { + Ok(selected) => Ok(selected), + Err( + ModelSelectionError::ProviderUnavailable { .. } + | ModelSelectionError::NoEligibleOffering { .. } + | ModelSelectionError::NoDefaultModel { .. }, + ) => self.resolve_selection(selector, explicit_provider, fallback_providers), + Err(error) => Err(error), + } + } + #[must_use] pub fn is_model_selector(&self, selector: &str) -> bool { self.candidate_indices(selector).is_some() @@ -3989,6 +4015,41 @@ adapter = "openai_compatible" )); } + #[test] + fn selection_fallback_preserves_ready_preference_per_request() { + let catalog = portable_model_catalog(); + let openai = ProviderId::openai(); + let openrouter = ProviderId::new("openrouter"); + let ready = HashSet::from([openrouter.clone()]); + let catalog_providers = HashSet::from([openai.clone(), openrouter.clone()]); + + let shared = catalog + .resolve_selection_with_fallback(Some("portable"), None, &ready, &catalog_providers) + .unwrap(); + assert_eq!(shared.provider, openrouter); + + let pinned = catalog + .resolve_selection_with_fallback( + Some("portable"), + Some(&openai), + &ready, + &catalog_providers, + ) + .unwrap(); + assert_eq!(pinned.provider, openai); + + let unknown = catalog + .resolve_selection_with_fallback( + Some("provider-private-preview"), + None, + &ready, + &catalog_providers, + ) + .unwrap(); + assert_eq!(unknown.provider, ProviderId::new("openrouter")); + assert_eq!(unknown.model, "provider-private-preview"); + } + #[test] fn legacy_builtin_selector_uses_readiness_priority_and_explicit_pins() { let catalog = portable_model_catalog(); From 142862f3422d72ab4106796342fdb4bbd0245da8 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 24 Jul 2026 07:36:56 -0400 Subject: [PATCH 12/24] Expose detailed completion token usage --- docs/public/api-reference/fabro-api.yaml | 25 ++++++- .../src/server/handler/completions.rs | 18 ++--- lib/apps/fabro-server/src/server/tests.rs | 67 +++++++++++++++++++ lib/foundation/fabro-api/build.rs | 1 + lib/foundation/fabro-api/src/lib.rs | 1 + .../tests/completion_usage_round_trip.rs | 59 ++++++++++++++++ .../src/models/completion-usage.ts | 21 ++++++ 7 files changed, 179 insertions(+), 13 deletions(-) create mode 100644 lib/foundation/fabro-api/tests/completion_usage_round_trip.rs diff --git a/docs/public/api-reference/fabro-api.yaml b/docs/public/api-reference/fabro-api.yaml index 519ee4188..342ef745a 100644 --- a/docs/public/api-reference/fabro-api.yaml +++ b/docs/public/api-reference/fabro-api.yaml @@ -8507,15 +8507,38 @@ components: description: Provider-specific options. CompletionUsage: + description: > + Five disjoint token buckets for one completion. `input_tokens` excludes + cache reads and writes, while `output_tokens` excludes reasoning tokens + when the provider reports them separately. type: object - required: [input_tokens, output_tokens] + required: + - input_tokens + - output_tokens + - reasoning_tokens + - cache_read_tokens + - cache_write_tokens properties: input_tokens: type: integer format: int64 + description: Number of uncached input tokens consumed. output_tokens: type: integer format: int64 + description: Number of non-reasoning output tokens generated. + reasoning_tokens: + type: integer + format: int64 + description: Number of separately reported reasoning tokens. + cache_read_tokens: + type: integer + format: int64 + description: Number of input tokens served from a provider cache. + cache_write_tokens: + type: integer + format: int64 + description: Number of input tokens written to a provider cache. CompletionResponse: type: object diff --git a/lib/apps/fabro-server/src/server/handler/completions.rs b/lib/apps/fabro-server/src/server/handler/completions.rs index b53e56d31..a8d7c3bb4 100644 --- a/lib/apps/fabro-server/src/server/handler/completions.rs +++ b/lib/apps/fabro-server/src/server/handler/completions.rs @@ -4,10 +4,10 @@ use std::sync::Arc; use fabro_model::{Catalog, ModelSelectionError}; use super::super::{ - ApiError, AppState, CompletionResponse, CompletionToolChoiceMode, CompletionUsage, - CreateCompletionRequest, FinishReason, GenerateParams, IntoResponse, Json, LlmMessage, - LlmRequest, ProviderId, RequiredUser, Response, Router, State, StatusCode, ToolChoice, - ToolDefinition, Ulid, error, generate_object, info, post, warn, + ApiError, AppState, CompletionResponse, CompletionToolChoiceMode, CreateCompletionRequest, + FinishReason, GenerateParams, IntoResponse, Json, LlmMessage, LlmRequest, ProviderId, + RequiredUser, Response, Router, State, StatusCode, ToolChoice, ToolDefinition, Ulid, error, + generate_object, info, post, warn, }; use super::llm_sse; @@ -169,10 +169,7 @@ async fn create_completion( provider: selected_provider, message: response.message, stop_reason, - usage: CompletionUsage { - input_tokens: response.usage.input_tokens, - output_tokens: response.usage.output_tokens, - }, + usage: response.usage, output, cost_usd: response.cost_usd, cost_source: response.cost_source, @@ -192,10 +189,7 @@ async fn create_completion( provider: ProviderId::new(response.provider), message: response.message, stop_reason, - usage: CompletionUsage { - input_tokens: response.usage.input_tokens, - output_tokens: response.usage.output_tokens, - }, + usage: response.usage, output: None, cost_usd: response.cost_usd, cost_source: response.cost_source, diff --git a/lib/apps/fabro-server/src/server/tests.rs b/lib/apps/fabro-server/src/server/tests.rs index 454455be0..982b270f6 100644 --- a/lib/apps/fabro-server/src/server/tests.rs +++ b/lib/apps/fabro-server/src/server/tests.rs @@ -15272,6 +15272,73 @@ async fn create_completion_unknown_provider_returns_clear_error() { ); } +#[tokio::test] +async fn create_completion_returns_disjoint_usage_buckets() { + let upstream = MockServer::start(); + let completion = upstream.mock(|when, then| { + when.method(POST).path("/chat/completions"); + then.status(200) + .header("content-type", "application/json") + .json_body(json!({ + "id": "chatcmpl-usage", + "model": "kimi-k3", + "choices": [{ + "message": {"role": "assistant", "content": "OK"}, + "finish_reason": "stop" + }], + "usage": { + "prompt_tokens": 200, + "completion_tokens": 30, + "total_tokens": 230, + "prompt_tokens_details": { + "cached_tokens": 50, + "cache_write_tokens": 100 + }, + "completion_tokens_details": { + "reasoning_tokens": 20 + } + } + })); + }); + let state = TestAppStateBuilder::new() + .provider_base_url("kimi", upstream.base_url()) + .vault_entries([(EnvVars::KIMI_API_KEY, "test-kimi-api-key")]) + .build(); + let app = crate::test_support::build_test_router(state); + + let req = Request::builder() + .method("POST") + .uri(api("/completions")) + .header("content-type", "application/json") + .body(Body::from( + json!({ + "provider": "kimi", + "model": "kimi-k3", + "stream": false, + "messages": [{ + "role": "user", + "content": [{"kind": "text", "data": "hi"}] + }] + }) + .to_string(), + )) + .unwrap(); + + let response = app.oneshot(req).await.unwrap(); + let body = response_json!(response, StatusCode::OK).await; + assert_eq!( + body["usage"], + json!({ + "input_tokens": 50, + "output_tokens": 10, + "reasoning_tokens": 20, + "cache_read_tokens": 50, + "cache_write_tokens": 100 + }) + ); + completion.assert(); +} + #[tokio::test] async fn create_completion_default_model_uses_app_state_catalog() { let upstream = MockServer::start(); diff --git a/lib/foundation/fabro-api/build.rs b/lib/foundation/fabro-api/build.rs index cdec7f07a..f979a22d1 100644 --- a/lib/foundation/fabro-api/build.rs +++ b/lib/foundation/fabro-api/build.rs @@ -461,6 +461,7 @@ fn main() { "fabro_types::PendingInterviewRecord", &[], ), + ("CompletionUsage", "fabro_model::TokenCounts", &[]), ("BilledTokenCounts", "fabro_types::BilledTokenCounts", &[]), ("BillingModelRef", "fabro_model::ModelRef", &[]), ("BillingSpeed", "fabro_model::Speed", &[]), diff --git a/lib/foundation/fabro-api/src/lib.rs b/lib/foundation/fabro-api/src/lib.rs index 6d68d57b4..4a7a65bcd 100644 --- a/lib/foundation/fabro-api/src/lib.rs +++ b/lib/foundation/fabro-api/src/lib.rs @@ -22,6 +22,7 @@ pub mod types { pub use fabro_model::{ CostSource, Model, ModelCosts, ModelFeatures, ModelLimits, ModelRef as BillingModelRef, ModelTestMode, Provider, ReasoningEffort, ReasoningEffortFeature, Speed as BillingSpeed, + TokenCounts as CompletionUsage, }; pub use fabro_types::run_event::AgentSessionActivatedProps; pub use fabro_types::settings::run::McpHttpProtocol; diff --git a/lib/foundation/fabro-api/tests/completion_usage_round_trip.rs b/lib/foundation/fabro-api/tests/completion_usage_round_trip.rs new file mode 100644 index 000000000..58da0288b --- /dev/null +++ b/lib/foundation/fabro-api/tests/completion_usage_round_trip.rs @@ -0,0 +1,59 @@ +use std::any::{TypeId, type_name}; + +use fabro_api::types::CompletionUsage as ApiCompletionUsage; +use fabro_model::TokenCounts; +use serde_json::json; + +#[test] +fn completion_usage_reuses_canonical_type() { + assert_same_type::(); +} + +#[test] +fn completion_usage_json_matches_openapi_shape() { + let usage = TokenCounts { + input_tokens: 10, + output_tokens: 20, + reasoning_tokens: 3, + cache_read_tokens: 4, + cache_write_tokens: 5, + }; + + let json = serde_json::to_value(&usage).unwrap(); + assert_eq!(json["input_tokens"], 10); + assert_eq!(json["output_tokens"], 20); + assert_eq!(json["reasoning_tokens"], 3); + assert_eq!(json["cache_read_tokens"], 4); + assert_eq!(json["cache_write_tokens"], 5); + + let round_trip: ApiCompletionUsage = serde_json::from_value(json).unwrap(); + assert_eq!(round_trip, usage); +} + +#[test] +fn completion_usage_keeps_zero_counts_present() { + let json = serde_json::to_value(TokenCounts::default()).unwrap(); + assert_eq!( + json, + json!({ + "input_tokens": 0, + "output_tokens": 0, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }) + ); + + let round_trip: ApiCompletionUsage = serde_json::from_value(json).unwrap(); + assert_eq!(round_trip, TokenCounts::default()); +} + +fn assert_same_type() { + assert_eq!( + TypeId::of::(), + TypeId::of::(), + "{} should be the same type as {}", + type_name::(), + type_name::() + ); +} diff --git a/lib/packages/fabro-api-client/src/models/completion-usage.ts b/lib/packages/fabro-api-client/src/models/completion-usage.ts index d10caeb7d..056007b1b 100644 --- a/lib/packages/fabro-api-client/src/models/completion-usage.ts +++ b/lib/packages/fabro-api-client/src/models/completion-usage.ts @@ -14,7 +14,28 @@ +/** + * Five disjoint token buckets for one completion. `input_tokens` excludes cache reads and writes, while `output_tokens` excludes reasoning tokens when the provider reports them separately. + */ export interface CompletionUsage { + /** + * Number of uncached input tokens consumed. + */ 'input_tokens': number; + /** + * Number of non-reasoning output tokens generated. + */ 'output_tokens': number; + /** + * Number of separately reported reasoning tokens. + */ + 'reasoning_tokens': number; + /** + * Number of input tokens served from a provider cache. + */ + 'cache_read_tokens': number; + /** + * Number of input tokens written to a provider cache. + */ + 'cache_write_tokens': number; } From 4bd975321773a5ede5cf834b4dcecf678da038f3 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 24 Jul 2026 07:44:40 -0400 Subject: [PATCH 13/24] Expose model reasoning effort controls --- docs/public/api-reference/fabro-api.yaml | 17 +++++ lib/apps/fabro-cli/src/commands/model.rs | 6 +- lib/apps/fabro-server/src/server/tests.rs | 29 +++++++++ lib/components/fabro-llm/src/model_test.rs | 5 +- .../fabro-api/tests/model_round_trip.rs | 14 ++++- .../fabro-api/tests/provider_id_round_trip.rs | 4 +- lib/foundation/fabro-model/src/catalog.rs | 63 ++++++++++++++++++- lib/foundation/fabro-model/src/lib.rs | 4 +- lib/foundation/fabro-model/src/types.rs | 12 ++++ .../src/.openapi-generator/FILES | 1 + .../fabro-api-client/src/models/index.ts | 1 + .../src/models/model-controls.ts | 28 +++++++++ .../fabro-api-client/src/models/model.ts | 4 ++ 13 files changed, 182 insertions(+), 6 deletions(-) create mode 100644 lib/packages/fabro-api-client/src/models/model-controls.ts diff --git a/docs/public/api-reference/fabro-api.yaml b/docs/public/api-reference/fabro-api.yaml index 519ee4188..1260db2b4 100644 --- a/docs/public/api-reference/fabro-api.yaml +++ b/docs/public/api-reference/fabro-api.yaml @@ -8289,6 +8289,20 @@ components: description: Cost per million cached input tokens in USD. example: 1.50 + ModelControls: + description: Request-control values accepted by a provider/model offering. + type: object + required: + - reasoning_effort + properties: + reasoning_effort: + type: array + description: >- + Exact reasoning-effort values accepted by this offering. An empty + array means the request control is unsupported. + items: + $ref: "#/components/schemas/ReasoningEffort" + Model: description: | One provider's offering of an LLM model. The `id` is unique within @@ -8303,6 +8317,7 @@ components: - training - knowledge_cutoff - features + - controls - costs - estimated_output_tps - aliases @@ -8336,6 +8351,8 @@ components: example: "May 2025" features: $ref: "#/components/schemas/ModelFeatures" + controls: + $ref: "#/components/schemas/ModelControls" costs: $ref: "#/components/schemas/ModelCosts" estimated_output_tps: diff --git a/lib/apps/fabro-cli/src/commands/model.rs b/lib/apps/fabro-cli/src/commands/model.rs index f84bd44e2..774feae42 100644 --- a/lib/apps/fabro-cli/src/commands/model.rs +++ b/lib/apps/fabro-cli/src/commands/model.rs @@ -517,7 +517,9 @@ impl Default for ModelsCommand { #[cfg(test)] mod tests { - use fabro_model::{ModelCosts, ModelFeatures, ModelLimits, ReasoningEffortFeature}; + use fabro_model::{ + ModelControls, ModelCosts, ModelFeatures, ModelLimits, ReasoningEffortFeature, + }; use super::*; @@ -546,6 +548,7 @@ mod tests { cache_control_breakpoints: false, sampling_params: true, }, + controls: ModelControls::default(), costs: ModelCosts { input_cost_per_mtok: Some(1.0), output_cost_per_mtok: Some(2.0), @@ -581,6 +584,7 @@ mod tests { cache_control_breakpoints: false, sampling_params: true, }, + controls: ModelControls::default(), costs: ModelCosts { input_cost_per_mtok: Some(1.0), output_cost_per_mtok: Some(2.0), diff --git a/lib/apps/fabro-server/src/server/tests.rs b/lib/apps/fabro-server/src/server/tests.rs index 454455be0..cf59270ce 100644 --- a/lib/apps/fabro-server/src/server/tests.rs +++ b/lib/apps/fabro-server/src/server/tests.rs @@ -6768,6 +6768,35 @@ async fn list_models_filters_by_provider() { ); } +#[tokio::test] +async fn list_models_exposes_reasoning_effort_controls() { + let app = test_app_with(); + + let req = Request::builder() + .method("GET") + .uri(api("/models?provider=kimi")) + .body(Body::empty()) + .unwrap(); + + let response = app.oneshot(req).await.unwrap(); + let body = response_json!(response, StatusCode::OK).await; + let models = body["data"].as_array().unwrap(); + let kimi_k3 = models + .iter() + .find(|model| model["id"] == "kimi-k3") + .expect("Kimi K3 should be listed"); + let kimi_k2_5 = models + .iter() + .find(|model| model["id"] == "kimi-k2.5") + .expect("Kimi K2.5 should be listed"); + + assert_eq!( + kimi_k3["controls"]["reasoning_effort"], + json!(["low", "high", "max"]) + ); + assert_eq!(kimi_k2_5["controls"]["reasoning_effort"], json!([])); +} + #[tokio::test] async fn list_models_marks_configured_true_when_provider_has_credential_material() { let state = test_app_state_with_env_lookup( diff --git a/lib/components/fabro-llm/src/model_test.rs b/lib/components/fabro-llm/src/model_test.rs index aec0a83a7..6a985fb1b 100644 --- a/lib/components/fabro-llm/src/model_test.rs +++ b/lib/components/fabro-llm/src/model_test.rs @@ -169,7 +169,9 @@ fn validate_deep_result(result: &GenerateResult) -> Result<(), String> { mod tests { use std::collections::HashMap; - use fabro_model::{ModelCosts, ModelFeatures, ModelLimits, ProviderId, ReasoningEffortFeature}; + use fabro_model::{ + ModelControls, ModelCosts, ModelFeatures, ModelLimits, ProviderId, ReasoningEffortFeature, + }; use super::*; use crate::types::{FinishReason, Message, Response, StepResult, TokenCounts, ToolResult}; @@ -187,6 +189,7 @@ mod tests { training: None, knowledge_cutoff: None, features, + controls: ModelControls::default(), costs: ModelCosts { input_cost_per_mtok: None, output_cost_per_mtok: None, diff --git a/lib/foundation/fabro-api/tests/model_round_trip.rs b/lib/foundation/fabro-api/tests/model_round_trip.rs index 57d3db557..5f39eb6ab 100644 --- a/lib/foundation/fabro-api/tests/model_round_trip.rs +++ b/lib/foundation/fabro-api/tests/model_round_trip.rs @@ -2,7 +2,8 @@ use std::any::{TypeId, type_name}; use fabro_api::types::Model as ApiModel; use fabro_model::{ - Model, ModelCosts, ModelFeatures, ModelLimits, ProviderId, ReasoningEffortFeature, + Model, ModelControls, ModelCosts, ModelFeatures, ModelLimits, ProviderId, ReasoningEffort, + ReasoningEffortFeature, }; #[test] @@ -32,6 +33,13 @@ fn model_json_matches_openapi_shape() { cache_control_breakpoints: false, sampling_params: true, }, + controls: ModelControls { + reasoning_effort: vec![ + ReasoningEffort::Low, + ReasoningEffort::High, + ReasoningEffort::Max, + ], + }, costs: ModelCosts { input_cost_per_mtok: Some(5.0), output_cost_per_mtok: Some(25.0), @@ -50,6 +58,10 @@ fn model_json_matches_openapi_shape() { assert_eq!(json["knowledge_cutoff"], "May 2025"); assert_eq!(json["features"]["reasoning_effort"], "levels"); assert_eq!(json["features"]["prompt_cache"], true); + assert_eq!( + json["controls"]["reasoning_effort"], + serde_json::json!(["low", "high", "max"]) + ); assert_eq!(json["estimated_output_tps"], 25.0); assert_eq!(json["small_default"], true); assert_eq!(json["configured"], true); diff --git a/lib/foundation/fabro-api/tests/provider_id_round_trip.rs b/lib/foundation/fabro-api/tests/provider_id_round_trip.rs index 970c7158f..2c834ac3f 100644 --- a/lib/foundation/fabro-api/tests/provider_id_round_trip.rs +++ b/lib/foundation/fabro-api/tests/provider_id_round_trip.rs @@ -2,7 +2,8 @@ use std::any::{TypeId, type_name}; use fabro_api::types::Model as ApiModel; use fabro_model::{ - Model, ModelCosts, ModelFeatures, ModelLimits, ProviderId, ReasoningEffortFeature, + Model, ModelControls, ModelCosts, ModelFeatures, ModelLimits, ProviderId, + ReasoningEffortFeature, }; use serde_json::json; @@ -42,6 +43,7 @@ fn provider_id_json_matches_openapi_shape_through_model() { cache_control_breakpoints: false, sampling_params: true, }, + controls: ModelControls::default(), costs: ModelCosts { input_cost_per_mtok: None, output_cost_per_mtok: None, diff --git a/lib/foundation/fabro-model/src/catalog.rs b/lib/foundation/fabro-model/src/catalog.rs index cac9f878f..b0f57712f 100644 --- a/lib/foundation/fabro-model/src/catalog.rs +++ b/lib/foundation/fabro-model/src/catalog.rs @@ -15,7 +15,9 @@ use crate::codec::CodecKind; use crate::ids::{ModelId, ProviderId}; use crate::provider::Provider; use crate::reasoning::ReasoningEffort; -use crate::types::{Model, ModelCosts, ModelFeatures, ModelLimits, ReasoningEffortFeature}; +use crate::types::{ + Model, ModelControls, ModelCosts, ModelFeatures, ModelLimits, ReasoningEffortFeature, +}; #[derive(RustEmbed)] #[folder = "src/catalog/providers"] @@ -2203,6 +2205,9 @@ fn build_model( training: settings.training.clone(), knowledge_cutoff: settings.knowledge_cutoff.clone(), features: model_features, + controls: ModelControls { + reasoning_effort: controls.reasoning_effort.clone(), + }, costs, estimated_output_tps: settings.estimated_output_tps, aliases: settings.aliases.clone().unwrap_or_default(), @@ -3245,6 +3250,12 @@ enabled = true cache_control_breakpoints: false, sampling_params: true, }, + controls: ModelControls { + reasoning_effort: [ + High, + XHigh, + ], + }, costs: ModelCosts { input_cost_per_mtok: Some( 0.784, @@ -3310,6 +3321,13 @@ enabled = true cache_control_breakpoints: false, sampling_params: false, }, + controls: ModelControls { + reasoning_effort: [ + Low, + High, + Max, + ], + }, costs: ModelCosts { input_cost_per_mtok: Some( 3.0, @@ -5910,6 +5928,15 @@ sampling_params = false cache_control_breakpoints: false, sampling_params: true, }, + controls: ModelControls { + reasoning_effort: [ + Low, + Medium, + High, + XHigh, + Max, + ], + }, costs: ModelCosts { input_cost_per_mtok: Some( 5.0, @@ -5967,6 +5994,9 @@ sampling_params = false cache_control_breakpoints: false, sampling_params: false, }, + controls: ModelControls { + reasoning_effort: [], + }, costs: ModelCosts { input_cost_per_mtok: Some( 0.6, @@ -6016,6 +6046,13 @@ sampling_params = false cache_control_breakpoints: false, sampling_params: false, }, + controls: ModelControls { + reasoning_effort: [ + Low, + High, + Max, + ], + }, costs: ModelCosts { input_cost_per_mtok: Some( 3.0, @@ -6089,6 +6126,12 @@ sampling_params = false cache_control_breakpoints: false, sampling_params: true, }, + controls: ModelControls { + reasoning_effort: [ + High, + Max, + ], + }, costs: ModelCosts { input_cost_per_mtok: Some( 1.4, @@ -6155,6 +6198,15 @@ sampling_params = false cache_control_breakpoints: false, sampling_params: true, }, + controls: ModelControls { + reasoning_effort: [ + Low, + Medium, + High, + XHigh, + Max, + ], + }, costs: ModelCosts { input_cost_per_mtok: Some( 0.25, @@ -6212,6 +6264,15 @@ sampling_params = false cache_control_breakpoints: false, sampling_params: true, }, + controls: ModelControls { + reasoning_effort: [ + Low, + Medium, + High, + XHigh, + Max, + ], + }, costs: ModelCosts { input_cost_per_mtok: Some( 30.0, diff --git a/lib/foundation/fabro-model/src/lib.rs b/lib/foundation/fabro-model/src/lib.rs index 5f3511096..fc43c95a7 100644 --- a/lib/foundation/fabro-model/src/lib.rs +++ b/lib/foundation/fabro-model/src/lib.rs @@ -27,4 +27,6 @@ pub use model_ref::ModelHandle; pub use model_test::ModelTestMode; pub use provider::Provider; pub use reasoning::ReasoningEffort; -pub use types::{Model, ModelCosts, ModelFeatures, ModelLimits, ReasoningEffortFeature}; +pub use types::{ + Model, ModelControls, ModelCosts, ModelFeatures, ModelLimits, ReasoningEffortFeature, +}; diff --git a/lib/foundation/fabro-model/src/types.rs b/lib/foundation/fabro-model/src/types.rs index ae3804784..9a0d27148 100644 --- a/lib/foundation/fabro-model/src/types.rs +++ b/lib/foundation/fabro-model/src/types.rs @@ -1,6 +1,7 @@ use serde::{Deserialize, Serialize}; use crate::ids::{ModelId, ProviderId}; +use crate::reasoning::ReasoningEffort; // --- 2.9 Model --- @@ -83,6 +84,14 @@ pub struct ModelCosts { pub cache_input_cost_per_mtok: Option, } +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct ModelControls { + /// Exact reasoning-effort values accepted by this provider/model offering. + /// An empty list means the request control is unsupported. + #[serde(default)] + pub reasoning_effort: Vec, +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct Model { pub id: ModelId, @@ -93,6 +102,8 @@ pub struct Model { pub training: Option, pub knowledge_cutoff: Option, pub features: ModelFeatures, + #[serde(default)] + pub controls: ModelControls, pub costs: ModelCosts, pub estimated_output_tps: Option, pub aliases: Vec, @@ -236,6 +247,7 @@ mod tests { cache_control_breakpoints: false, sampling_params: true, }, + controls: ModelControls::default(), costs: ModelCosts { input_cost_per_mtok: Some(1.0), output_cost_per_mtok: Some(2.0), diff --git a/lib/packages/fabro-api-client/src/.openapi-generator/FILES b/lib/packages/fabro-api-client/src/.openapi-generator/FILES index 52393c07a..bc64eab3f 100644 --- a/lib/packages/fabro-api-client/src/.openapi-generator/FILES +++ b/lib/packages/fabro-api-client/src/.openapi-generator/FILES @@ -226,6 +226,7 @@ models/mcp-transport.ts models/merge-method.ts models/merge-run-pull-request-request.ts models/merge-run-pull-request-response.ts +models/model-controls.ts models/model-costs.ts models/model-features.ts models/model-limits.ts diff --git a/lib/packages/fabro-api-client/src/models/index.ts b/lib/packages/fabro-api-client/src/models/index.ts index 88f4a75d2..8a3bfd568 100644 --- a/lib/packages/fabro-api-client/src/models/index.ts +++ b/lib/packages/fabro-api-client/src/models/index.ts @@ -197,6 +197,7 @@ export * from './merge-method'; export * from './merge-run-pull-request-request'; export * from './merge-run-pull-request-response'; export * from './model'; +export * from './model-controls'; export * from './model-costs'; export * from './model-features'; export * from './model-limits'; diff --git a/lib/packages/fabro-api-client/src/models/model-controls.ts b/lib/packages/fabro-api-client/src/models/model-controls.ts new file mode 100644 index 000000000..59f22ce2a --- /dev/null +++ b/lib/packages/fabro-api-client/src/models/model-controls.ts @@ -0,0 +1,28 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * Fabro Run API + * HTTP API for managing Fabro workflow run executions. + * + * The version of the OpenAPI document: 0.1.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + +// May contain unused imports in some cases +// @ts-ignore +import type { ReasoningEffort } from './reasoning-effort'; + +/** + * Request-control values accepted by a provider/model offering. + */ +export interface ModelControls { + /** + * Exact reasoning-effort values accepted by this offering. An empty array means the request control is unsupported. + */ + 'reasoning_effort': Array; +} diff --git a/lib/packages/fabro-api-client/src/models/model.ts b/lib/packages/fabro-api-client/src/models/model.ts index d3326e2db..0bf86bee5 100644 --- a/lib/packages/fabro-api-client/src/models/model.ts +++ b/lib/packages/fabro-api-client/src/models/model.ts @@ -13,6 +13,9 @@ */ +// May contain unused imports in some cases +// @ts-ignore +import type { ModelControls } from './model-controls'; // May contain unused imports in some cases // @ts-ignore import type { ModelCosts } from './model-costs'; @@ -53,6 +56,7 @@ export interface Model { */ 'knowledge_cutoff': string | null; 'features': ModelFeatures; + 'controls': ModelControls; 'costs': ModelCosts; /** * Estimated output tokens per second. From 40d6992148f26a5c0b872d197a947cf03b1beb05 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 24 Jul 2026 08:13:54 -0400 Subject: [PATCH 14/24] Trim redundant reasoning effort request tests The unknown-value rejection test duplicated strum coverage in fabro-model and the HTTP 422 test in fabro-server. Keep only the field-type assertion, using the same field-pinning idiom as stage_model_usage_round_trip. Co-Authored-By: Claude Fable 5 --- .../tests/create_completion_request_round_trip.rs | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/lib/foundation/fabro-api/tests/create_completion_request_round_trip.rs b/lib/foundation/fabro-api/tests/create_completion_request_round_trip.rs index 913947453..084827149 100644 --- a/lib/foundation/fabro-api/tests/create_completion_request_round_trip.rs +++ b/lib/foundation/fabro-api/tests/create_completion_request_round_trip.rs @@ -10,16 +10,5 @@ fn create_completion_request_reuses_canonical_reasoning_effort() { })) .unwrap(); - let reasoning_effort: Option = request.reasoning_effort; - assert_eq!(reasoning_effort, Some(ReasoningEffort::High)); -} - -#[test] -fn create_completion_request_rejects_unknown_reasoning_effort() { - let result = serde_json::from_value::(json!({ - "messages": [], - "reasoning_effort": "bogus" - })); - - assert!(result.is_err()); + assert_eq!(request.reasoning_effort, Some(ReasoningEffort::High)); } From af27e98e1cade1d2225a63eddfcdb3022be177bd Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 24 Jul 2026 08:28:58 -0400 Subject: [PATCH 15/24] refactor: simplify parallel handler and overview parsing - Extract emit_branch_completed() to replace three near-identical ParallelBranchCompleted constructions; status now reads consistently from outcome.status - Add context_diff_public() so parallel.rs and manager_loop.rs share the diff-minus-engine-internal-keys step; move context_diff tests next to the function in context.rs - Replace fan_in's dead BranchShape struct with the canonical Vec (from_value moves, so no payload cloning) - Narrow parseParallelOverview to ParallelBranchSummary {id, status}; its only consumer renders just those fields - Drop helpers.test.ts's duplicate envelope() fixture in favor of the shared makeEventEnvelope Co-Authored-By: Claude Fable 5 --- .../stage-renderers/helpers.test.ts | 66 +++++---------- .../app/components/stage-renderers/helpers.ts | 25 +++--- lib/components/fabro-workflow/src/context.rs | 77 +++++++++++++++++ .../fabro-workflow/src/handler/fan_in.rs | 15 +--- .../src/handler/manager_loop.rs | 79 +----------------- .../fabro-workflow/src/handler/parallel.rs | 83 +++++++++++-------- 6 files changed, 166 insertions(+), 179 deletions(-) diff --git a/apps/fabro-web/app/components/stage-renderers/helpers.test.ts b/apps/fabro-web/app/components/stage-renderers/helpers.test.ts index 4fcf47e95..0982cdc99 100644 --- a/apps/fabro-web/app/components/stage-renderers/helpers.test.ts +++ b/apps/fabro-web/app/components/stage-renderers/helpers.test.ts @@ -1,6 +1,7 @@ import { describe, expect, test } from "bun:test"; import type { EventEnvelope } from "@qltysh/fabro-api-client"; +import { makeEventEnvelope } from "../../lib/test-utils"; import { extractStageContext, parseHumanInterviewPairs, @@ -8,21 +9,10 @@ import { parseReducerTranscript, } from "./helpers"; -function envelope(seq: number, partial: Partial): EventEnvelope { - return { - seq, - id: `evt-${seq}`, - ts: `2026-04-09T12:00:0${seq}Z`, - run_id: "run-1", - event: "stage.prompt", - ...partial, - } as EventEnvelope; -} - describe("parseHumanInterviewPairs", () => { test("pairs interview.started with interview.completed by question_id", () => { const events: EventEnvelope[] = [ - envelope(1, { + makeEventEnvelope(1, { event: "interview.started", properties: { question_id: "q-1", @@ -35,7 +25,7 @@ describe("parseHumanInterviewPairs", () => { allow_freeform: false, }, }), - envelope(2, { + makeEventEnvelope(2, { event: "interview.completed", properties: { question_id: "q-1", @@ -65,7 +55,7 @@ describe("parseHumanInterviewPairs", () => { test("leaves resolution null for unanswered (still pending) questions", () => { const events: EventEnvelope[] = [ - envelope(1, { + makeEventEnvelope(1, { event: "interview.started", properties: { question_id: "q-1", @@ -80,7 +70,7 @@ describe("parseHumanInterviewPairs", () => { test("preserves option description and preview metadata from started events", () => { const events: EventEnvelope[] = [ - envelope(1, { + makeEventEnvelope(1, { event: "interview.started", properties: { question_id: "q-1", @@ -110,19 +100,19 @@ describe("parseHumanInterviewPairs", () => { test("captures timeout and interrupted resolutions", () => { const events: EventEnvelope[] = [ - envelope(1, { + makeEventEnvelope(1, { event: "interview.started", properties: { question_id: "q-1", question: "?", question_type: "freeform" }, }), - envelope(2, { + makeEventEnvelope(2, { event: "interview.timeout", properties: { question_id: "q-1", duration_ms: 30000 }, }), - envelope(3, { + makeEventEnvelope(3, { event: "interview.started", properties: { question_id: "q-2", question: "?", question_type: "freeform" }, }), - envelope(4, { + makeEventEnvelope(4, { event: "interview.interrupted", properties: { question_id: "q-2", @@ -145,11 +135,11 @@ describe("parseHumanInterviewPairs", () => { describe("parseParallelOverview", () => { test("rolls up branch_count and status-only results", () => { const events: EventEnvelope[] = [ - envelope(1, { + makeEventEnvelope(1, { event: "parallel.started", properties: { branch_count: 3 }, }), - envelope(2, { + makeEventEnvelope(2, { event: "parallel.completed", properties: { duration_ms: 12000, @@ -182,21 +172,9 @@ describe("parseParallelOverview", () => { failureCount: 1, durationMs: 12000, results: [ - { - id: "branch-a", - status: "succeeded", - context_updates: { "response.branch-a": "A" }, - }, - { - id: "branch-b", - status: "succeeded", - context_updates: { "command.output": { stdout: "B" } }, - }, - { - id: "branch-c", - status: "failed", - context_updates: { "response.branch-c": "C" }, - }, + { id: "branch-a", status: "succeeded" }, + { id: "branch-b", status: "succeeded" }, + { id: "branch-c", status: "failed" }, ], isComplete: true, }); @@ -204,7 +182,7 @@ describe("parseParallelOverview", () => { test("reports in-flight when only the started event is present", () => { const events: EventEnvelope[] = [ - envelope(1, { + makeEventEnvelope(1, { event: "parallel.started", properties: { branch_count: 4 }, }), @@ -223,7 +201,7 @@ describe("parseReducerTranscript", () => { test("parses the standard prompt transcript when a reducer ran", () => { const events: EventEnvelope[] = [ - envelope(1, { + makeEventEnvelope(1, { event: "stage.prompt", properties: { mode: "prompt", @@ -231,7 +209,7 @@ describe("parseReducerTranscript", () => { model: "claude-sonnet-4-6", }, }), - envelope(2, { + makeEventEnvelope(2, { event: "prompt.completed", properties: { response: "The branch results are joined.", @@ -251,11 +229,11 @@ describe("parseReducerTranscript", () => { test("uses normal prompt mode for the reducer transcript", () => { const events: EventEnvelope[] = [ - envelope(1, { + makeEventEnvelope(1, { event: "stage.prompt", properties: { mode: "prompt", text: "Standard reducer" }, }), - envelope(2, { + makeEventEnvelope(2, { event: "prompt.completed", properties: { response: "Standard response" }, }), @@ -268,7 +246,7 @@ describe("parseReducerTranscript", () => { describe("extractStageContext", () => { test("keeps author-set keys and drops engine bookkeeping keys", () => { const events: EventEnvelope[] = [ - envelope(1, { + makeEventEnvelope(1, { event: "stage.completed", properties: { context_updates: { @@ -296,7 +274,7 @@ describe("extractStageContext", () => { test("extracts routing hints from preferred_label and suggested_next_ids", () => { const events: EventEnvelope[] = [ - envelope(1, { + makeEventEnvelope(1, { event: "stage.completed", properties: { preferred_label: "approve", @@ -312,7 +290,7 @@ describe("extractStageContext", () => { test("returns null when the stage only wrote engine keys", () => { const events: EventEnvelope[] = [ - envelope(1, { + makeEventEnvelope(1, { event: "stage.completed", properties: { context_updates: { last_stage: "implement", "command.output": "blob:x" }, diff --git a/apps/fabro-web/app/components/stage-renderers/helpers.ts b/apps/fabro-web/app/components/stage-renderers/helpers.ts index 3c631a9e2..2ea42b1f4 100644 --- a/apps/fabro-web/app/components/stage-renderers/helpers.ts +++ b/apps/fabro-web/app/components/stage-renderers/helpers.ts @@ -1,7 +1,5 @@ import { StageOutcome } from "@qltysh/fabro-api-client"; -import type { EventEnvelope, ParallelBranchResult } from "@qltysh/fabro-api-client"; - -export type { ParallelBranchResult }; +import type { EventEnvelope } from "@qltysh/fabro-api-client"; import { getArray, getNumber, getObject, getString, type UnknownRecord } from "../../lib/unknown"; @@ -146,12 +144,18 @@ export function parseHumanInterviewPairs(events: EventEnvelope[]): HumanIntervie return Array.from(pairs.values()).sort((a, b) => a.question.ts.localeCompare(b.question.ts)); } +/** Identity and outcome of one branch, parsed from `parallel.completed`. */ +export interface ParallelBranchSummary { + id: string; + status: StageOutcome; +} + export interface ParallelOverview { branchCount: number | null; successCount: number | null; failureCount: number | null; durationMs: number | null; - results: ParallelBranchResult[]; + results: ParallelBranchSummary[]; isComplete: boolean; } @@ -165,7 +169,7 @@ export function parseParallelOverview(events: EventEnvelope[]): ParallelOverview let successCount: number | null = null; let failureCount: number | null = null; let durationMs: number | null = null; - let results: ParallelBranchResult[] = []; + let results: ParallelBranchSummary[] = []; let isComplete = false; for (const event of events) { @@ -184,15 +188,10 @@ export function parseParallelOverview(events: EventEnvelope[]): ParallelOverview if (!record) return null; const id = getString(record, "id"); const status = asStageOutcome(getString(record, "status")); - const contextUpdates = getObject(record, "context_updates"); - if (!id || !status || !contextUpdates) return null; - return { - id, - status, - context_updates: contextUpdates, - } satisfies ParallelBranchResult; + if (!id || !status) return null; + return { id, status } satisfies ParallelBranchSummary; }) - .filter((r): r is ParallelBranchResult => r != null); + .filter((r): r is ParallelBranchSummary => r != null); if (branchCount == null) branchCount = results.length; } } diff --git a/lib/components/fabro-workflow/src/context.rs b/lib/components/fabro-workflow/src/context.rs index 87c93e4c7..86c5ea79e 100644 --- a/lib/components/fabro-workflow/src/context.rs +++ b/lib/components/fabro-workflow/src/context.rs @@ -159,6 +159,19 @@ pub(crate) fn context_diff( .collect() } +/// [`context_diff`] restricted to user-visible keys: the diff that should +/// propagate outside the executing scope (to a parent workflow or across a +/// parallel fork), with engine-internal keys removed. +pub(crate) fn context_diff_public( + before: &HashMap, + after: HashMap, +) -> HashMap { + context_diff(before, after) + .into_iter() + .filter(|(key, _)| !keys::is_engine_internal_key(key)) + .collect() +} + /// One entry of the [`keys::INTERNAL_PARALLEL_BRANCH_PREAMBLES`] stash. /// /// The stash is a JSON array indexed by the parallel node's outgoing-edge @@ -256,6 +269,70 @@ mod tests { assert_eq!(ctx.get("missing"), None); } + #[test] + fn context_diff_detects_additions() { + let before = HashMap::new(); + let mut after = HashMap::new(); + after.insert("key".to_string(), serde_json::json!("value")); + let diff = context_diff(&before, after); + assert_eq!(diff.len(), 1); + assert_eq!(diff.get("key"), Some(&serde_json::json!("value"))); + } + + #[test] + fn context_diff_detects_changes() { + let mut before = HashMap::new(); + before.insert("key".to_string(), serde_json::json!("old")); + let mut after = HashMap::new(); + after.insert("key".to_string(), serde_json::json!("new")); + let diff = context_diff(&before, after); + assert_eq!(diff.len(), 1); + assert_eq!(diff.get("key"), Some(&serde_json::json!("new"))); + } + + #[test] + fn context_diff_ignores_unchanged() { + let mut before = HashMap::new(); + before.insert("key".to_string(), serde_json::json!("same")); + let mut after = HashMap::new(); + after.insert("key".to_string(), serde_json::json!("same")); + let diff = context_diff(&before, after); + assert!(diff.is_empty()); + } + + #[test] + fn context_diff_ignores_deletions() { + let mut before = HashMap::new(); + before.insert("removed".to_string(), serde_json::json!("gone")); + let after = HashMap::new(); + let diff = context_diff(&before, after); + assert!(diff.is_empty()); + } + + #[test] + fn context_diff_public_excludes_engine_internal_keys() { + let before = HashMap::new(); + let mut after = HashMap::new(); + after.insert("graph.goal".to_string(), serde_json::json!("child goal")); + after.insert( + "internal.run_id".to_string(), + serde_json::json!("child-run"), + ); + after.insert( + "thread.main.current_node".to_string(), + serde_json::json!("exit"), + ); + after.insert("current_node".to_string(), serde_json::json!("exit")); + after.insert("response.plan".to_string(), serde_json::json!("the plan")); + after.insert("review.result".to_string(), serde_json::json!("approved")); + + let filtered = context_diff_public(&before, after); + + assert_eq!(filtered.len(), 2); + assert!(filtered.contains_key("response.plan")); + assert!(filtered.contains_key("review.result")); + } + #[test] fn get_string_with_value() { let ctx = Context::new(); diff --git a/lib/components/fabro-workflow/src/handler/fan_in.rs b/lib/components/fabro-workflow/src/handler/fan_in.rs index b6b73db05..2665744fc 100644 --- a/lib/components/fabro-workflow/src/handler/fan_in.rs +++ b/lib/components/fabro-workflow/src/handler/fan_in.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use async_trait::async_trait; use fabro_graphviz::graph::{Graph, Node}; +use fabro_types::ParallelBranchResult; use super::agent::CodergenBackend; use super::prompt::PromptHandler; @@ -90,22 +91,12 @@ impl Handler for FanInHandler { } } -/// Validate that `parallel.results` exists and has the typed shape without -/// cloning the (potentially hydrated) branch payloads into a full -/// [`ParallelBranchResult`] vec that would go unused. +/// Validate that `parallel.results` exists and has the typed shape. fn validated_branch_count(context: &Context) -> Result { - #[derive(serde::Deserialize)] - struct BranchShape { - #[expect(dead_code, reason = "deserialized only to validate the shape")] - id: String, - #[expect(dead_code, reason = "deserialized only to validate the shape")] - status: fabro_types::StageOutcome, - } - let value = context .get(keys::PARALLEL_RESULTS) .ok_or_else(|| Error::handler("No parallel results to join"))?; - let results: Vec = serde_json::from_value(value) + let results: Vec = serde_json::from_value(value) .map_err(|err| Error::handler_with_source("Invalid parallel results", err))?; Ok(results.len()) } diff --git a/lib/components/fabro-workflow/src/handler/manager_loop.rs b/lib/components/fabro-workflow/src/handler/manager_loop.rs index 431ed5d76..13644e997 100644 --- a/lib/components/fabro-workflow/src/handler/manager_loop.rs +++ b/lib/components/fabro-workflow/src/handler/manager_loop.rs @@ -14,7 +14,7 @@ use tokio::time::{sleep, timeout}; use super::{EngineServices, Handler}; use crate::artifact_upload::ArtifactSink; use crate::condition::evaluate_condition; -use crate::context::{Context, WorkflowContext, context_diff, keys}; +use crate::context::{Context, WorkflowContext, context_diff_public, keys}; use crate::error::Error; use crate::operations::{ValidateInput, WorkflowInput, validate}; use crate::outcome::{Outcome, OutcomeExt, StageOutcome}; @@ -282,13 +282,8 @@ impl Handler for SubWorkflowHandler { Err(e) => return Ok(Outcome::fail_classify(format!("Child task panicked: {e}"))), }; - // Compute context diff, filtering engine-internal keys - let raw_diff = - context_diff(&before_snapshot, child_final_context.snapshot()); - let diff: HashMap = raw_diff - .into_iter() - .filter(|(key, _)| !keys::is_engine_internal_key(key)) - .collect(); + let diff = + context_diff_public(&before_snapshot, child_final_context.snapshot()); tracing::debug!( node = %node.id, @@ -803,74 +798,6 @@ mod tests { assert_eq!(parse_duration_str("bad"), Duration::from_secs(45)); } - #[test] - fn context_diff_detects_additions() { - let before = HashMap::new(); - let mut after = HashMap::new(); - after.insert("key".to_string(), serde_json::json!("value")); - let diff = context_diff(&before, after); - assert_eq!(diff.len(), 1); - assert_eq!(diff.get("key"), Some(&serde_json::json!("value"))); - } - - #[test] - fn context_diff_detects_changes() { - let mut before = HashMap::new(); - before.insert("key".to_string(), serde_json::json!("old")); - let mut after = HashMap::new(); - after.insert("key".to_string(), serde_json::json!("new")); - let diff = context_diff(&before, after); - assert_eq!(diff.len(), 1); - assert_eq!(diff.get("key"), Some(&serde_json::json!("new"))); - } - - #[test] - fn context_diff_ignores_unchanged() { - let mut before = HashMap::new(); - before.insert("key".to_string(), serde_json::json!("same")); - let mut after = HashMap::new(); - after.insert("key".to_string(), serde_json::json!("same")); - let diff = context_diff(&before, after); - assert!(diff.is_empty()); - } - - #[test] - fn context_diff_ignores_deletions() { - let mut before = HashMap::new(); - before.insert("removed".to_string(), serde_json::json!("gone")); - let after = HashMap::new(); - let diff = context_diff(&before, after); - assert!(diff.is_empty()); - } - - #[test] - fn context_diff_excludes_engine_internal_keys() { - let before = HashMap::new(); - let mut after = HashMap::new(); - after.insert("graph.goal".to_string(), serde_json::json!("child goal")); - after.insert( - "internal.run_id".to_string(), - serde_json::json!("child-run"), - ); - after.insert( - "thread.main.current_node".to_string(), - serde_json::json!("exit"), - ); - after.insert("current_node".to_string(), serde_json::json!("exit")); - after.insert("response.plan".to_string(), serde_json::json!("the plan")); - after.insert("review.result".to_string(), serde_json::json!("approved")); - - let raw_diff = context_diff(&before, after); - let filtered: HashMap = raw_diff - .into_iter() - .filter(|(key, _)| !keys::is_engine_internal_key(key)) - .collect(); - - assert_eq!(filtered.len(), 2); - assert!(filtered.contains_key("response.plan")); - assert!(filtered.contains_key("review.result")); - } - #[tokio::test] async fn context_flows_parent_to_child_and_back_excludes_internals() { struct ContextEchoHandler; diff --git a/lib/components/fabro-workflow/src/handler/parallel.rs b/lib/components/fabro-workflow/src/handler/parallel.rs index 6102f1ec4..3efd0e14d 100644 --- a/lib/components/fabro-workflow/src/handler/parallel.rs +++ b/lib/components/fabro-workflow/src/handler/parallel.rs @@ -12,9 +12,9 @@ use tokio::sync::Semaphore; use tokio::task::JoinHandle; use super::{EngineServices, Handler}; -use crate::context::{Context, ParallelBranchPreamble, WorkflowContext, context_diff, keys}; +use crate::context::{Context, ParallelBranchPreamble, WorkflowContext, context_diff_public, keys}; use crate::error::Error; -use crate::event::{Event, RunNoticeCode, RunNoticeLevel, StageScope}; +use crate::event::{Emitter, Event, RunNoticeCode, RunNoticeLevel, StageScope}; use crate::hook_context::set_hook_node; use crate::outcome::{FailureCategory, FailureDetail, Outcome, OutcomeExt}; use crate::{artifact, millis_u64}; @@ -238,16 +238,14 @@ async fn run_branches( status: outcome.status, context_updates, }; - branch_services.run.emitter.emit_scoped( - &Event::ParallelBranchCompleted { - parallel_group_id: group_id.clone(), - parallel_branch_id: parallel_branch_id.clone(), - branch: target_id.clone(), - index: branch_index, - duration_ms: millis_u64(branch_start.elapsed()), - status: result.status, - }, + emit_branch_completed( + &branch_services.run.emitter, &branch_scope, + group_id.clone(), + parallel_branch_id.clone(), + branch_index, + millis_u64(branch_start.elapsed()), + outcome.status, ); Ok::(BranchResult { result, outcome }) }; @@ -257,16 +255,14 @@ async fn run_branches( Err(payload) => { let result = failed_branch_result(&target_id, super::format_panic_message(&payload)); - branch_services.run.emitter.emit_scoped( - &Event::ParallelBranchCompleted { - parallel_group_id: group_id, - parallel_branch_id, - branch: target_id, - index: branch_index, - duration_ms: millis_u64(branch_start.elapsed()), - status: result.result.status, - }, + emit_branch_completed( + &branch_services.run.emitter, &branch_scope, + group_id, + parallel_branch_id, + branch_index, + millis_u64(branch_start.elapsed()), + result.outcome.status, ); Ok(result) } @@ -299,16 +295,14 @@ async fn run_branches( ), }; if emit_completion { - services.run.emitter.emit_scoped( - &Event::ParallelBranchCompleted { - parallel_group_id: parallel_group_id.clone(), - parallel_branch_id: dispatch.branch_id, - branch: dispatch.target_id, - index: dispatch.index, - duration_ms: 0, - status: result.result.status, - }, + emit_branch_completed( + &services.run.emitter, &dispatch.scope, + parallel_group_id.clone(), + dispatch.branch_id, + dispatch.index, + 0, + result.outcome.status, ); } if result.outcome.failure_category() == Some(FailureCategory::Canceled) { @@ -421,14 +415,35 @@ fn branch_context_updates( .iter() .map(|(key, value)| (key.clone(), value.clone())) .collect::>(); - updates.extend( - context_diff(before, after) - .into_iter() - .filter(|(key, _)| !keys::is_engine_internal_key(key)), - ); + updates.extend(context_diff_public(before, after)); updates } +/// Emit `ParallelBranchCompleted` for the branch that `scope` identifies; +/// `scope.node_id` is the branch target by construction +/// ([`StageScope::for_parallel_branch`]). +fn emit_branch_completed( + emitter: &Emitter, + scope: &StageScope, + parallel_group_id: StageId, + parallel_branch_id: ParallelBranchId, + index: usize, + duration_ms: u64, + status: StageOutcome, +) { + emitter.emit_scoped( + &Event::ParallelBranchCompleted { + parallel_group_id, + parallel_branch_id, + branch: scope.node_id.clone(), + index, + duration_ms, + status, + }, + scope, + ); +} + fn failed_branch_result(id: &str, reason: impl Into) -> BranchResult { let outcome = Outcome::fail_classify(reason); BranchResult { From 0e4244a24a5aa1f1fade6fb5551d8040312bba3d Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 24 Jul 2026 08:29:06 -0400 Subject: [PATCH 16/24] refactor: simplify and harden the seek-based event listing - Unify list_events_from with list_events_from_with_limit so projection replay shares the seek path instead of duplicating the decode loop - Bound the event scan with keys::run_events_range instead of an unbounded range plus a manual prefix break, so slatedb never touches SSTs belonging to other runs or namespaces - Store reader event_seq as None instead of a valid-looking sentinel of 1, so appends through a reader-built inner fail as ReadOnly rather than writing duplicate sequences - Borrow keys during scans instead of allocating a String per entry, drop a dead branch in cached_events_from, collapse recover_next_seq's single-caller parameters, and document the zero-padded key ordering invariant the seek depends on Co-Authored-By: Claude Fable 5 --- lib/components/fabro-store/src/keys.rs | 40 +++++++++ .../fabro-store/src/slate/run_store.rs | 86 +++++++------------ 2 files changed, 72 insertions(+), 54 deletions(-) diff --git a/lib/components/fabro-store/src/keys.rs b/lib/components/fabro-store/src/keys.rs index 2451df632..f63bb27b7 100644 --- a/lib/components/fabro-store/src/keys.rs +++ b/lib/components/fabro-store/src/keys.rs @@ -1,4 +1,5 @@ use std::fmt::{self, Write}; +use std::ops::Range; use fabro_types::{RunBlobId, RunId, SessionId}; @@ -23,6 +24,13 @@ impl SlateKey { self } + /// Exclusive end bound of this key's prefix keyspace: every key under + /// `self.into_prefix()` sorts below it and no other key sorts between. + fn into_prefix_end(mut self) -> Self { + self.0.push('\u{1}'); + self + } + #[cfg(test)] fn as_str(&self) -> &str { &self.0 @@ -52,6 +60,9 @@ pub(crate) fn run_events_prefix(run_id: &RunId) -> SlateKey { .into_prefix() } +// Sequence keys zero-pad `seq` to six digits so lexicographic key order +// matches numeric seq order for up to 999,999 events per run. Seek-based +// event listing (`run_events_range`) depends on this invariant. pub(crate) fn run_event_key(run_id: &RunId, seq: u32, epoch_ms: i64) -> SlateKey { SlateKey::new("runs") .with(run_id) @@ -66,6 +77,17 @@ pub(crate) fn run_event_seq_prefix(run_id: &RunId, seq: u32) -> SlateKey { .with(format!("{seq:06}-")) } +/// Scan range covering the run's event keys from `start_seq` to the end of +/// the run's event namespace, so seek-based listing never touches keys of +/// other runs or namespaces. +pub(crate) fn run_events_range(run_id: &RunId, start_seq: u32) -> Range { + let end = SlateKey::new("runs") + .with(run_id) + .with("events") + .into_prefix_end(); + run_event_seq_prefix(run_id, start_seq)..end +} + pub(crate) fn blobs_prefix() -> SlateKey { SlateKey::new("blobs").with("sha256").into_prefix() } @@ -152,6 +174,24 @@ mod tests { assert_eq!(leaf, "000007-123"); } + #[test] + fn run_events_range_bounds_the_event_namespace() { + let run_id: RunId = "01JT56VE4Z5NZ814GZN2JZD65A".parse().unwrap(); + let range = run_events_range(&run_id, 2); + let contains = |key: &SlateKey| { + range.start.as_ref() <= key.as_ref() && key.as_ref() < range.end.as_ref() + }; + + assert!(!contains(&run_event_key(&run_id, 1, 123))); + assert!(contains(&run_event_key(&run_id, 2, 123))); + assert!(contains(&run_event_key(&run_id, 999_999, 123))); + // Sibling namespaces of the same run sort outside the range. + assert!(!contains(&SlateKey::new("runs").with(run_id).with("state"))); + assert!(!contains( + &session_by_id_key(&fabro_types::SessionId::new()) + )); + } + #[test] fn parse_helpers_roundtrip() { let run_id: RunId = "01JT56VE4Z5NZ814GZN2JZD65A".parse().unwrap(); diff --git a/lib/components/fabro-store/src/slate/run_store.rs b/lib/components/fabro-store/src/slate/run_store.rs index 83b0a087a..664ce3162 100644 --- a/lib/components/fabro-store/src/slate/run_store.rs +++ b/lib/components/fabro-store/src/slate/run_store.rs @@ -38,7 +38,9 @@ pub(crate) struct RunDatabaseInner { run_id: RunId, db: Db, blob_store: BlobStore, - event_seq: AtomicU32, + // `None` for reader-built inners: readers never append, so they carry no + // next-write sequence and any append through them fails as read-only. + event_seq: Option, close_lock: Mutex<()>, state_lock: Mutex<()>, projection_cache: Mutex, @@ -87,9 +89,9 @@ impl RunDatabase { let event_seq = if read_only { // Readers never append, so they do not need to scan the full event // history to recover the next write sequence. - 1 + None } else { - recover_next_seq(&db, keys::run_events_prefix(&run_id), keys::parse_event_seq).await? + Some(AtomicU32::new(recover_next_seq(&db, &run_id).await?)) }; let (event_tx, _) = broadcast::channel(DEFAULT_EVENT_TAIL_LIMIT.max(16)); let blob_store = BlobStore::new(Arc::new(db.clone())); @@ -98,7 +100,7 @@ impl RunDatabase { run_id, db, blob_store, - event_seq: AtomicU32::new(event_seq), + event_seq, close_lock: Mutex::new(()), state_lock: Mutex::new(()), projection_cache: Mutex::new(EventProjectionCache::default()), @@ -245,15 +247,12 @@ impl RunDatabase { if start_seq < oldest_seq { return None; } - let mut events = recent_events + let events = recent_events .iter() .filter(|event| event.seq >= start_seq) .take(limit.saturating_add(1)) .cloned() .collect::>(); - if events.is_empty() && start_seq <= self.inner.event_seq.load(Ordering::SeqCst) { - events = Vec::new(); - } Some(events) } } @@ -292,7 +291,8 @@ impl RunDatabase { } async fn append_event_envelope_locked(&self, payload: &EventPayload) -> Result { - let seq = self.inner.event_seq.fetch_add(1, Ordering::SeqCst); + let event_seq = self.inner.event_seq.as_ref().ok_or(Error::ReadOnly)?; + let seq = event_seq.fetch_add(1, Ordering::SeqCst); let event = EventEnvelope { seq, event: RunEvent::try_from(payload)?, @@ -365,9 +365,11 @@ impl RunDatabase { } pub async fn list_events(&self) -> Result> { - self.list_events_from_with_limit(1, usize::MAX / 2).await + self.list_events_from_with_limit(1, usize::MAX).await } + /// Returns up to `limit + 1` events starting at `start_seq`. The extra + /// item lets callers compute `has_more` without a second read. pub async fn list_events_from_with_limit( &self, start_seq: u32, @@ -517,19 +519,15 @@ fn apply_cached_projection_event( Ok(()) } -async fn recover_next_seq( - db: &R, - prefix: keys::SlateKey, - parse: fn(&str) -> Option, -) -> Result +async fn recover_next_seq(db: &R, run_id: &RunId) -> Result where R: DbRead + Sync, { - let mut iter = db.scan_prefix(prefix).await?; + let mut iter = db.scan_prefix(keys::run_events_prefix(run_id)).await?; let mut max_seq = 0; while let Some(entry) = iter.next().await? { - let key = key_to_string(&entry.key)?; - if let Some(seq) = parse(&key) { + let key = key_to_str(&entry.key)?; + if let Some(seq) = keys::parse_event_seq(key) { max_seq = max_seq.max(seq); } } @@ -540,25 +538,11 @@ async fn list_events_from(db: &R, run_id: &RunId, start_seq: u32) -> Result( db: &R, run_id: &RunId, @@ -568,23 +552,17 @@ async fn list_events_from_with_limit( where R: DbRead + Sync, { - let event_prefix = keys::run_events_prefix(run_id); let max_events = limit.saturating_add(1); - // Seek to the page cursor and decode only the requested page plus the - // sentinel used to compute `has_more`. - let mut iter = db - .scan(keys::run_event_seq_prefix(run_id, start_seq)..) - .await?; + // Seek to the page cursor and decode only the requested page. Zero-padded + // sequence keys scan in seq order, so no post-scan sort is needed. + let mut iter = db.scan(keys::run_events_range(run_id, start_seq)).await?; let mut events = Vec::new(); while events.len() < max_events { let Some(entry) = iter.next().await? else { break; }; - if !entry.key.starts_with(event_prefix.as_ref()) { - break; - } - let key = key_to_string(&entry.key)?; - let Some(seq) = keys::parse_event_seq(&key) else { + let key = key_to_str(&entry.key)?; + let Some(seq) = keys::parse_event_seq(key) else { continue; }; if seq < start_seq { @@ -645,8 +623,8 @@ where let mut iter = db.scan_prefix(keys::run_events_prefix(run_id)).await?; let mut events: Vec = Vec::new(); while let Some(entry) = iter.next().await? { - let key = key_to_string(&entry.key)?; - let Some(seq) = keys::parse_event_seq(&key) else { + let key = key_to_str(&entry.key)?; + let Some(seq) = keys::parse_event_seq(key) else { continue; }; if seq < start_seq { @@ -705,8 +683,8 @@ where let mut iter = db.scan_prefix(keys::run_events_prefix(run_id)).await?; let mut events = Vec::new(); while let Some(entry) = iter.next().await? { - let key = key_to_string(&entry.key)?; - let Some(seq) = keys::parse_event_seq(&key) else { + let key = key_to_str(&entry.key)?; + let Some(seq) = keys::parse_event_seq(key) else { continue; }; if seq < start_seq { @@ -740,8 +718,8 @@ where let mut iter = db.scan_prefix(keys::blobs_prefix()).await?; let mut blob_ids = Vec::new(); while let Some(entry) = iter.next().await? { - let key = key_to_string(&entry.key)?; - let Some(blob_id) = keys::parse_blob_id(&key) else { + let key = key_to_str(&entry.key)?; + let Some(blob_id) = keys::parse_blob_id(key) else { continue; }; blob_ids.push(blob_id); @@ -750,8 +728,8 @@ where Ok(blob_ids) } -fn key_to_string(key: &Bytes) -> Result { - String::from_utf8(key.to_vec()) +fn key_to_str(key: &Bytes) -> Result<&str> { + std::str::from_utf8(key) .map_err(|err| Error::Other(format!("stored key is not valid UTF-8: {err}"))) } From 59d5b317dcced86241f45c1871b9ea5b8c93c739 Mon Sep 17 00:00:00 2001 From: Release Repro Date: Fri, 24 Jul 2026 08:29:12 -0400 Subject: [PATCH 17/24] Simplify LLM error mapping and validate reasoning_effort parsing - Return InvalidRequest (400) for unsupported speed too, matching the reasoning_effort check and the complete()/stream() doc comments - Centralize fabro_llm::Error -> ApiError mapping in a From impl so the completions handler, playground handler, and Error::Llm arm agree on the InvalidRequest -> 400 / else -> 502 split - Reject unparseable reasoning_effort values with 400 instead of silently dropping them - Add classify_sdk_invalid_request test per fabro-workflow convention Co-Authored-By: Claude Fable 5 --- lib/apps/fabro-server/src/error.rs | 13 +++++- .../src/server/handler/completions.rs | 39 +++++++++------- .../src/server/handler/playground.rs | 3 +- lib/apps/fabro-server/src/server/tests.rs | 46 ++++++++++++++++++- lib/components/fabro-llm/src/client.rs | 9 ++-- lib/components/fabro-workflow/src/error.rs | 8 ++++ 6 files changed, 92 insertions(+), 26 deletions(-) diff --git a/lib/apps/fabro-server/src/error.rs b/lib/apps/fabro-server/src/error.rs index 356d68af1..70537f714 100644 --- a/lib/apps/fabro-server/src/error.rs +++ b/lib/apps/fabro-server/src/error.rs @@ -161,7 +161,7 @@ impl From for ApiError { Error::BadGateway(msg) => Self::new(StatusCode::BAD_GATEWAY, msg), Error::Workflow(err) => Self::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()), Error::Agent(err) => Self::new(StatusCode::BAD_GATEWAY, err.to_string()), - Error::Llm(err) => Self::new(StatusCode::BAD_GATEWAY, err.to_string()), + Error::Llm(err) => Self::from(err), Error::Store(err) => Self::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()), Error::Config(err) => Self::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()), Error::Vault(err) => Self::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()), @@ -170,6 +170,17 @@ impl From for ApiError { } } +/// LLM errors split at the HTTP boundary: request-validation failures are the +/// caller's fault (400); everything else is an upstream failure (502). +impl From for ApiError { + fn from(err: fabro_llm::Error) -> Self { + match err { + fabro_llm::Error::InvalidRequest { message } => Self::bad_request(message), + err => Self::new(StatusCode::BAD_GATEWAY, format!("LLM error: {err}")), + } + } +} + impl IntoResponse for ApiError { fn into_response(self) -> Response { let title = self diff --git a/lib/apps/fabro-server/src/server/handler/completions.rs b/lib/apps/fabro-server/src/server/handler/completions.rs index 726847d5a..e8f06a5d2 100644 --- a/lib/apps/fabro-server/src/server/handler/completions.rs +++ b/lib/apps/fabro-server/src/server/handler/completions.rs @@ -1,7 +1,7 @@ use std::collections::HashSet; use std::sync::Arc; -use fabro_model::{Catalog, ModelSelectionError}; +use fabro_model::{Catalog, ModelSelectionError, ReasoningEffort}; use super::super::{ ApiError, AppState, CompletionResponse, CompletionToolChoiceMode, CompletionUsage, @@ -26,17 +26,6 @@ fn finish_reason_to_api_stop_reason(reason: &FinishReason) -> String { } } -fn llm_error_response(error: fabro_llm::Error) -> Response { - match error { - fabro_llm::Error::InvalidRequest { message } => { - ApiError::bad_request(message).into_response() - } - error => { - ApiError::new(StatusCode::BAD_GATEWAY, format!("LLM error: {error}")).into_response() - } - } -} - async fn create_completion( _auth: RequiredUser, State(state): State>, @@ -104,6 +93,24 @@ async fn create_completion( CompletionToolChoiceMode::Named => ToolChoice::named(tc.tool_name.unwrap_or_default()), }); + let reasoning_effort = match req.reasoning_effort.as_deref() { + None => None, + Some(value) => match value.parse::() { + Ok(effort) => Some(effort), + Err(_) => { + return ApiError::bad_request(format!( + "invalid reasoning_effort '{value}'; allowed values: {}", + ReasoningEffort::variants() + .iter() + .map(|v| <&'static str>::from(*v)) + .collect::>() + .join(", ") + )) + .into_response(); + } + }, + }; + // Build the LLM request let request = LlmRequest { model: model_id.clone(), @@ -120,7 +127,7 @@ async fn create_completion( } else { Some(req.stop_sequences) }, - reasoning_effort: req.reasoning_effort.as_deref().and_then(|s| s.parse().ok()), + reasoning_effort, speed: None, metadata: None, provider_options: req.provider_options, @@ -138,7 +145,7 @@ async fn create_completion( // Streaming path: forward all StreamEvents as SSE let stream_result = match client.stream(&request).await { Ok(s) => s, - Err(error) => return llm_error_response(error), + Err(error) => return ApiError::from(error).into_response(), }; llm_sse::stream_response(stream_result, state.shutdown_token()) @@ -187,7 +194,7 @@ async fn create_completion( }) .into_response() } - Err(error) => llm_error_response(error), + Err(error) => ApiError::from(error).into_response(), } } else { match client.complete(&request).await { @@ -209,7 +216,7 @@ async fn create_completion( }) .into_response() } - Err(error) => llm_error_response(error), + Err(error) => ApiError::from(error).into_response(), } } } diff --git a/lib/apps/fabro-server/src/server/handler/playground.rs b/lib/apps/fabro-server/src/server/handler/playground.rs index d404c2b58..ee52c56e0 100644 --- a/lib/apps/fabro-server/src/server/handler/playground.rs +++ b/lib/apps/fabro-server/src/server/handler/playground.rs @@ -169,8 +169,7 @@ async fn create_playground_chat( Ok(s) => s, Err(e) => { error!(error = ?e, "playground: LLM stream call failed"); - return ApiError::new(StatusCode::BAD_GATEWAY, format!("LLM error: {e}")) - .into_response(); + return ApiError::from(e).into_response(); } }; diff --git a/lib/apps/fabro-server/src/server/tests.rs b/lib/apps/fabro-server/src/server/tests.rs index 372cc1e6c..13d276ca8 100644 --- a/lib/apps/fabro-server/src/server/tests.rs +++ b/lib/apps/fabro-server/src/server/tests.rs @@ -15314,7 +15314,8 @@ async fn create_completion_unsupported_reasoning_efforts_return_bad_request() { body["errors"][0]["detail"], format!( "model 'kimi-k3' does not support reasoning_effort '{effort}'; allowed values: low, high, max" - ) + ), + "stream={stream}" ); } } @@ -15322,6 +15323,49 @@ async fn create_completion_unsupported_reasoning_efforts_return_bad_request() { completion.assert_calls(0); } +#[tokio::test] +async fn create_completion_unparseable_reasoning_effort_returns_bad_request() { + let upstream = MockServer::start(); + let completion = upstream.mock(|when, then| { + when.method(POST); + then.status(500); + }); + let state = TestAppStateBuilder::new() + .provider_base_url("kimi", upstream.url("/v1")) + .vault_entries([(EnvVars::KIMI_API_KEY, "test-kimi-api-key")]) + .build(); + let app = crate::test_support::build_test_router(state); + + let req = Request::builder() + .method("POST") + .uri(api("/completions")) + .header("content-type", "application/json") + .body(Body::from( + serde_json::json!({ + "provider": "kimi", + "model": "kimi-k3", + "reasoning_effort": "bananas", + "messages": [ + { + "role": "user", + "content": [{"kind": "text", "data": "hi"}] + } + ] + }) + .to_string(), + )) + .unwrap(); + + let response = app.oneshot(req).await.unwrap(); + let body = response_json!(response, StatusCode::BAD_REQUEST).await; + assert_eq!( + body["errors"][0]["detail"], + "invalid reasoning_effort 'bananas'; allowed values: low, medium, high, xhigh, max" + ); + + completion.assert_calls(0); +} + #[tokio::test] async fn create_completion_default_model_uses_app_state_catalog() { let upstream = MockServer::start(); diff --git a/lib/components/fabro-llm/src/client.rs b/lib/components/fabro-llm/src/client.rs index 8cfaadf1b..30d5c05a0 100644 --- a/lib/components/fabro-llm/src/client.rs +++ b/lib/components/fabro-llm/src/client.rs @@ -421,12 +421,11 @@ impl Client { if let Some(speed) = request.speed { if speed != Speed::Standard && !settings.controls.speed.contains(&speed) { - return Err(Error::Configuration { + return Err(Error::InvalidRequest { message: format!( "model '{model_id}' does not support speed '{speed}'; allowed values: standard{}", format_additional_speeds(&settings.controls.speed), ), - source: None, }); } } @@ -1527,9 +1526,8 @@ output_cost_per_mtok = 20.0 assert!(matches!( err, - Error::Configuration { + Error::InvalidRequest { ref message, - .. } if message.contains("model 'gpt-5.4' does not support speed 'fast'") )); } @@ -1616,9 +1614,8 @@ output_cost_per_mtok = 20.0 assert!(matches!( err, - Error::Configuration { + Error::InvalidRequest { ref message, - .. } if message.contains("model 'gpt-5.4' does not support speed 'fast'") )); } diff --git a/lib/components/fabro-workflow/src/error.rs b/lib/components/fabro-workflow/src/error.rs index 128134480..f52079a29 100644 --- a/lib/components/fabro-workflow/src/error.rs +++ b/lib/components/fabro-workflow/src/error.rs @@ -1223,6 +1223,14 @@ mod tests { assert_eq!(classify_sdk_error(&err), FailureCategory::Deterministic); } + #[test] + fn classify_sdk_invalid_request() { + let err = SdkError::InvalidRequest { + message: "unsupported reasoning effort".into(), + }; + assert_eq!(classify_sdk_error(&err), FailureCategory::Deterministic); + } + // --- hints count guards --- #[test] From 0cd22ebd7586680b9b4e4f1b8ea762d83ba806ce Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 24 Jul 2026 08:40:55 -0400 Subject: [PATCH 18/24] refactor: build structured-output GenerateParams via struct update Replaces the per-field if-let cascade in the structured completion path with a single struct-update expression. The cascade had to be extended by hand for every request field and silently dropped stop_sequences and provider_options, which the non-structured path already forwarded. Co-Authored-By: Claude Fable 5 --- .../src/server/handler/completions.rs | 36 +++++++++---------- lib/apps/fabro-server/src/server/tests.rs | 2 +- 2 files changed, 18 insertions(+), 20 deletions(-) diff --git a/lib/apps/fabro-server/src/server/handler/completions.rs b/lib/apps/fabro-server/src/server/handler/completions.rs index eb6dafaf4..146d20808 100644 --- a/lib/apps/fabro-server/src/server/handler/completions.rs +++ b/lib/apps/fabro-server/src/server/handler/completions.rs @@ -139,25 +139,23 @@ async fn create_completion( let msg_id = Ulid::new().to_string(); if let Some(schema) = req.schema { - // Structured output uses generate_object for JSON parsing logic - let mut params = - GenerateParams::new(&request.model, std::sync::Arc::new(client.clone())) - .messages(request.messages); - if let Some(ref p) = request.provider { - params = params.provider(p); - } - if let Some(temp) = request.temperature { - params = params.temperature(temp); - } - if let Some(max_tokens) = request.max_tokens { - params = params.max_tokens(max_tokens); - } - if let Some(top_p) = request.top_p { - params = params.top_p(top_p); - } - if let Some(reasoning_effort) = request.reasoning_effort { - params = params.reasoning_effort(reasoning_effort); - } + // Structured output uses generate_object for JSON parsing logic. + // tools/tool_choice are not forwarded: GenerateParams carries + // executable Arcs, not wire ToolDefinitions, and + // generate_object sets response_format from the schema itself. + let params = GenerateParams { + messages: Some(request.messages), + provider: request.provider, + temperature: request.temperature, + top_p: request.top_p, + max_tokens: request.max_tokens, + stop_sequences: request.stop_sequences, + reasoning_effort: request.reasoning_effort, + speed: request.speed, + metadata: request.metadata, + provider_options: request.provider_options, + ..GenerateParams::new(request.model, std::sync::Arc::new(client.clone())) + }; match generate_object(params, schema).await { Ok(result) => { // `result.finish_reason` / `result.usage` resolve through diff --git a/lib/apps/fabro-server/src/server/tests.rs b/lib/apps/fabro-server/src/server/tests.rs index 8e7b124e7..eb0083948 100644 --- a/lib/apps/fabro-server/src/server/tests.rs +++ b/lib/apps/fabro-server/src/server/tests.rs @@ -15431,7 +15431,7 @@ async fn create_completion_structured_output_forwards_reasoning_effort() { let response = app.oneshot(req).await.unwrap(); let body = response_json!(response, StatusCode::OK).await; assert_eq!(body["output"], json!({"answer": 42})); - completion.assert_calls(1); + completion.assert(); } #[tokio::test] From c06c60214aa478d89bf53d4472c10ce27ea5596d Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 24 Jul 2026 08:49:01 -0400 Subject: [PATCH 19/24] refactor: simplify readiness-fallback plumbing The fallback provider set was always catalog.all_provider_ids(), computed at every call site and threaded through five layers alongside the catalog itself. Fold it into Catalog::resolve_selection_with_catalog_fallback and carry only a catalog_fallback flag through the transform/validate/ materialize entry points. - materialize_run delegates to resolve_run_model again instead of re-inlining its provider normalization and selection - run_preflight derives ready providers from llm_result instead of taking both, so callers cannot pass inconsistent pairs; the legacy tests now exercise the production ready-first routing path - AppState::resolve_llm_client_with_ready_ids replaces three copies of resolve-then-extract-provider-ids, and ready_llm_provider_ids delegates to it - the unreachable "model resolution failed" preflight check becomes an invariant error where the materialized run is produced - validate_prepared_manifest_with_vars/_for_preflight share the ValidateInput construction Co-Authored-By: Claude Fable 5 --- lib/apps/fabro-server/src/run_manifest.rs | 153 +++++++----------- lib/apps/fabro-server/src/server.rs | 26 ++- .../fabro-server/src/server/handler/runs.rs | 49 ++---- .../fabro-workflow/src/operations/create.rs | 12 +- .../fabro-workflow/src/operations/mod.rs | 2 +- .../fabro-workflow/src/operations/start.rs | 1 + .../fabro-workflow/src/operations/validate.rs | 20 +-- .../fabro-workflow/src/pipeline/transform.rs | 10 +- .../fabro-workflow/src/pipeline/types.rs | 4 +- .../fabro-workflow/src/pipeline/validate.rs | 2 +- .../fabro-workflow/src/run_materialization.rs | 59 +++---- .../src/transforms/model_resolution.rs | 32 ++-- .../fabro-workflow/tests/it/integration.rs | 2 +- lib/foundation/fabro-model/src/catalog.rs | 26 +-- 14 files changed, 166 insertions(+), 232 deletions(-) diff --git a/lib/apps/fabro-server/src/run_manifest.rs b/lib/apps/fabro-server/src/run_manifest.rs index aff59b0c5..65cc6d36e 100644 --- a/lib/apps/fabro-server/src/run_manifest.rs +++ b/lib/apps/fabro-server/src/run_manifest.rs @@ -35,12 +35,10 @@ use fabro_util::check_report::{CheckDetail, CheckReport, CheckResult, CheckSecti use fabro_validate::Severity; use fabro_workflow::Error as WorkflowError; use fabro_workflow::operations::{ - CreateRunInput, ValidateInput, WorkflowInput, validate, validate_with_provider_fallback, + CreateRunInput, ValidateInput, WorkflowInput, validate, validate_with_ready_providers, }; use fabro_workflow::pipeline::Validated; -#[cfg(test)] -use fabro_workflow::run_materialization::materialize_run; -use fabro_workflow::run_materialization::materialize_run_with_provider_fallback; +use fabro_workflow::run_materialization::materialize_run_with_ready_providers; use fabro_workflow::workflow_bundle::{BundledWorkflow, ParsedWorkflowConfig, WorkflowBundle}; use futures_util::stream::{self, StreamExt}; use tokio::process::Command; @@ -199,14 +197,7 @@ pub(crate) fn validate_prepared_manifest_with_vars( catalog: Arc, vars: HashMap, ) -> Result { - validate(ValidateInput { - workflow: WorkflowInput::Bundled(prepared.workflow_input.clone()), - settings: prepared.settings.clone(), - vars, - cwd: prepared.cwd.clone(), - custom_transforms: Vec::new(), - catalog, - }) + validate(manifest_validate_input(prepared, catalog, vars)) } pub(crate) fn validate_prepared_manifest_for_preflight( @@ -215,21 +206,27 @@ pub(crate) fn validate_prepared_manifest_for_preflight( vars: HashMap, ready_providers: &[ProviderId], ) -> Result { - let fallback_providers = catalog.all_provider_ids().into_iter().collect::>(); - validate_with_provider_fallback( - ValidateInput { - workflow: WorkflowInput::Bundled(prepared.workflow_input.clone()), - settings: prepared.settings.clone(), - vars, - cwd: prepared.cwd.clone(), - custom_transforms: Vec::new(), - catalog, - }, + validate_with_ready_providers( + manifest_validate_input(prepared, catalog, vars), ready_providers, - &fallback_providers, ) } +fn manifest_validate_input( + prepared: &PreparedManifest, + catalog: Arc, + vars: HashMap, +) -> ValidateInput { + ValidateInput { + workflow: WorkflowInput::Bundled(prepared.workflow_input.clone()), + settings: prepared.settings.clone(), + vars, + cwd: prepared.cwd.clone(), + custom_transforms: Vec::new(), + catalog, + } +} + pub(crate) fn create_run_input( prepared: PreparedManifest, configured_providers: Vec, @@ -262,11 +259,10 @@ pub(crate) async fn run_preflight( state: &AppState, prepared: &PreparedManifest, validated: &Validated, - preferred_providers: &[ProviderId], llm_result: Result, ) -> Result<(types::PreflightResponse, bool)> { let (report, checks_ok) = - build_preflight_report(state, prepared, validated, preferred_providers, llm_result).await?; + build_preflight_report(state, prepared, validated, llm_result).await?; let preflight_ok = !validated.has_errors() && checks_ok; Ok(( preflight_response( @@ -485,7 +481,6 @@ async fn build_preflight_report( state: &AppState, prepared: &PreparedManifest, validated: &Validated, - preferred_providers: &[ProviderId], llm_result: Result, ) -> Result<(CheckReport, bool)> { let graph = validated.graph(); @@ -504,15 +499,23 @@ async fn build_preflight_report( } let catalog = state.catalog(); - let fallback_providers = catalog.all_provider_ids().into_iter().collect::>(); - let materialized = materialize_run_with_provider_fallback( + let ready_providers = llm_result + .as_ref() + .map(LlmClientResult::provider_ids) + .unwrap_or_default(); + let materialized = materialize_run_with_ready_providers( prepared.settings.clone(), graph, catalog.as_ref(), - preferred_providers, - &fallback_providers, + &ready_providers, )?; let resolved_run = materialized.run; + let (Some(run_model), Some(run_provider)) = ( + resolved_run.model.name.as_deref(), + resolved_run.model.provider.as_deref(), + ) else { + bail!("materialized run is missing a resolved model or provider"); + }; let server_settings = state.server_settings(); let github_integration = &server_settings.server.integrations.github; let sandbox_provider = effective_sandbox_provider(&resolved_run); @@ -575,7 +578,8 @@ async fn build_preflight_report( let llm_ok = run_llm_check( &mut checks, graph, - &resolved_run, + run_model, + run_provider, catalog.as_ref(), llm_result, ) @@ -1051,25 +1055,11 @@ struct PendingModelProbe { async fn run_llm_check( checks: &mut Vec, graph: &Graph, - settings: &RunNamespace, + model: &str, + default_provider: &str, catalog: &Catalog, llm_result: Result, ) -> bool { - let (Some(model), Some(default_provider)) = ( - settings.model.name.as_deref(), - settings.model.provider.as_deref(), - ) else { - checks.push(CheckResult { - name: "LLM".into(), - status: CheckStatus::Error, - summary: "model resolution failed".into(), - details: Vec::new(), - remediation: Some( - "Preflight did not produce a resolved run model and provider".to_string(), - ), - }); - return false; - }; let mut model_providers = std::collections::BTreeSet::new(); let mut has_llm_nodes = false; @@ -1388,6 +1378,7 @@ fn report_to_api(report: &CheckReport) -> types::PreflightCheckReport { mod tests { use fabro_model::ProviderId; use fabro_model::catalog::LlmCatalogSettings; + use fabro_workflow::run_materialization::materialize_run; use super::*; @@ -1563,29 +1554,18 @@ digraph Demo {{ ) .unwrap(); - run_preflight( - state.as_ref(), - &prepared, - &validated, - &ready_providers, - llm_result, - ) - .await - .unwrap() + run_preflight(state.as_ref(), &prepared, &validated, llm_result) + .await + .unwrap() } - async fn run_preflight_with_catalog_routes( + async fn resolve_and_run_preflight( state: &AppState, prepared: &PreparedManifest, validated: &Validated, ) -> Result<(types::PreflightResponse, bool)> { let llm_result = state.resolve_llm_client().await; - let preferred_providers = state - .catalog() - .all_provider_ids() - .into_iter() - .collect::>(); - run_preflight(state, prepared, validated, &preferred_providers, llm_result).await + run_preflight(state, prepared, validated, llm_result).await } fn manifest_workflow() -> types::ManifestWorkflow { @@ -2217,10 +2197,9 @@ name = "Control Plane" assert!(validated.has_errors()); - let (response, ok) = - run_preflight_with_catalog_routes(state.as_ref(), &prepared, &validated) - .await - .unwrap(); + let (response, ok) = resolve_and_run_preflight(state.as_ref(), &prepared, &validated) + .await + .unwrap(); assert!(!ok); assert_eq!(response.workflow.name, "Invalid"); @@ -2262,10 +2241,9 @@ issues = "read" let validated = validate_prepared_manifest(&prepared, test_catalog()).unwrap(); assert!(!validated.has_errors()); - let (response, _ok) = - run_preflight_with_catalog_routes(state.as_ref(), &prepared, &validated) - .await - .unwrap(); + let (response, _ok) = resolve_and_run_preflight(state.as_ref(), &prepared, &validated) + .await + .unwrap(); assert!( response.checks.sections[0] @@ -2314,10 +2292,9 @@ id = "local" assert!(!validated.has_errors()); - let (response, ok) = - run_preflight_with_catalog_routes(state.as_ref(), &prepared, &validated) - .await - .unwrap(); + let (response, ok) = resolve_and_run_preflight(state.as_ref(), &prepared, &validated) + .await + .unwrap(); assert!(ok); assert!(response.workflow.diagnostics.is_empty()); @@ -2422,10 +2399,9 @@ id = "daytona" .unwrap(); let validated = validate_prepared_manifest(&prepared, test_catalog()).unwrap(); - let (response, _ok) = - run_preflight_with_catalog_routes(state.as_ref(), &prepared, &validated) - .await - .unwrap(); + let (response, _ok) = resolve_and_run_preflight(state.as_ref(), &prepared, &validated) + .await + .unwrap(); assert!(response.workflow.diagnostics.is_empty()); assert!( @@ -2491,10 +2467,9 @@ digraph Demo { .unwrap(); let validated = validate_prepared_manifest(&prepared, test_catalog()).unwrap(); - let (response, ok) = - run_preflight_with_catalog_routes(state.as_ref(), &prepared, &validated) - .await - .unwrap(); + let (response, ok) = resolve_and_run_preflight(state.as_ref(), &prepared, &validated) + .await + .unwrap(); assert!(!ok); let llm_check = response.checks.sections[0] @@ -2677,15 +2652,9 @@ digraph Demo { ) .unwrap(); - let (response, ok) = run_preflight( - state.as_ref(), - &prepared, - &validated, - &ready_providers, - llm_result, - ) - .await - .unwrap(); + let (response, ok) = run_preflight(state.as_ref(), &prepared, &validated, llm_result) + .await + .unwrap(); assert!(!ok); let llm_check = response.checks.sections[0] diff --git a/lib/apps/fabro-server/src/server.rs b/lib/apps/fabro-server/src/server.rs index fac2dd51a..47b57a462 100644 --- a/lib/apps/fabro-server/src/server.rs +++ b/lib/apps/fabro-server/src/server.rs @@ -1445,14 +1445,26 @@ impl AppState { self.llm_source.configured_providers(catalog.as_ref()).await } - pub(crate) async fn ready_llm_provider_ids(&self) -> Vec { - match self.resolve_llm_client().await { - Ok(result) => result.provider_ids(), - Err(err) => { - warn!(error = ?err, "Failed to resolve LLM client while checking ready providers"); - Vec::new() - } + /// Resolve the LLM client once and derive the ready provider IDs from it, + /// logging a warning when resolution fails. Callers that need both values + /// must use this instead of `ready_llm_provider_ids` so the client is not + /// resolved twice. + pub(crate) async fn resolve_llm_client_with_ready_ids( + &self, + ) -> (anyhow::Result, Vec) { + let llm_result = self.resolve_llm_client().await; + if let Err(err) = &llm_result { + warn!(error = ?err, "Failed to resolve LLM client while checking ready providers"); } + let ready_provider_ids = llm_result + .as_ref() + .map(LlmClientResult::provider_ids) + .unwrap_or_default(); + (llm_result, ready_provider_ids) + } + + pub(crate) async fn ready_llm_provider_ids(&self) -> Vec { + self.resolve_llm_client_with_ready_ids().await.1 } pub(crate) async fn decorate_run_summary(&self, run: fabro_types::Run) -> fabro_types::Run { diff --git a/lib/apps/fabro-server/src/server/handler/runs.rs b/lib/apps/fabro-server/src/server/handler/runs.rs index 20a4b3f9d..fdb9c1c3e 100644 --- a/lib/apps/fabro-server/src/server/handler/runs.rs +++ b/lib/apps/fabro-server/src/server/handler/runs.rs @@ -51,7 +51,6 @@ use crate::run_files::{list_run_commits, list_run_files}; use crate::run_manifest; use crate::run_selector::{ResolveRunError, resolve_run_by_selector}; use crate::run_title_generation::{self, GenerateTitleInput, TitlePromptInput, WorkflowSummary}; -use crate::server_secrets::LlmClientResult; #[cfg(any(test, feature = "test-support"))] use crate::test_support as server_test_support; @@ -591,17 +590,8 @@ pub(crate) async fn create_run_from_manifest( // and ask-fabro-readiness) and the LLM client itself (for the spawned // title-generation task). `ready_llm_provider_ids` would otherwise call // `resolve_llm_client` a second time and discard the client. - let llm_client_for_title = match state.resolve_llm_client().await { - Ok(result) => Some(result), - Err(err) => { - tracing::warn!(error = ?err, "Failed to resolve LLM client while creating run"); - None - } - }; - let ready_provider_ids = llm_client_for_title - .as_ref() - .map(LlmClientResult::provider_ids) - .unwrap_or_default(); + let (llm_result, ready_provider_ids) = state.resolve_llm_client_with_ready_ids().await; + let llm_client_for_title = llm_result.ok(); let run_materialization_provider_ids = { #[cfg(any(test, feature = "test-support"))] { @@ -835,17 +825,7 @@ async fn run_preflight( return ApiError::bad_request(format!("Run config variable interpolation failed: {err}")) .into_response(); } - let llm_result = state.resolve_llm_client().await; - if let Err(error) = &llm_result { - tracing::warn!( - error = ?error, - "Failed to resolve LLM client while checking ready providers" - ); - } - let ready_providers = llm_result - .as_ref() - .map(LlmClientResult::provider_ids) - .unwrap_or_default(); + let (llm_result, ready_providers) = state.resolve_llm_client_with_ready_ids().await; let mut validated = match run_manifest::validate_prepared_manifest_for_preflight( &prepared, state.catalog(), @@ -859,21 +839,14 @@ async fn run_preflight( Err(err) => return ApiError::bad_request(err.to_string()).into_response(), }; validated.promote_template_undefined_variables_to_errors(); - let response = match run_manifest::run_preflight( - &state, - &prepared, - &validated, - &ready_providers, - llm_result, - ) - .await - { - Ok((response, _ok)) => response, - Err(err) => { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) - .into_response(); - } - }; + let response = + match run_manifest::run_preflight(&state, &prepared, &validated, llm_result).await { + Ok((response, _ok)) => response, + Err(err) => { + return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) + .into_response(); + } + }; (StatusCode::OK, Json(response)).into_response() } diff --git a/lib/components/fabro-workflow/src/operations/create.rs b/lib/components/fabro-workflow/src/operations/create.rs index 39bd097bf..2ec40d443 100644 --- a/lib/components/fabro-workflow/src/operations/create.rs +++ b/lib/components/fabro-workflow/src/operations/create.rs @@ -312,7 +312,7 @@ fn create_from_source( .filter(|provider| !provider.is_empty()) .map(ProviderId::new), &options.configured_providers, - None, + false, &options.catalog, )?; @@ -337,7 +337,7 @@ pub(super) fn preprocess_and_validate( render_mode: RenderMode, default_provider: Option, eligible_providers: &[ProviderId], - fallback_providers: Option<&[ProviderId]>, + catalog_fallback: bool, catalog: &Arc, ) -> Result { let mut parsed = pipeline::parse(dot_source)?; @@ -353,7 +353,7 @@ pub(super) fn preprocess_and_validate( catalog: Arc::clone(catalog), default_provider, eligible_providers: eligible_providers.iter().cloned().collect(), - fallback_providers: fallback_providers.map(|providers| providers.iter().cloned().collect()), + catalog_fallback, })?; Ok(pipeline::validate(transformed, catalog.as_ref(), &[])) } @@ -582,7 +582,7 @@ reasoning = false RenderMode::Structural, None, &test_provider_ids(), - None, + false, &test_catalog(), ) .unwrap() @@ -753,7 +753,7 @@ reasoning = false RenderMode::Strict, None, &test_provider_ids(), - None, + false, &test_catalog(), ); let Err(err) = result else { @@ -792,7 +792,7 @@ reasoning = false RenderMode::Strict, None, &test_provider_ids(), - None, + false, &test_catalog(), ); let Err(err) = result else { diff --git a/lib/components/fabro-workflow/src/operations/mod.rs b/lib/components/fabro-workflow/src/operations/mod.rs index 9e5726515..38b7d9aaa 100644 --- a/lib/components/fabro-workflow/src/operations/mod.rs +++ b/lib/components/fabro-workflow/src/operations/mod.rs @@ -22,7 +22,7 @@ pub use rewind::{RewindInput, RewindOutcome, rewind}; pub use source::WorkflowInput; pub use start::{StartServices, Started, start}; pub use timeline::{ForkTarget, RunTimeline, TimelineEntry, build_timeline, timeline}; -pub use validate::{ValidateInput, validate, validate_with_provider_fallback}; +pub use validate::{ValidateInput, validate, validate_with_ready_providers}; pub use crate::pipeline::{LlmSpec, SandboxEnvSpec}; pub use crate::transforms::RenderMode; diff --git a/lib/components/fabro-workflow/src/operations/start.rs b/lib/components/fabro-workflow/src/operations/start.rs index 9c2e3ed8d..ba720b11f 100644 --- a/lib/components/fabro-workflow/src/operations/start.rs +++ b/lib/components/fabro-workflow/src/operations/start.rs @@ -622,6 +622,7 @@ fn resolve_start_llm( &eligible, settings.model.name.as_deref(), settings.model.provider.as_deref(), + false, )?; let fallback_chain = resolve_fallback_chain(catalog, &provider_id, &model, &settings.model, &eligible)?; diff --git a/lib/components/fabro-workflow/src/operations/validate.rs b/lib/components/fabro-workflow/src/operations/validate.rs index 2fb7235b0..c2b990f5c 100644 --- a/lib/components/fabro-workflow/src/operations/validate.rs +++ b/lib/components/fabro-workflow/src/operations/validate.rs @@ -33,23 +33,23 @@ pub fn validate(input: ValidateInput) -> Result { .all_provider_ids() .into_iter() .collect::>(); - validate_with_provider_sets(input, &eligible_providers, None) + validate_with_eligible_providers(input, &eligible_providers, false) } -/// Parse, transform, and validate while preferring one provider snapshot and -/// falling back to another only for provider-readiness selection failures. -pub fn validate_with_provider_fallback( +/// Parse, transform, and validate, resolving models against the ready +/// providers first and falling back to the full catalog only for +/// provider-readiness selection failures. +pub fn validate_with_ready_providers( input: ValidateInput, - preferred_providers: &[ProviderId], - fallback_providers: &[ProviderId], + ready_providers: &[ProviderId], ) -> Result { - validate_with_provider_sets(input, preferred_providers, Some(fallback_providers)) + validate_with_eligible_providers(input, ready_providers, true) } -fn validate_with_provider_sets( +fn validate_with_eligible_providers( input: ValidateInput, eligible_providers: &[ProviderId], - fallback_providers: Option<&[ProviderId]>, + catalog_fallback: bool, ) -> Result { let resolved = resolve_workflow(ResolveWorkflowInput { workflow: input.workflow, @@ -79,7 +79,7 @@ fn validate_with_provider_sets( .filter(|provider| !provider.is_empty()) .map(fabro_model::ProviderId::new), eligible_providers, - fallback_providers, + catalog_fallback, &input.catalog, ) } diff --git a/lib/components/fabro-workflow/src/pipeline/transform.rs b/lib/components/fabro-workflow/src/pipeline/transform.rs index cc3fe089d..b399d6637 100644 --- a/lib/components/fabro-workflow/src/pipeline/transform.rs +++ b/lib/components/fabro-workflow/src/pipeline/transform.rs @@ -68,7 +68,7 @@ pub fn transform(parsed: Parsed, options: &TransformOptions) -> Result, pub default_provider: Option, pub eligible_providers: HashSet, - pub fallback_providers: Option>, + /// Fall back to the full catalog when the eligible providers cannot + /// supply a requested model, instead of erroring. + pub catalog_fallback: bool, } /// Options for the FINALIZE phase. diff --git a/lib/components/fabro-workflow/src/pipeline/validate.rs b/lib/components/fabro-workflow/src/pipeline/validate.rs index 8e19bb5a6..f0cd51dbd 100644 --- a/lib/components/fabro-workflow/src/pipeline/validate.rs +++ b/lib/components/fabro-workflow/src/pipeline/validate.rs @@ -51,7 +51,7 @@ mod tests { catalog: std::sync::Arc::clone(&catalog), default_provider: None, eligible_providers: catalog.all_provider_ids(), - fallback_providers: None, + catalog_fallback: false, }) .unwrap(); validate(transformed, catalog.as_ref(), &[]) diff --git a/lib/components/fabro-workflow/src/run_materialization.rs b/lib/components/fabro-workflow/src/run_materialization.rs index 27183bcf5..d1118b076 100644 --- a/lib/components/fabro-workflow/src/run_materialization.rs +++ b/lib/components/fabro-workflow/src/run_materialization.rs @@ -14,31 +14,27 @@ pub fn materialize_run( catalog: &Catalog, configured_providers: &[ProviderId], ) -> Result { - materialize_run_with_provider_sets(settings, graph, catalog, configured_providers, None) + materialize_run_with_eligible_providers(settings, graph, catalog, configured_providers, false) } -pub fn materialize_run_with_provider_fallback( +/// Materialize while resolving the run model against the ready providers +/// first, falling back to the full catalog only for provider-readiness +/// selection failures. +pub fn materialize_run_with_ready_providers( settings: WorkflowSettings, graph: &Graph, catalog: &Catalog, - preferred_providers: &[ProviderId], - fallback_providers: &[ProviderId], + ready_providers: &[ProviderId], ) -> Result { - materialize_run_with_provider_sets( - settings, - graph, - catalog, - preferred_providers, - Some(fallback_providers), - ) + materialize_run_with_eligible_providers(settings, graph, catalog, ready_providers, true) } -fn materialize_run_with_provider_sets( +fn materialize_run_with_eligible_providers( mut settings: WorkflowSettings, graph: &Graph, catalog: &Catalog, - configured_providers: &[ProviderId], - fallback_providers: Option<&[ProviderId]>, + eligible_providers: &[ProviderId], + catalog_fallback: bool, ) -> Result { let configured_model = settings.run.model.name.take(); let configured_provider = settings.run.model.provider.take(); @@ -55,25 +51,17 @@ fn materialize_run_with_provider_sets( let provider = configured_provider.or(graph_provider); let model = configured_model.or(graph_model); - let eligible = configured_providers.iter().cloned().collect::>(); - let fallback = - fallback_providers.map(|providers| providers.iter().cloned().collect::>()); - let provider = provider - .as_deref() - .filter(|provider| !provider.is_empty()) - .map(ProviderId::new); - let selected = match fallback { - Some(fallback) => catalog.resolve_selection_with_fallback( - model.as_deref(), - provider.as_ref(), - &eligible, - &fallback, - ), - None => catalog.resolve_selection(model.as_deref(), provider.as_ref(), &eligible), - }?; + let eligible = eligible_providers.iter().cloned().collect::>(); + let (resolved_model, resolved_provider) = resolve_run_model( + catalog, + &eligible, + model.as_deref(), + provider.as_deref(), + catalog_fallback, + )?; - settings.run.model.name = Some(selected.model); - settings.run.model.provider = Some(selected.provider.into_inner()); + settings.run.model.name = Some(resolved_model); + settings.run.model.provider = Some(resolved_provider.into_inner()); let goal = graph.goal().to_string(); settings.run.goal = if goal.is_empty() { @@ -99,10 +87,15 @@ pub(crate) fn resolve_run_model( eligible: &HashSet, model: Option<&str>, provider: Option<&str>, + catalog_fallback: bool, ) -> Result<(String, ProviderId), ModelSelectionError> { let provider = provider .filter(|provider| !provider.is_empty()) .map(ProviderId::new); - let selected = catalog.resolve_selection(model, provider.as_ref(), eligible)?; + let selected = if catalog_fallback { + catalog.resolve_selection_with_catalog_fallback(model, provider.as_ref(), eligible)? + } else { + catalog.resolve_selection(model, provider.as_ref(), eligible)? + }; Ok((selected.model, selected.provider)) } diff --git a/lib/components/fabro-workflow/src/transforms/model_resolution.rs b/lib/components/fabro-workflow/src/transforms/model_resolution.rs index 00e71ba74..12f29ac5a 100644 --- a/lib/components/fabro-workflow/src/transforms/model_resolution.rs +++ b/lib/components/fabro-workflow/src/transforms/model_resolution.rs @@ -13,7 +13,7 @@ pub struct ModelResolutionTransform { catalog: Arc, default_provider: Option, eligible_providers: HashSet, - fallback_providers: Option>, + catalog_fallback: bool, } impl ModelResolutionTransform { @@ -24,7 +24,7 @@ impl ModelResolutionTransform { catalog, default_provider: None, eligible_providers, - fallback_providers: None, + catalog_fallback: false, } } @@ -34,7 +34,7 @@ impl ModelResolutionTransform { catalog, default_provider: None, eligible_providers, - fallback_providers: None, + catalog_fallback: false, } } @@ -44,12 +44,11 @@ impl ModelResolutionTransform { self } + /// When enabled, provider-readiness selection failures fall back to the + /// full catalog instead of erroring. #[must_use] - pub fn with_fallback_providers( - mut self, - fallback_providers: Option>, - ) -> Self { - self.fallback_providers = fallback_providers; + pub fn with_catalog_fallback(mut self, catalog_fallback: bool) -> Self { + self.catalog_fallback = catalog_fallback; self } @@ -58,18 +57,15 @@ impl ModelResolutionTransform { model: &str, explicit_provider: Option<&ProviderId>, ) -> Result<(String, ProviderId), Error> { - let selected = match &self.fallback_providers { - Some(fallback_providers) => self.catalog.resolve_selection_with_fallback( + let selected = if self.catalog_fallback { + self.catalog.resolve_selection_with_catalog_fallback( Some(model), explicit_provider, &self.eligible_providers, - fallback_providers, - ), - None => self.catalog.resolve_selection( - Some(model), - explicit_provider, - &self.eligible_providers, - ), + ) + } else { + self.catalog + .resolve_selection(Some(model), explicit_provider, &self.eligible_providers) }?; Ok((selected.model, selected.provider)) } @@ -378,7 +374,7 @@ enabled = true Arc::clone(&catalog), HashSet::from([ProviderId::new("openrouter")]), ) - .with_fallback_providers(Some(catalog.all_provider_ids())) + .with_catalog_fallback(true) .apply(graph) .unwrap(); diff --git a/lib/components/fabro-workflow/tests/it/integration.rs b/lib/components/fabro-workflow/tests/it/integration.rs index 44307ca81..8e415c81a 100644 --- a/lib/components/fabro-workflow/tests/it/integration.rs +++ b/lib/components/fabro-workflow/tests/it/integration.rs @@ -4908,7 +4908,7 @@ async fn import_e2e_through_engine() { catalog: std::sync::Arc::clone(&catalog), default_provider: None, eligible_providers: catalog.all_provider_ids(), - fallback_providers: None, + catalog_fallback: false, }) .unwrap(); let validated = validate(transformed, catalog.as_ref(), &[]); diff --git a/lib/foundation/fabro-model/src/catalog.rs b/lib/foundation/fabro-model/src/catalog.rs index da9ab96fd..db59dcb98 100644 --- a/lib/foundation/fabro-model/src/catalog.rs +++ b/lib/foundation/fabro-model/src/catalog.rs @@ -1102,19 +1102,18 @@ impl Catalog { } /// Resolve a selection against a preferred provider snapshot, falling back - /// to a broader eligible set only when the preferred set cannot supply the - /// requested provider or model. + /// to every provider in the catalog only when the preferred set cannot + /// supply the requested provider or model. /// /// This is useful for readiness checks: ready providers remain preferred, /// while a catalog-only offering can still be selected so the caller can /// report why its provider is unavailable. Semantic failures such as an /// unknown provider do not fall back. - pub fn resolve_selection_with_fallback( + pub fn resolve_selection_with_catalog_fallback( &self, selector: Option<&str>, explicit_provider: Option<&ProviderId>, preferred_providers: &HashSet, - fallback_providers: &HashSet, ) -> Result { match self.resolve_selection(selector, explicit_provider, preferred_providers) { Ok(selected) => Ok(selected), @@ -1122,7 +1121,7 @@ impl Catalog { ModelSelectionError::ProviderUnavailable { .. } | ModelSelectionError::NoEligibleOffering { .. } | ModelSelectionError::NoDefaultModel { .. }, - ) => self.resolve_selection(selector, explicit_provider, fallback_providers), + ) => self.resolve_selection(selector, explicit_provider, &self.all_provider_ids()), Err(error) => Err(error), } } @@ -4021,30 +4020,19 @@ adapter = "openai_compatible" let openai = ProviderId::openai(); let openrouter = ProviderId::new("openrouter"); let ready = HashSet::from([openrouter.clone()]); - let catalog_providers = HashSet::from([openai.clone(), openrouter.clone()]); let shared = catalog - .resolve_selection_with_fallback(Some("portable"), None, &ready, &catalog_providers) + .resolve_selection_with_catalog_fallback(Some("portable"), None, &ready) .unwrap(); assert_eq!(shared.provider, openrouter); let pinned = catalog - .resolve_selection_with_fallback( - Some("portable"), - Some(&openai), - &ready, - &catalog_providers, - ) + .resolve_selection_with_catalog_fallback(Some("portable"), Some(&openai), &ready) .unwrap(); assert_eq!(pinned.provider, openai); let unknown = catalog - .resolve_selection_with_fallback( - Some("provider-private-preview"), - None, - &ready, - &catalog_providers, - ) + .resolve_selection_with_catalog_fallback(Some("provider-private-preview"), None, &ready) .unwrap(); assert_eq!(unknown.provider, ProviderId::new("openrouter")); assert_eq!(unknown.model, "provider-private-preview"); From d611da345b2d2294022a5c75b69c336d03e1e818 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 24 Jul 2026 09:13:26 -0400 Subject: [PATCH 20/24] Clarify non-validation LLM error mapping --- lib/apps/fabro-server/src/error.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/apps/fabro-server/src/error.rs b/lib/apps/fabro-server/src/error.rs index 70537f714..62831c983 100644 --- a/lib/apps/fabro-server/src/error.rs +++ b/lib/apps/fabro-server/src/error.rs @@ -171,7 +171,8 @@ impl From for ApiError { } /// LLM errors split at the HTTP boundary: request-validation failures are the -/// caller's fault (400); everything else is an upstream failure (502). +/// caller's fault (400); non-validation LLM failures, including provider, +/// middleware, and local configuration failures, return 502. impl From for ApiError { fn from(err: fabro_llm::Error) -> Self { match err { From 5d8befa6acd443372b4e80c293d912d756cea426 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 24 Jul 2026 09:15:18 -0400 Subject: [PATCH 21/24] Reuse canonical ModelControls in fabro-api and tighten serde contract Add the missing with_replacement for ModelControls so progenitor reuses fabro_model::ModelControls instead of generating a dead parallel DTO, re-export it from fabro_api::types, and assert type identity in the round-trip test. Drop #[serde(default)] from Model.controls and ModelControls.reasoning_effort: the OpenAPI spec marks both required, matching the strict deserialization of the sibling features/costs fields. Update CLI stub payloads to include the now-required field. Co-Authored-By: Claude Fable 5 --- lib/apps/fabro-cli/tests/it/cmd/model.rs | 12 ++++++++++++ lib/apps/fabro-cli/tests/it/cmd/model_test.rs | 3 +++ lib/foundation/fabro-api/build.rs | 1 + lib/foundation/fabro-api/src/lib.rs | 5 +++-- lib/foundation/fabro-api/tests/model_round_trip.rs | 3 ++- lib/foundation/fabro-model/src/types.rs | 4 +--- 6 files changed, 22 insertions(+), 6 deletions(-) diff --git a/lib/apps/fabro-cli/tests/it/cmd/model.rs b/lib/apps/fabro-cli/tests/it/cmd/model.rs index 18d3bce01..28fe1fa01 100644 --- a/lib/apps/fabro-cli/tests/it/cmd/model.rs +++ b/lib/apps/fabro-cli/tests/it/cmd/model.rs @@ -108,6 +108,9 @@ fn list_with_filters_renders_server_models_table() { "vision": false, "reasoning": false }, + "controls": { + "reasoning_effort": [] + }, "costs": { "input_cost_per_mtok": 1.2, "output_cost_per_mtok": 3.4, @@ -134,6 +137,9 @@ fn list_with_filters_renders_server_models_table() { "vision": true, "reasoning": true }, + "controls": { + "reasoning_effort": [] + }, "costs": { "input_cost_per_mtok": null, "output_cost_per_mtok": null, @@ -206,6 +212,9 @@ fn list_uses_configured_server_target_without_server_flag() { "vision": false, "reasoning": false }, + "controls": { + "reasoning_effort": [] + }, "costs": { "input_cost_per_mtok": 1.0, "output_cost_per_mtok": 2.0, @@ -260,6 +269,9 @@ fn list_uses_fabro_config_for_machine_settings() { "vision": false, "reasoning": false }, + "controls": { + "reasoning_effort": [] + }, "costs": { "input_cost_per_mtok": 1.0, "output_cost_per_mtok": 2.0, diff --git a/lib/apps/fabro-cli/tests/it/cmd/model_test.rs b/lib/apps/fabro-cli/tests/it/cmd/model_test.rs index 70e64ac77..acaae9851 100644 --- a/lib/apps/fabro-cli/tests/it/cmd/model_test.rs +++ b/lib/apps/fabro-cli/tests/it/cmd/model_test.rs @@ -42,6 +42,9 @@ fn model_json(id: &str, provider: &str, configured: bool) -> serde_json::Value { "vision": false, "reasoning": false }, + "controls": { + "reasoning_effort": [] + }, "costs": { "input_cost_per_mtok": 1.0, "output_cost_per_mtok": 2.0, diff --git a/lib/foundation/fabro-api/build.rs b/lib/foundation/fabro-api/build.rs index cdec7f07a..e560335ae 100644 --- a/lib/foundation/fabro-api/build.rs +++ b/lib/foundation/fabro-api/build.rs @@ -478,6 +478,7 @@ fn main() { ), ("ReasoningEffort", "fabro_model::ReasoningEffort", &[]), ("ModelFeatures", "fabro_model::ModelFeatures", &[]), + ("ModelControls", "fabro_model::ModelControls", &[]), ("ModelCosts", "fabro_model::ModelCosts", &[]), ("ModelTestMode", "fabro_model::ModelTestMode", &[]), ("RunProjection", "fabro_types::RunProjection", &[]), diff --git a/lib/foundation/fabro-api/src/lib.rs b/lib/foundation/fabro-api/src/lib.rs index 6d68d57b4..17622a6ca 100644 --- a/lib/foundation/fabro-api/src/lib.rs +++ b/lib/foundation/fabro-api/src/lib.rs @@ -20,8 +20,9 @@ pub mod types { }; pub use fabro_environment::Environment; pub use fabro_model::{ - CostSource, Model, ModelCosts, ModelFeatures, ModelLimits, ModelRef as BillingModelRef, - ModelTestMode, Provider, ReasoningEffort, ReasoningEffortFeature, Speed as BillingSpeed, + CostSource, Model, ModelControls, ModelCosts, ModelFeatures, ModelLimits, + ModelRef as BillingModelRef, ModelTestMode, Provider, ReasoningEffort, + ReasoningEffortFeature, Speed as BillingSpeed, }; pub use fabro_types::run_event::AgentSessionActivatedProps; pub use fabro_types::settings::run::McpHttpProtocol; diff --git a/lib/foundation/fabro-api/tests/model_round_trip.rs b/lib/foundation/fabro-api/tests/model_round_trip.rs index 5f39eb6ab..0decc177f 100644 --- a/lib/foundation/fabro-api/tests/model_round_trip.rs +++ b/lib/foundation/fabro-api/tests/model_round_trip.rs @@ -1,6 +1,6 @@ use std::any::{TypeId, type_name}; -use fabro_api::types::Model as ApiModel; +use fabro_api::types::{Model as ApiModel, ModelControls as ApiModelControls}; use fabro_model::{ Model, ModelControls, ModelCosts, ModelFeatures, ModelLimits, ProviderId, ReasoningEffort, ReasoningEffortFeature, @@ -9,6 +9,7 @@ use fabro_model::{ #[test] fn model_reuses_canonical_type() { assert_same_type::(); + assert_same_type::(); } #[test] diff --git a/lib/foundation/fabro-model/src/types.rs b/lib/foundation/fabro-model/src/types.rs index 9a0d27148..9e4a9e96d 100644 --- a/lib/foundation/fabro-model/src/types.rs +++ b/lib/foundation/fabro-model/src/types.rs @@ -84,11 +84,10 @@ pub struct ModelCosts { pub cache_input_cost_per_mtok: Option, } -#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] pub struct ModelControls { /// Exact reasoning-effort values accepted by this provider/model offering. /// An empty list means the request control is unsupported. - #[serde(default)] pub reasoning_effort: Vec, } @@ -102,7 +101,6 @@ pub struct Model { pub training: Option, pub knowledge_cutoff: Option, pub features: ModelFeatures, - #[serde(default)] pub controls: ModelControls, pub costs: ModelCosts, pub estimated_output_tps: Option, From 4c7d13aff0f8188bc1d967880ff5541c5de39fbc Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 24 Jul 2026 09:18:43 -0400 Subject: [PATCH 22/24] fix(store): enforce event sequence key limit --- lib/components/fabro-store/src/error.rs | 2 + lib/components/fabro-store/src/keys.rs | 9 ++-- .../fabro-store/src/slate/run_store.rs | 48 ++++++++++++++++++- 3 files changed, 54 insertions(+), 5 deletions(-) diff --git a/lib/components/fabro-store/src/error.rs b/lib/components/fabro-store/src/error.rs index 33126d474..43c26b44a 100644 --- a/lib/components/fabro-store/src/error.rs +++ b/lib/components/fabro-store/src/error.rs @@ -24,6 +24,8 @@ pub enum Error { SessionAlreadyExists(String), #[error("run store is read-only")] ReadOnly, + #[error("event sequence limit of {max_seq} reached")] + EventSequenceExhausted { max_seq: u32 }, #[error("invalid key segment: {segment:?}")] InvalidKeySegment { segment: String }, #[error("failed to parse key: {0}")] diff --git a/lib/components/fabro-store/src/keys.rs b/lib/components/fabro-store/src/keys.rs index f63bb27b7..343cdc1d0 100644 --- a/lib/components/fabro-store/src/keys.rs +++ b/lib/components/fabro-store/src/keys.rs @@ -3,6 +3,8 @@ use std::ops::Range; use fabro_types::{RunBlobId, RunId, SessionId}; +pub(crate) const MAX_EVENT_SEQ: u32 = 999_999; + #[derive(Debug, PartialEq, Eq)] pub(crate) struct SlateKey(String); @@ -61,8 +63,9 @@ pub(crate) fn run_events_prefix(run_id: &RunId) -> SlateKey { } // Sequence keys zero-pad `seq` to six digits so lexicographic key order -// matches numeric seq order for up to 999,999 events per run. Seek-based -// event listing (`run_events_range`) depends on this invariant. +// matches numeric seq order through `MAX_EVENT_SEQ`. Seek-based event listing +// (`run_events_range`) depends on this invariant, so event allocation rejects +// larger sequences. pub(crate) fn run_event_key(run_id: &RunId, seq: u32, epoch_ms: i64) -> SlateKey { SlateKey::new("runs") .with(run_id) @@ -184,7 +187,7 @@ mod tests { assert!(!contains(&run_event_key(&run_id, 1, 123))); assert!(contains(&run_event_key(&run_id, 2, 123))); - assert!(contains(&run_event_key(&run_id, 999_999, 123))); + assert!(contains(&run_event_key(&run_id, MAX_EVENT_SEQ, 123))); // Sibling namespaces of the same run sort outside the range. assert!(!contains(&SlateKey::new("runs").with(run_id).with("state"))); assert!(!contains( diff --git a/lib/components/fabro-store/src/slate/run_store.rs b/lib/components/fabro-store/src/slate/run_store.rs index 664ce3162..7efa68aba 100644 --- a/lib/components/fabro-store/src/slate/run_store.rs +++ b/lib/components/fabro-store/src/slate/run_store.rs @@ -292,7 +292,7 @@ impl RunDatabase { async fn append_event_envelope_locked(&self, payload: &EventPayload) -> Result { let event_seq = self.inner.event_seq.as_ref().ok_or(Error::ReadOnly)?; - let seq = event_seq.fetch_add(1, Ordering::SeqCst); + let seq = allocate_event_seq(event_seq)?; let event = EventEnvelope { seq, event: RunEvent::try_from(payload)?, @@ -507,6 +507,16 @@ impl RunDatabase { } } +fn allocate_event_seq(event_seq: &AtomicU32) -> Result { + event_seq + .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |seq| { + (seq <= keys::MAX_EVENT_SEQ).then_some(seq + 1) + }) + .map_err(|_| Error::EventSequenceExhausted { + max_seq: keys::MAX_EVENT_SEQ, + }) +} + fn apply_cached_projection_event( state: &mut Option, event: &EventEnvelope, @@ -736,13 +746,14 @@ fn key_to_str(key: &Bytes) -> Result<&str> { #[cfg(test)] mod tests { use std::sync::Arc; + use std::sync::atomic::Ordering; use std::time::Duration; use fabro_types::{Graph, RunId, SessionId, StageId, WorkflowSettings, test_support}; use object_store::memory::InMemory; use serde_json::json; - use crate::{Database, EventPayload, keys}; + use crate::{Database, Error, EventPayload, keys}; #[tokio::test] async fn list_blobs_reads_global_cas_namespace() { @@ -894,6 +905,39 @@ mod tests { assert_eq!(seqs, vec![3]); } + #[tokio::test] + async fn append_event_rejects_sequences_beyond_key_order_limit() { + let run = fresh_run().await; + let run_id = run.run_id(); + run.inner + .event_seq + .as_ref() + .unwrap() + .store(keys::MAX_EVENT_SEQ, Ordering::SeqCst); + + let seq = run + .append_event(&stage_prompt_payload(&run_id, 1, Some("alpha"))) + .await + .unwrap(); + assert_eq!(seq, keys::MAX_EVENT_SEQ); + + let err = run + .append_event(&stage_prompt_payload(&run_id, 2, Some("beta"))) + .await + .unwrap_err(); + assert!(matches!( + err, + Error::EventSequenceExhausted { max_seq } + if max_seq == keys::MAX_EVENT_SEQ + )); + assert!( + run.get_event(keys::MAX_EVENT_SEQ + 1) + .await + .unwrap() + .is_none() + ); + } + #[tokio::test] async fn list_events_for_stage_returns_only_matching_events_in_seq_order() { let run = fresh_run().await; From 3970c9f545d9fc91bd2c486ade5eb0b5750e2dac Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 24 Jul 2026 09:23:18 -0400 Subject: [PATCH 23/24] Restore serde defaults on Model.controls for older-server compatibility Copilot review flagged that dropping #[serde(default)] makes newer clients hard-fail against servers that predate the controls field. The late-added Model fields (default, small_default, configured) set the precedent: required in the OpenAPI spec, defaulted on deserialization. An empty controls list already means "unsupported", so the degraded value is semantically correct. Co-Authored-By: Claude Fable 5 --- lib/foundation/fabro-model/src/types.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/foundation/fabro-model/src/types.rs b/lib/foundation/fabro-model/src/types.rs index 9e4a9e96d..50ae020dc 100644 --- a/lib/foundation/fabro-model/src/types.rs +++ b/lib/foundation/fabro-model/src/types.rs @@ -88,6 +88,7 @@ pub struct ModelCosts { pub struct ModelControls { /// Exact reasoning-effort values accepted by this provider/model offering. /// An empty list means the request control is unsupported. + #[serde(default)] pub reasoning_effort: Vec, } @@ -101,6 +102,9 @@ pub struct Model { pub training: Option, pub knowledge_cutoff: Option, pub features: ModelFeatures, + /// Required in API responses; defaulted on deserialization so newer + /// clients tolerate older servers that predate this field. + #[serde(default)] pub controls: ModelControls, pub costs: ModelCosts, pub estimated_output_tps: Option, From ac62585eb7057b9fa003dc60017fcb95d4df5e31 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 24 Jul 2026 09:26:22 -0400 Subject: [PATCH 24/24] fix parallel result artifact handling --- docs/internal/parallel-strategy.md | 6 +- lib/components/fabro-workflow/src/artifact.rs | 133 ++++++++++++------ 2 files changed, 90 insertions(+), 49 deletions(-) diff --git a/docs/internal/parallel-strategy.md b/docs/internal/parallel-strategy.md index a7f9ed77e..fb577cf6c 100644 --- a/docs/internal/parallel-strategy.md +++ b/docs/internal/parallel-strategy.md @@ -83,9 +83,9 @@ The parallel stage outcome is: ## 4. Artifacts and downstream context -Large context values use the normal artifact store. Offloading recursively -replaces oversized leaf values while retaining the object and array structure -of `parallel.results`. +Large context values use the normal artifact store. Offloading replaces +oversized values within each branch's `context_updates` while retaining the +outer object and array structure of `parallel.results`. When Fabro constructs execution or prompt context, it resolves nested textual blob references under `response.*` and `command.output`, including those keys diff --git a/lib/components/fabro-workflow/src/artifact.rs b/lib/components/fabro-workflow/src/artifact.rs index 31b98f6c5..9b7b1b0c5 100644 --- a/lib/components/fabro-workflow/src/artifact.rs +++ b/lib/components/fabro-workflow/src/artifact.rs @@ -29,9 +29,9 @@ const ARTIFACT_POINTER_PREFIX: &str = "file://"; /// and replaced with a `"blob://sha256/{blob_id}"` reference. /// Small values are left untouched. /// -/// `parallel.results` is offloaded leaf-wise instead of as one value so it -/// stays a structured array that fan-in prompts, projections, and the UI can -/// read without hydrating the whole payload. +/// `parallel.results` is offloaded at each branch context-update boundary +/// instead of as one value so it stays a structured array that fan-in prompts, +/// projections, and the UI can read without hydrating the whole payload. /// /// # Errors /// @@ -42,7 +42,7 @@ pub async fn offload_large_values( ) -> Result<()> { for (key, value) in updates { if key == context::keys::PARALLEL_RESULTS { - offload_large_leaves(value, run_store).await?; + offload_parallel_result_updates(value, run_store).await?; } else { offload_value(value, run_store).await?; } @@ -50,8 +50,9 @@ pub async fn offload_large_values( Ok(()) } -/// Offload large leaves of typed parallel branch results before they are -/// emitted through `parallel.completed` and stored in projections. +/// Offload large context-update values from typed parallel branch results +/// before they are emitted through `parallel.completed` and stored in +/// projections. /// /// # Errors /// @@ -62,34 +63,31 @@ pub async fn offload_parallel_branch_updates( ) -> Result<()> { for result in results.iter_mut() { for value in result.context_updates.values_mut() { - offload_large_leaves(value, run_store).await?; + offload_value(value, run_store).await?; } } Ok(()) } -fn offload_large_leaves<'a>( - value: &'a mut Value, - run_store: &'a RunStoreHandle, -) -> BoxFuture<'a, Result<()>> { - Box::pin(async move { - match value { - Value::Array(items) => { - for item in items { - offload_large_leaves(item, run_store).await?; - } - } - Value::Object(map) => { - for item in map.values_mut() { - offload_large_leaves(item, run_store).await?; - } - } - Value::String(_) | Value::Null | Value::Bool(_) | Value::Number(_) => { - offload_value(value, run_store).await?; - } +async fn offload_parallel_result_updates( + value: &mut Value, + run_store: &RunStoreHandle, +) -> Result<()> { + let Some(results) = value.as_array_mut() else { + return Ok(()); + }; + for result in results { + let Some(context_updates) = result + .get_mut("context_updates") + .and_then(Value::as_object_mut) + else { + continue; + }; + for value in context_updates.values_mut() { + offload_value(value, run_store).await?; } - Ok(()) - }) + } + Ok(()) } async fn offload_value(value: &mut Value, run_store: &RunStoreHandle) -> Result<()> { @@ -364,14 +362,13 @@ fn resolve_execution_value<'a>( } Value::Object(map) => { for (child_key, item) in map.iter_mut() { - resolve_execution_value( - Some(child_key.as_str()), - item, - run_store, - env, - run_dir, - ) - .await?; + let child_context_key = if key.is_some_and(is_text_context_key) { + key + } else { + Some(child_key.as_str()) + }; + resolve_execution_value(child_context_key, item, run_store, env, run_dir) + .await?; } } Value::Null | Value::Bool(_) | Value::Number(_) => {} @@ -556,23 +553,42 @@ mod tests { } #[tokio::test] - async fn offload_preserves_parallel_results_and_replaces_only_large_leaves() { + async fn offload_preserves_parallel_results_and_replaces_large_context_updates() { let run_store = make_run_store("parallel-result-artifact-offload").await; let large_response = "r".repeat(BLOB_OFFLOAD_THRESHOLD + 1); let large_output = "o".repeat(BLOB_OFFLOAD_THRESHOLD + 1); + let large_report = Value::Array(vec![ + Value::String("small".to_string()); + BLOB_OFFLOAD_THRESHOLD / 4 + ]); + let expected_report_blob = RunBlobId::new(&serde_json::to_vec(&large_report).unwrap()); + let mut typed_results = vec![ParallelBranchResult { + id: "branch_a".to_string(), + status: fabro_types::StageOutcome::Succeeded, + context_updates: std::collections::BTreeMap::from([ + ( + "response.branch_a".to_string(), + serde_json::json!(large_response), + ), + ( + context::keys::COMMAND_OUTPUT.to_string(), + serde_json::json!(large_output), + ), + ("report".to_string(), large_report.clone()), + ("small".to_string(), serde_json::json!("kept inline")), + ]), + }]; + + offload_parallel_branch_updates(&mut typed_results, &run_store.clone().into()) + .await + .unwrap(); let mut updates = HashMap::from([( context::keys::PARALLEL_RESULTS.to_string(), - serde_json::json!([{ - "id": "branch_a", - "status": "failed", - "context_updates": { - "response.branch_a": large_response, - "command.output": large_output, - "small": "kept inline", - } - }]), + serde_json::to_value(typed_results).unwrap(), )]); + // The ordinary lifecycle pass must preserve the typed result structure + // and the values already offloaded before the completion event. offload_large_values(&mut updates, &run_store.clone().into()) .await .unwrap(); @@ -593,6 +609,19 @@ mod tests { .as_str() .is_some_and(|value| fabro_types::parse_blob_ref(value).is_some()) ); + assert_eq!( + branch_updates["report"], + serde_json::json!(format_blob_ref(&expected_report_blob)) + ); + let stored_report = run_store + .read_blob(&expected_report_blob) + .await + .unwrap() + .expect("structured report blob should exist"); + assert_eq!( + serde_json::from_slice::(&stored_report).unwrap(), + large_report + ); assert_eq!(branch_updates["small"], serde_json::json!("kept inline")); } @@ -642,6 +671,10 @@ mod tests { "status": "succeeded", "context_updates": { "response.branch_a": fabro_types::format_blob_ref(&response_blob), + "response.nested": { + "text": fabro_types::format_blob_ref(&response_blob), + "items": [fabro_types::format_blob_ref(&output_blob)], + }, "command.output": fabro_types::format_blob_ref(&output_blob), "report": fabro_types::format_blob_ref(&unrelated_blob), } @@ -657,6 +690,14 @@ mod tests { let updates = &resolved[context::keys::PARALLEL_RESULTS][0]["context_updates"]; assert_eq!(updates["response.branch_a"], serde_json::json!(response)); + assert_eq!( + updates["response.nested"]["text"], + serde_json::json!(response) + ); + assert_eq!( + updates["response.nested"]["items"][0], + serde_json::json!(output) + ); assert_eq!( updates[context::keys::COMMAND_OUTPUT], serde_json::json!(output)