scriptoriumadmin/fabro
1
0
Fork 0
mirror of https://github.com/fabro-sh/fabro.git synced 2026-10-10 03:30:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

checkpoint

Browse source 
⚒️ Generated with [Fabro](https://fabro.sh)
This commit is contained in:
Fabro 2026-07-30 00:46:04 +00:00
parent 7425261f11
commit d05213a2b3
6 changed files with 1770 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

637
run.json
View file

File diff suppressed because one or more lines are too long

755
stages/008-verify@1/diff.patch Normal file
View file

@ -0,0 +1,755 @@
diff --git a/docs/public/reference/cli.mdx b/docs/public/reference/cli.mdx
index f264f7bab..530107c6a 100644
--- a/docs/public/reference/cli.mdx
+++ b/docs/public/reference/cli.mdx
@@ -186,7 +186,8 @@ fabro artifact cp [OPTIONS] <SOURCE> [DEST]
| `--node <node>` | Filter to artifacts from a specific node |
| `--retry <retry>` | Filter to artifacts from a specific retry attempt |
| `--server <server>` | Fabro server target: http(s) URL or absolute Unix socket path |
-| `--tree` | Preserve {node_slug}/retry_{N}/ directory structure |
+| `--stage <stage>` | Filter to artifacts from a specific stage visit (node@visit) |
+| `--tree` | Preserve node[/visit_{N}]/retry_{N}/ directory structure |
#### `fabro artifact list`
@@ -209,6 +210,7 @@ fabro artifact list [OPTIONS] <RUN_ID>
| `--node <node>` | Filter to artifacts from a specific node |
| `--retry <retry>` | Filter to artifacts from a specific retry attempt |
| `--server <server>` | Fabro server target: http(s) URL or absolute Unix socket path |
+| `--stage <stage>` | Filter to artifacts from a specific stage visit (node@visit) |
### `fabro ask`
diff --git a/docs/public/workflows/stages-and-nodes.mdx b/docs/public/workflows/stages-and-nodes.mdx
index 79ae0b00f..59f48df69 100644
--- a/docs/public/workflows/stages-and-nodes.mdx
+++ b/docs/public/workflows/stages-and-nodes.mdx
@@ -117,7 +117,7 @@ When a node has no explicit `shape` or `type`, the presence of `script` makes it
For `stdin_source`, strings are passed unchanged. Other JSON values use compact
JSON. Fabro does not add a newline. A missing source, or a value larger than
-10 MiB, fails before the command starts.
+30 MiB, fails before the command starts.
### Human
diff --git a/lib/apps/fabro-cli/src/args.rs b/lib/apps/fabro-cli/src/args.rs
index fb20b1158..d9b487670 100644
--- a/lib/apps/fabro-cli/src/args.rs
+++ b/lib/apps/fabro-cli/src/args.rs
@@ -531,6 +531,10 @@ pub(crate) struct ArtifactListArgs {
#[arg(long)]
pub(crate) node: Option<String>,
+ /// Filter to artifacts from a specific stage visit (node@visit)
+ #[arg(long)]
+ pub(crate) stage: Option<String>,
+
/// Filter to artifacts from a specific retry attempt
#[arg(long)]
pub(crate) retry: Option<u32>,
@@ -552,11 +556,15 @@ pub(crate) struct ArtifactCpArgs {
#[arg(long)]
pub(crate) node: Option<String>,
+ /// Filter to artifacts from a specific stage visit (node@visit)
+ #[arg(long)]
+ pub(crate) stage: Option<String>,
+
/// Filter to artifacts from a specific retry attempt
#[arg(long)]
pub(crate) retry: Option<u32>,
- /// Preserve {node_slug}/retry_{N}/ directory structure
+ /// Preserve node[/visit_{N}]/retry_{N}/ directory structure
#[arg(long)]
pub(crate) tree: bool,
}
diff --git a/lib/apps/fabro-cli/src/commands/artifact/cp.rs b/lib/apps/fabro-cli/src/commands/artifact/cp.rs
index e33ec2cfc..76e755549 100644
--- a/lib/apps/fabro-cli/src/commands/artifact/cp.rs
+++ b/lib/apps/fabro-cli/src/commands/artifact/cp.rs
@@ -3,6 +3,7 @@
reason = "CLI `artifact cp` command: sync file I/O in command handler; not on a Tokio hot path"
)]
+use std::collections::{HashMap, HashSet};
use std::path::{Path, PathBuf};
use anyhow::{Context, Result, bail};
@@ -20,6 +21,7 @@ pub(super) async fn cp_command(args: &ArtifactCpArgs, base_ctx: &CommandContext)
&args.server,
run_id_selector,
args.node.as_deref(),
+ args.stage.as_deref(),
args.retry,
)
.await?;
@@ -45,7 +47,7 @@ pub(super) async fn cp_command(args: &ArtifactCpArgs, base_ctx: &CommandContext)
.map(|entry| format_candidate(entry))
.collect();
bail!(
- "Path '{path}' matches multiple artifacts: {}. Use --node and/or --retry to disambiguate.",
+ "Path '{path}' matches multiple artifacts: {}. Use --stage and/or --retry to disambiguate.",
candidates.join(", ")
);
}
@@ -77,10 +79,9 @@ pub(super) async fn cp_command(args: &ArtifactCpArgs, base_ctx: &CommandContext)
let mut copied = Vec::new();
if args.tree {
+ let multi_visit_nodes = multi_visit_nodes(&entries);
for entry in &entries {
- let relative_dest = PathBuf::from(&entry.node_slug)
- .join(format!("retry_{}", entry.retry))
- .join(&entry.relative_path);
+ let relative_dest = artifact_tree_path(entry, &multi_visit_nodes);
let dest_file = args.dest.join(relative_dest);
write_artifact_file(&client, &run_id, entry, &dest_file).await?;
copied.push(serde_json::json!({
@@ -99,7 +100,7 @@ pub(super) async fn cp_command(args: &ArtifactCpArgs, base_ctx: &CommandContext)
.into_owned();
if let Some((_, existing)) = by_filename.iter().find(|(name, _)| name == &filename) {
bail!(
- "Filename collision: '{}' exists in both {} and {}. Use --tree to preserve directory structure, or --node and/or --retry to filter.",
+ "Filename collision: '{}' exists in both {} and {}. Use --tree to preserve directory structure, or --stage and/or --retry to filter.",
filename,
format_candidate(existing),
format_candidate(entry)
@@ -157,7 +158,34 @@ fn parse_source(source: &str) -> (&str, Option<&str>) {
}
fn format_candidate(entry: &super::ArtifactEntry) -> String {
- format!("{}:retry_{}", entry.node_slug, entry.retry)
+ format!("{}:retry_{}", entry.stage_id, entry.retry)
+}
+
+fn multi_visit_nodes(entries: &[super::ArtifactEntry]) -> HashSet<&str> {
+ let mut first_visit_by_node = HashMap::new();
+ let mut multi_visit_nodes = HashSet::new();
+ for entry in entries {
+ let node_slug = entry.node_slug.as_str();
+ let visit = entry.stage_id.visit();
+ if first_visit_by_node
+ .get(node_slug)
+ .is_some_and(|first_visit| *first_visit != visit)
+ {
+ multi_visit_nodes.insert(node_slug);
+ } else {
+ first_visit_by_node.entry(node_slug).or_insert(visit);
+ }
+ }
+ multi_visit_nodes
+}
+
+fn artifact_tree_path(entry: &super::ArtifactEntry, multi_visit_nodes: &HashSet<&str>) -> PathBuf {
+ let mut path = PathBuf::from(&entry.node_slug);
+ if entry.stage_id.visit() != 1 || multi_visit_nodes.contains(entry.node_slug.as_str()) {
+ path.push(format!("visit_{}", entry.stage_id.visit()));
+ }
+ path.join(format!("retry_{}", entry.retry))
+ .join(&entry.relative_path)
}
#[cfg(test)]
@@ -202,6 +230,6 @@ mod tests {
size: 6,
};
- assert_eq!(format_candidate(&entry), "retry_assets:retry_2");
+ assert_eq!(format_candidate(&entry), "retry_assets@2:retry_2");
}
}
diff --git a/lib/apps/fabro-cli/src/commands/artifact/list.rs b/lib/apps/fabro-cli/src/commands/artifact/list.rs
index 782e323c9..4c533115f 100644
--- a/lib/apps/fabro-cli/src/commands/artifact/list.rs
+++ b/lib/apps/fabro-cli/src/commands/artifact/list.rs
@@ -13,6 +13,7 @@ pub(super) async fn list_command(args: &ArtifactListArgs, base_ctx: &CommandCont
&args.server,
&args.run_id,
args.node.as_deref(),
+ args.stage.as_deref(),
args.retry,
)
.await?;
@@ -31,7 +32,7 @@ pub(super) async fn list_command(args: &ArtifactListArgs, base_ctx: &CommandCont
let use_color = styles.use_color;
let title: Vec<CellStruct> = vec![
- "NODE".cell().bold(use_color),
+ "STAGE".cell().bold(use_color),
"RETRY".cell().bold(use_color).justify(Justify::Right),
"PATH".cell().bold(use_color),
];
@@ -40,7 +41,7 @@ pub(super) async fn list_command(args: &ArtifactListArgs, base_ctx: &CommandCont
.iter()
.map(|entry| {
vec![
- entry.node_slug.clone().cell().bold(use_color),
+ entry.stage_id.to_string().cell().bold(use_color),
entry.retry.cell().justify(Justify::Right),
entry.relative_path.clone().cell(),
]
diff --git a/lib/apps/fabro-cli/src/commands/artifact/mod.rs b/lib/apps/fabro-cli/src/commands/artifact/mod.rs
index f0325fd49..a4bdfedf3 100644
--- a/lib/apps/fabro-cli/src/commands/artifact/mod.rs
+++ b/lib/apps/fabro-cli/src/commands/artifact/mod.rs
@@ -10,7 +10,6 @@ use crate::server_client::Client;
#[derive(Clone, Debug, serde::Serialize)]
pub(super) struct ArtifactEntry {
- #[serde(skip_serializing)]
pub(super) stage_id: StageId,
pub(super) node_slug: String,
pub(super) retry: u32,
@@ -23,8 +22,13 @@ pub(super) async fn resolve_artifacts(
server: &ServerTargetArgs,
run_selector: &str,
node: Option<&str>,
+ stage: Option<&str>,
retry: Option<u32>,
) -> Result<(RunId, Client, Vec<ArtifactEntry>)> {
+ let stage = stage
+ .map(str::parse::<StageId>)
+ .transpose()
+ .context("invalid artifact stage filter")?;
let ctx = base_ctx.with_target(server)?;
let client = ctx.server().await?;
let run_id = client.resolve_run(run_selector).await?.id;
@@ -33,6 +37,10 @@ pub(super) async fn resolve_artifacts(
if node.is_some_and(|value| entry.node_slug != value) {
continue;
}
+ let stage_id = parse_server_stage_id(&entry.stage_id)?;
+ if stage.as_ref().is_some_and(|value| &stage_id != value) {
+ continue;
+ }
let entry_retry = u32::try_from(entry.retry)
.context("server returned invalid negative artifact retry")?;
if retry.is_some_and(|value| entry_retry != value) {
@@ -41,7 +49,7 @@ pub(super) async fn resolve_artifacts(
let size =
u64::try_from(entry.size).context("server returned invalid negative artifact size")?;
entries.push(ArtifactEntry {
- stage_id: entry.stage_id.parse()?,
+ stage_id,
node_slug: entry.node_slug,
retry: entry_retry,
relative_path: entry.relative_path,
@@ -59,9 +67,31 @@ pub(super) async fn resolve_artifacts(
Ok((run_id, client.clone_for_reuse(), entries))
}
+fn parse_server_stage_id(value: &str) -> Result<StageId> {
+ value
+ .parse()
+ .context("server returned invalid artifact stage ID")
+}
+
pub(crate) async fn dispatch(ns: ArtifactNamespace, base_ctx: &CommandContext) -> Result<()> {
match ns.command {
ArtifactCommand::List(args) => list::list_command(&args, base_ctx).await,
ArtifactCommand::Cp(args) => cp::cp_command(&args, base_ctx).await,
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::parse_server_stage_id;
+
+ #[test]
+ fn invalid_server_stage_id_preserves_parse_error_context() {
+ let err = parse_server_stage_id("invalid").unwrap_err();
+ let chain = err.chain().map(ToString::to_string).collect::<Vec<_>>();
+
+ assert_eq!(chain, [
+ "server returned invalid artifact stage ID",
+ "stage id must contain '@'",
+ ]);
+ }
+}
diff --git a/lib/apps/fabro-cli/src/commands/run/output.rs b/lib/apps/fabro-cli/src/commands/run/output.rs
index 65e60d85c..136d2b506 100644
--- a/lib/apps/fabro-cli/src/commands/run/output.rs
+++ b/lib/apps/fabro-cli/src/commands/run/output.rs
@@ -5,7 +5,7 @@ use anyhow::{Context as _, Result};
use cli_table::format::{Border, Justify, Separator};
use cli_table::{Cell, CellStruct, Style, Table};
use fabro_api::types;
-use fabro_types::{PullRequestLink, RunBlobId, RunId, parse_blob_ref};
+use fabro_types::{PullRequestLink, RunBlobId, RunId, StageId, parse_blob_ref};
use fabro_util::check_report::{CheckDetail, CheckReport, CheckResult, CheckSection, CheckStatus};
use fabro_util::error::render_with_causes;
use fabro_util::printer::Printer;
@@ -348,12 +348,16 @@ fn blob_id_from_response(response: &str) -> Option<RunBlobId> {
async fn list_artifact_display_entries_with_client(
client: &server_client::Client,
run_id: &RunId,
-) -> Result<Vec<(String, u32, String)>> {
+) -> Result<Vec<(StageId, u32, String)>> {
let mut entries = Vec::new();
for entry in client.list_run_artifacts(run_id).await? {
let retry = u32::try_from(entry.retry)
.context("server returned invalid negative artifact retry")?;
- entries.push((entry.node_slug, retry, entry.relative_path));
+ let stage_id = entry
+ .stage_id
+ .parse()
+ .context("server returned invalid artifact stage ID")?;
+ entries.push((stage_id, retry, entry.relative_path));
}
entries.sort();
Ok(entries)
@@ -373,16 +377,16 @@ async fn print_assets_with_client(
let use_color = styles.use_color;
let title: Vec<CellStruct> = vec![
- "NODE".cell().bold(use_color),
+ "STAGE".cell().bold(use_color),
"RETRY".cell().bold(use_color).justify(Justify::Right),
"PATH".cell().bold(use_color),
];
let rows: Vec<Vec<CellStruct>> = entries
.iter()
- .map(|(node_slug, retry, relative_path)| {
+ .map(|(stage_id, retry, relative_path)| {
vec![
- node_slug.clone().cell().bold(use_color),
+ stage_id.to_string().cell().bold(use_color),
retry.cell().justify(Justify::Right),
relative_path.clone().cell(),
]
@@ -413,7 +417,7 @@ async fn print_assets_with_client(
printer,
"{}",
styles.dim.apply_to(format!(
- "Copy with: fabro artifact cp {run_id}:<path> <dest> --node <node_slug> --retry <retry>"
+ "Copy with: fabro artifact cp {run_id}:<path> <dest> --stage <node@visit> --retry <retry>"
))
);
Ok(())
diff --git a/lib/apps/fabro-cli/tests/it/cmd/support.rs b/lib/apps/fabro-cli/tests/it/cmd/support.rs
index 7d462eb1c..b3c133695 100644
--- a/lib/apps/fabro-cli/tests/it/cmd/support.rs
+++ b/lib/apps/fabro-cli/tests/it/cmd/support.rs
@@ -931,6 +931,7 @@ async fn seed_artifact_run(context: &TestContext) -> RunSetup {
for (stage_id, retry, path, contents) in [
("create_assets@1", 1, "assets/node_a/summary.txt", "alpha"),
("create_assets@1", 1, "assets/shared/report.txt", "one"),
+ ("create_assets@2", 1, "assets/shared/report.txt", "two"),
("create_colliding@1", 1, "assets/other/summary.txt", "beta"),
("create_colliding@1", 1, "assets/retry/report.txt", "second"),
("retry_assets@1", 1, "assets/retry/report.txt", "first"),
@@ -1455,7 +1456,7 @@ async fn append_seeded_artifact_run_events(
"run.completed",
serde_json::json!({
"timing": {"wall_time_ms": 123, "inference_time_ms": 0, "tool_time_ms": 0, "active_time_ms": 0},
- "artifact_count": 6,
+ "artifact_count": 7,
"status": "succeeded",
"reason": "completed",
"total_usd_micros": null,
diff --git a/lib/apps/fabro-cli/tests/it/scenario/artifacts.rs b/lib/apps/fabro-cli/tests/it/scenario/artifacts.rs
index b5d5b006f..d35e3ec53 100644
--- a/lib/apps/fabro-cli/tests/it/scenario/artifacts.rs
+++ b/lib/apps/fabro-cli/tests/it/scenario/artifacts.rs
@@ -27,36 +27,49 @@ fn artifact_commands_share_populated_run_fixture() {
----- stdout -----
[
{
+ "stage_id": "create_assets@1",
"node_slug": "create_assets",
"retry": 1,
"relative_path": "assets/node_a/summary.txt",
"size": 5
},
{
+ "stage_id": "create_assets@1",
"node_slug": "create_assets",
"retry": 1,
"relative_path": "assets/shared/report.txt",
"size": 3
},
{
+ "stage_id": "create_assets@2",
+ "node_slug": "create_assets",
+ "retry": 1,
+ "relative_path": "assets/shared/report.txt",
+ "size": 3
+ },
+ {
+ "stage_id": "create_colliding@1",
"node_slug": "create_colliding",
"retry": 1,
"relative_path": "assets/other/summary.txt",
"size": 4
},
{
+ "stage_id": "create_colliding@1",
"node_slug": "create_colliding",
"retry": 1,
"relative_path": "assets/retry/report.txt",
"size": 6
},
{
+ "stage_id": "retry_assets@1",
"node_slug": "retry_assets",
"retry": 1,
"relative_path": "assets/retry/report.txt",
"size": 5
},
{
+ "stage_id": "retry_assets@1",
"node_slug": "retry_assets",
"retry": 2,
"relative_path": "assets/retry/report.txt",
@@ -83,6 +96,7 @@ fn artifact_commands_share_populated_run_fixture() {
----- stdout -----
[
{
+ "stage_id": "retry_assets@1",
"node_slug": "retry_assets",
"retry": 2,
"relative_path": "assets/retry/report.txt",
@@ -92,6 +106,31 @@ fn artifact_commands_share_populated_run_fixture() {
----- stderr -----
"#);
+ let mut list_stage_filtered = context.command();
+ list_stage_filtered.args([
+ "artifact",
+ "list",
+ &run.run_id,
+ "--stage",
+ "create_assets@2",
+ "--json",
+ ]);
+ fabro_snapshot!(filters.clone(), list_stage_filtered, @r#"
+ success: true
+ exit_code: 0
+ ----- stdout -----
+ [
+ {
+ "stage_id": "create_assets@2",
+ "node_slug": "create_assets",
+ "retry": 1,
+ "relative_path": "assets/shared/report.txt",
+ "size": 3
+ }
+ ]
+ ----- stderr -----
+ "#);
+
let single_dest = context.temp_dir.join("artifact-one");
let mut cp_single = context.command();
cp_single.args([
@@ -99,8 +138,8 @@ fn artifact_commands_share_populated_run_fixture() {
"cp",
&format!("{}:assets/shared/report.txt", run.run_id),
single_dest.to_str().unwrap(),
- "--node",
- "create_assets",
+ "--stage",
+ "create_assets@2",
]);
fabro_snapshot!(context.filters(), cp_single, @"
success: true
@@ -109,7 +148,50 @@ fn artifact_commands_share_populated_run_fixture() {
Copied assets/shared/report.txt to [TEMP_DIR]/artifact-one/report.txt
----- stderr -----
");
- assert_eq!(read_text(&single_dest.join("report.txt")), "one");
+ assert_eq!(read_text(&single_dest.join("report.txt")), "two");
+
+ let stage_tree_dest = context.temp_dir.join("artifact-stage-tree");
+ let mut cp_stage_tree = context.command();
+ cp_stage_tree.args([
+ "artifact",
+ "cp",
+ &run.run_id,
+ stage_tree_dest.to_str().unwrap(),
+ "--stage",
+ "create_assets@2",
+ "--tree",
+ ]);
+ fabro_snapshot!(context.filters(), cp_stage_tree, @"
+ success: true
+ exit_code: 0
+ ----- stdout -----
+ Copied 1 artifact(s) to [TEMP_DIR]/artifact-stage-tree
+ ----- stderr -----
+ ");
+ insta::assert_snapshot!(
+ text_tree(&stage_tree_dest).join("\n"),
+ @"create_assets/visit_2/retry_1/assets/shared/report.txt = two"
+ );
+
+ let repeated_visit_dest = context.temp_dir.join("artifact-repeated-visit");
+ let mut cp_repeated_visit = context.command();
+ cp_repeated_visit.args([
+ "artifact",
+ "cp",
+ &format!("{}:assets/shared/report.txt", run.run_id),
+ repeated_visit_dest.to_str().unwrap(),
+ "--node",
+ "create_assets",
+ "--retry",
+ "1",
+ ]);
+ fabro_snapshot!(context.filters(), cp_repeated_visit, @"
+ success: false
+ exit_code: 1
+ ----- stdout -----
+ ----- stderr -----
+ × Path 'assets/shared/report.txt' matches multiple artifacts: create_assets@1:retry_1, create_assets@2:retry_1. Use --stage and/or --retry to disambiguate.
+ ");
let tree_dest = context.temp_dir.join("artifact-tree");
let mut cp_tree = context.command();
@@ -125,14 +207,15 @@ fn artifact_commands_share_populated_run_fixture() {
success: true
exit_code: 0
----- stdout -----
- Copied 6 artifact(s) to [TEMP_DIR]/artifact-tree
+ Copied 7 artifact(s) to [TEMP_DIR]/artifact-tree
----- stderr -----
");
insta::assert_snapshot!(
text_tree(&tree_dest).join("\n"),
@r"
- create_assets/retry_1/assets/node_a/summary.txt = alpha
- create_assets/retry_1/assets/shared/report.txt = one
+ create_assets/visit_1/retry_1/assets/node_a/summary.txt = alpha
+ create_assets/visit_1/retry_1/assets/shared/report.txt = one
+ create_assets/visit_2/retry_1/assets/shared/report.txt = two
create_colliding/retry_1/assets/other/summary.txt = beta
create_colliding/retry_1/assets/retry/report.txt = second
retry_assets/retry_1/assets/retry/report.txt = first
@@ -153,7 +236,7 @@ fn artifact_commands_share_populated_run_fixture() {
exit_code: 1
----- stdout -----
----- stderr -----
- × Path 'assets/retry/report.txt' matches multiple artifacts: create_colliding:retry_1, retry_assets:retry_1, retry_assets:retry_2. Use --node and/or --retry to disambiguate.
+ × Path 'assets/retry/report.txt' matches multiple artifacts: create_colliding@1:retry_1, retry_assets@1:retry_1, retry_assets@1:retry_2. Use --stage and/or --retry to disambiguate.
");
let flat_dest = context.temp_dir.join("artifact-flat");
@@ -164,6 +247,6 @@ fn artifact_commands_share_populated_run_fixture() {
exit_code: 1
----- stdout -----
----- stderr -----
- × Filename collision: 'summary.txt' exists in both create_assets:retry_1 and create_colliding:retry_1. Use --tree to preserve directory structure, or --node and/or --retry to filter.
+ × Filename collision: 'report.txt' exists in both create_assets@1:retry_1 and create_assets@2:retry_1. Use --tree to preserve directory structure, or --stage and/or --retry to filter.
");
}
diff --git a/lib/apps/fabro-cli/tests/it/scenario/smoke.rs b/lib/apps/fabro-cli/tests/it/scenario/smoke.rs
index 08e56acb8..aac6eb33f 100644
--- a/lib/apps/fabro-cli/tests/it/scenario/smoke.rs
+++ b/lib/apps/fabro-cli/tests/it/scenario/smoke.rs
@@ -79,8 +79,9 @@ fn help_smoke_covers_high_cost_commands() {
--debug Enable DEBUG-level logging (default is INFO) [env: FABRO_DEBUG=]
--node <NODE> Filter to artifacts from a specific node
--no-upgrade-check Disable automatic upgrade check [env: FABRO_NO_UPGRADE_CHECK=true]
- --retry <RETRY> Filter to artifacts from a specific retry attempt
+ --stage <STAGE> Filter to artifacts from a specific stage visit (node@visit)
--quiet Suppress non-essential output [env: FABRO_QUIET=]
+ --retry <RETRY> Filter to artifacts from a specific retry attempt
--verbose Enable verbose output [env: FABRO_VERBOSE=]
-h, --help Print help
----- stderr -----
@@ -106,9 +107,10 @@ fn help_smoke_covers_high_cost_commands() {
--debug Enable DEBUG-level logging (default is INFO) [env: FABRO_DEBUG=]
--node <NODE> Filter to artifacts from a specific node
--no-upgrade-check Disable automatic upgrade check [env: FABRO_NO_UPGRADE_CHECK=true]
- --retry <RETRY> Filter to artifacts from a specific retry attempt
+ --stage <STAGE> Filter to artifacts from a specific stage visit (node@visit)
--quiet Suppress non-essential output [env: FABRO_QUIET=]
- --tree Preserve {node_slug}/retry_{N}/ directory structure
+ --retry <RETRY> Filter to artifacts from a specific retry attempt
+ --tree Preserve node[/visit_{N}]/retry_{N}/ directory structure
--verbose Enable verbose output [env: FABRO_VERBOSE=]
-h, --help Print help
----- stderr -----
diff --git a/lib/components/fabro-workflow/src/handler/command.rs b/lib/components/fabro-workflow/src/handler/command.rs
index 0de483869..6cf7d5741 100644
--- a/lib/components/fabro-workflow/src/handler/command.rs
+++ b/lib/components/fabro-workflow/src/handler/command.rs
@@ -220,8 +220,10 @@ impl Handler for CommandHandler {
/// Ceiling on encoded stdin bytes. `stdin_source` values are runtime data —
/// often model-produced — so their size is not something a workflow author
/// reviewed; this bounds peak memory and remote uploads the same way
-/// `MAX_FOR_EACH_ITEMS` bounds `for_each` fan-out.
-const MAX_STDIN_BYTES: usize = 10 * 1024 * 1024;
+/// `MAX_FOR_EACH_ITEMS` bounds `for_each` fan-out. Sized for wide fan-in:
+/// a `context.parallel.results` batch from a large `for_each` round easily
+/// carries tens of structured agent outputs.
+const MAX_STDIN_BYTES: usize = 30 * 1024 * 1024;
fn validated_stdin_source(node: &Node) -> Result<Option<&str>, String> {
match node.context_key_attr("stdin_source") {
diff --git a/lib/components/fabro-workflow/src/handler/llm/api.rs b/lib/components/fabro-workflow/src/handler/llm/api.rs
index 178448bc4..6523b8477 100644
--- a/lib/components/fabro-workflow/src/handler/llm/api.rs
+++ b/lib/components/fabro-workflow/src/handler/llm/api.rs
@@ -838,13 +838,17 @@ impl AgentApiBackend {
let supervisor = SubAgentSupervisor::new(config.max_subagent_depth);
let supervisor_for_session = supervisor.clone();
- // Build factory that creates child sessions WITHOUT subagent tools
+ // Build factory that creates child sessions WITHOUT subagent tools.
+ // Child sessions inherit the parent's tool hooks: blocking
+ // pre_tool_use hooks are the only policy boundary workflow agents
+ // have, so a subagent's tool calls must pass through them too.
let factory_client = client.clone();
let factory_profile_builder = profile_builder;
let factory_env = Arc::clone(sandbox);
let factory_tool_env = tool_env.cloned();
let factory_fabro_run_tools = fabro_run_tools.clone();
let factory_permission_level = config.permission_level;
+ let factory_tool_hooks = config.tool_hooks.clone();
let factory: SessionFactory = Arc::new(move || {
let mut child_profile = factory_profile_builder.build();
if let Some(services) = factory_fabro_run_tools.clone() {
@@ -858,6 +862,7 @@ impl AgentApiBackend {
SessionOptions {
reasoning_effort: controls.reasoning_effort,
speed: controls.speed,
+ tool_hooks: factory_tool_hooks.clone(),
permission_level: factory_permission_level,
..SessionOptions::default()
},
@@ -2706,6 +2711,133 @@ reasoning = false
assert!(names.contains(&"close_agent".to_string()));
}
+ /// Records every `pre_tool_use` call it sees and lets them all proceed.
+ struct RecordingHooks(Arc<Mutex<Vec<String>>>);
+
+ #[async_trait]
+ impl fabro_agent::ToolHookCallback for RecordingHooks {
+ async fn pre_tool_use(
+ &self,
+ tool_name: &str,
+ _tool_input: &serde_json::Value,
+ ) -> fabro_agent::ToolHookDecision {
+ self.0.lock().unwrap().push(tool_name.to_string());
+ fabro_agent::ToolHookDecision::Proceed
+ }
+
+ async fn post_tool_use(&self, _tool_name: &str, _tool_call_id: &str, _output: &str) {}
+
+ async fn post_tool_use_failure(&self, _tool_name: &str, _tool_call_id: &str, _error: &str) {
+ }
+ }
+
+ /// Blocking `pre_tool_use` hooks are the only policy boundary a workflow
+ /// agent has: workflow sessions run at `PermissionLevel::Full` with the
+ /// whole tool registry exposed. A child session created for `spawn_agent`
+ /// therefore has to run under the same `tool_hooks` as its parent —
+ /// otherwise any agent that can spawn a subagent gets an unguarded
+ /// read-write-shell escape from every hook-enforced policy.
+ #[tokio::test]
+ async fn subagent_tool_calls_pass_through_session_tool_hooks() {
+ let server = MockServer::start();
+ // Parent turn 1: spawn a subagent.
+ let parent_spawn = server.mock(|when, then| {
+ when.method(POST)
+ .path("/chat/completions")
+ .body_includes("PARENT_PROMPT_MARKER")
+ .body_excludes(r#""role":"tool""#);
+ then.status(200)
+ .header("content-type", "text/event-stream")
+ .body(chat_completion_tool_call_stream(
+ "spawn_agent",
+ "call_spawn_helper",
+ r#"{"task":"CHILD_TASK_MARKER: read data.txt and report its contents"}"#,
+ ));
+ });
+ // Parent turn 2: the spawn result is back; finish the parent turn
+ // while the child keeps running in the background.
+ let parent_final = server.mock(|when, then| {
+ when.method(POST)
+ .path("/chat/completions")
+ .body_includes("call_spawn_helper");
+ then.status(200)
+ .header("content-type", "text/event-stream")
+ .body(chat_completion_stream("parent done", 10, 1));
+ });
+ // Child turn 1: the child session uses a tool.
+ let child_read = server.mock(|when, then| {
+ when.method(POST)
+ .path("/chat/completions")
+ .body_includes("CHILD_TASK_MARKER")
+ .body_excludes("PARENT_PROMPT_MARKER")
+ .body_excludes(r#""role":"tool""#);
+ then.status(200)
+ .header("content-type", "text/event-stream")
+ .body(chat_completion_tool_call_stream(
+ "read_file",
+ "call_child_read",
+ r#"{"file_path":"data.txt"}"#,
+ ));
+ });
+ // Child turn 2: the tool result is back; the child completes.
+ let child_final = server.mock(|when, then| {
+ when.method(POST)
+ .path("/chat/completions")
+ .body_includes("call_child_read");
+ then.status(200)
+ .header("content-type", "text/event-stream")
+ .body(chat_completion_stream("child done", 10, 1));
+ });
+
+ let hook_calls: Arc<Mutex<Vec<String>>> = Arc::new(Mutex::new(Vec::new()));
+ let hooks: Arc<dyn fabro_agent::ToolHookCallback> =
+ Arc::new(RecordingHooks(Arc::clone(&hook_calls)));
+
+ let backend = mock_api_backend(&server);
+ let node = Node::new("researcher");
+ let workspace = tempfile::tempdir().unwrap();
+ tokio::fs::write(workspace.path().join("data.txt"), "hello\n")
+ .await
+ .unwrap();
+ let sandbox: Arc<dyn fabro_agent::Sandbox> =
+ Arc::new(LocalSandbox::new(workspace.path().to_path_buf()));
+
+ let mut session = backend
+ .create_session(&node, &sandbox, Some(hooks))
+ .await
+ .unwrap();
+ session
+ .process_input("PARENT_PROMPT_MARKER: spawn a helper subagent")
+ .await
+ .unwrap();
+
+ // The child runs on background tasks owned by the still-alive parent
+ // session; wait until its final turn has been served.
+ let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(10);
+ while child_final.calls() == 0 {
+ assert!(
+ tokio::time::Instant::now() < deadline,
+ "the spawned subagent never completed its turns against the mock provider"
+ );
+ tokio::time::sleep(std::time::Duration::from_millis(25)).await;
+ }
+ parent_spawn.assert_calls(1);
+ parent_final.assert_calls(1);
+ child_read.assert_calls(1);
+ child_final.assert_calls(1);
+
+ let recorded = hook_calls.lock().unwrap().clone();
+ assert!(
+ recorded.iter().any(|name| name == "spawn_agent"),
+ "the parent's own tool calls should reach the hooks; hooks saw: {recorded:?}"
+ );
+ assert!(
+ recorded.iter().any(|name| name == "read_file"),
+ "the child subagent's tool calls must pass through the same tool hooks as the \
+ parent's, but the hooks never saw the child's read_file; hooks saw: {recorded:?}"
+ );
+ }
+
#[test]
fn api_backend_provider_pin_wins_over_priority_selection() {
let settings: LlmCatalogSettings = toml::from_str(

6
stages/008-verify@1/status.json Normal file
View file

@ -0,0 +1,6 @@
{
"outcome": "failed",
"notes": null,
"failure_reason": "Script failed with exit code: 1\n\n## output\nto link package: @remotion/compositor-linux-x64-musl@4.0.437 (link)\nENOENT: No such file or directory: failed to link package: @babel/types@7.29.0 (link)\nerror: failed to download css-tree@3.2.1: NoSpaceLeft\n https://registry.npmjs.org/css-tree/-/css-tree-3.2.1.tgz\nerror: failed to download node-fetch-native@1.6.7: NoSpaceLeft\n https://registry.npmjs.org/node-fetch-native/-/node-fetch-native-1.6.7.tgz\nerror: failed to download date-fns@2.30.0: NoSpaceLeft\n https://registry.npmjs.org/date-fns/-/date-fns-2.30.0.tgz\nerror: NoSpaceLeft when create temporary directory named \".f9cfefbddfffb77f-0000031F.chokidar\" (while extracting \"chokidar\")\nerror: failed to download chokidar@5.0.0: InstallFailed\n https://registry.npmjs.org/chokidar/-/chokidar-5.0.0.tgz\nerror: NoSpaceLeft when create temporary directory named \".fad6ebb7dbfa37ef-0000031E.anymatch\" (while extracting \"anymatch\")\nerror: failed to download anymatch@3.1.3: InstallFailed\n https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz\nerror: NoSpaceLeft when create temporary directory named \".bfdefbd7dfbff9ff-00000321.css-what\" (while extracting \"css-what\")\nerror: failed to download css-what@6.2.2: InstallFailed\n https://registry.npmjs.org/css-what/-/css-what-6.2.2.tgz\nENOENT: No such file or directory: failed to link package: @radix-ui/react-dialog@1.1.15 (link)\nerror: NoSpaceLeft when create temporary directory named \".9bd7fdd5ff3be1ce-00000322.h3\" (while extracting \"h3\")\nerror: failed to download h3@1.15.11: InstallFailed\n https://registry.npmjs.org/h3/-/h3-1.15.11.tgz\nENOENT: No such file or directory: failed to link package: @jridgewell/sourcemap-codec@1.5.5 (link)\nENOENT: No such file or directory: failed to link package: eventemitter3@5.0.4 (link)\nerror: NoSpaceLeft when create temporary directory named \".bcceff9dfbfadfbf-00000320.ansi-styles\" (while extracting \"ansi-styles\")\nerror: failed to download ansi-styles@4.3.0: InstallFailed\n https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz\nENOENT: No such file or directory: failed to link package: css-select@5.2.2 (link)\nerror: NoSpaceLeft when create temporary directory named \".1ec7e9d7dfbefe63-00000324.lru-cache\" (while extracting \"lru-cache\")\nerror: failed to download lru-cache@11.3.5: InstallFailed\n https://registry.npmjs.org/lru-cache/-/lru-cache-11.3.5.tgz\nENOENT: No such file or directory: failed to link package: rehype-parse@9.0.1 (link)\nerror: NoSpaceLeft when create temporary directory named \".7bceff95fe5f1dd6-00000323.core\" (while extracting \"@clack/core\")\nerror: failed to download @clack/core@1.2.0: InstallFailed\n https://registry.npmjs.org/@clack/core/-/core-1.2.0.tgz\nENOENT: failed to link binaries for package: @assistant-ui/react@0.14.5\nerror: NoSpaceLeft when create temporary directory named \".bbeef9dddf6f7b75-00000325.destr\" (while extracting \"destr\")\nerror: failed to download destr@2.0.5: InstallFailed\n https://registry.npmjs.org/destr/-/destr-2.0.5.tgz\nENOENT: failed to link binaries for package: svgo@4.0.1\nerror: NoSpaceLeft when create temporary directory named \".3feff9f7fb33fdff-00000326.ofetch\" (while extracting \"ofetch\")\nerror: failed to download ofetch@1.5.1: InstallFailed\n https://registry.npmjs.org/ofetch/-/ofetch-1.5.1.tgz\nerror: NoSpaceLeft when create temporary directory named \".d8e7fdd7fbb7f7bf-00000327.yocto-queue\" (while extracting \"yocto-queue\")\nerror: failed to download yocto-queue@1.2.2: InstallFailed\n https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.2.2.tgz\nENOENT: failed to link binaries for package: @tailwindcss/node@4.2.error: ENOSPC extracting tarball for \"@img/sharp-libvips-linuxmusl-x64\"\nerror: failed to download @img/sharp-libvips-linuxmusl-x64@1.2.4: ENOSPC\n https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.2.4.tgz\nerror: ENOSPC extracting tarball for \"@rspack/binding-linux-x64-gnu\"\nerror: failed to download @rspack/binding-linux-x64-gnu@1.7.6: ENOSPC\n https://registry.npmjs.org/@rspack/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.7.6.tgz\n\n1519 packages installed [3.76s]\nFailed to install 55 packages\n",
"timestamp": "2026-07-30T00:25:33.454398275Z"
}

358
stages/009-fixup@1/prompt.md Normal file
View file

@ -0,0 +1,358 @@
Goal: # PR 1 — Make run-event appends validate before write and report commit status unambiguously
**Self-contained implementation plan.** Everything needed to implement this
is in this file plus the repository.
**Precondition:** none — this is foundational work with no dependency on
other in-flight changes. Re-verify the "Verified current state" section
against HEAD before starting; if the append path in
`lib/components/fabro-store/src/slate/run_store.rs` has been materially
restructured since the pinned commit, stop and state that in the PR
description instead of adapting blindly.
> **Token notation.** Interpolation tokens are written in this file without
> their enclosing double curly braces, so the file is safe to pass directly
> as a workflow goal (the goal templater would otherwise try to expand them).
> Read `secrets.NAME`, `env.NAME`, `vars.NAME` as the double-curly-brace
> token form used in the codebase, and write the real double-brace syntax in
> the code, tests, and docs you produce.
## Context and goal
Fabro's run state is event-sourced: each run has an append-only event log in
a shared SlateDB store (`fabro-store`), a reduced in-memory projection
(`RunProjection`), and a derived SQLite summary row used by all listing
endpoints. Run status transitions are enforced by a state machine
(`RunStatus::can_transition_to` / `transition_to` in
`lib/foundation/fabro-types/src/status.rs`) — for example, a run whose
durable status is `Runnable` may legally move to `Failed` only with reason
`Cancelled`; a `Failed { WorkflowError }` from `Runnable` is an invalid
transition and the reducer hard-errors on it.
The append path has two defects, and this PR fixes both at the store layer:
**Defect 1 — poison events.** `append_event_envelope_locked` writes the
event bytes to SlateDB *before* any reduction happens. If the event turns
out to be transition-invalid, the caller gets an error — but the invalid
event is already durably in the log. From then on the run's projection can
never be rebuilt: replay hits the same invalid transition every time. The
user-visible consequence is severe: at startup, projection warmup skips the
unreadable run, and the SQLite reconciler then *deletes its summary row*
because it is absent from the authoritative entries — the run disappears
from every listing, and get/cancel return 404. This is a real shipped bug:
several server failure helpers attempt exactly such illegal appends today
(e.g. a worker-launch failure helper appends `Failed { LaunchFailed }`
while the durable status is still `Runnable`). Those call sites are being
fixed in separate planned work — this PR's job is to make the store refuse
to write the poison event in the first place.
**Defect 2 — ambiguous append errors.** After the SlateDB put succeeds, the
append still does derived work: applying the event to the shared projection
cache and upserting the SQLite summary row. Failures in either currently
propagate as `Err` from the append — so callers cannot distinguish "the
event was not committed, safe to retry" from "the event IS committed but a
derived update failed." Worse, when the projection-cache update fails, the
current code removes the cache entry entirely. Upcoming scheduler work will
retry appends that report failure, so this ambiguity must be resolved
before it exists: retrying a committed append would attempt a duplicate
event.
**Goal:** after this PR, the append contract is unambiguous:
1. An event that the current projection cannot legally reduce is **rejected
before anything is written** — the log, the projection cache, and the
summary row are all untouched, and the caller gets a typed rejection
error.
2. A failure of the authoritative SlateDB put (or of event-sequence
allocation) returns a typed **not-committed** error — safe to retry.
3. Once the authoritative put succeeds, the append **is committed** and
reports success. Derived-state updates (projection cache install, event
cache, SQLite summary upsert) are best-effort: failures are logged
loudly with the run id but never surface as an append error. Derived
state is repairable (startup reconciliation rebuilds it; the summary
upsert is already guarded to be monotonic by event seq, so a later
successful append also repairs it).
Design rules (fixed — do not re-litigate):
- **Validation must reuse the same reduction code that replay uses.** The
invariant is "an event is written iff replay can reduce it." Any
divergence between the pre-write check and replay reintroduces poison
events. Apply the candidate event to a clone of the current projection
using the existing reducer entry points; do not write a parallel
validity checker.
- **No event schema changes and no public API changes.** This is a store
contract fix, not a wire change.
- **Do not rework the failing call sites.** Server helpers that attempt
illegal appends will now receive a clean rejection with nothing written —
that is the intended intermediate state. Fixing their logic is separate
planned work.
- **The rejection error must be a distinct variant** from the existing
`Error::InvalidEvent` (which means "malformed payload") so callers can
tell "rejected by the run's state machine" apart from "bad input" and
from "not committed, retry."
- **Do not attempt to repair logs that already contain poison events.**
Pre-existing corrupted logs remain unreadable and continue to be surfaced
by the existing unreadable-runs listing; repair tooling is out of scope.
## Verified current state (as of origin/main `1aa7a153b`, 2026-07-28 — re-verify before starting)
- `lib/components/fabro-store/src/slate/run_store.rs`:
- `append_event(&EventPayload)` → `append_event_envelope` → validates the
payload shape (`payload.validate(&run_id)`), takes the per-run
`state_lock`, then calls `append_event_envelope_locked` (≈ lines
273-305).
- `append_event_if(payload, predicate)` — same, but loads the current
projection under the lock and returns `Ok(None)` when the predicate
rejects (≈ 279-294). This method's contract must be preserved.
- `append_event_envelope_locked` (≈ 305-324): allocates the event seq
(can fail with `Error::EventSequenceExhausted`), builds the
`EventEnvelope` (`RunEvent::try_from(payload)?`), then **puts the event
bytes into SlateDB first**, then `cache_event`, then
`update_summary_projection_after_append`.
- `update_summary_projection_after_append` (≈ 325-377): applies the event
to the shared projection cache; on failure it attempts a full rebuild
from the db (which, for a just-written invalid event, fails again
because the poison event is in the log), **removes the cache entry**,
warns, and returns `Err`. If the SQLite summary store is attached
(`run_summary_store` is an `OnceLock` — absent in some deployments),
an upsert failure also returns `Err`. Both paths make a committed
append look failed.
- `lib/components/fabro-store/src/error.rs`: `Error` enum with
`InvalidEvent(String)`, `EventSequenceExhausted { max_seq }`,
`Slate(..)`, `Sqlite(..)`, etc. No variant distinguishes
state-machine rejection or commit status.
- `lib/foundation/fabro-types/src/status.rs` (:132-202): the transition
table; `transition_to` returns `Err(InvalidTransition)`. From `Runnable`,
`Failed` is legal only with reason `Cancelled`.
- `lib/foundation/fabro-types/src/run_projection.rs`: `try_apply_status`
(≈ :1025) is where reduction enforces transitions; the reducer dispatch
lives in `lib/components/fabro-store/src/run_state.rs`
(`apply_event` / `apply_events`, plus `projection_from_created` for the
first event). Both files were recently extended for new event kinds —
re-derive exact line numbers rather than trusting the ones here.
- Startup behavior that makes poison events user-visible:
`warm_projection_cache` in `lib/components/fabro-store/src/slate/mod.rs`
skips runs whose replay fails (per-run `warn!`), and
`RunSummaryStore::reconcile` deletes summary rows absent from the
authoritative entries (pinned by the existing test
`reconcile_removes_rows_absent_from_authoritative_entries` in
`run_summary_store.rs`). `list_unreadable_runs` (slate/mod.rs) surfaces
skipped runs.
- The summary upsert is monotonic by event seq (`WHERE excluded.source_last_seq > runs.source_last_seq`
in `run_summary_store.rs`), which is what makes "later append repairs the
row" true.
- Existing test pinning seq exhaustion:
`append_event_rejects_sequences_beyond_key_order_limit`
(run_store.rs ≈ :1292).
## Implementation
1. **Add the typed errors** in `lib/components/fabro-store/src/error.rs`.
Read `docs/internal/error-handling-strategy.md` first (required by
project convention when touching error types). Two additions, named to
read well at call sites — suggested shapes:
- `EventRejected { reason: String }` (or carrying the
`InvalidTransition` detail) — the event cannot be legally reduced by
the run's current projection; nothing was written.
- A way for callers to know an `Err` means not-committed. Simplest
honest contract: after this PR, **every** `Err` from append means
not-committed (rejection included), because post-put failures no
longer return `Err`. Prefer that global simplification over a wrapper
enum; document it on the append methods' doc comments explicitly.
2. **Validate before the put** in `append_event_envelope_locked` (all under
the already-held `state_lock`):
- Obtain the current projection: the cheapest correct source is the
same one `append_event_if` uses (`projected_state_locked`); for a run
with no events yet, the candidate must be validated through the
first-event path (`projection_from_created` route in
`run_state.rs`) — mirror however `apply_events` treats the initial
event so validation ≡ replay exactly.
- Apply the candidate envelope to a **clone** of that projection via the
existing reducer entry point. On reduction failure → return
`EventRejected`, having written nothing.
- Keep the pre-existing `payload.validate(...)` shape check where it is.
3. **Reorder the post-put work to be best-effort.** After a successful
SlateDB put:
- Install the already-validated clone into the shared projection cache
(replacing the apply-then-rebuild-then-remove dance — the clone IS the
correct post-append projection, computed before the write). Keep the
cache's seq bookkeeping consistent with the existing
`apply_event`/`replace` semantics.
- `cache_event` and the SQLite upsert stay in place but become
log-only on failure (`warn!`/`error!` with run id and seq, matching
the logging style already present in this file). The append returns
`Ok(envelope)` regardless of derived-state failures.
- Do NOT remove the projection-cache entry on derived failure paths
anymore; a stale entry that a later append or startup reconciliation
repairs is strictly better than an absent one.
4. **Seq allocation and put failures** already return `Err` before any
derived work — with step 3 in place these are now unambiguously
not-committed. Verify `EventSequenceExhausted` still propagates (the
existing test pins it).
5. **Audit append callers for compile-only impact.** Call sites that
currently treat any `Err` as "append failed" remain correct under the
new contract (their errors now genuinely mean not-committed). No caller
behavior changes in this PR. `append_event_if`'s `Ok(None)` predicate
contract is unchanged.
6. **Doc comments.** State the three-outcome contract (rejected-nothing-
written / not-committed / committed-with-best-effort-derived) on
`append_event`, `append_event_if`, and `append_event_envelope`.
## Scope boundaries — deliberately NOT in this PR
- **The server failure helpers that attempt illegal appends** (e.g. the
worker-launch failure path appending `Failed { LaunchFailed }` from
durable `Runnable`, and similar pre-worker failure sites in
`fabro-server`) — leave their logic as-is. They will now receive a clean
`EventRejected` and write nothing, which is the intended intermediate
state; reworking when/what they append is separate planned work. Do not
"fix" them to append legal events.
- **Admission/scheduler changes** (durable claims, retry/backoff, startup
re-admission of queued runs) — known follow-up work, deliberately
excluded here.
- **Repairing already-poisoned logs** or adding repair/diagnostic tooling —
known gap, addressed separately if needed. Pre-existing unreadable runs
keep their current behavior (skipped at warmup, surfaced by the
unreadable-runs listing).
- **Event schema, OpenAPI, or public API changes** — none. This PR is
entirely inside `fabro-store` (plus its error type).
- **SQLite schema changes** — none; the monotonic upsert and startup
reconcile already provide the repair path.
If work outside these boundaries seems genuinely required for this PR to
compile or pass its tests, stop and state that in the PR description rather
than expanding scope.
## Tests (write failing-first; hermetic — temp-dir fixtures, no ambient provider keys)
Existing store tests in `run_store.rs` / `run_summary_store.rs` show the
fixture style (temp-dir object store, in-memory SQLite). Add:
1. **Rejected transition writes nothing** — create a run, drive it to
durable `Runnable` (append the events the lifecycle uses today:
created/submitted/start-requested/runnable), then append a
`run.failed { WorkflowError }`-shaped event. Assert: the append returns
the rejection variant; `list_events` shows no new event; `state()` still
reduces successfully; the projection cache still holds an entry for the
run (not removed). *Property pinned: an event is written iff replay can
reduce it.*
2. **Rejected transition leaves listings consistent** — after the rejected
append, run the summary reconcile path and assert the run's summary row
still exists. *Property: no more vanishing runs from rejected appends.*
3. **Committed append survives derived-state failure** — attach a SQLite
summary store, then make its pool unusable (e.g. close the pool or drop
the underlying file) before appending a legal event. Assert: append
returns `Ok`; the event is in `list_events`; a warning/error was the
only symptom. Then restore/reopen the summary store and assert the row
is repairable (via reconcile or a subsequent append). If pool-closing
proves impractical through public seams, an injected failing summary
store behind the existing test-support feature is acceptable — but do
not weaken the assertion that append reports success. *Property:
committed is committed.*
4. **Not-committed errors are retryable** — the existing
seq-exhaustion test keeps passing; extend it (or add a sibling) to
assert the log is unchanged after the error, pinning "Err ⇒ nothing
written."
5. **First-event validation** — a malformed first event (one the reducer
cannot initialize a projection from) is rejected with nothing written;
a valid `run.created` still works. *Property: the empty-log path
validates like replay too.*
6. **append_event_if contract unchanged** — predicate-false still returns
`Ok(None)` with nothing written.
Run the full workspace suite; the reducer and lifecycle tests in
`fabro-store`, `fabro-workflow`, and `fabro-server` are the regression net
for "legal appends behave exactly as before."
## Acceptance / verification
- `cargo +nightly-2026-04-14 fmt --check --all`
- `cargo +nightly-2026-04-14 clippy --workspace --all-targets -- -D warnings`
- `cargo nextest run --workspace`
- No OpenAPI/wire change (do not touch `docs/public/api-reference/`).
- `cargo build --workspace` without the `test-support` feature still
succeeds if any test helper was added behind it.
## Conventions
- Read `docs/internal/error-handling-strategy.md` before changing the error
enum, and `docs/internal/events-strategy.md` before touching anything
that emits or documents events.
- Never print or log a resolved secret value, including from tests.
- Plain-English commit messages, PR text, and comments — describe what the
change does; no internal planning identifiers or plan-file names in
anything that ships.
- PR description must state plainly: (1) the vanishing-runs failure mode
this fixes (invalid append → unreadable projection → summary row deleted
→ run 404s) and that call sites attempting such appends now get a clean
error with nothing written; (2) the new append contract, including that
a failed SQLite summary update after a committed append now logs loudly
and reports success instead of returning an error — operators see a
warning where they previously saw a failed operation; (3) that
pre-existing corrupted run logs are not repaired by this change.
- If implementation uncovers a caller that genuinely depends on the old
"Err after committed write" behavior, stop and surface it in the PR
description rather than working around it.
## Completed stages
- **toolchain**: succeeded
- Script: `command -v cargo >/dev/null || { curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && sudo ln -sf $HOME/.cargo/bin/* /usr/local/bin/; }; cargo --version 2>&1`
- Output:
```
cargo 1.95.0 (f2d3ce0bd 2026-03-21)
```
- **preflight_compile**: succeeded
- Script: `cargo check -q --workspace 2>&1`
- Output: (empty)
- **preflight_lint**: succeeded
- Script: `cargo +nightly-2026-04-14 clippy -q --workspace --all-targets -- -D warnings 2>&1`
- Output: (empty)
- **implement**: succeeded
- Model: gpt-5.6-sol
- Files: /home/daytona/workspace/fabro/lib/components/fabro-store/src/error.rs, /home/daytona/workspace/fabro/lib/components/fabro-store/src/run_summary_store.rs, /home/daytona/workspace/fabro/lib/components/fabro-store/src/slate/mod.rs, /home/daytona/workspace/fabro/lib/components/fabro-store/src/slate/projection_cache.rs, /home/daytona/workspace/fabro/lib/components/fabro-store/src/slate/run_store.rs, /home/daytona/workspace/fabro/lib/components/fabro-store/src/types.rs
- **simplify_fable**: succeeded
- Model: claude-fable-5
- Files: /home/daytona/workspace/fabro/lib/apps/fabro-server/Cargo.toml, /home/daytona/workspace/fabro/lib/apps/fabro-server/src/server/tests.rs, /home/daytona/workspace/fabro/lib/components/fabro-store/Cargo.toml, /home/daytona/workspace/fabro/lib/components/fabro-store/src/error.rs, /home/daytona/workspace/fabro/lib/components/fabro-store/src/slate/mod.rs, /home/daytona/workspace/fabro/lib/components/fabro-store/src/slate/run_store.rs, /home/daytona/workspace/fabro/lib/components/fabro-store/src/test_util.rs, /home/daytona/workspace/fabro/lib/foundation/fabro-types/src/run_event/mod.rs
- **simplify_sol**: succeeded
- Model: gpt-5.6-sol
- Files: /home/daytona/workspace/fabro/lib/apps/fabro-server/src/server/tests.rs, /home/daytona/workspace/fabro/lib/components/fabro-store/src/error.rs, /home/daytona/workspace/fabro/lib/components/fabro-store/src/lib.rs, /home/daytona/workspace/fabro/lib/components/fabro-store/src/run_summary_store.rs, /home/daytona/workspace/fabro/lib/components/fabro-store/src/slate/mod.rs, /home/daytona/workspace/fabro/lib/components/fabro-store/src/slate/run_store.rs, /home/daytona/workspace/fabro/lib/components/fabro-store/src/test_support.rs, /home/daytona/workspace/fabro/lib/components/fabro-store/src/test_support/mod.rs, /home/daytona/workspace/fabro/lib/components/fabro-store/src/types.rs, /home/daytona/workspace/fabro/lib/foundation/fabro-types/src/run_event/mod.rs
- **verify**: failed
- Script: `git fetch origin main 2>&1 && git merge --no-edit --no-stat origin/main 2>&1 && cargo +nightly-2026-04-14 fmt --all 2>&1 && cargo dev docs refresh 2>&1 && cargo +nightly-2026-04-14 fmt --check --all 2>&1 && { command -v rg >/dev/null 2>&1 || { echo 'rg is required for verify'; exit 127; }; } && ! rg -n 'AuthMode::Disabled|RunAuthMethod|RunSubjectProvenance|\bActorRef\b|\bActorKind\b|AuthenticatedSubject|AuthenticatedService|AuthorizeRunScoped|AuthorizeRunBlob|AuthorizeStageArtifact|AuthorizeCommandLog|auth_method\s*==\s*"disabled"' lib/crates apps lib/packages docs/public/api-reference/fabro-api.yaml 2>&1 && cargo +nightly-2026-04-14 clippy --workspace --all-targets -- -D warnings 2>&1 && cargo nextest run --workspace --status-level slow --profile ci 2>&1 && cargo dev docs check 2>&1 && bun install --frozen-lockfile 2>&1 && (cd apps/fabro-web && bun run typecheck) 2>&1 && (cd apps/fabro-web && bun run test) 2>&1 && (cd lib/packages/fabro-api-client && bun run typecheck) 2>&1 && cargo dev build -- -p fabro-cli --release 2>&1`
- Output:
```
(101 lines omitted)
https://registry.npmjs.org/lru-cache/-/lru-cache-11.3.5.tgz
ENOENT: No such file or directory: failed to link package: rehype-parse@9.0.1 (link)
error: NoSpaceLeft when create temporary directory named ".7bceff95fe5f1dd6-00000323.core" (while extracting "@clack/core")
error: failed to download @clack/core@1.2.0: InstallFailed
https://registry.npmjs.org/@clack/core/-/core-1.2.0.tgz
ENOENT: failed to link binaries for package: @assistant-ui/react@0.14.5
error: NoSpaceLeft when create temporary directory named ".bbeef9dddf6f7b75-00000325.destr" (while extracting "destr")
error: failed to download destr@2.0.5: InstallFailed
https://registry.npmjs.org/destr/-/destr-2.0.5.tgz
ENOENT: failed to link binaries for package: svgo@4.0.1
error: NoSpaceLeft when create temporary directory named ".3feff9f7fb33fdff-00000326.ofetch" (while extracting "ofetch")
error: failed to download ofetch@1.5.1: InstallFailed
https://registry.npmjs.org/ofetch/-/ofetch-1.5.1.tgz
error: NoSpaceLeft when create temporary directory named ".d8e7fdd7fbb7f7bf-00000327.yocto-queue" (while extracting "yocto-queue")
error: failed to download yocto-queue@1.2.2: InstallFailed
https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.2.2.tgz
ENOENT: failed to link binaries for package: @tailwindcss/node@4.2.error: ENOSPC extracting tarball for "@img/sharp-libvips-linuxmusl-x64"
error: failed to download @img/sharp-libvips-linuxmusl-x64@1.2.4: ENOSPC
https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.2.4.tgz
error: ENOSPC extracting tarball for "@rspack/binding-linux-x64-gnu"
error: failed to download @rspack/binding-linux-x64-gnu@1.7.6: ENOSPC
https://registry.npmjs.org/@rspack/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.7.6.tgz
1519 packages installed [3.76s]
Failed to install 55 packages
```
## Context
- failure_class: deterministic
- failure_signature: verify|deterministic|script failed with exit code: <n> ## output to link package: @remotion/compositor-linux-x64-musl@<n>.<n>.<n> (link) enoent: no such file or directory: failed to link package: @babel/types@<n>.<n>.<n> (link) error: failed to download css-tre
The verify step failed. Read the build output from context and fix all format, clippy, Rust test, docs, TypeScript typecheck/test, and build failures.

6
stages/009-fixup@1/provider_used.json Normal file
View file

@ -0,0 +1,6 @@
{
"mode": "agent",
"provider": "openrouter",
"model": "claude-fable-5",
"reasoning_effort": "xhigh"
}

22
stages/009-fixup@1/response.md Normal file
View file

@ -0,0 +1,22 @@
The verify failure is fixed — the entire pipeline now passes end to end. No code changes were needed.
## Root cause
The failure was environmental, not a code defect. The disk was 100% full (`20G/20G` used), so `bun install --frozen-lockfile` failed with `ENOSPC`/`NoSpaceLeft` while extracting packages, and it also left `node_modules` in a broken half-linked state. The space was consumed by 18 GB of accumulated debug artifacts in `target/debug/deps` (many stale duplicate compilation units from repeated workflow stages) plus ~120 MB of leftover `/tmp/fabro-test-*` fixtures.
## What I did
1. Deleted `target/debug`, stale `/tmp/fabro-test-*` dirs, and the corrupted `node_modules` trees (freed ~18 GB).
2. Re-ran every step of the verify script in order:
- `git fetch` / `git merge origin/main` — already up to date.
- `cargo +nightly-2026-04-14 fmt --all` then `fmt --check --all` — clean.
- `cargo dev docs refresh` + `cargo dev docs check` — both generated docs up to date, no diff.
- Forbidden-pattern `rg` check — passed (note: the script's `lib/crates` path doesn't exist in the repo; `rg` warns but the check still evaluates correctly since no forbidden patterns match).
- `cargo +nightly-2026-04-14 clippy --workspace --all-targets -- -D warnings` — clean.
- `cargo nextest run --workspace --profile ci` — **7658 tests passed, 0 failed** (203 skipped), including the new fabro-store append-contract tests.
- `bun install --frozen-lockfile` — 1625 packages installed successfully.
- `apps/fabro-web`: typecheck clean, **799 tests passed, 0 failed**.
- `lib/packages/fabro-api-client`: typecheck clean.
- `cargo dev build -- -p fabro-cli --release` — succeeded, SPA asset budgets verified.
The working tree is clean (no uncommitted changes — nothing in the PR's code needed modification), and the disk now has 5.7 GB of headroom after the release build.