mirror of
https://github.com/fabro-sh/fabro.git
synced 2026-09-21 00:21:27 +00:00
e901cd3a81
### Summary Silent fallback paths now emit stable warnings instead of degrading without a user-visible signal. The fallback behavior is unchanged; runs still continue, but worktree, Git, checkpoint, and LLM failover issues now show up in the run feed and logs. ### Plan Summary - Emit run notices for workflow Git/worktree fallback paths. - Reuse the existing failover event for one-shot LLM provider fallback. - Add tracing for sandbox pipe drain failures. ### What changed - Added `worktree_skipped_no_git` and gated `sandbox_git_unavailable` notices during initialization. - Added `git_push_failed` and `parallel_base_checkpoint_failed` notices, including redacted output tails where available. - Logged GitHub token mint failures with a structured `error` field before the existing notice. - Plumbed `Emitter` and `StageScope` through `CodergenBackend::one_shot` so the API backend emits the existing `agent.failover` event instead of a duplicate tracing-only warning. - Extracted sandbox pipe draining into a helper that warns on stdout/stderr read failures, with unit coverage for the error path. - Updated CLI snapshots for the new worktree warning in stderr and JSON event output. ⚒️ Generated with [Fabro](https://fabro.sh) --------- Co-authored-by: Fabro <noreply@fabro.sh> Co-authored-by: Bryan Helmkamp <bryan@brynary.com>
363 lines
11 KiB
Rust
363 lines
11 KiB
Rust
#![expect(
|
||
clippy::disallowed_methods,
|
||
reason = "integration tests stage fixtures with sync std::fs; test infrastructure, not Tokio-hot path"
|
||
)]
|
||
|
||
use std::fs;
|
||
use std::time::Duration;
|
||
|
||
use fabro_client::ServerTarget;
|
||
use fabro_test::{fabro_snapshot, test_context};
|
||
use insta::assert_snapshot;
|
||
|
||
use super::support::{
|
||
local_dev_token, server_target, setup_completed_dry_run, setup_seeded_completed_dry_run,
|
||
setup_seeded_created_dry_run,
|
||
};
|
||
use crate::support::{LightweightCli, seed_dev_token_auth, unique_run_id};
|
||
|
||
#[test]
|
||
fn help() {
|
||
let context = test_context!();
|
||
let mut cmd = context.command();
|
||
cmd.args(["dump", "--help"]);
|
||
fabro_snapshot!(context.filters(), cmd, @"
|
||
success: true
|
||
exit_code: 0
|
||
----- stdout -----
|
||
Export a run's durable state to a directory
|
||
|
||
Usage: fabro dump [OPTIONS] --output <OUTPUT> <RUN>
|
||
|
||
Arguments:
|
||
<RUN> Run ID prefix or workflow name
|
||
|
||
Options:
|
||
--json Output as JSON [env: FABRO_JSON=]
|
||
--server <SERVER> Fabro server target: http(s) URL or absolute Unix socket path [env: FABRO_SERVER=]
|
||
--debug Enable DEBUG-level logging (default is INFO) [env: FABRO_DEBUG=]
|
||
-o, --output <OUTPUT> Output directory (must not exist or be empty)
|
||
--no-upgrade-check Disable automatic upgrade check [env: FABRO_NO_UPGRADE_CHECK=true]
|
||
--quiet Suppress non-essential output [env: FABRO_QUIET=]
|
||
--verbose Enable verbose output [env: FABRO_VERBOSE=]
|
||
-h, --help Print help
|
||
----- stderr -----
|
||
");
|
||
}
|
||
|
||
#[test]
|
||
fn dump_accepts_server_target_from_separate_home() {
|
||
let context = test_context!();
|
||
let run = setup_seeded_completed_dry_run(&context);
|
||
let cli = LightweightCli::new();
|
||
let output_dir = context.temp_dir.join("remote-export");
|
||
let server = server_target(&context.storage_dir);
|
||
if let Some(dev_token) = local_dev_token(&context.storage_dir) {
|
||
let target = server
|
||
.parse::<ServerTarget>()
|
||
.expect("server target should parse");
|
||
seed_dev_token_auth(cli.home(), &target, &dev_token);
|
||
}
|
||
|
||
let mut cmd = cli.command();
|
||
cmd.args([
|
||
"dump",
|
||
"--server",
|
||
&server,
|
||
"--output",
|
||
output_dir.to_str().unwrap(),
|
||
&run.run_id,
|
||
]);
|
||
|
||
let output = cmd.output().expect("dump should execute");
|
||
assert!(
|
||
output.status.success(),
|
||
"dump via remote server target failed\nstdout:\n{}\nstderr:\n{}",
|
||
String::from_utf8_lossy(&output.stdout),
|
||
String::from_utf8_lossy(&output.stderr)
|
||
);
|
||
assert!(output_dir.join("run.json").is_file());
|
||
}
|
||
|
||
#[test]
|
||
fn dump_exports_large_command_output_backed_by_blob_refs() {
|
||
let context = test_context!();
|
||
let workflow = context.temp_dir.join("large-output.fabro");
|
||
fs::write(
|
||
&workflow,
|
||
r#"digraph LargeOutput {
|
||
graph [goal="Generate oversized command output"]
|
||
rankdir=LR
|
||
|
||
start [shape=Mdiamond, label="Start"]
|
||
exit [shape=Msquare, label="Exit"]
|
||
big [shape=parallelogram, label="Big", script="printf '%*s' 120000 '' | tr ' ' x"]
|
||
|
||
start -> big -> exit
|
||
}
|
||
"#,
|
||
)
|
||
.unwrap();
|
||
|
||
let run_id = unique_run_id();
|
||
let mut run_cmd = context.run_cmd();
|
||
run_cmd.current_dir(&context.temp_dir);
|
||
run_cmd.timeout(Duration::from_secs(30));
|
||
run_cmd.args([
|
||
"--run-id",
|
||
run_id.as_str(),
|
||
"--no-retro",
|
||
"--sandbox",
|
||
"local",
|
||
]);
|
||
run_cmd.arg(&workflow);
|
||
let run_output = run_cmd.output().expect("command should execute");
|
||
assert!(
|
||
run_output.status.success(),
|
||
"workflow run failed\nstdout:\n{}\nstderr:\n{}",
|
||
String::from_utf8_lossy(&run_output.stdout),
|
||
String::from_utf8_lossy(&run_output.stderr)
|
||
);
|
||
|
||
let mut inspect_cmd = context.command();
|
||
inspect_cmd.args(["inspect", "--json", &run_id]);
|
||
let inspect_output = inspect_cmd.output().expect("inspect should execute");
|
||
assert!(
|
||
inspect_output.status.success(),
|
||
"inspect failed\nstdout:\n{}\nstderr:\n{}",
|
||
String::from_utf8_lossy(&inspect_output.stdout),
|
||
String::from_utf8_lossy(&inspect_output.stderr)
|
||
);
|
||
let inspect_json = String::from_utf8(inspect_output.stdout).unwrap();
|
||
assert!(
|
||
inspect_json.contains("blob://sha256/"),
|
||
"inspect output should contain blob refs to exercise hydration\n{inspect_json}"
|
||
);
|
||
|
||
let output_dir = context.temp_dir.join("export");
|
||
let mut dump_cmd = context.command();
|
||
dump_cmd.args(["dump", "--output", output_dir.to_str().unwrap(), &run_id]);
|
||
let dump_output = dump_cmd.output().expect("dump should execute");
|
||
assert!(
|
||
dump_output.status.success(),
|
||
"dump failed\nstdout:\n{}\nstderr:\n{}",
|
||
String::from_utf8_lossy(&dump_output.stdout),
|
||
String::from_utf8_lossy(&dump_output.stderr)
|
||
);
|
||
|
||
let run_json = fs::read_to_string(output_dir.join("run.json")).unwrap();
|
||
assert!(
|
||
!run_json.contains("blob://sha256/"),
|
||
"run export should hydrate blob refs\n{run_json}"
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn dump_exports_blob_refs_and_artifacts_together() {
|
||
let context = test_context!();
|
||
let workspace_dir = context.temp_dir.join("mixed-export");
|
||
fs::create_dir_all(&workspace_dir).unwrap();
|
||
|
||
fs::write(
|
||
workspace_dir.join("mixed-export.fabro"),
|
||
r#"digraph MixedExport {
|
||
graph [goal="Generate oversized command output and artifacts"]
|
||
rankdir=LR
|
||
|
||
start [shape=Mdiamond, label="Start"]
|
||
exit [shape=Msquare, label="Exit"]
|
||
big [shape=parallelogram, label="Big", script="mkdir -p assets/shared && printf exported > assets/shared/report.txt && printf '%*s' 120000 '' | tr ' ' x"]
|
||
|
||
start -> big -> exit
|
||
}
|
||
"#,
|
||
)
|
||
.unwrap();
|
||
fs::write(
|
||
workspace_dir.join("run.toml"),
|
||
r#"_version = 1
|
||
|
||
[workflow]
|
||
graph = "mixed-export.fabro"
|
||
|
||
[run]
|
||
goal = "Generate oversized command output and artifacts"
|
||
|
||
[run.sandbox]
|
||
provider = "local"
|
||
preserve = true
|
||
|
||
[run.sandbox.local]
|
||
worktree_mode = "never"
|
||
|
||
[run.artifacts]
|
||
include = ["assets/**"]
|
||
"#,
|
||
)
|
||
.unwrap();
|
||
|
||
let run_id = unique_run_id();
|
||
let mut run_cmd = context.run_cmd();
|
||
run_cmd.current_dir(&workspace_dir);
|
||
run_cmd.timeout(Duration::from_secs(30));
|
||
run_cmd.args([
|
||
"--run-id",
|
||
run_id.as_str(),
|
||
"--no-retro",
|
||
"--sandbox",
|
||
"local",
|
||
"run.toml",
|
||
]);
|
||
let run_output = run_cmd.output().expect("command should execute");
|
||
assert!(
|
||
run_output.status.success(),
|
||
"workflow run failed\nstdout:\n{}\nstderr:\n{}",
|
||
String::from_utf8_lossy(&run_output.stdout),
|
||
String::from_utf8_lossy(&run_output.stderr)
|
||
);
|
||
|
||
let mut inspect_cmd = context.command();
|
||
inspect_cmd.args(["inspect", "--json", &run_id]);
|
||
let inspect_output = inspect_cmd.output().expect("inspect should execute");
|
||
assert!(
|
||
inspect_output.status.success(),
|
||
"inspect failed\nstdout:\n{}\nstderr:\n{}",
|
||
String::from_utf8_lossy(&inspect_output.stdout),
|
||
String::from_utf8_lossy(&inspect_output.stderr)
|
||
);
|
||
let inspect_json = String::from_utf8(inspect_output.stdout).unwrap();
|
||
assert!(
|
||
inspect_json.contains("blob://sha256/"),
|
||
"inspect output should contain blob refs to exercise hydration\n{inspect_json}"
|
||
);
|
||
|
||
let output_dir = context.temp_dir.join("export-mixed");
|
||
let mut dump_cmd = context.command();
|
||
dump_cmd.args(["dump", "--output", output_dir.to_str().unwrap(), &run_id]);
|
||
let dump_output = dump_cmd.output().expect("dump should execute");
|
||
assert!(
|
||
dump_output.status.success(),
|
||
"dump failed\nstdout:\n{}\nstderr:\n{}",
|
||
String::from_utf8_lossy(&dump_output.stdout),
|
||
String::from_utf8_lossy(&dump_output.stderr)
|
||
);
|
||
|
||
let run_json = fs::read_to_string(output_dir.join("run.json")).unwrap();
|
||
assert!(
|
||
!run_json.contains("blob://sha256/"),
|
||
"run export should hydrate blob refs\n{run_json}"
|
||
);
|
||
assert_eq!(
|
||
fs::read_to_string(
|
||
output_dir.join("artifacts/002-big@1/retry-0001/assets/shared/report.txt")
|
||
)
|
||
.unwrap(),
|
||
"exported"
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn dump_exports_completed_run_snapshot() {
|
||
let context = test_context!();
|
||
let run = setup_completed_dry_run(&context);
|
||
let output_dir = context.temp_dir.join("export");
|
||
|
||
let mut cmd = context.command();
|
||
cmd.args([
|
||
"dump",
|
||
"--output",
|
||
output_dir.to_str().unwrap(),
|
||
&run.run_id,
|
||
]);
|
||
fabro_snapshot!(context.filters(), cmd, @"
|
||
success: true
|
||
exit_code: 0
|
||
----- stdout -----
|
||
Exported 13 files for run [ULID] to [TEMP_DIR]/export
|
||
----- stderr -----
|
||
");
|
||
|
||
assert_snapshot!(dump_file_summary(&output_dir), @"
|
||
checkpoints/0014.json
|
||
checkpoints/0018.json
|
||
checkpoints/0022.json
|
||
events.jsonl
|
||
graph.fabro
|
||
run.json
|
||
run.log
|
||
stages/001-start@1/status.json
|
||
stages/002-run_tests@1/response.md
|
||
stages/002-run_tests@1/status.json
|
||
stages/003-report@1/response.md
|
||
stages/003-report@1/status.json
|
||
stages/004-exit@1/status.json
|
||
");
|
||
}
|
||
|
||
#[test]
|
||
fn dump_succeeds_when_run_log_is_missing() {
|
||
let context = test_context!();
|
||
let run = setup_seeded_created_dry_run(&context);
|
||
let output_dir = context.temp_dir.join("export-missing-log");
|
||
|
||
let mut cmd = context.command();
|
||
cmd.args([
|
||
"dump",
|
||
"--output",
|
||
output_dir.to_str().unwrap(),
|
||
&run.run_id,
|
||
]);
|
||
let output = cmd.output().expect("dump should execute");
|
||
assert!(
|
||
output.status.success(),
|
||
"dump failed\nstdout:\n{}\nstderr:\n{}",
|
||
String::from_utf8_lossy(&output.stdout),
|
||
String::from_utf8_lossy(&output.stderr)
|
||
);
|
||
assert!(
|
||
!output_dir.join("run.log").exists(),
|
||
"dump should skip run.log when the server has no run log"
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn dump_rejects_non_empty_output_dir() {
|
||
let context = test_context!();
|
||
let run = setup_seeded_completed_dry_run(&context);
|
||
let output_dir = context.temp_dir.join("nonempty");
|
||
std::fs::create_dir_all(&output_dir).unwrap();
|
||
std::fs::write(output_dir.join("file.txt"), "x").unwrap();
|
||
|
||
let mut cmd = context.command();
|
||
cmd.args([
|
||
"dump",
|
||
"--output",
|
||
output_dir.to_str().unwrap(),
|
||
&run.run_id,
|
||
]);
|
||
fabro_snapshot!(context.filters(), cmd, @"
|
||
success: false
|
||
exit_code: 1
|
||
----- stdout -----
|
||
----- stderr -----
|
||
× output path [TEMP_DIR]/nonempty already exists and is not an empty directory; remove it first or choose a different path
|
||
");
|
||
}
|
||
|
||
fn dump_file_summary(output_dir: &std::path::Path) -> String {
|
||
let mut files: Vec<String> = walkdir::WalkDir::new(output_dir)
|
||
.into_iter()
|
||
.filter_map(Result::ok)
|
||
.filter(|entry| entry.file_type().is_file())
|
||
.map(|entry| {
|
||
entry
|
||
.path()
|
||
.strip_prefix(output_dir)
|
||
.expect("walked file should stay under the output directory")
|
||
.to_string_lossy()
|
||
.replace('\\', "/")
|
||
})
|
||
.collect();
|
||
files.sort();
|
||
files.join("\n") + "\n"
|
||
}
|