scriptoriumadmin/fabro
1
0
Fork 0
mirror of https://github.com/fabro-sh/fabro.git synced 2026-08-28 05:27:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Split scenario.rs into scenario/ directory

Browse source 
Break the monolithic scenario.rs into three focused files:
- scenario/workflows.rs — 6 parametrized E2E workflow scenarios
- scenario/lifecycle.rs — run lifecycle (ps, inspect, logs, assets, rm)
- scenario/exec.rs — exec creates file scenario
- scenario/mod.rs — shared helpers, macro, and timeout_for

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Bryan Helmkamp 2026-03-30 09:23:11 -04:00
parent f01d74c692
commit 90fc680609
No known key found for this signature in database
5 changed files with 621 additions and 613 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

613
lib/crates/fabro-cli/tests/it/scenario.rs
View file

@ -1,613 +0,0 @@
use std::path::{Path, PathBuf};
use std::time::Duration;
use fabro_store::RuntimeState;
use fabro_test::test_context;
use serde_json::Value;
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
fn fixture(name: &str) -> PathBuf {
Path::new(env!("CARGO_MANIFEST_DIR"))
.join("../../../test/scenario")
.join(name)
}
fn read_json(path: &Path) -> Value {
let content = std::fs::read_to_string(path)
.unwrap_or_else(|e| panic!("failed to read {}: {e}", path.display()));
serde_json::from_str(&content)
.unwrap_or_else(|e| panic!("failed to parse {}: {e}", path.display()))
}
fn read_checkpoint(run_dir: &Path) -> Value {
read_json(&run_dir.join("checkpoint.json"))
}
fn read_conclusion(run_dir: &Path) -> Value {
read_json(&run_dir.join("conclusion.json"))
}
/// Find the single run directory under `storage_dir/runs/`.
fn find_run_dir(storage_dir: &Path) -> PathBuf {
let runs_base = storage_dir.join("runs");
let entries: Vec<_> = std::fs::read_dir(&runs_base)
.unwrap_or_else(|e| panic!("failed to read {}: {e}", runs_base.display()))
.filter_map(|e| e.ok())
.filter(|e| e.path().is_dir())
.collect();
assert_eq!(
entries.len(),
1,
"expected exactly one run directory under {}",
runs_base.display()
);
entries[0].path()
}
fn completed_nodes(run_dir: &Path) -> Vec<String> {
let cp = read_checkpoint(run_dir);
cp["completed_nodes"]
.as_array()
.expect("completed_nodes should be an array")
.iter()
.map(|v| v.as_str().unwrap().to_string())
.collect()
}
fn has_event(run_dir: &Path, event_name: &str) -> bool {
let path = run_dir.join("progress.jsonl");
let content = std::fs::read_to_string(&path)
.unwrap_or_else(|e| panic!("failed to read progress.jsonl: {e}"));
content.lines().any(|line| {
if let Ok(v) = serde_json::from_str::<Value>(line) {
v["event"].as_str() == Some(event_name)
} else {
false
}
})
}
// ---------------------------------------------------------------------------
// Macro: generate local_* and daytona_* variants for each scenario
// ---------------------------------------------------------------------------
macro_rules! scenario_tests {
($name:ident) => {
paste::paste! {
#[test]
#[ignore = "scenario: requires local sandbox"]
fn [<local_ $name>]() {
[<scenario_ $name>]("local");
}
#[test]
#[ignore = "scenario: requires DAYTONA_API_KEY"]
fn [<daytona_ $name>]() {
[<scenario_ $name>]("daytona");
}
}
};
}
fn timeout_for(sandbox: &str) -> Duration {
match sandbox {
"daytona" => Duration::from_secs(600),
_ => Duration::from_secs(180),
}
}
// ---------------------------------------------------------------------------
// Scenarios
// ---------------------------------------------------------------------------
// 1. command_pipeline — two command nodes in sequence, no LLM
scenario_tests!(command_pipeline);
fn scenario_command_pipeline(sandbox: &str) {
dotenvy::dotenv().ok();
let context = test_context!();
context
.validate()
.arg(fixture("command_pipeline.fabro"))
.assert()
.success();
context
.run_cmd()
.args(["--auto-approve", "--no-retro", "--sandbox", sandbox])
.arg(fixture("command_pipeline.fabro"))
.timeout(timeout_for(sandbox))
.assert()
.success();
let run_dir = find_run_dir(&context.storage_dir);
let conclusion = read_conclusion(&run_dir);
assert_eq!(
conclusion["status"].as_str(),
Some("success"),
"conclusion status should be success"
);
let nodes = completed_nodes(&run_dir);
assert!(
nodes.contains(&"step1".to_string()),
"step1 should be completed"
);
assert!(
nodes.contains(&"step2".to_string()),
"step2 should be completed"
);
// Verify step1 stdout
let stdout1 = std::fs::read_to_string(run_dir.join("nodes/step1/stdout.log"))
.expect("step1 stdout.log should exist");
assert!(
stdout1.contains("hello-from-step1"),
"step1 stdout should contain hello-from-step1, got: {stdout1}"
);
}
// 2. conditional_branching — command + diamond gate, success path taken
scenario_tests!(conditional_branching);
fn scenario_conditional_branching(sandbox: &str) {
dotenvy::dotenv().ok();
let context = test_context!();
context
.run_cmd()
.args(["--auto-approve", "--no-retro", "--sandbox", sandbox])
.arg(fixture("conditional_branching.fabro"))
.timeout(timeout_for(sandbox))
.assert()
.success();
let run_dir = find_run_dir(&context.storage_dir);
let conclusion = read_conclusion(&run_dir);
assert_eq!(conclusion["status"].as_str(), Some("success"));
let nodes = completed_nodes(&run_dir);
assert!(
nodes.contains(&"passed".to_string()),
"passed node should be in completed_nodes: {nodes:?}"
);
assert!(
!nodes.contains(&"failed".to_string()),
"failed node should NOT be in completed_nodes: {nodes:?}"
);
}
// 3. agent_linear — single agent node with LLM
scenario_tests!(agent_linear);
fn scenario_agent_linear(sandbox: &str) {
dotenvy::dotenv().ok();
let context = test_context!();
context
.run_cmd()
.args([
"--auto-approve",
"--no-retro",
"--sandbox",
sandbox,
"--model",
"claude-haiku-4-5",
])
.arg(fixture("agent_linear.fabro"))
.timeout(timeout_for(sandbox))
.assert()
.success();
let run_dir = find_run_dir(&context.storage_dir);
let conclusion = read_conclusion(&run_dir);
assert_eq!(conclusion["status"].as_str(), Some("success"));
let nodes = completed_nodes(&run_dir);
assert!(
nodes.contains(&"work".to_string()),
"work should be completed"
);
// Agent node should produce prompt.md and response.md
let prompt_path = run_dir.join("nodes/work/prompt.md");
assert!(prompt_path.exists(), "nodes/work/prompt.md should exist");
let response_path = run_dir.join("nodes/work/response.md");
assert!(
response_path.exists(),
"nodes/work/response.md should exist"
);
let response = std::fs::read_to_string(&response_path).unwrap();
assert!(!response.is_empty(), "response.md should not be empty");
}
// 4. human_gate — human gate with --auto-approve selects first edge
scenario_tests!(human_gate);
fn scenario_human_gate(sandbox: &str) {
dotenvy::dotenv().ok();
let context = test_context!();
context
.run_cmd()
.args([
"--auto-approve",
"--no-retro",
"--sandbox",
sandbox,
"--model",
"claude-haiku-4-5",
])
.arg(fixture("human_gate.fabro"))
.timeout(timeout_for(sandbox))
.assert()
.success();
let run_dir = find_run_dir(&context.storage_dir);
let conclusion = read_conclusion(&run_dir);
assert_eq!(conclusion["status"].as_str(), Some("success"));
let nodes = completed_nodes(&run_dir);
assert!(
nodes.contains(&"ship".to_string()),
"ship should be in completed_nodes (auto-approve picks first edge): {nodes:?}"
);
assert!(
!nodes.contains(&"revise".to_string()),
"revise should NOT be in completed_nodes: {nodes:?}"
);
}
// 5. command_agent_mixed — command writes file, agent reads it, command verifies
scenario_tests!(command_agent_mixed);
fn scenario_command_agent_mixed(sandbox: &str) {
dotenvy::dotenv().ok();
let context = test_context!();
context
.run_cmd()
.args([
"--auto-approve",
"--no-retro",
"--sandbox",
sandbox,
"--model",
"claude-haiku-4-5",
])
.arg(fixture("command_agent_mixed.fabro"))
.timeout(timeout_for(sandbox))
.assert()
.success();
let run_dir = find_run_dir(&context.storage_dir);
let conclusion = read_conclusion(&run_dir);
assert_eq!(conclusion["status"].as_str(), Some("success"));
let nodes = completed_nodes(&run_dir);
assert!(
nodes.contains(&"setup".to_string()),
"setup should be completed"
);
assert!(
nodes.contains(&"work".to_string()),
"work should be completed"
);
assert!(
nodes.contains(&"verify".to_string()),
"verify should be completed"
);
// Verify command node saw the flag
let stdout = std::fs::read_to_string(run_dir.join("nodes/verify/stdout.log"))
.expect("verify stdout.log should exist");
assert!(
stdout.contains("SCENARIO_FLAG_42"),
"verify stdout should contain SCENARIO_FLAG_42, got: {stdout}"
);
}
// 6. full_stack — command + agent + human gate + goal_gate, kitchen sink
scenario_tests!(full_stack);
fn scenario_full_stack(sandbox: &str) {
dotenvy::dotenv().ok();
let context = test_context!();
context
.run_cmd()
.args([
"--auto-approve",
"--no-retro",
"--sandbox",
sandbox,
"--model",
"claude-haiku-4-5",
])
.arg(fixture("full_stack.fabro"))
.timeout(timeout_for(sandbox))
.assert()
.success();
let run_dir = find_run_dir(&context.storage_dir);
let conclusion = read_conclusion(&run_dir);
assert_eq!(
conclusion["status"].as_str(),
Some("success"),
"conclusion: {conclusion}"
);
assert!(
conclusion["duration_ms"].as_u64().unwrap_or(0) > 0,
"duration_ms should be > 0"
);
// RunRecord should have key fields
let run_record = read_json(&run_dir.join("run.json"));
assert!(
run_record["run_id"].as_str().is_some(),
"run record should have run_id"
);
assert!(
run_record["graph"]["name"].as_str().is_some(),
"run record should have graph.name"
);
// Progress events
assert!(
has_event(&run_dir, "WorkflowRunStarted"),
"progress should contain WorkflowRunStarted"
);
assert!(
has_event(&run_dir, "WorkflowRunCompleted"),
"progress should contain WorkflowRunCompleted"
);
// All expected nodes completed
let nodes = completed_nodes(&run_dir);
for expected in &["setup", "plan", "approve", "impl", "verify"] {
assert!(
nodes.contains(&expected.to_string()),
"{expected} should be in completed_nodes: {nodes:?}"
);
}
// Verify node stdout should contain PASS
let stdout = std::fs::read_to_string(run_dir.join("nodes/verify/stdout.log"))
.expect("verify stdout.log should exist");
assert!(
stdout.contains("PASS"),
"verify stdout should contain PASS, got: {stdout}"
);
}
// Run lifecycle: ps / inspect / logs / rm / system df
// ---------------------------------------------------------------------------
#[test]
#[ignore = "scenario: requires local sandbox"]
fn local_run_lifecycle() {
dotenvy::dotenv().ok();
let context = test_context!();
let cmd = |args: &[&str]| -> assert_cmd::assert::Assert {
context
.command()
.args(args)
.timeout(timeout_for("local"))
.assert()
};
// 1. Run a workflow
cmd(&[
"run",
"--auto-approve",
"--no-retro",
"--sandbox",
"local",
fixture("command_pipeline.fabro").to_str().unwrap(),
])
.success();
// 2. ps -a --json — should list exactly one run
let ps_out = cmd(&["ps", "-a", "--json"]).success();
let ps_stdout = String::from_utf8(ps_out.get_output().stdout.clone()).unwrap();
let runs: Vec<Value> =
serde_json::from_str(&ps_stdout).expect("ps --json should produce a JSON array");
assert_eq!(runs.len(), 1, "should have exactly one run: {ps_stdout}");
let run_id = runs[0]["run_id"]
.as_str()
.expect("run should have run_id")
.to_string();
assert_eq!(
runs[0]["workflow_name"].as_str(),
Some("CommandPipeline"),
"workflow_name should be CommandPipeline"
);
// 3. inspect <run_id> — JSON array with run_record and conclusion
let inspect_out = cmd(&["inspect", &run_id]).success();
let inspect_stdout = String::from_utf8(inspect_out.get_output().stdout.clone()).unwrap();
let items: Vec<Value> =
serde_json::from_str(&inspect_stdout).expect("inspect should produce a JSON array");
assert!(!items.is_empty(), "inspect should return at least one item");
assert!(
items[0]["run_record"].is_object(),
"inspect should include run_record"
);
assert!(
items[0]["conclusion"].is_object(),
"inspect should include conclusion"
);
let run_dir = PathBuf::from(
items[0]["run_dir"]
.as_str()
.expect("inspect should include run_dir"),
);
// 4. logs <run_id> — non-empty, first line is valid JSONL with event field
let logs_out = cmd(&["logs", &run_id]).success();
let logs_stdout = String::from_utf8(logs_out.get_output().stdout.clone()).unwrap();
assert!(!logs_stdout.is_empty(), "logs should not be empty");
let first_line = logs_stdout.lines().next().unwrap();
let log_entry: Value =
serde_json::from_str(first_line).expect("first log line should be valid JSON");
assert!(
log_entry["event"].is_string(),
"first log line should have an event field"
);
// 5. asset list — no assets yet, should succeed with empty message
let asset_list_out = cmd(&["asset", "list", &run_id]).success();
let asset_list_stdout = String::from_utf8(asset_list_out.get_output().stdout.clone()).unwrap();
assert!(
asset_list_stdout.contains("No assets found"),
"asset list should report no assets: {asset_list_stdout}"
);
// 6. Seed a synthetic asset so asset list/cp have something to work with.
let asset_dir = RuntimeState::new(&run_dir).asset_stage_dir("step1", 1);
std::fs::create_dir_all(&asset_dir).unwrap();
std::fs::write(asset_dir.join("output.txt"), "asset-content-42").unwrap();
std::fs::write(
asset_dir.join("manifest.json"),
r#"{"files_copied":1,"total_bytes":16,"files_skipped":0,"download_errors":0,"hash_errors":0,"captured_assets":[{"path":"output.txt","mime":"text/plain","content_md5":"f02439728c0a94b7bfc465acb1201a1f","content_sha256":"0af9dea3e1c2dec968531c18c9331659b8268e8c9cf24b01cda7b8ce51d2ff00","bytes":16}]}"#,
)
.unwrap();
let retry_two_dir = RuntimeState::new(&run_dir).asset_stage_dir("step1", 2);
std::fs::create_dir_all(&retry_two_dir).unwrap();
std::fs::write(retry_two_dir.join("output.txt"), "asset-content-84").unwrap();
std::fs::write(
retry_two_dir.join("manifest.json"),
r#"{"files_copied":1,"total_bytes":16,"files_skipped":0,"download_errors":0,"hash_errors":0,"captured_assets":[{"path":"output.txt","mime":"text/plain","content_md5":"5b4e23e40a1630f9caa15a4cb6cfb79b","content_sha256":"1f71e0df61fc3b4e1ee3aba7ceac9ae391af22595b5b5630d97d34cf33d4d540","bytes":16}]}"#,
)
.unwrap();
// 7. asset list — now shows the seeded assets
let asset_list_out2 = cmd(&["asset", "list", &run_id, "--json"]).success();
let asset_list_stdout2 =
String::from_utf8(asset_list_out2.get_output().stdout.clone()).unwrap();
let assets: Vec<Value> = serde_json::from_str(&asset_list_stdout2)
.expect("asset list --json should produce a JSON array");
assert_eq!(
assets.len(),
2,
"should have two assets: {asset_list_stdout2}"
);
assert_eq!(assets[0]["relative_path"].as_str(), Some("output.txt"));
assert_eq!(assets[0]["node_slug"].as_str(), Some("step1"));
let retry_filtered_out = cmd(&["asset", "list", &run_id, "--retry", "1", "--json"]).success();
let retry_filtered_stdout =
String::from_utf8(retry_filtered_out.get_output().stdout.clone()).unwrap();
let retry_filtered_assets: Vec<Value> = serde_json::from_str(&retry_filtered_stdout)
.expect("asset list --json should produce a JSON array");
assert_eq!(retry_filtered_assets.len(), 1);
assert_eq!(retry_filtered_assets[0]["retry"].as_u64(), Some(1));
// 8. asset cp — ambiguous without --retry when multiple retries captured the same path
let asset_dest = context.temp_dir.join("asset_copy");
cmd(&[
"asset",
"cp",
&format!("{run_id}:output.txt"),
asset_dest.to_str().unwrap(),
])
.failure();
cmd(&[
"asset",
"cp",
&format!("{run_id}:output.txt"),
asset_dest.to_str().unwrap(),
"--retry",
"1",
])
.success();
let copied = std::fs::read_to_string(asset_dest.join("output.txt")).unwrap();
assert_eq!(
copied, "asset-content-42",
"asset cp should copy file content"
);
// 9. cp — download a file from the local sandbox workdir
let sandbox_json: Value = read_json(&run_dir.join("sandbox.json"));
let workdir = sandbox_json["working_directory"]
.as_str()
.expect("sandbox.json should have working_directory");
// Plant a file in the sandbox workdir so we can download it
std::fs::write(
PathBuf::from(workdir).join("cp_test.txt"),
"downloaded-via-cp",
)
.unwrap();
let cp_dest = context.temp_dir.join("cp_download.txt");
cmd(&[
"cp",
&format!("{run_id}:cp_test.txt"),
cp_dest.to_str().unwrap(),
])
.success();
let cp_content = std::fs::read_to_string(&cp_dest).unwrap();
assert_eq!(
cp_content, "downloaded-via-cp",
"cp should download file from sandbox"
);
// 10. system df — mentions "Runs"
let df_out = cmd(&["system", "df"]).success();
let df_stdout = String::from_utf8(df_out.get_output().stdout.clone()).unwrap();
assert!(
df_stdout.contains("Runs"),
"system df should mention Runs: {df_stdout}"
);
// 11. rm <run_id> — remove the run
cmd(&["rm", &run_id]).success();
// 12. ps -a --json — should be empty
let ps_out2 = cmd(&["ps", "-a", "--json"]).success();
let ps_stdout2 = String::from_utf8(ps_out2.get_output().stdout.clone()).unwrap();
let runs2: Vec<Value> =
serde_json::from_str(&ps_stdout2).expect("ps --json should produce a JSON array");
assert!(
runs2.is_empty(),
"runs should be empty after rm: {ps_stdout2}"
);
}
// ---------------------------------------------------------------------------
// exec — exercises `fabro exec`
// ---------------------------------------------------------------------------
#[test]
#[ignore = "scenario: requires ANTHROPIC_API_KEY"]
fn test_exec_creates_file() {
dotenvy::dotenv().ok();
let context = test_context!();
let mut cmd = context.exec_cmd();
cmd.args([
"--auto-approve",
"--permissions",
"full",
"--provider",
"anthropic",
"--model",
"claude-haiku-4-5",
"Create a file called hello.txt containing exactly 'Hello from exec scenario'",
]);
cmd.current_dir(&context.temp_dir);
cmd.timeout(Duration::from_secs(120));
cmd.assert().success();
let hello = context.temp_dir.join("hello.txt");
assert!(hello.exists(), "hello.txt should exist after exec");
let content = std::fs::read_to_string(&hello).unwrap();
assert!(
content.contains("Hello from exec scenario"),
"hello.txt should contain greeting, got: {content}"
);
}

33
lib/crates/fabro-cli/tests/it/scenario/exec.rs Normal file
View file

@ -0,0 +1,33 @@
use std::time::Duration;
use fabro_test::test_context;
#[test]
#[ignore = "scenario: requires ANTHROPIC_API_KEY"]
fn test_exec_creates_file() {
dotenvy::dotenv().ok();
let context = test_context!();
let mut cmd = context.exec_cmd();
cmd.args([
"--auto-approve",
"--permissions",
"full",
"--provider",
"anthropic",
"--model",
"claude-haiku-4-5",
"Create a file called hello.txt containing exactly 'Hello from exec scenario'",
]);
cmd.current_dir(&context.temp_dir);
cmd.timeout(Duration::from_secs(120));
cmd.assert().success();
let hello = context.temp_dir.join("hello.txt");
assert!(hello.exists(), "hello.txt should exist after exec");
let content = std::fs::read_to_string(&hello).unwrap();
assert!(
content.contains("Hello from exec scenario"),
"hello.txt should contain greeting, got: {content}"
);
}

197
lib/crates/fabro-cli/tests/it/scenario/lifecycle.rs Normal file
View file

@ -0,0 +1,197 @@
use std::path::PathBuf;
use fabro_store::RuntimeState;
use fabro_test::test_context;
use serde_json::Value;
use super::{fixture, read_json, timeout_for};
#[test]
#[ignore = "scenario: requires local sandbox"]
fn local_run_lifecycle() {
dotenvy::dotenv().ok();
let context = test_context!();
let cmd = |args: &[&str]| -> assert_cmd::assert::Assert {
context
.command()
.args(args)
.timeout(timeout_for("local"))
.assert()
};
// 1. Run a workflow
cmd(&[
"run",
"--auto-approve",
"--no-retro",
"--sandbox",
"local",
fixture("command_pipeline.fabro").to_str().unwrap(),
])
.success();
// 2. ps -a --json — should list exactly one run
let ps_out = cmd(&["ps", "-a", "--json"]).success();
let ps_stdout = String::from_utf8(ps_out.get_output().stdout.clone()).unwrap();
let runs: Vec<Value> =
serde_json::from_str(&ps_stdout).expect("ps --json should produce a JSON array");
assert_eq!(runs.len(), 1, "should have exactly one run: {ps_stdout}");
let run_id = runs[0]["run_id"]
.as_str()
.expect("run should have run_id")
.to_string();
assert_eq!(
runs[0]["workflow_name"].as_str(),
Some("CommandPipeline"),
"workflow_name should be CommandPipeline"
);
// 3. inspect <run_id> — JSON array with run_record and conclusion
let inspect_out = cmd(&["inspect", &run_id]).success();
let inspect_stdout = String::from_utf8(inspect_out.get_output().stdout.clone()).unwrap();
let items: Vec<Value> =
serde_json::from_str(&inspect_stdout).expect("inspect should produce a JSON array");
assert!(!items.is_empty(), "inspect should return at least one item");
assert!(
items[0]["run_record"].is_object(),
"inspect should include run_record"
);
assert!(
items[0]["conclusion"].is_object(),
"inspect should include conclusion"
);
let run_dir = PathBuf::from(
items[0]["run_dir"]
.as_str()
.expect("inspect should include run_dir"),
);
// 4. logs <run_id> — non-empty, first line is valid JSONL with event field
let logs_out = cmd(&["logs", &run_id]).success();
let logs_stdout = String::from_utf8(logs_out.get_output().stdout.clone()).unwrap();
assert!(!logs_stdout.is_empty(), "logs should not be empty");
let first_line = logs_stdout.lines().next().unwrap();
let log_entry: Value =
serde_json::from_str(first_line).expect("first log line should be valid JSON");
assert!(
log_entry["event"].is_string(),
"first log line should have an event field"
);
// 5. asset list — no assets yet, should succeed with empty message
let asset_list_out = cmd(&["asset", "list", &run_id]).success();
let asset_list_stdout = String::from_utf8(asset_list_out.get_output().stdout.clone()).unwrap();
assert!(
asset_list_stdout.contains("No assets found"),
"asset list should report no assets: {asset_list_stdout}"
);
// 6. Seed a synthetic asset so asset list/cp have something to work with.
let asset_dir = RuntimeState::new(&run_dir).asset_stage_dir("step1", 1);
std::fs::create_dir_all(&asset_dir).unwrap();
std::fs::write(asset_dir.join("output.txt"), "asset-content-42").unwrap();
std::fs::write(
asset_dir.join("manifest.json"),
r#"{"files_copied":1,"total_bytes":16,"files_skipped":0,"download_errors":0,"hash_errors":0,"captured_assets":[{"path":"output.txt","mime":"text/plain","content_md5":"f02439728c0a94b7bfc465acb1201a1f","content_sha256":"0af9dea3e1c2dec968531c18c9331659b8268e8c9cf24b01cda7b8ce51d2ff00","bytes":16}]}"#,
)
.unwrap();
let retry_two_dir = RuntimeState::new(&run_dir).asset_stage_dir("step1", 2);
std::fs::create_dir_all(&retry_two_dir).unwrap();
std::fs::write(retry_two_dir.join("output.txt"), "asset-content-84").unwrap();
std::fs::write(
retry_two_dir.join("manifest.json"),
r#"{"files_copied":1,"total_bytes":16,"files_skipped":0,"download_errors":0,"hash_errors":0,"captured_assets":[{"path":"output.txt","mime":"text/plain","content_md5":"5b4e23e40a1630f9caa15a4cb6cfb79b","content_sha256":"1f71e0df61fc3b4e1ee3aba7ceac9ae391af22595b5b5630d97d34cf33d4d540","bytes":16}]}"#,
)
.unwrap();
// 7. asset list — now shows the seeded assets
let asset_list_out2 = cmd(&["asset", "list", &run_id, "--json"]).success();
let asset_list_stdout2 =
String::from_utf8(asset_list_out2.get_output().stdout.clone()).unwrap();
let assets: Vec<Value> = serde_json::from_str(&asset_list_stdout2)
.expect("asset list --json should produce a JSON array");
assert_eq!(
assets.len(),
2,
"should have two assets: {asset_list_stdout2}"
);
assert_eq!(assets[0]["relative_path"].as_str(), Some("output.txt"));
assert_eq!(assets[0]["node_slug"].as_str(), Some("step1"));
let retry_filtered_out = cmd(&["asset", "list", &run_id, "--retry", "1", "--json"]).success();
let retry_filtered_stdout =
String::from_utf8(retry_filtered_out.get_output().stdout.clone()).unwrap();
let retry_filtered_assets: Vec<Value> = serde_json::from_str(&retry_filtered_stdout)
.expect("asset list --json should produce a JSON array");
assert_eq!(retry_filtered_assets.len(), 1);
assert_eq!(retry_filtered_assets[0]["retry"].as_u64(), Some(1));
// 8. asset cp — ambiguous without --retry when multiple retries captured the same path
let asset_dest = context.temp_dir.join("asset_copy");
cmd(&[
"asset",
"cp",
&format!("{run_id}:output.txt"),
asset_dest.to_str().unwrap(),
])
.failure();
cmd(&[
"asset",
"cp",
&format!("{run_id}:output.txt"),
asset_dest.to_str().unwrap(),
"--retry",
"1",
])
.success();
let copied = std::fs::read_to_string(asset_dest.join("output.txt")).unwrap();
assert_eq!(
copied, "asset-content-42",
"asset cp should copy file content"
);
// 9. cp — download a file from the local sandbox workdir
let sandbox_json: Value = read_json(&run_dir.join("sandbox.json"));
let workdir = sandbox_json["working_directory"]
.as_str()
.expect("sandbox.json should have working_directory");
// Plant a file in the sandbox workdir so we can download it
std::fs::write(
PathBuf::from(workdir).join("cp_test.txt"),
"downloaded-via-cp",
)
.unwrap();
let cp_dest = context.temp_dir.join("cp_download.txt");
cmd(&[
"cp",
&format!("{run_id}:cp_test.txt"),
cp_dest.to_str().unwrap(),
])
.success();
let cp_content = std::fs::read_to_string(&cp_dest).unwrap();
assert_eq!(
cp_content, "downloaded-via-cp",
"cp should download file from sandbox"
);
// 10. system df — mentions "Runs"
let df_out = cmd(&["system", "df"]).success();
let df_stdout = String::from_utf8(df_out.get_output().stdout.clone()).unwrap();
assert!(
df_stdout.contains("Runs"),
"system df should mention Runs: {df_stdout}"
);
// 11. rm <run_id> — remove the run
cmd(&["rm", &run_id]).success();
// 12. ps -a --json — should be empty
let ps_out2 = cmd(&["ps", "-a", "--json"]).success();
let ps_stdout2 = String::from_utf8(ps_out2.get_output().stdout.clone()).unwrap();
let runs2: Vec<Value> =
serde_json::from_str(&ps_stdout2).expect("ps --json should produce a JSON array");
assert!(
runs2.is_empty(),
"runs should be empty after rm: {ps_stdout2}"
);
}

103
lib/crates/fabro-cli/tests/it/scenario/mod.rs Normal file
View file

@ -0,0 +1,103 @@
mod exec;
mod lifecycle;
mod workflows;
use std::path::{Path, PathBuf};
use std::time::Duration;
use serde_json::Value;
// ---------------------------------------------------------------------------
// Shared helpers
// ---------------------------------------------------------------------------
pub(super) fn fixture(name: &str) -> PathBuf {
Path::new(env!("CARGO_MANIFEST_DIR"))
.join("../../../test/scenario")
.join(name)
}
pub(super) fn read_json(path: &Path) -> Value {
let content = std::fs::read_to_string(path)
.unwrap_or_else(|e| panic!("failed to read {}: {e}", path.display()));
serde_json::from_str(&content)
.unwrap_or_else(|e| panic!("failed to parse {}: {e}", path.display()))
}
pub(super) fn read_checkpoint(run_dir: &Path) -> Value {
read_json(&run_dir.join("checkpoint.json"))
}
pub(super) fn read_conclusion(run_dir: &Path) -> Value {
read_json(&run_dir.join("conclusion.json"))
}
/// Find the single run directory under `storage_dir/runs/`.
pub(super) fn find_run_dir(storage_dir: &Path) -> PathBuf {
let runs_base = storage_dir.join("runs");
let entries: Vec<_> = std::fs::read_dir(&runs_base)
.unwrap_or_else(|e| panic!("failed to read {}: {e}", runs_base.display()))
.filter_map(|e| e.ok())
.filter(|e| e.path().is_dir())
.collect();
assert_eq!(
entries.len(),
1,
"expected exactly one run directory under {}",
runs_base.display()
);
entries[0].path()
}
pub(super) fn completed_nodes(run_dir: &Path) -> Vec<String> {
let cp = read_checkpoint(run_dir);
cp["completed_nodes"]
.as_array()
.expect("completed_nodes should be an array")
.iter()
.map(|v| v.as_str().unwrap().to_string())
.collect()
}
pub(super) fn has_event(run_dir: &Path, event_name: &str) -> bool {
let path = run_dir.join("progress.jsonl");
let content = std::fs::read_to_string(&path)
.unwrap_or_else(|e| panic!("failed to read progress.jsonl: {e}"));
content.lines().any(|line| {
if let Ok(v) = serde_json::from_str::<Value>(line) {
v["event"].as_str() == Some(event_name)
} else {
false
}
})
}
// ---------------------------------------------------------------------------
// Macro: generate local_* and daytona_* variants for each scenario
// ---------------------------------------------------------------------------
macro_rules! scenario_tests {
($name:ident) => {
paste::paste! {
#[test]
#[ignore = "scenario: requires local sandbox"]
fn [<local_ $name>]() {
[<scenario_ $name>]("local");
}
#[test]
#[ignore = "scenario: requires DAYTONA_API_KEY"]
fn [<daytona_ $name>]() {
[<scenario_ $name>]("daytona");
}
}
};
}
pub(super) use scenario_tests;
pub(super) fn timeout_for(sandbox: &str) -> Duration {
match sandbox {
"daytona" => Duration::from_secs(600),
_ => Duration::from_secs(180),
}
}

288
lib/crates/fabro-cli/tests/it/scenario/workflows.rs Normal file
View file

@ -0,0 +1,288 @@
use fabro_test::test_context;
use super::{
completed_nodes, find_run_dir, fixture, has_event, read_conclusion, read_json, scenario_tests,
timeout_for,
};
// 1. command_pipeline — two command nodes in sequence, no LLM
scenario_tests!(command_pipeline);
fn scenario_command_pipeline(sandbox: &str) {
dotenvy::dotenv().ok();
let context = test_context!();
context
.validate()
.arg(fixture("command_pipeline.fabro"))
.assert()
.success();
context
.run_cmd()
.args(["--auto-approve", "--no-retro", "--sandbox", sandbox])
.arg(fixture("command_pipeline.fabro"))
.timeout(timeout_for(sandbox))
.assert()
.success();
let run_dir = find_run_dir(&context.storage_dir);
let conclusion = read_conclusion(&run_dir);
assert_eq!(
conclusion["status"].as_str(),
Some("success"),
"conclusion status should be success"
);
let nodes = completed_nodes(&run_dir);
assert!(
nodes.contains(&"step1".to_string()),
"step1 should be completed"
);
assert!(
nodes.contains(&"step2".to_string()),
"step2 should be completed"
);
// Verify step1 stdout
let stdout1 = std::fs::read_to_string(run_dir.join("nodes/step1/stdout.log"))
.expect("step1 stdout.log should exist");
assert!(
stdout1.contains("hello-from-step1"),
"step1 stdout should contain hello-from-step1, got: {stdout1}"
);
}
// 2. conditional_branching — command + diamond gate, success path taken
scenario_tests!(conditional_branching);
fn scenario_conditional_branching(sandbox: &str) {
dotenvy::dotenv().ok();
let context = test_context!();
context
.run_cmd()
.args(["--auto-approve", "--no-retro", "--sandbox", sandbox])
.arg(fixture("conditional_branching.fabro"))
.timeout(timeout_for(sandbox))
.assert()
.success();
let run_dir = find_run_dir(&context.storage_dir);
let conclusion = read_conclusion(&run_dir);
assert_eq!(conclusion["status"].as_str(), Some("success"));
let nodes = completed_nodes(&run_dir);
assert!(
nodes.contains(&"passed".to_string()),
"passed node should be in completed_nodes: {nodes:?}"
);
assert!(
!nodes.contains(&"failed".to_string()),
"failed node should NOT be in completed_nodes: {nodes:?}"
);
}
// 3. agent_linear — single agent node with LLM
scenario_tests!(agent_linear);
fn scenario_agent_linear(sandbox: &str) {
dotenvy::dotenv().ok();
let context = test_context!();
context
.run_cmd()
.args([
"--auto-approve",
"--no-retro",
"--sandbox",
sandbox,
"--model",
"claude-haiku-4-5",
])
.arg(fixture("agent_linear.fabro"))
.timeout(timeout_for(sandbox))
.assert()
.success();
let run_dir = find_run_dir(&context.storage_dir);
let conclusion = read_conclusion(&run_dir);
assert_eq!(conclusion["status"].as_str(), Some("success"));
let nodes = completed_nodes(&run_dir);
assert!(
nodes.contains(&"work".to_string()),
"work should be completed"
);
// Agent node should produce prompt.md and response.md
let prompt_path = run_dir.join("nodes/work/prompt.md");
assert!(prompt_path.exists(), "nodes/work/prompt.md should exist");
let response_path = run_dir.join("nodes/work/response.md");
assert!(
response_path.exists(),
"nodes/work/response.md should exist"
);
let response = std::fs::read_to_string(&response_path).unwrap();
assert!(!response.is_empty(), "response.md should not be empty");
}
// 4. human_gate — human gate with --auto-approve selects first edge
scenario_tests!(human_gate);
fn scenario_human_gate(sandbox: &str) {
dotenvy::dotenv().ok();
let context = test_context!();
context
.run_cmd()
.args([
"--auto-approve",
"--no-retro",
"--sandbox",
sandbox,
"--model",
"claude-haiku-4-5",
])
.arg(fixture("human_gate.fabro"))
.timeout(timeout_for(sandbox))
.assert()
.success();
let run_dir = find_run_dir(&context.storage_dir);
let conclusion = read_conclusion(&run_dir);
assert_eq!(conclusion["status"].as_str(), Some("success"));
let nodes = completed_nodes(&run_dir);
assert!(
nodes.contains(&"ship".to_string()),
"ship should be in completed_nodes (auto-approve picks first edge): {nodes:?}"
);
assert!(
!nodes.contains(&"revise".to_string()),
"revise should NOT be in completed_nodes: {nodes:?}"
);
}
// 5. command_agent_mixed — command writes file, agent reads it, command verifies
scenario_tests!(command_agent_mixed);
fn scenario_command_agent_mixed(sandbox: &str) {
dotenvy::dotenv().ok();
let context = test_context!();
context
.run_cmd()
.args([
"--auto-approve",
"--no-retro",
"--sandbox",
sandbox,
"--model",
"claude-haiku-4-5",
])
.arg(fixture("command_agent_mixed.fabro"))
.timeout(timeout_for(sandbox))
.assert()
.success();
let run_dir = find_run_dir(&context.storage_dir);
let conclusion = read_conclusion(&run_dir);
assert_eq!(conclusion["status"].as_str(), Some("success"));
let nodes = completed_nodes(&run_dir);
assert!(
nodes.contains(&"setup".to_string()),
"setup should be completed"
);
assert!(
nodes.contains(&"work".to_string()),
"work should be completed"
);
assert!(
nodes.contains(&"verify".to_string()),
"verify should be completed"
);
// Verify command node saw the flag
let stdout = std::fs::read_to_string(run_dir.join("nodes/verify/stdout.log"))
.expect("verify stdout.log should exist");
assert!(
stdout.contains("SCENARIO_FLAG_42"),
"verify stdout should contain SCENARIO_FLAG_42, got: {stdout}"
);
}
// 6. full_stack — command + agent + human gate + goal_gate, kitchen sink
scenario_tests!(full_stack);
fn scenario_full_stack(sandbox: &str) {
dotenvy::dotenv().ok();
let context = test_context!();
context
.run_cmd()
.args([
"--auto-approve",
"--no-retro",
"--sandbox",
sandbox,
"--model",
"claude-haiku-4-5",
])
.arg(fixture("full_stack.fabro"))
.timeout(timeout_for(sandbox))
.assert()
.success();
let run_dir = find_run_dir(&context.storage_dir);
let conclusion = read_conclusion(&run_dir);
assert_eq!(
conclusion["status"].as_str(),
Some("success"),
"conclusion: {conclusion}"
);
assert!(
conclusion["duration_ms"].as_u64().unwrap_or(0) > 0,
"duration_ms should be > 0"
);
// RunRecord should have key fields
let run_record = read_json(&run_dir.join("run.json"));
assert!(
run_record["run_id"].as_str().is_some(),
"run record should have run_id"
);
assert!(
run_record["graph"]["name"].as_str().is_some(),
"run record should have graph.name"
);
// Progress events
assert!(
has_event(&run_dir, "WorkflowRunStarted"),
"progress should contain WorkflowRunStarted"
);
assert!(
has_event(&run_dir, "WorkflowRunCompleted"),
"progress should contain WorkflowRunCompleted"
);
// All expected nodes completed
let nodes = completed_nodes(&run_dir);
for expected in &["setup", "plan", "approve", "impl", "verify"] {
assert!(
nodes.contains(&expected.to_string()),
"{expected} should be in completed_nodes: {nodes:?}"
);
}
// Verify node stdout should contain PASS
let stdout = std::fs::read_to_string(run_dir.join("nodes/verify/stdout.log"))
.expect("verify stdout.log should exist");
assert!(
stdout.contains("PASS"),
"verify stdout should contain PASS, got: {stdout}"
);
}