Expand scenario tests to cover more CLI subcommands

Add coverage for validate, model list, workflow list, doctor, exec,
ps, inspect, logs, rm, system df, asset list, asset cp, and cp.
Uses HOME isolation for run lifecycle tests and synthetic assets
for asset/cp testing.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Bryan Helmkamp 2026-03-18 16:05:13 -04:00
parent e12157cb57
commit f7e07cb827
No known key found for this signature in database
9 changed files with 762 additions and 0 deletions

1
Cargo.lock generated
View file

@ -1378,6 +1378,7 @@ dependencies = [
"jsonwebtoken",
"libc",
"open",
"paste",
"predicates",
"rand 0.8.5",
"regex",

View file

@ -83,7 +83,9 @@ chrono = { workspace = true }
[dev-dependencies]
assert_cmd = "2"
insta = { workspace = true }
paste = "1"
predicates = "3"
serde_json.workspace = true
tempfile = "3"
httpmock = "0.8"
trycmd = "0.15"

View file

@ -0,0 +1,669 @@
use std::path::{Path, PathBuf};
use std::time::Duration;
use assert_cmd::Command;
use predicates;
use serde_json::Value;
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
#[allow(deprecated)]
fn fabro() -> Command {
let mut cmd = Command::cargo_bin("fabro").unwrap();
cmd.env("NO_COLOR", "1");
cmd
}
fn fixture(name: &str) -> PathBuf {
Path::new(env!("CARGO_MANIFEST_DIR"))
.join("../../../test/scenario")
.join(name)
}
fn fixture_root(name: &str) -> PathBuf {
Path::new(env!("CARGO_MANIFEST_DIR"))
.join("../../../test")
.join(name)
}
fn read_json(path: &Path) -> Value {
let content = std::fs::read_to_string(path)
.unwrap_or_else(|e| panic!("failed to read {}: {e}", path.display()));
serde_json::from_str(&content)
.unwrap_or_else(|e| panic!("failed to parse {}: {e}", path.display()))
}
fn read_checkpoint(run_dir: &Path) -> Value {
read_json(&run_dir.join("checkpoint.json"))
}
fn read_conclusion(run_dir: &Path) -> Value {
read_json(&run_dir.join("conclusion.json"))
}
fn completed_nodes(run_dir: &Path) -> Vec<String> {
let cp = read_checkpoint(run_dir);
cp["completed_nodes"]
.as_array()
.expect("completed_nodes should be an array")
.iter()
.map(|v| v.as_str().unwrap().to_string())
.collect()
}
fn has_event(run_dir: &Path, event_name: &str) -> bool {
let path = run_dir.join("progress.jsonl");
let content = std::fs::read_to_string(&path)
.unwrap_or_else(|e| panic!("failed to read progress.jsonl: {e}"));
content.lines().any(|line| {
if let Ok(v) = serde_json::from_str::<Value>(line) {
v["event"].as_str() == Some(event_name)
} else {
false
}
})
}
// ---------------------------------------------------------------------------
// Macro: generate local_* and daytona_* variants for each scenario
// ---------------------------------------------------------------------------
macro_rules! scenario_tests {
($name:ident) => {
paste::paste! {
#[test]
#[ignore = "scenario: requires local sandbox"]
fn [<local_ $name>]() {
[<scenario_ $name>]("local");
}
#[test]
#[ignore = "scenario: requires DAYTONA_API_KEY"]
fn [<daytona_ $name>]() {
[<scenario_ $name>]("daytona");
}
}
};
}
fn timeout_for(sandbox: &str) -> Duration {
match sandbox {
"daytona" => Duration::from_secs(600),
_ => Duration::from_secs(180),
}
}
// ---------------------------------------------------------------------------
// Scenarios
// ---------------------------------------------------------------------------
// 1. command_pipeline — two command nodes in sequence, no LLM
scenario_tests!(command_pipeline);
fn scenario_command_pipeline(sandbox: &str) {
dotenvy::dotenv().ok();
let tmp = tempfile::tempdir().unwrap();
let run_dir = tmp.path().join("run");
// Validate the workflow before running it
fabro()
.args([
"validate",
fixture("command_pipeline.fabro").to_str().unwrap(),
])
.assert()
.success();
fabro()
.args([
"run",
"--auto-approve",
"--no-retro",
"--sandbox",
sandbox,
"--run-dir",
run_dir.to_str().unwrap(),
fixture("command_pipeline.fabro").to_str().unwrap(),
])
.timeout(timeout_for(sandbox))
.assert()
.success();
let conclusion = read_conclusion(&run_dir);
assert_eq!(
conclusion["status"].as_str(),
Some("success"),
"conclusion status should be success"
);
let nodes = completed_nodes(&run_dir);
assert!(
nodes.contains(&"step1".to_string()),
"step1 should be completed"
);
assert!(
nodes.contains(&"step2".to_string()),
"step2 should be completed"
);
// Verify step1 stdout
let stdout1 = std::fs::read_to_string(run_dir.join("nodes/step1/stdout.log"))
.expect("step1 stdout.log should exist");
assert!(
stdout1.contains("hello-from-step1"),
"step1 stdout should contain hello-from-step1, got: {stdout1}"
);
}
// 2. conditional_branching — command + diamond gate, success path taken
scenario_tests!(conditional_branching);
fn scenario_conditional_branching(sandbox: &str) {
dotenvy::dotenv().ok();
let tmp = tempfile::tempdir().unwrap();
let run_dir = tmp.path().join("run");
fabro()
.args([
"run",
"--auto-approve",
"--no-retro",
"--sandbox",
sandbox,
"--run-dir",
run_dir.to_str().unwrap(),
fixture("conditional_branching.fabro").to_str().unwrap(),
])
.timeout(timeout_for(sandbox))
.assert()
.success();
let conclusion = read_conclusion(&run_dir);
assert_eq!(conclusion["status"].as_str(), Some("success"));
let nodes = completed_nodes(&run_dir);
assert!(
nodes.contains(&"passed".to_string()),
"passed node should be in completed_nodes: {nodes:?}"
);
assert!(
!nodes.contains(&"failed".to_string()),
"failed node should NOT be in completed_nodes: {nodes:?}"
);
}
// 3. agent_linear — single agent node with LLM
scenario_tests!(agent_linear);
fn scenario_agent_linear(sandbox: &str) {
dotenvy::dotenv().ok();
let tmp = tempfile::tempdir().unwrap();
let run_dir = tmp.path().join("run");
fabro()
.args([
"run",
"--auto-approve",
"--no-retro",
"--sandbox",
sandbox,
"--model",
"claude-haiku-4-5",
"--run-dir",
run_dir.to_str().unwrap(),
fixture("agent_linear.fabro").to_str().unwrap(),
])
.timeout(timeout_for(sandbox))
.assert()
.success();
let conclusion = read_conclusion(&run_dir);
assert_eq!(conclusion["status"].as_str(), Some("success"));
let nodes = completed_nodes(&run_dir);
assert!(
nodes.contains(&"work".to_string()),
"work should be completed"
);
// Agent node should produce prompt.md and response.md
let prompt_path = run_dir.join("nodes/work/prompt.md");
assert!(prompt_path.exists(), "nodes/work/prompt.md should exist");
let response_path = run_dir.join("nodes/work/response.md");
assert!(
response_path.exists(),
"nodes/work/response.md should exist"
);
let response = std::fs::read_to_string(&response_path).unwrap();
assert!(!response.is_empty(), "response.md should not be empty");
}
// 4. human_gate — human gate with --auto-approve selects first edge
scenario_tests!(human_gate);
fn scenario_human_gate(sandbox: &str) {
dotenvy::dotenv().ok();
let tmp = tempfile::tempdir().unwrap();
let run_dir = tmp.path().join("run");
fabro()
.args([
"run",
"--auto-approve",
"--no-retro",
"--sandbox",
sandbox,
"--model",
"claude-haiku-4-5",
"--run-dir",
run_dir.to_str().unwrap(),
fixture("human_gate.fabro").to_str().unwrap(),
])
.timeout(timeout_for(sandbox))
.assert()
.success();
let conclusion = read_conclusion(&run_dir);
assert_eq!(conclusion["status"].as_str(), Some("success"));
let nodes = completed_nodes(&run_dir);
assert!(
nodes.contains(&"ship".to_string()),
"ship should be in completed_nodes (auto-approve picks first edge): {nodes:?}"
);
assert!(
!nodes.contains(&"revise".to_string()),
"revise should NOT be in completed_nodes: {nodes:?}"
);
}
// 5. command_agent_mixed — command writes file, agent reads it, command verifies
scenario_tests!(command_agent_mixed);
fn scenario_command_agent_mixed(sandbox: &str) {
dotenvy::dotenv().ok();
let tmp = tempfile::tempdir().unwrap();
let run_dir = tmp.path().join("run");
fabro()
.args([
"run",
"--auto-approve",
"--no-retro",
"--sandbox",
sandbox,
"--model",
"claude-haiku-4-5",
"--run-dir",
run_dir.to_str().unwrap(),
fixture("command_agent_mixed.fabro").to_str().unwrap(),
])
.timeout(timeout_for(sandbox))
.assert()
.success();
let conclusion = read_conclusion(&run_dir);
assert_eq!(conclusion["status"].as_str(), Some("success"));
let nodes = completed_nodes(&run_dir);
assert!(
nodes.contains(&"setup".to_string()),
"setup should be completed"
);
assert!(
nodes.contains(&"work".to_string()),
"work should be completed"
);
assert!(
nodes.contains(&"verify".to_string()),
"verify should be completed"
);
// Verify command node saw the flag
let stdout = std::fs::read_to_string(run_dir.join("nodes/verify/stdout.log"))
.expect("verify stdout.log should exist");
assert!(
stdout.contains("SCENARIO_FLAG_42"),
"verify stdout should contain SCENARIO_FLAG_42, got: {stdout}"
);
}
// 6. full_stack — command + agent + human gate + goal_gate, kitchen sink
scenario_tests!(full_stack);
fn scenario_full_stack(sandbox: &str) {
dotenvy::dotenv().ok();
let tmp = tempfile::tempdir().unwrap();
let run_dir = tmp.path().join("run");
fabro()
.args([
"run",
"--auto-approve",
"--no-retro",
"--sandbox",
sandbox,
"--model",
"claude-haiku-4-5",
"--run-dir",
run_dir.to_str().unwrap(),
fixture("full_stack.fabro").to_str().unwrap(),
])
.timeout(timeout_for(sandbox))
.assert()
.success();
let conclusion = read_conclusion(&run_dir);
assert_eq!(
conclusion["status"].as_str(),
Some("success"),
"conclusion: {conclusion}"
);
assert!(
conclusion["duration_ms"].as_u64().unwrap_or(0) > 0,
"duration_ms should be > 0"
);
// Manifest should have key fields
let manifest = read_json(&run_dir.join("manifest.json"));
assert!(
manifest["run_id"].as_str().is_some(),
"manifest should have run_id"
);
assert!(
manifest["goal"].as_str().is_some(),
"manifest should have goal"
);
assert!(
manifest["workflow_name"].as_str().is_some(),
"manifest should have workflow_name"
);
// Progress events
assert!(
has_event(&run_dir, "WorkflowRunStarted"),
"progress should contain WorkflowRunStarted"
);
assert!(
has_event(&run_dir, "WorkflowRunCompleted"),
"progress should contain WorkflowRunCompleted"
);
// All expected nodes completed
let nodes = completed_nodes(&run_dir);
for expected in &["setup", "plan", "approve", "impl", "verify"] {
assert!(
nodes.contains(&expected.to_string()),
"{expected} should be in completed_nodes: {nodes:?}"
);
}
// Verify node stdout should contain PASS
let stdout = std::fs::read_to_string(run_dir.join("nodes/verify/stdout.log"))
.expect("verify stdout.log should exist");
assert!(
stdout.contains("PASS"),
"verify stdout should contain PASS, got: {stdout}"
);
}
// ---------------------------------------------------------------------------
// Standalone tests (no sandbox parametrization)
// ---------------------------------------------------------------------------
#[test]
fn test_validate_rejects_invalid() {
fabro()
.args(["validate", fixture_root("invalid.fabro").to_str().unwrap()])
.assert()
.failure();
}
#[test]
fn test_model_list() {
fabro()
.args(["model", "list"])
.assert()
.success()
.stdout(predicates::str::contains("claude-haiku"));
}
#[test]
fn test_workflow_list() {
let tmp = tempfile::tempdir().unwrap();
// Minimal project structure: fabro.toml + a workflow
std::fs::write(tmp.path().join("fabro.toml"), "version = 1\n").unwrap();
let wf_dir = tmp.path().join("workflows/my_test_wf");
std::fs::create_dir_all(&wf_dir).unwrap();
std::fs::write(
wf_dir.join("workflow.toml"),
"version = 1\ngoal = \"A test workflow\"\n",
)
.unwrap();
fabro()
.args(["workflow", "list"])
.current_dir(tmp.path())
.assert()
.success()
// workflow list prints to stderr
.stderr(predicates::str::contains("my_test_wf"));
}
#[test]
#[ignore = "scenario: requires ANTHROPIC_API_KEY"]
fn test_doctor() {
dotenvy::dotenv().ok();
fabro().args(["doctor"]).assert().success();
}
// ---------------------------------------------------------------------------
// Run lifecycle: ps / inspect / logs / rm / system df
// ---------------------------------------------------------------------------
#[test]
#[ignore = "scenario: requires local sandbox"]
fn local_run_lifecycle() {
dotenvy::dotenv().ok();
let tmp = tempfile::tempdir().unwrap();
let fabro_home = |args: &[&str]| -> assert_cmd::assert::Assert {
fabro()
.env("HOME", tmp.path())
.args(args)
.timeout(timeout_for("local"))
.assert()
};
// 1. Run a workflow — run lands under <tmp>/.fabro/runs/
fabro_home(&[
"run",
"--auto-approve",
"--no-retro",
"--no-upgrade-check",
"--sandbox",
"local",
fixture("command_pipeline.fabro").to_str().unwrap(),
])
.success();
// 2. ps -a --json — should list exactly one run
let ps_out = fabro_home(&["ps", "-a", "--json"]).success();
let ps_stdout = String::from_utf8(ps_out.get_output().stdout.clone()).unwrap();
let runs: Vec<Value> =
serde_json::from_str(&ps_stdout).expect("ps --json should produce a JSON array");
assert_eq!(runs.len(), 1, "should have exactly one run: {ps_stdout}");
let run_id = runs[0]["run_id"]
.as_str()
.expect("run should have run_id")
.to_string();
assert_eq!(
runs[0]["workflow_name"].as_str(),
Some("CommandPipeline"),
"workflow_name should be CommandPipeline"
);
// 3. inspect <run_id> — JSON array with manifest and conclusion
let inspect_out = fabro_home(&["inspect", &run_id]).success();
let inspect_stdout = String::from_utf8(inspect_out.get_output().stdout.clone()).unwrap();
let items: Vec<Value> =
serde_json::from_str(&inspect_stdout).expect("inspect should produce a JSON array");
assert!(!items.is_empty(), "inspect should return at least one item");
assert!(
items[0]["manifest"].is_object(),
"inspect should include manifest"
);
assert!(
items[0]["conclusion"].is_object(),
"inspect should include conclusion"
);
let run_dir = PathBuf::from(
items[0]["run_dir"]
.as_str()
.expect("inspect should include run_dir"),
);
// 4. logs <run_id> — non-empty, first line is valid JSONL with event field
let logs_out = fabro_home(&["logs", &run_id]).success();
let logs_stdout = String::from_utf8(logs_out.get_output().stdout.clone()).unwrap();
assert!(!logs_stdout.is_empty(), "logs should not be empty");
let first_line = logs_stdout.lines().next().unwrap();
let log_entry: Value =
serde_json::from_str(first_line).expect("first log line should be valid JSON");
assert!(
log_entry["event"].is_string(),
"first log line should have an event field"
);
// 5. asset list — no assets yet, should succeed with empty message
let asset_list_out = fabro_home(&["asset", "list", &run_id]).success();
let asset_list_stdout = String::from_utf8(asset_list_out.get_output().stdout.clone()).unwrap();
assert!(
asset_list_stdout.contains("No assets found"),
"asset list should report no assets: {asset_list_stdout}"
);
// 6. Seed a synthetic asset so asset list/cp have something to work with.
let asset_dir = run_dir.join("artifacts/assets/step1/retry_0");
std::fs::create_dir_all(&asset_dir).unwrap();
std::fs::write(asset_dir.join("output.txt"), "asset-content-42").unwrap();
std::fs::write(
asset_dir.join("manifest.json"),
r#"{"files_copied":1,"total_bytes":16,"files_skipped":0,"download_errors":0,"copied_paths":["output.txt"]}"#,
)
.unwrap();
// 7. asset list — now shows the seeded asset
let asset_list_out2 = fabro_home(&["asset", "list", &run_id, "--json"]).success();
let asset_list_stdout2 =
String::from_utf8(asset_list_out2.get_output().stdout.clone()).unwrap();
let assets: Vec<Value> = serde_json::from_str(&asset_list_stdout2)
.expect("asset list --json should produce a JSON array");
assert_eq!(
assets.len(),
1,
"should have one asset: {asset_list_stdout2}"
);
assert_eq!(assets[0]["relative_path"].as_str(), Some("output.txt"));
assert_eq!(assets[0]["node_slug"].as_str(), Some("step1"));
// 8. asset cp — copy the asset out
let asset_dest = tmp.path().join("asset_copy");
fabro_home(&[
"asset",
"cp",
&format!("{run_id}:output.txt"),
asset_dest.to_str().unwrap(),
])
.success();
let copied = std::fs::read_to_string(asset_dest.join("output.txt")).unwrap();
assert_eq!(
copied, "asset-content-42",
"asset cp should copy file content"
);
// 9. cp — download a file from the local sandbox workdir
let sandbox_json: Value = read_json(&run_dir.join("sandbox.json"));
let workdir = sandbox_json["working_directory"]
.as_str()
.expect("sandbox.json should have working_directory");
// Plant a file in the sandbox workdir so we can download it
std::fs::write(
PathBuf::from(workdir).join("cp_test.txt"),
"downloaded-via-cp",
)
.unwrap();
let cp_dest = tmp.path().join("cp_download.txt");
fabro_home(&[
"cp",
&format!("{run_id}:cp_test.txt"),
cp_dest.to_str().unwrap(),
])
.success();
let cp_content = std::fs::read_to_string(&cp_dest).unwrap();
assert_eq!(
cp_content, "downloaded-via-cp",
"cp should download file from sandbox"
);
// 10. system df — mentions "Runs"
let df_out = fabro_home(&["system", "df"]).success();
let df_stdout = String::from_utf8(df_out.get_output().stdout.clone()).unwrap();
assert!(
df_stdout.contains("Runs"),
"system df should mention Runs: {df_stdout}"
);
// 11. rm <run_id> — remove the run
fabro_home(&["rm", &run_id]).success();
// 12. ps -a --json — should be empty
let ps_out2 = fabro_home(&["ps", "-a", "--json"]).success();
let ps_stdout2 = String::from_utf8(ps_out2.get_output().stdout.clone()).unwrap();
let runs2: Vec<Value> =
serde_json::from_str(&ps_stdout2).expect("ps --json should produce a JSON array");
assert!(
runs2.is_empty(),
"runs should be empty after rm: {ps_stdout2}"
);
}
// ---------------------------------------------------------------------------
// exec — exercises `fabro exec`
// ---------------------------------------------------------------------------
#[test]
#[ignore = "scenario: requires ANTHROPIC_API_KEY"]
fn test_exec_creates_file() {
dotenvy::dotenv().ok();
let tmp = tempfile::tempdir().unwrap();
fabro()
.args([
"exec",
"--auto-approve",
"--permissions",
"full",
"--provider",
"anthropic",
"--model",
"claude-haiku-4-5",
"Create a file called hello.txt containing exactly 'Hello from exec scenario'",
])
.current_dir(tmp.path())
.timeout(Duration::from_secs(120))
.assert()
.success();
let hello = tmp.path().join("hello.txt");
assert!(hello.exists(), "hello.txt should exist after exec");
let content = std::fs::read_to_string(&hello).unwrap();
assert!(
content.contains("Hello from exec scenario"),
"hello.txt should contain greeting, got: {content}"
);
}

View file

@ -0,0 +1,11 @@
digraph AgentLinear {
graph [goal="Write hello to a file using an agent"]
rankdir=LR
start [shape=Mdiamond, label="Start"]
exit [shape=Msquare, label="Exit"]
work [label="Work", prompt="Write the word 'hello' to a file called scenario_test.txt"]
start -> work -> exit
}

View file

@ -0,0 +1,13 @@
digraph CommandAgentMixed {
graph [goal="Verify sandbox filesystem is shared across command and agent nodes"]
rankdir=LR
start [shape=Mdiamond, label="Start"]
exit [shape=Msquare, label="Exit"]
setup [label="Setup", shape=parallelogram, script="echo SCENARIO_FLAG_42 > /tmp/scenario_flag.txt"]
work [label="Work", prompt="Read the file /tmp/scenario_flag.txt and tell me what it contains."]
verify [label="Verify", shape=parallelogram, script="cat /tmp/scenario_flag.txt"]
start -> setup -> work -> verify -> exit
}

View file

@ -0,0 +1,12 @@
digraph CommandPipeline {
graph [goal="Run two commands in sequence"]
rankdir=LR
start [shape=Mdiamond, label="Start"]
exit [shape=Msquare, label="Exit"]
step1 [label="Step 1", shape=parallelogram, script="echo hello-from-step1"]
step2 [label="Step 2", shape=parallelogram, script="echo hello-from-step2"]
start -> step1 -> step2 -> exit
}

View file

@ -0,0 +1,18 @@
digraph ConditionalBranching {
graph [goal="Test conditional edge selection after a command"]
rankdir=LR
start [shape=Mdiamond, label="Start"]
exit [shape=Msquare, label="Exit"]
cmd [label="Run Command", shape=parallelogram, script="echo pass"]
gate [label="Check Result", shape=diamond]
passed [label="Passed", shape=parallelogram, script="echo branch-passed"]
failed [label="Failed", shape=parallelogram, script="echo branch-failed"]
start -> cmd -> gate
gate -> passed [label="Pass", condition="outcome=success"]
gate -> failed [label="Fail"]
passed -> exit
failed -> exit
}

View file

@ -0,0 +1,18 @@
digraph FullStack {
graph [goal="Full-stack scenario: command, agent, human gate, and goal gate verification"]
rankdir=LR
start [shape=Mdiamond, label="Start"]
exit [shape=Msquare, label="Exit"]
setup [label="Setup", shape=parallelogram, script="mkdir -p /tmp/scenario_full && echo ready > /tmp/scenario_full/flag.txt"]
plan [label="Plan", prompt="Read /tmp/scenario_full/flag.txt. Then create a file /tmp/scenario_full/result.txt containing the word PASS.", shape=tab]
approve [label="Approve?", shape=hexagon]
impl [label="Implement", prompt="Create the file /tmp/scenario_full/result.txt containing exactly the word PASS on the first line."]
verify [label="Verify", shape=parallelogram, script="cat /tmp/scenario_full/result.txt", goal_gate=true, retry_target="impl"]
start -> setup -> plan -> approve
approve -> impl [label="[A] Approve"]
approve -> plan [label="[R] Revise"]
impl -> verify -> exit
}

View file

@ -0,0 +1,18 @@
digraph HumanGate {
graph [goal="Test human gate with auto-approve"]
rankdir=LR
start [shape=Mdiamond, label="Start"]
exit [shape=Msquare, label="Exit"]
draft [label="Draft", prompt="Write a short greeting message.", shape=tab]
approve [label="Approve?", shape=hexagon]
ship [label="Ship", shape=parallelogram, script="echo shipped"]
revise [label="Revise", prompt="Revise the greeting.", shape=tab]
start -> draft -> approve
approve -> ship [label="[A] Approve"]
approve -> revise [label="[R] Revise"]
ship -> exit
revise -> draft
}