fabro/lib/components/fabro-workflow/tests/it/integration.rs

13871 lines
476 KiB
Rust

#![allow(
clippy::absolute_paths,
clippy::get_unwrap,
clippy::ignore_without_reason,
clippy::items_after_statements,
clippy::large_futures,
clippy::manual_let_else,
clippy::print_stderr,
clippy::unnecessary_box_returns,
clippy::unnecessary_literal_bound,
clippy::unreadable_literal,
reason = "These workflow integration tests value explicit scenarios over pedantic style lints."
)]
#![expect(
clippy::disallowed_methods,
reason = "These end-to-end workflow integration tests use the real git CLI to verify checkpoint and branch behavior."
)]
use std::collections::VecDeque;
use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::Duration;
use fabro_config::RunScratch;
use fabro_graphviz::graph::{AttrValue, Edge, Graph, Node};
use fabro_graphviz::parser::parse;
use fabro_interview::{
Answer, AnswerValue, AutoApproveInterviewer, CallbackInterviewer, Interviewer,
QueueInterviewer, RecordingInterviewer,
};
use fabro_model::catalog::{LlmCatalogSettings, ProviderCatalogSettings};
use fabro_model::{Catalog, ProviderId};
use fabro_store::{ArtifactKey, ArtifactStore};
use fabro_types::{EventBody, RunEvent, RunId, StageId, WorkflowSettings, parse_blob_ref};
use fabro_validate::{Severity, validate, validate_or_raise};
use fabro_workflow::artifact;
use fabro_workflow::context::Context;
use fabro_workflow::error::{Error, FailureSignatureExt};
use fabro_workflow::event::{Emitter, Event};
use fabro_workflow::handler::agent::{
AgentHandler, CodergenBackend, CodergenResult, CodergenRunRequest,
};
use fabro_workflow::handler::command::CommandHandler;
use fabro_workflow::handler::conditional::ConditionalHandler;
use fabro_workflow::handler::exit::ExitHandler;
use fabro_workflow::handler::human::HumanHandler;
use fabro_workflow::handler::llm::AgentApiBackend;
use fabro_workflow::handler::manager_loop::SubWorkflowHandler;
use fabro_workflow::handler::start::StartHandler;
use fabro_workflow::handler::wait::WaitHandler;
use fabro_workflow::handler::{Handler, HandlerRegistry};
use fabro_workflow::model_fallback::ModelFallbackPolicy;
use fabro_workflow::outcome::{Outcome, OutcomeExt, StageOutcome};
use fabro_workflow::records::{Checkpoint, CheckpointExt};
use fabro_workflow::run_options::{GitCheckpointOptions, RunOptions};
use fabro_workflow::runtime_store::RunStoreHandle;
use fabro_workflow::test_support::{
WorkflowRunner, collect_events, run_graph_with_hooks, test_store_dir,
};
use fabro_workflow::transforms::stylesheet::{apply_stylesheet, parse_stylesheet};
use fabro_workflow::transforms::{StylesheetApplicationTransform, TemplateTransform, Transform};
use object_store::local::LocalFileSystem;
use tokio_util::sync::CancellationToken;
use ulid::Ulid;
fn default_catalog() -> Arc<Catalog> {
Arc::new(Catalog::from_builtin().expect("default catalog should build"))
}
fn catalog_with_provider_base_url(provider: &str, base_url: &str) -> Arc<Catalog> {
let mut settings = LlmCatalogSettings::default();
settings
.providers
.insert(provider.to_string(), ProviderCatalogSettings {
base_url: Some(base_url.to_string()),
..ProviderCatalogSettings::default()
});
Arc::new(
Catalog::from_builtin_with_overrides(&settings)
.expect("catalog with custom base_url should build"),
)
}
fn local_env() -> Arc<dyn fabro_agent::Sandbox> {
Arc::new(fabro_agent::LocalSandbox::new(
std::env::current_dir().unwrap_or_else(|_| std::path::PathBuf::from(".")),
))
}
fn test_run_id(label: &str) -> RunId {
let mut hasher = DefaultHasher::new();
label.hash(&mut hasher);
RunId::from(Ulid(u128::from(hasher.finish())))
}
fn load_checkpoint(path: &Path) -> Result<Checkpoint, Box<dyn std::error::Error>> {
let data = std::fs::read_to_string(path)?;
Ok(serde_json::from_str(&data)?)
}
#[expect(
clippy::disallowed_methods,
reason = "This helper spins up a dedicated current-thread runtime when called from inside an existing Tokio runtime."
)]
fn load_run_checkpoint(run_dir: &Path) -> Result<Checkpoint, Box<dyn std::error::Error>> {
let run_dir = run_dir.to_path_buf();
let uses_shared_store = run_dir
.parent()
.and_then(Path::file_name)
.is_some_and(|name| name == "scratch");
let store_dir = if uses_shared_store {
let runs_dir = run_dir.parent().ok_or("run dir should have parent")?;
let storage_dir = runs_dir.parent().ok_or("runs dir should have parent")?;
storage_dir.join("store")
} else {
test_store_dir(&run_dir)
};
let object_store = Arc::new(LocalFileSystem::new_with_prefix(&store_dir)?);
let store = Arc::new(fabro_store::test_support::test_database_at(
object_store,
"",
Duration::from_millis(1),
None,
&store_dir,
));
let state = if tokio::runtime::Handle::try_current().is_ok() {
std::thread::spawn(
move || -> Result<_, Box<dyn std::error::Error + Send + Sync>> {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()?;
let run_id = if uses_shared_store {
run_dir
.file_name()
.ok_or("run dir should have file name")?
.to_string_lossy()
.rsplit('-')
.next()
.ok_or("run dir should contain run id suffix")?
.parse()?
} else {
runtime
.block_on(store.run_summary_store().list_all(chrono::Utc::now()))?
.into_iter()
.next()
.ok_or("test store should contain one run")?
.id
};
let run = runtime.block_on(store.open_run_reader(&run_id))?;
let state = runtime.block_on(async {
for attempt in 0..20 {
let state = run.state().await?;
if state.current_checkpoint().is_some() || attempt == 19 {
return Ok::<_, fabro_store::Error>(state);
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
unreachable!()
})?;
Ok(state)
},
)
.join()
.map_err(|_| "checkpoint loader thread panicked")?
.map_err(|err| err.to_string())?
} else {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()?;
let run_id = if uses_shared_store {
run_dir
.file_name()
.ok_or("run dir should have file name")?
.to_string_lossy()
.rsplit('-')
.next()
.ok_or("run dir should contain run id suffix")?
.parse()?
} else {
runtime
.block_on(store.run_summary_store().list_all(chrono::Utc::now()))?
.into_iter()
.next()
.ok_or("test store should contain one run")?
.id
};
let run = runtime.block_on(store.open_run_reader(&run_id))?;
runtime.block_on(async {
for attempt in 0..20 {
let state = run.state().await?;
if state.current_checkpoint().is_some() || attempt == 19 {
return Ok::<_, fabro_store::Error>(state);
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
unreachable!()
})?
};
state
.current_checkpoint()
.cloned()
.ok_or_else(|| "checkpoint should exist in run store".into())
}
fn run_store_dir_and_mode(run_dir: &Path) -> Result<(PathBuf, bool), Box<dyn std::error::Error>> {
let uses_shared_store = run_dir
.parent()
.and_then(Path::file_name)
.is_some_and(|name| name == "scratch");
let store_dir = if uses_shared_store {
let runs_dir = run_dir.parent().ok_or("run dir should have parent")?;
let storage_dir = runs_dir.parent().ok_or("runs dir should have parent")?;
storage_dir.join("store")
} else {
test_store_dir(run_dir)
};
Ok((store_dir, uses_shared_store))
}
#[expect(
clippy::disallowed_methods,
reason = "This helper spins up a dedicated current-thread runtime when called from inside an existing Tokio runtime."
)]
fn resolve_checkpoint_text(
run_dir: &Path,
value: &serde_json::Value,
) -> Result<String, Box<dyn std::error::Error>> {
let Some(current) = value.as_str() else {
return Ok(value.to_string());
};
if parse_blob_ref(current).is_none() {
return Ok(current.to_string());
}
let current = current.to_string();
let run_dir = run_dir.to_path_buf();
let (store_dir, uses_shared_store) = run_store_dir_and_mode(&run_dir)?;
std::thread::spawn(
move || -> Result<_, Box<dyn std::error::Error + Send + Sync>> {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()?;
let object_store = Arc::new(LocalFileSystem::new_with_prefix(&store_dir)?);
let store = Arc::new(fabro_store::test_support::test_database_at(
object_store,
"",
Duration::from_millis(1),
None,
&store_dir,
));
let run_id = if uses_shared_store {
run_dir
.file_name()
.ok_or("run dir should have file name")?
.to_string_lossy()
.rsplit('-')
.next()
.ok_or("run dir should contain run id suffix")?
.parse()?
} else {
runtime
.block_on(store.run_summary_store().list_all(chrono::Utc::now()))?
.into_iter()
.next()
.ok_or("test store should contain one run")?
.id
};
let run = runtime.block_on(store.open_run_reader(&run_id))?;
let run_store = RunStoreHandle::from(run);
Ok(runtime.block_on(artifact::resolve_text_or_blob_ref_str(&current, &run_store))?)
},
)
.join()
.map_err(|_| "checkpoint text resolver thread panicked")?
.map_err(|err| err.to_string().into())
}
fn save_checkpoint(path: &Path, checkpoint: &Checkpoint) {
let serialized_checkpoint =
serde_json::to_string_pretty(checkpoint).expect("checkpoint should serialize to JSON");
std::fs::write(path, serialized_checkpoint).expect("checkpoint file should be written");
}
fn test_artifact_store(run_dir: &Path) -> ArtifactStore {
let object_store = Arc::new(
LocalFileSystem::new_with_prefix(test_store_dir(run_dir))
.expect("failed to create local artifact store"),
);
ArtifactStore::new(object_store, "artifacts")
}
// ---------------------------------------------------------------------------
// 1. Parse and validate all 3 spec examples (Section 2.13)
// ---------------------------------------------------------------------------
#[test]
fn parse_and_validate_simple_linear() {
let input = r#"digraph Simple {
graph [goal="Run tests and report"]
rankdir=LR
start [shape=Mdiamond, label="Start"]
exit [shape=Msquare, label="Exit"]
run_tests [label="Run Tests", prompt="Run the test suite and report results"]
report [label="Report", prompt="Summarize the test results"]
start -> run_tests -> report -> exit
}"#;
let graph = parse(input).expect("parsing should succeed");
assert_eq!(graph.name, "Simple");
assert_eq!(graph.goal(), "Run tests and report");
assert_eq!(graph.nodes.len(), 4);
assert_eq!(graph.edges.len(), 3);
assert!(graph.find_start_node().is_some());
assert!(graph.find_exit_node().is_some());
let diagnostics = validate_or_raise(&graph, &[]).expect("validation should pass");
let errors: Vec<_> = diagnostics
.iter()
.filter(|d| d.severity == fabro_validate::Severity::Error)
.collect();
assert!(errors.is_empty(), "expected no validation errors");
}
#[test]
fn parse_and_validate_branching_with_conditions() {
let input = r#"digraph Branch {
graph [goal="Implement and validate a feature"]
rankdir=LR
node [shape=box, timeout="900s"]
start [shape=Mdiamond, label="Start"]
exit [shape=Msquare, label="Exit"]
plan [label="Plan", prompt="Plan the implementation"]
implement [label="Implement", prompt="Implement the plan"]
validate [label="Validate", prompt="Run tests"]
gate [shape=diamond, label="Tests passing?"]
start -> plan -> implement -> validate -> gate
gate -> exit [label="Yes", condition="outcome=succeeded"]
gate -> implement [label="No"]
}"#;
let graph = parse(input).expect("parsing should succeed");
assert_eq!(graph.name, "Branch");
assert_eq!(graph.nodes.len(), 6);
assert_eq!(graph.edges.len(), 6);
let gate_exit = graph
.edges
.iter()
.find(|e| e.from == "gate" && e.to == "exit")
.expect("gate -> exit edge should exist");
assert_eq!(gate_exit.condition(), Some("outcome=succeeded"));
let gate_impl = graph
.edges
.iter()
.find(|e| e.from == "gate" && e.to == "implement")
.expect("gate -> implement edge should exist");
assert_eq!(gate_impl.condition(), None);
let diagnostics = validate_or_raise(&graph, &[]).expect("validation should pass");
let errors: Vec<_> = diagnostics
.iter()
.filter(|d| d.severity == fabro_validate::Severity::Error)
.collect();
assert!(errors.is_empty(), "expected no validation errors");
}
#[test]
fn parse_and_validate_human_gate() {
let input = r#"digraph Review {
rankdir=LR
start [shape=Mdiamond, label="Start"]
exit [shape=Msquare, label="Exit"]
review_gate [
shape=hexagon,
label="Review Changes",
type="human"
]
ship_it [prompt="Ship the change"]
fixes [prompt="Apply the requested fixes"]
start -> review_gate
review_gate -> ship_it [label="[A] Approve"]
review_gate -> fixes [label="[F] Fix"]
ship_it -> exit
fixes -> review_gate
}"#;
let graph = parse(input).expect("parsing should succeed");
assert_eq!(graph.name, "Review");
assert_eq!(graph.nodes.len(), 5);
assert_eq!(graph.edges.len(), 5);
let gate = &graph.nodes["review_gate"];
assert_eq!(gate.node_type(), Some("human"));
assert_eq!(gate.shape(), "hexagon");
assert_eq!(gate.label(), "Review Changes");
let diagnostics = validate_or_raise(&graph, &[]).expect("validation should pass");
let errors: Vec<_> = diagnostics
.iter()
.filter(|d| d.severity == fabro_validate::Severity::Error)
.collect();
assert!(errors.is_empty(), "expected no validation errors");
}
// ---------------------------------------------------------------------------
// 2. End-to-end linear pipeline
// ---------------------------------------------------------------------------
fn make_linear_registry() -> HandlerRegistry {
let mut registry = HandlerRegistry::new(Box::new(AgentHandler::new(None)));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register("agent", Box::new(AgentHandler::new(None)));
registry
}
#[tokio::test]
async fn end_to_end_linear_pipeline() {
let input = r#"digraph Linear {
graph [goal="Build the feature"]
start [shape=Mdiamond]
exit [shape=Msquare]
codergen_step [shape=box, label="Code", prompt="Implement the feature"]
start -> codergen_step -> exit
}"#;
let graph = parse(input).expect("parse should succeed");
validate_or_raise(&graph, &[]).expect("validation should pass");
let dir = tempfile::tempdir().expect("temporary run dir should be created");
let engine = WorkflowRunner::new(
make_linear_registry(),
Arc::new(Emitter::default()),
local_env(),
);
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (outcome, state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("run should succeed");
assert_eq!(outcome.status, StageOutcome::Succeeded);
let checkpoint = load_run_checkpoint(dir.path()).expect("checkpoint should load");
assert!(checkpoint.completed_nodes.contains(&"start".to_string()));
assert!(
checkpoint
.completed_nodes
.contains(&"codergen_step".to_string())
);
let node_state = state
.stage(&fabro_types::StageId::new("codergen_step", 1))
.unwrap();
assert!(
node_state.response.is_some(),
"response should be projected"
);
assert!(
node_state.completion.is_some(),
"completion should be projected"
);
let prompt_content = node_state.prompt.as_deref().unwrap();
assert!(
prompt_content.ends_with("Implement the feature"),
"prompt should end with original prompt, got: {prompt_content}"
);
}
// ---------------------------------------------------------------------------
// 3. End-to-end branching pipeline
// ---------------------------------------------------------------------------
#[tokio::test]
async fn end_to_end_branching_pipeline() {
// Build a graph:
// start -> work -> gate (diamond)
// gate -> success_path [condition="outcome=succeeded"]
// gate -> fail_path [condition="outcome=failed"]
// success_path -> exit
// fail_path -> exit
//
// Since work defaults to codergen (shape=box) which returns SUCCESS,
// the engine should route gate -> success_path via condition match.
let mut graph = Graph::new("BranchTest");
graph.attrs.insert(
"goal".to_string(),
AttrValue::String("Test branching".to_string()),
);
let mut start = Node::new("start");
start.attrs.insert(
"shape".to_string(),
AttrValue::String("Mdiamond".to_string()),
);
graph.nodes.insert("start".to_string(), start);
let mut exit = Node::new("exit");
exit.attrs.insert(
"shape".to_string(),
AttrValue::String("Msquare".to_string()),
);
graph.nodes.insert("exit".to_string(), exit);
let mut work = Node::new("work");
work.attrs
.insert("shape".to_string(), AttrValue::String("box".to_string()));
work.attrs.insert(
"prompt".to_string(),
AttrValue::String("Do work".to_string()),
);
graph.nodes.insert("work".to_string(), work);
let mut gate = Node::new("gate");
gate.attrs.insert(
"shape".to_string(),
AttrValue::String("diamond".to_string()),
);
graph.nodes.insert("gate".to_string(), gate);
graph
.nodes
.insert("success_path".to_string(), Node::new("success_path"));
graph
.nodes
.insert("fail_path".to_string(), Node::new("fail_path"));
graph.edges.push(Edge::new("start", "work"));
graph.edges.push(Edge::new("work", "gate"));
let mut gate_success = Edge::new("gate", "success_path");
gate_success.attrs.insert(
"condition".to_string(),
AttrValue::String("outcome=succeeded".to_string()),
);
graph.edges.push(gate_success);
let mut gate_fail = Edge::new("gate", "fail_path");
gate_fail.attrs.insert(
"condition".to_string(),
AttrValue::String("outcome=failed".to_string()),
);
graph.edges.push(gate_fail);
graph.edges.push(Edge::new("success_path", "exit"));
graph.edges.push(Edge::new("fail_path", "exit"));
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(AgentHandler::new(None)));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register("agent", Box::new(AgentHandler::new(None)));
registry.register("conditional", Box::new(ConditionalHandler));
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (outcome, state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("run should succeed");
assert_eq!(outcome.status, StageOutcome::Succeeded);
let checkpoint = state
.current_checkpoint()
.cloned()
.expect("checkpoint should be captured");
assert!(
checkpoint
.completed_nodes
.contains(&"success_path".to_string()),
"should have traversed success_path"
);
assert!(
!checkpoint
.completed_nodes
.contains(&"fail_path".to_string()),
"should NOT have traversed fail_path"
);
}
// ---------------------------------------------------------------------------
// 4. End-to-end human gate pipeline with QueueInterviewer
// ---------------------------------------------------------------------------
#[tokio::test]
async fn end_to_end_human_gate_pipeline() {
// Build a graph:
// start -> gate (hexagon, type=wait.human)
// gate -> approve [label="[A] Approve"]
// gate -> reject [label="[R] Reject"]
// approve -> exit
// reject -> exit
//
// QueueInterviewer pre-filled to select "R" -> should route to reject
let mut graph = Graph::new("HumanGateTest");
let mut start = Node::new("start");
start.attrs.insert(
"shape".to_string(),
AttrValue::String("Mdiamond".to_string()),
);
graph.nodes.insert("start".to_string(), start);
let mut exit = Node::new("exit");
exit.attrs.insert(
"shape".to_string(),
AttrValue::String("Msquare".to_string()),
);
graph.nodes.insert("exit".to_string(), exit);
let mut gate = Node::new("gate");
gate.attrs.insert(
"shape".to_string(),
AttrValue::String("hexagon".to_string()),
);
gate.attrs
.insert("type".to_string(), AttrValue::String("human".to_string()));
gate.attrs.insert(
"label".to_string(),
AttrValue::String("Review Changes".to_string()),
);
graph.nodes.insert("gate".to_string(), gate);
graph
.nodes
.insert("approve".to_string(), Node::new("approve"));
graph
.nodes
.insert("reject".to_string(), Node::new("reject"));
graph.edges.push(Edge::new("start", "gate"));
let mut e_approve = Edge::new("gate", "approve");
e_approve.attrs.insert(
"label".to_string(),
AttrValue::String("[A] Approve".to_string()),
);
graph.edges.push(e_approve);
let mut e_reject = Edge::new("gate", "reject");
e_reject.attrs.insert(
"label".to_string(),
AttrValue::String("[R] Reject".to_string()),
);
graph.edges.push(e_reject);
graph.edges.push(Edge::new("approve", "exit"));
graph.edges.push(Edge::new("reject", "exit"));
// Pre-fill the queue with an answer selecting "R"
let answers = VecDeque::from([Answer {
value: AnswerValue::Selected("R".to_string()),
selected_option: None,
text: None,
}]);
let interviewer = Arc::new(QueueInterviewer::new(answers));
let dir = tempfile::tempdir().expect("temporary run dir should be created");
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register("human", Box::new(HumanHandler::new(interviewer)));
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (outcome, state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("run should succeed");
assert_eq!(outcome.status, StageOutcome::Succeeded);
let checkpoint = state
.current_checkpoint()
.cloned()
.expect("checkpoint should be captured");
assert!(
checkpoint.completed_nodes.contains(&"reject".to_string()),
"should have traversed reject path"
);
assert!(
!checkpoint.completed_nodes.contains(&"approve".to_string()),
"should NOT have traversed approve path"
);
}
#[tokio::test]
async fn human_gate_interrupted_input_fails_closed_without_fail_route() {
let mut graph = Graph::new("HumanGateInterruptedClosed");
let mut start = Node::new("start");
start.attrs.insert(
"shape".to_string(),
AttrValue::String("Mdiamond".to_string()),
);
graph.nodes.insert("start".to_string(), start);
let mut exit = Node::new("exit");
exit.attrs.insert(
"shape".to_string(),
AttrValue::String("Msquare".to_string()),
);
graph.nodes.insert("exit".to_string(), exit);
let mut gate = Node::new("gate");
gate.attrs.insert(
"shape".to_string(),
AttrValue::String("hexagon".to_string()),
);
gate.attrs
.insert("type".to_string(), AttrValue::String("human".to_string()));
gate.attrs.insert(
"label".to_string(),
AttrValue::String("Approve release?".to_string()),
);
graph.nodes.insert("gate".to_string(), gate);
graph
.nodes
.insert("approve".to_string(), Node::new("approve"));
graph
.nodes
.insert("revise".to_string(), Node::new("revise"));
graph.edges.push(Edge::new("start", "gate"));
let mut approve_edge = Edge::new("gate", "approve");
approve_edge.attrs.insert(
"label".to_string(),
AttrValue::String("[A] Approve".to_string()),
);
graph.edges.push(approve_edge);
let mut revise_edge = Edge::new("gate", "revise");
revise_edge.attrs.insert(
"label".to_string(),
AttrValue::String("[R] Revise".to_string()),
);
graph.edges.push(revise_edge);
graph.edges.push(Edge::new("approve", "exit"));
graph.edges.push(Edge::new("revise", "exit"));
let interviewer = Arc::new(CallbackInterviewer::new(|_| Answer::interrupted()));
let dir = tempfile::tempdir().expect("temporary run dir should be created");
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register("human", Box::new(HumanHandler::new(interviewer)));
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (outcome, state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("engine should return Ok with fail outcome");
assert_eq!(
outcome.status,
StageOutcome::Failed {
retry_requested: false,
},
"interrupted human gate should fail closed"
);
assert!(
outcome
.failure_reason()
.unwrap_or("")
.contains("no outgoing fail edge"),
"unexpected outcome: {outcome:?}"
);
let checkpoint = state
.current_checkpoint()
.cloned()
.expect("checkpoint should be captured");
assert!(
checkpoint.node_outcomes.contains_key("gate"),
"gate outcome should be checkpointed before termination"
);
assert!(
!checkpoint.completed_nodes.contains(&"approve".to_string()),
"approval path must not execute on interrupted input"
);
assert!(
!checkpoint.completed_nodes.contains(&"revise".to_string()),
"other unconditional choice edges must not execute on interrupted input"
);
}
struct NeverAnswerInterviewer;
#[async_trait::async_trait]
impl Interviewer for NeverAnswerInterviewer {
async fn ask(&self, _question: fabro_interview::Question) -> fabro_interview::AnswerSubmission {
tokio::time::sleep(Duration::from_mins(1)).await;
fabro_interview::AnswerSubmission::system(
Answer::interrupted(),
fabro_types::SystemActorKind::Engine,
)
}
}
#[tokio::test]
async fn human_gate_timeout_routes_to_default_choice_when_unanswered() {
let mut graph = Graph::new("HumanGateTimeoutDefaultChoice");
let mut start = Node::new("start");
start.attrs.insert(
"shape".to_string(),
AttrValue::String("Mdiamond".to_string()),
);
graph.nodes.insert("start".to_string(), start);
let mut exit = Node::new("exit");
exit.attrs.insert(
"shape".to_string(),
AttrValue::String("Msquare".to_string()),
);
graph.nodes.insert("exit".to_string(), exit);
let mut gate = Node::new("gate");
gate.attrs.insert(
"shape".to_string(),
AttrValue::String("hexagon".to_string()),
);
gate.attrs
.insert("type".to_string(), AttrValue::String("human".to_string()));
gate.attrs.insert(
"label".to_string(),
AttrValue::String("Approve release?".to_string()),
);
gate.attrs.insert(
"question_type".to_string(),
AttrValue::String("multiple_choice".to_string()),
);
gate.attrs.insert(
"human.default_choice".to_string(),
AttrValue::String("approve".to_string()),
);
gate.attrs.insert(
"timeout".to_string(),
AttrValue::Duration(Duration::from_millis(20)),
);
graph.nodes.insert("gate".to_string(), gate);
graph
.nodes
.insert("approve".to_string(), Node::new("approve"));
graph
.nodes
.insert("revise".to_string(), Node::new("revise"));
graph.edges.push(Edge::new("start", "gate"));
let mut approve_edge = Edge::new("gate", "approve");
approve_edge.attrs.insert(
"label".to_string(),
AttrValue::String("[A] Approve".to_string()),
);
graph.edges.push(approve_edge);
let mut revise_edge = Edge::new("gate", "revise");
revise_edge.attrs.insert(
"label".to_string(),
AttrValue::String("[R] Revise".to_string()),
);
graph.edges.push(revise_edge);
graph.edges.push(Edge::new("approve", "exit"));
graph.edges.push(Edge::new("revise", "exit"));
let interviewer = Arc::new(NeverAnswerInterviewer);
let emitter = Emitter::default();
let events = collect_events(&emitter);
let dir = tempfile::tempdir().expect("temporary run dir should be created");
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register("human", Box::new(HumanHandler::new(interviewer)));
let engine = WorkflowRunner::new(registry, Arc::new(emitter), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (outcome, state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("human timeout should route through default choice");
if outcome.status != StageOutcome::Succeeded {
let gate_outcome = state
.current_checkpoint()
.and_then(|checkpoint| checkpoint.node_outcomes.get("gate"));
panic!(
"human timeout should have selected default choice; outcome: {outcome:?}; gate outcome: {gate_outcome:?}"
);
}
let checkpoint = state
.current_checkpoint()
.cloned()
.expect("checkpoint should be captured");
assert!(
checkpoint.completed_nodes.contains(&"approve".to_string()),
"default choice target should have completed"
);
assert!(
!checkpoint.completed_nodes.contains(&"revise".to_string()),
"non-default choice target should not have completed"
);
let captured_events = events.lock().expect("event log lock poisoned");
assert!(
captured_events
.iter()
.any(|event| event.event_name() == "interview.timeout"),
"interview.timeout should be emitted on human gate timeout"
);
}
#[tokio::test]
async fn human_gate_interrupted_input_routes_via_outcome_fail_condition() {
let mut graph = Graph::new("HumanGateInterruptedFailRoute");
let mut start = Node::new("start");
start.attrs.insert(
"shape".to_string(),
AttrValue::String("Mdiamond".to_string()),
);
graph.nodes.insert("start".to_string(), start);
let mut exit = Node::new("exit");
exit.attrs.insert(
"shape".to_string(),
AttrValue::String("Msquare".to_string()),
);
graph.nodes.insert("exit".to_string(), exit);
let mut gate = Node::new("gate");
gate.attrs.insert(
"shape".to_string(),
AttrValue::String("hexagon".to_string()),
);
gate.attrs
.insert("type".to_string(), AttrValue::String("human".to_string()));
gate.attrs.insert(
"label".to_string(),
AttrValue::String("Approve release?".to_string()),
);
graph.nodes.insert("gate".to_string(), gate);
graph
.nodes
.insert("approve".to_string(), Node::new("approve"));
graph
.nodes
.insert("manual_review".to_string(), Node::new("manual_review"));
graph.edges.push(Edge::new("start", "gate"));
let mut approve_edge = Edge::new("gate", "approve");
approve_edge.attrs.insert(
"label".to_string(),
AttrValue::String("[A] Approve".to_string()),
);
graph.edges.push(approve_edge);
let mut fail_edge = Edge::new("gate", "manual_review");
fail_edge.attrs.insert(
"condition".to_string(),
AttrValue::String("outcome=failed".to_string()),
);
graph.edges.push(fail_edge);
graph.edges.push(Edge::new("approve", "exit"));
graph.edges.push(Edge::new("manual_review", "exit"));
let interviewer = Arc::new(CallbackInterviewer::new(|_| Answer::interrupted()));
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register("human", Box::new(HumanHandler::new(interviewer)));
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (outcome, state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("interrupted human gate should follow explicit fail route");
assert_eq!(outcome.status, StageOutcome::Succeeded);
let checkpoint = state
.current_checkpoint()
.cloned()
.expect("checkpoint should be captured");
assert!(
checkpoint
.completed_nodes
.contains(&"manual_review".to_string()),
"explicit fail route should handle unanswered human gates"
);
assert!(
!checkpoint.completed_nodes.contains(&"approve".to_string()),
"approval path must not execute on interrupted input"
);
}
// ---------------------------------------------------------------------------
// 5. Goal gate enforcement
// ---------------------------------------------------------------------------
/// A custom handler that always returns FAIL for testing goal gate enforcement.
struct AlwaysFailHandler;
#[async_trait::async_trait]
impl Handler for AlwaysFailHandler {
async fn execute(
&self,
node: &Node,
_context: &fabro_workflow::context::Context,
_graph: &Graph,
_run_dir: &Path,
_services: &fabro_workflow::handler::EngineServices,
) -> Result<Outcome, fabro_workflow::error::Error> {
Ok(Outcome::fail_classify(format!(
"forced failure for {}",
node.id
)))
}
}
struct OnFailureRecordingHandler {
visits: Arc<std::sync::Mutex<Vec<String>>>,
}
#[async_trait::async_trait]
impl Handler for OnFailureRecordingHandler {
async fn execute(
&self,
node: &Node,
_context: &fabro_workflow::context::Context,
_graph: &Graph,
_run_dir: &Path,
_services: &fabro_workflow::handler::EngineServices,
) -> Result<Outcome, fabro_workflow::error::Error> {
self.visits.lock().unwrap().push(node.id.clone());
if node.id == "work" {
let mut outcome = Outcome::fail_classify("forced work failure");
outcome
.context_updates
.insert("recovery_ready".to_string(), serde_json::json!(true));
Ok(outcome)
} else {
Ok(Outcome::success())
}
}
}
/// Builds the linear on_failure test graph, splicing `extra` statements
/// (policy attribute, recovery edges, node attributes) into the DOT source so
/// tests exercise the real parser path for the `on_failure` attribute.
fn on_failure_graph(extra: &str) -> Graph {
let input = format!(
r"digraph OnFailureTest {{
{extra}
start [shape=Mdiamond]
exit [shape=Msquare]
work
downstream
start -> work -> downstream -> exit
}}"
);
parse(&input).expect("on_failure test graph should parse")
}
fn on_failure_registry(visits: Arc<std::sync::Mutex<Vec<String>>>) -> HandlerRegistry {
let mut registry = HandlerRegistry::new(Box::new(OnFailureRecordingHandler { visits }));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry
}
struct OnFailureRun {
outcome: Outcome,
state: fabro_store::RunProjection,
visits: Arc<std::sync::Mutex<Vec<String>>>,
_run_dir: tempfile::TempDir,
}
async fn run_on_failure(graph: &Graph, emitter: Emitter) -> OnFailureRun {
let visits = Arc::new(std::sync::Mutex::new(Vec::new()));
let engine = WorkflowRunner::new(
on_failure_registry(Arc::clone(&visits)),
Arc::new(emitter),
local_env(),
);
let run_dir = tempfile::tempdir().expect("temporary run dir should be created");
let (outcome, state) = engine
.run_with_state(graph, &make_run_options(run_dir.path()))
.await
.expect("on_failure run should complete without engine errors");
OnFailureRun {
outcome,
state,
visits,
_run_dir: run_dir,
}
}
#[tokio::test]
async fn on_failure_exit_stops_linear_workflow_and_records_failed_lifecycle() {
let graph = on_failure_graph(r#"graph [on_failure="exit"]"#);
let emitter = Emitter::default();
let events = collect_events(&emitter);
let run = run_on_failure(&graph, emitter).await;
assert_eq!(run.outcome.status, StageOutcome::Failed {
retry_requested: false,
});
assert_eq!(
run.outcome.failure_reason(),
Some("stage work failed and graph on_failure=exit stopped routing")
);
assert_eq!(*run.visits.lock().unwrap(), vec!["work"]);
let checkpoint = run
.state
.current_checkpoint()
.expect("failed work should be checkpointed");
assert_eq!(checkpoint.current_node, "work");
assert_eq!(checkpoint.next_node_id, None);
assert!(!checkpoint.node_outcomes.contains_key("downstream"));
let events = events.lock().unwrap();
assert!(
events
.iter()
.any(|event| matches!(&event.body, EventBody::RunFailed(_)))
);
assert!(events.iter().any(|event| {
matches!(
&event.body,
EventBody::CheckpointCompleted(properties)
if properties.current_node == "work" && properties.next_node_id.is_none()
)
}));
assert!(!events.iter().any(|event| {
matches!(
&event.body,
EventBody::EdgeSelected(properties) if properties.from_node == "work"
)
}));
}
#[tokio::test]
async fn on_failure_route_and_absent_policy_preserve_unconditional_fallback() {
for policy_attr in ["", r#"graph [on_failure="route"]"#] {
let graph = on_failure_graph(policy_attr);
let run = run_on_failure(&graph, Emitter::default()).await;
assert_eq!(run.outcome.status, StageOutcome::Succeeded);
assert_eq!(*run.visits.lock().unwrap(), vec!["work", "downstream"]);
assert!(
run.state
.current_checkpoint()
.expect("downstream should be checkpointed")
.node_outcomes
.contains_key("downstream")
);
}
}
#[tokio::test]
async fn on_failure_exit_allows_explicit_failure_recovery_edge() {
let graph = on_failure_graph(
r#"graph [on_failure="exit"]
recovery
work -> recovery [condition="outcome=failed"]
recovery -> exit"#,
);
let run = run_on_failure(&graph, Emitter::default()).await;
assert_eq!(run.outcome.status, StageOutcome::Succeeded);
assert_eq!(*run.visits.lock().unwrap(), vec!["work", "recovery"]);
}
#[tokio::test]
async fn on_failure_exit_uses_retry_target_instead_of_unconditional_edge() {
let graph = on_failure_graph(
r#"graph [on_failure="exit"]
work [retry_target="recovery"]
recovery
recovery -> exit"#,
);
let run = run_on_failure(&graph, Emitter::default()).await;
assert_eq!(run.outcome.status, StageOutcome::Succeeded);
assert_eq!(*run.visits.lock().unwrap(), vec!["work", "recovery"]);
}
#[tokio::test]
async fn node_on_failure_exit_overrides_graph_route_policy() {
let graph = on_failure_graph(r#"work [on_failure="exit"]"#);
let run = run_on_failure(&graph, Emitter::default()).await;
assert_eq!(run.outcome.status, StageOutcome::Failed {
retry_requested: false,
});
assert_eq!(
run.outcome.failure_reason(),
Some("stage work failed and node on_failure=exit stopped routing")
);
assert_eq!(*run.visits.lock().unwrap(), vec!["work"]);
}
#[tokio::test]
async fn node_on_failure_route_overrides_graph_exit_policy() {
let graph = on_failure_graph(
r#"graph [on_failure="exit"]
work [on_failure="route"]"#,
);
let run = run_on_failure(&graph, Emitter::default()).await;
assert_eq!(run.outcome.status, StageOutcome::Succeeded);
assert_eq!(*run.visits.lock().unwrap(), vec!["work", "downstream"]);
}
#[tokio::test]
async fn node_on_failure_succeed_promotes_failed_node_and_keeps_failure_in_events() {
let graph = on_failure_graph(
r#"graph [on_failure="exit"]
work [on_failure="succeed"]"#,
);
let emitter = Emitter::default();
let events = collect_events(&emitter);
let run = run_on_failure(&graph, emitter).await;
assert_eq!(run.outcome.status, StageOutcome::Succeeded);
assert_eq!(*run.visits.lock().unwrap(), vec!["work", "downstream"]);
let checkpoint = run
.state
.current_checkpoint()
.expect("downstream should be checkpointed");
let work = &checkpoint.node_outcomes["work"];
assert_eq!(work.status, StageOutcome::Succeeded);
assert_eq!(work.failure_reason(), Some("forced work failure"));
assert_eq!(
work.notes.as_deref(),
Some("node on_failure=succeed promoted a failed outcome to succeeded")
);
let events = events.lock().unwrap();
let completed = events
.iter()
.find_map(|event| match &event.body {
EventBody::StageCompleted(props) if event.node_id.as_deref() == Some("work") => {
Some(props.clone())
}
_ => None,
})
.expect("promoted work stage should emit stage.completed");
assert_eq!(completed.status, StageOutcome::Succeeded);
assert_eq!(
completed
.failure
.as_ref()
.map(|failure| failure.message.as_str()),
Some("forced work failure")
);
assert!(!events.iter().any(|event| {
matches!(&event.body, EventBody::StageFailed(_)) && event.node_id.as_deref() == Some("work")
}));
assert!(
events
.iter()
.any(|event| matches!(&event.body, EventBody::RunCompleted(_)))
);
}
#[tokio::test]
async fn auto_status_true_is_an_alias_for_on_failure_succeed() {
let graph = on_failure_graph(
r#"graph [on_failure="exit"]
work [auto_status=true]"#,
);
let run = run_on_failure(&graph, Emitter::default()).await;
assert_eq!(run.outcome.status, StageOutcome::Succeeded);
assert_eq!(*run.visits.lock().unwrap(), vec!["work", "downstream"]);
let checkpoint = run
.state
.current_checkpoint()
.expect("downstream should be checkpointed");
assert_eq!(
checkpoint.node_outcomes["work"].status,
StageOutcome::Succeeded
);
}
#[tokio::test]
async fn node_on_failure_succeed_prefers_explicit_failure_edge() {
let graph = on_failure_graph(
r#"work [on_failure="succeed"]
recovery
work -> recovery [condition="outcome=failed"]
recovery -> exit"#,
);
let run = run_on_failure(&graph, Emitter::default()).await;
assert_eq!(run.outcome.status, StageOutcome::Succeeded);
assert_eq!(*run.visits.lock().unwrap(), vec!["work", "recovery"]);
let checkpoint = run
.state
.current_checkpoint()
.expect("recovery should be checkpointed");
assert!(checkpoint.node_outcomes["work"].status.is_failure());
}
#[tokio::test]
async fn node_on_failure_succeed_tests_recovery_edge_with_pending_result_context() {
let graph = on_failure_graph(
r#"work [on_failure="succeed"]
recovery
work -> recovery [condition="outcome=failed && context.recovery_ready=true && context.failure_class=deterministic"]
recovery -> exit"#,
);
let run = run_on_failure(&graph, Emitter::default()).await;
assert_eq!(run.outcome.status, StageOutcome::Succeeded);
assert_eq!(*run.visits.lock().unwrap(), vec!["work", "recovery"]);
let checkpoint = run
.state
.current_checkpoint()
.expect("recovery should be checkpointed");
assert!(checkpoint.node_outcomes["work"].status.is_failure());
}
#[tokio::test]
async fn node_on_failure_succeed_satisfies_goal_gate_without_retry_target() {
let graph =
on_failure_graph(r#"work [on_failure="succeed" goal_gate=true retry_target="start"]"#);
let run = run_on_failure(&graph, Emitter::default()).await;
assert_eq!(run.outcome.status, StageOutcome::Succeeded);
assert_eq!(*run.visits.lock().unwrap(), vec!["work", "downstream"]);
}
#[tokio::test]
async fn goal_gate_routes_to_retry_target_on_failure() {
// Pipeline:
// start -> gated_work -> exit
// gated_work has goal_gate=true, retry_target=start
// gated_work always returns FAIL
//
// When engine reaches exit, it checks goal gates and finds gated_work failed.
// It should route back to retry_target (start).
//
// To avoid infinite loops, we set max_retries=0 on gated_work so it fails
// immediately each time. After looping once (start -> gated_work -> exit ->
// start -> gated_work -> exit), if goal gate is still unsatisfied and no
// retry_target changes, we need to limit iterations. The engine itself
// doesn't limit loops, so we test a simpler scenario: verify the error when
// retry_target is missing.
// Test: goal_gate with NO retry_target returns an error
let mut graph = Graph::new("GoalGateNoRetry");
let mut start = Node::new("start");
start.attrs.insert(
"shape".to_string(),
AttrValue::String("Mdiamond".to_string()),
);
graph.nodes.insert("start".to_string(), start);
let mut exit = Node::new("exit");
exit.attrs.insert(
"shape".to_string(),
AttrValue::String("Msquare".to_string()),
);
graph.nodes.insert("exit".to_string(), exit);
let mut gated_work = Node::new("gated_work");
gated_work
.attrs
.insert("goal_gate".to_string(), AttrValue::Boolean(true));
gated_work
.attrs
.insert("max_retries".to_string(), AttrValue::Integer(0));
gated_work.attrs.insert(
"type".to_string(),
AttrValue::String("always_fail".to_string()),
);
graph.nodes.insert("gated_work".to_string(), gated_work);
graph.edges.push(Edge::new("start", "gated_work"));
graph.edges.push(Edge::new("gated_work", "exit"));
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register("always_fail", Box::new(AlwaysFailHandler));
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let result = engine.run(&graph, &run_options).await;
assert!(
result.is_ok(),
"goal gate unsatisfied with no retry_target should return Ok(fail outcome)"
);
let outcome = result.unwrap();
assert_eq!(
outcome.status,
StageOutcome::Failed {
retry_requested: false,
},
"pipeline outcome should be 'fail' when goal gate unsatisfied"
);
let failure_reason = outcome.failure_reason().unwrap_or_default();
assert!(
failure_reason.contains("goal gate unsatisfied"),
"failure_reason should mention goal gate, got: {failure_reason}"
);
}
#[tokio::test]
async fn goal_gate_routes_to_retry_target_when_present() {
// Pipeline:
// start -> gated_work -> exit
// gated_work has goal_gate=true, retry_target=start
// gated_work always fails via AlwaysFailHandler.
//
// When engine reaches exit and finds goal gate unsatisfied, it should route
// to the retry_target. Since AlwaysFailHandler always fails, this creates a
// loop. However, the gated_work node will emit a FAIL outcome, and the
// edge gated_work -> exit is unconditional, so it still reaches exit. After
// the first retry (start -> gated_work -> exit), goal gate is still failed
// and retry_target is still start, so it loops. To prevent an infinite loop
// in tests, we use a custom handler that fails the first time and succeeds
// the second time.
struct FailThenSucceedHandler {
call_count: std::sync::atomic::AtomicU32,
}
#[async_trait::async_trait]
impl Handler for FailThenSucceedHandler {
async fn execute(
&self,
_node: &Node,
_context: &fabro_workflow::context::Context,
_graph: &Graph,
_run_dir: &Path,
_services: &fabro_workflow::handler::EngineServices,
) -> Result<Outcome, fabro_workflow::error::Error> {
let count = self
.call_count
.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
if count == 0 {
Ok(Outcome::fail_classify("first attempt fails"))
} else {
Ok(Outcome::success())
}
}
}
let mut graph = Graph::new("GoalGateRetry");
let mut start = Node::new("start");
start.attrs.insert(
"shape".to_string(),
AttrValue::String("Mdiamond".to_string()),
);
graph.nodes.insert("start".to_string(), start);
let mut exit = Node::new("exit");
exit.attrs.insert(
"shape".to_string(),
AttrValue::String("Msquare".to_string()),
);
graph.nodes.insert("exit".to_string(), exit);
let mut gated_work = Node::new("gated_work");
gated_work
.attrs
.insert("goal_gate".to_string(), AttrValue::Boolean(true));
gated_work
.attrs
.insert("max_retries".to_string(), AttrValue::Integer(0));
gated_work.attrs.insert(
"retry_target".to_string(),
AttrValue::String("start".to_string()),
);
gated_work.attrs.insert(
"type".to_string(),
AttrValue::String("fail_then_succeed".to_string()),
);
graph.nodes.insert("gated_work".to_string(), gated_work);
graph.edges.push(Edge::new("start", "gated_work"));
graph.edges.push(Edge::new("gated_work", "exit"));
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"fail_then_succeed",
Box::new(FailThenSucceedHandler {
call_count: std::sync::atomic::AtomicU32::new(0),
}),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (outcome, state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("run should eventually succeed after retry");
assert_eq!(outcome.status, StageOutcome::Succeeded);
let checkpoint = state
.current_checkpoint()
.cloned()
.expect("checkpoint should be captured");
// gated_work should appear in completed nodes (at least twice -- first fail,
// then succeed)
let gated_work_count = checkpoint
.completed_nodes
.iter()
.filter(|n| *n == "gated_work")
.count();
assert!(
gated_work_count >= 2,
"gated_work should have been executed at least twice, got {gated_work_count}"
);
}
// ---------------------------------------------------------------------------
// 6. Variable expansion transform
// ---------------------------------------------------------------------------
#[test]
fn variable_expansion_replaces_goal_in_prompts() {
let mut graph = Graph::new("test");
graph.attrs.insert(
"goal".to_string(),
AttrValue::String("Fix all bugs".to_string()),
);
let mut plan_node = Node::new("plan");
plan_node.attrs.insert(
"prompt".to_string(),
AttrValue::String("Plan to achieve: {{ goal }}".to_string()),
);
graph.nodes.insert("plan".to_string(), plan_node);
let mut impl_node = Node::new("implement");
impl_node.attrs.insert(
"prompt".to_string(),
AttrValue::String("Implement {{ goal }} now".to_string()),
);
graph.nodes.insert("implement".to_string(), impl_node);
let mut no_var_node = Node::new("report");
no_var_node.attrs.insert(
"prompt".to_string(),
AttrValue::String("Generate a report".to_string()),
);
graph.nodes.insert("report".to_string(), no_var_node);
let transform = TemplateTransform::new(std::collections::HashMap::new());
let graph = transform.apply(graph).unwrap();
let plan_prompt = graph.nodes["plan"]
.attrs
.get("prompt")
.and_then(AttrValue::as_str)
.expect("plan prompt should exist");
assert_eq!(plan_prompt, "Plan to achieve: Fix all bugs");
let impl_prompt = graph.nodes["implement"]
.attrs
.get("prompt")
.and_then(AttrValue::as_str)
.expect("implement prompt should exist");
assert_eq!(impl_prompt, "Implement Fix all bugs now");
let report_prompt = graph.nodes["report"]
.attrs
.get("prompt")
.and_then(AttrValue::as_str)
.expect("report prompt should exist");
assert_eq!(report_prompt, "Generate a report");
}
// ---------------------------------------------------------------------------
// 7. Stylesheet application
// ---------------------------------------------------------------------------
#[test]
fn stylesheet_application_by_specificity() {
let stylesheet_text = r"
* { model: claude-sonnet-4-5; provider: anthropic; }
.code { model: claude-opus-4-6; provider: anthropic; }
#critical_review { model: gpt-5.2; provider: openai; reasoning_effort: high; }
";
let mut graph = Graph::new("test");
graph.attrs.insert(
"model_stylesheet".to_string(),
AttrValue::String(stylesheet_text.to_string()),
);
// plan node: no class, should get universal defaults
let plan = Node::new("plan");
graph.nodes.insert("plan".to_string(), plan);
// implement node: class="code", should get .code overrides
let mut implement = Node::new("implement");
implement.classes.push("code".to_string());
graph.nodes.insert("implement".to_string(), implement);
// critical_review node: class="code" AND id="critical_review", id wins
let mut critical = Node::new("critical_review");
critical.classes.push("code".to_string());
graph.nodes.insert("critical_review".to_string(), critical);
// explicit node: has explicit model, should NOT be overridden
let mut explicit = Node::new("explicit_node");
explicit.attrs.insert(
"model".to_string(),
AttrValue::String("my-custom-model".to_string()),
);
graph.nodes.insert("explicit_node".to_string(), explicit);
let transform = StylesheetApplicationTransform;
let graph = transform.apply(graph).unwrap();
// plan: universal -> claude-sonnet-4-5
assert_eq!(
graph.nodes["plan"].attrs.get("model"),
Some(&AttrValue::String("claude-sonnet-4-5".to_string()))
);
assert_eq!(
graph.nodes["plan"].attrs.get("provider"),
Some(&AttrValue::String("anthropic".to_string()))
);
// implement: .code -> claude-opus-4-6
assert_eq!(
graph.nodes["implement"].attrs.get("model"),
Some(&AttrValue::String("claude-opus-4-6".to_string()))
);
assert_eq!(
graph.nodes["implement"].attrs.get("provider"),
Some(&AttrValue::String("anthropic".to_string()))
);
// critical_review: #critical_review -> gpt-5.2 (id overrides class)
assert_eq!(
graph.nodes["critical_review"].attrs.get("model"),
Some(&AttrValue::String("gpt-5.2".to_string()))
);
assert_eq!(
graph.nodes["critical_review"].attrs.get("provider"),
Some(&AttrValue::String("openai".to_string()))
);
assert_eq!(
graph.nodes["critical_review"].attrs.get("reasoning_effort"),
Some(&AttrValue::String("high".to_string()))
);
// explicit_node: explicit attr NOT overridden by universal
assert_eq!(
graph.nodes["explicit_node"].attrs.get("model"),
Some(&AttrValue::String("my-custom-model".to_string()))
);
}
#[test]
fn stylesheet_comments_apply_via_parsed_graph() {
let input = r#"digraph StyleTest {
graph [
goal="Test stylesheet",
model_stylesheet="
/* Apply Sonnet by default. */
* {
/* Comments can appear between declarations. */
model: sonnet;
}
"
]
start [shape=Mdiamond]
exit [shape=Msquare]
work [shape=box, prompt="Do work"]
start -> work -> exit
}"#;
let graph = parse(input).expect("parse should succeed");
validate_or_raise(&graph, &[]).expect("validation should pass");
let transform = StylesheetApplicationTransform;
let graph = transform.apply(graph).unwrap();
// All nodes without explicit model should get "sonnet"
assert_eq!(
graph.nodes["work"].attrs.get("model"),
Some(&AttrValue::String("sonnet".to_string()))
);
assert_eq!(
graph.nodes["start"].attrs.get("model"),
Some(&AttrValue::String("sonnet".to_string()))
);
assert_eq!(
graph.nodes["exit"].attrs.get("model"),
Some(&AttrValue::String("sonnet".to_string()))
);
}
#[test]
fn model_stylesheet_template_renders_through_pipeline() {
use fabro_workflow::pipeline::{TransformOptions, transform, validate};
let input = r#"digraph StyleTemplate {
graph [
goal="Review the change",
model_stylesheet="
* { reasoning_effort: low; }
{% for effort in inputs.efforts %}
.tier-{{ loop.index }} { reasoning_effort: {{ effort }}; }
{% endfor %}
"
]
start [shape=Mdiamond]
baseline [prompt="Baseline"]
selected [prompt="Selected", class="tier-2"]
exit [shape=Msquare]
start -> baseline -> selected -> exit
}"#;
let parsed = fabro_workflow::pipeline::parse(input).expect("parse should succeed");
let transformed = transform(parsed, &TransformOptions {
current_dir: None,
file_resolver: None,
template_context: fabro_template::TemplateContext::new().with_inputs(
std::collections::HashMap::from([(
"efforts".to_string(),
toml::Value::Array(vec![
toml::Value::String("medium".to_string()),
toml::Value::String("high".to_string()),
]),
)]),
),
source_name: Some("style-template.fabro".to_string()),
render_mode: fabro_workflow::operations::RenderMode::Structural,
custom_transforms: vec![],
model_resolution: None,
})
.expect("transform should succeed");
let validated = validate(transformed, None, &[]);
validated
.raise_on_errors()
.expect("rendered stylesheet should validate");
assert_eq!(
validated.graph().nodes["baseline"]
.attrs
.get("reasoning_effort")
.and_then(AttrValue::as_str),
Some("low")
);
assert_eq!(
validated.graph().nodes["selected"]
.attrs
.get("reasoning_effort")
.and_then(AttrValue::as_str),
Some("high")
);
}
#[test]
fn stylesheet_application_matches_space_separated_classes_from_dot() {
let input = r#"digraph StyleTest {
graph [
goal="Test class selectors",
model_stylesheet="
* { model: default-model; provider: default-provider; }
.research { reasoning_effort: high; }
.ensemble-a { model: ensemble-model; provider: openrouter; }
"
]
start [shape=Mdiamond]
work [shape=tab, class="research ensemble-a", prompt="Do work"]
exit [shape=Msquare]
start -> work -> exit
}"#;
let graph = parse(input).expect("parse should succeed");
validate_or_raise(&graph, &[]).expect("validation should pass");
let transform = StylesheetApplicationTransform;
let graph = transform.apply(graph).unwrap();
let work = &graph.nodes["work"];
assert_eq!(work.classes, vec!["research", "ensemble-a"]);
assert_eq!(work.model(), Some("ensemble-model"));
assert_eq!(work.provider(), Some("openrouter"));
assert_eq!(
work.attrs.get("reasoning_effort"),
Some(&AttrValue::String("high".to_string()))
);
}
#[test]
fn stylesheet_parse_and_apply_directly() {
let stylesheet_text = "* { model: base; } .fast { model: turbo; }";
let stylesheet = parse_stylesheet(stylesheet_text).expect("stylesheet parse should succeed");
assert_eq!(stylesheet.rules.len(), 2);
let mut graph = Graph::new("test");
let plain = Node::new("a");
graph.nodes.insert("a".to_string(), plain);
let mut fast_node = Node::new("b");
fast_node.classes.push("fast".to_string());
graph.nodes.insert("b".to_string(), fast_node);
apply_stylesheet(&stylesheet, &mut graph);
assert_eq!(
graph.nodes["a"].attrs.get("model"),
Some(&AttrValue::String("base".to_string()))
);
assert_eq!(
graph.nodes["b"].attrs.get("model"),
Some(&AttrValue::String("turbo".to_string()))
);
}
// ---------------------------------------------------------------------------
// 8. Retry on failure (Gap #35.1)
// ---------------------------------------------------------------------------
#[tokio::test]
async fn retry_on_failure_then_succeed() {
// A handler that fails the first call and succeeds on the second.
struct RetryHandler {
call_count: std::sync::atomic::AtomicU32,
}
#[async_trait::async_trait]
impl Handler for RetryHandler {
async fn execute(
&self,
_node: &Node,
_context: &Context,
_graph: &Graph,
_run_dir: &Path,
_services: &fabro_workflow::handler::EngineServices,
) -> Result<Outcome, Error> {
let count = self
.call_count
.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
if count == 0 {
Ok(Outcome::retry_classify("transient failure"))
} else {
Ok(Outcome::success())
}
}
}
let mut graph = Graph::new("RetryTest");
let mut start = Node::new("start");
start.attrs.insert(
"shape".to_string(),
AttrValue::String("Mdiamond".to_string()),
);
graph.nodes.insert("start".to_string(), start);
let mut exit = Node::new("exit");
exit.attrs.insert(
"shape".to_string(),
AttrValue::String("Msquare".to_string()),
);
graph.nodes.insert("exit".to_string(), exit);
let mut retry_node = Node::new("work");
retry_node.attrs.insert(
"type".to_string(),
AttrValue::String("retry_handler".to_string()),
);
retry_node
.attrs
.insert("max_retries".to_string(), AttrValue::Integer(3));
retry_node.attrs.insert(
"retry_policy".to_string(),
AttrValue::String("linear".to_string()),
);
graph.nodes.insert("work".to_string(), retry_node);
graph.edges.push(Edge::new("start", "work"));
graph.edges.push(Edge::new("work", "exit"));
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"retry_handler",
Box::new(RetryHandler {
call_count: std::sync::atomic::AtomicU32::new(0),
}),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let outcome = engine
.run(&graph, &run_options)
.await
.expect("should succeed after retry");
assert_eq!(outcome.status, StageOutcome::Succeeded);
}
// ---------------------------------------------------------------------------
// 9. Pipeline with 10+ nodes (Gap #35.2)
// ---------------------------------------------------------------------------
#[tokio::test]
async fn pipeline_with_many_nodes() {
// Build a linear pipeline: start -> n1 -> n2 -> ... -> n10 -> exit (12 nodes)
let mut graph = Graph::new("ManyNodes");
graph.attrs.insert(
"goal".to_string(),
AttrValue::String("Test large pipeline".to_string()),
);
let mut start = Node::new("start");
start.attrs.insert(
"shape".to_string(),
AttrValue::String("Mdiamond".to_string()),
);
graph.nodes.insert("start".to_string(), start);
let mut exit = Node::new("exit");
exit.attrs.insert(
"shape".to_string(),
AttrValue::String("Msquare".to_string()),
);
graph.nodes.insert("exit".to_string(), exit);
let node_names: Vec<String> = (1..=10).map(|i| format!("step_{i}")).collect();
for name in &node_names {
let mut node = Node::new(name.clone());
node.attrs
.insert("shape".to_string(), AttrValue::String("box".to_string()));
node.attrs.insert(
"prompt".to_string(),
AttrValue::String(format!("Execute {name}")),
);
graph.nodes.insert(name.clone(), node);
}
graph.edges.push(Edge::new("start", &node_names[0]));
for pair in node_names.windows(2) {
graph.edges.push(Edge::new(&pair[0], &pair[1]));
}
graph
.edges
.push(Edge::new(node_names.last().unwrap(), "exit"));
let dir = tempfile::tempdir().unwrap();
let engine = WorkflowRunner::new(
make_linear_registry(),
Arc::new(Emitter::default()),
local_env(),
);
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (outcome, state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("large pipeline should succeed");
assert_eq!(outcome.status, StageOutcome::Succeeded);
let checkpoint = state
.current_checkpoint()
.cloned()
.expect("checkpoint should be captured");
// All 10 step nodes should be in completed_nodes
for name in &node_names {
assert!(
checkpoint.completed_nodes.contains(name),
"{name} should be in completed_nodes"
);
}
}
// ---------------------------------------------------------------------------
// 10. Checkpoint save and load round-trip (Gap #35.3)
// ---------------------------------------------------------------------------
#[test]
fn checkpoint_save_and_resume_roundtrip() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("checkpoint_state.json");
let ctx = Context::new();
ctx.set("goal", serde_json::json!("Test checkpoint"));
ctx.set("progress", serde_json::json!(42));
let mut retries = std::collections::HashMap::new();
retries.insert("step_1".to_string(), 1u32);
let checkpoint = Checkpoint::from_context(
&ctx,
"step_2",
vec!["start".to_string(), "step_1".to_string()],
retries,
std::collections::HashMap::new(),
None,
std::collections::HashMap::new(),
std::collections::HashMap::new(),
std::collections::HashMap::new(),
);
save_checkpoint(&path, &checkpoint);
let loaded = load_checkpoint(&path).expect("load should succeed");
assert_eq!(loaded.current_node, "step_2");
assert_eq!(loaded.completed_nodes.len(), 2);
assert!(loaded.completed_nodes.contains(&"start".to_string()));
assert!(loaded.completed_nodes.contains(&"step_1".to_string()));
assert_eq!(loaded.node_retries.get("step_1"), Some(&1));
assert_eq!(
loaded.context_values.get("goal"),
Some(&serde_json::json!("Test checkpoint"))
);
assert_eq!(
loaded.context_values.get("progress"),
Some(&serde_json::json!(42))
);
}
// ---------------------------------------------------------------------------
// 11. Smoke test with mock CodergenBackend (Gap #36)
// ---------------------------------------------------------------------------
struct MockCodergenBackend;
#[async_trait::async_trait]
impl CodergenBackend for MockCodergenBackend {
async fn run(&self, request: CodergenRunRequest<'_>) -> Result<CodergenResult, Error> {
Ok(CodergenResult::Text {
text: format!(
"Response for {}: processed prompt '{}'",
request.node.id,
&request.prompt[..request.prompt.len().min(50)]
),
usage: None,
files_touched: Vec::new(),
last_file_touched: None,
timing: fabro_types::StageTiming::default(),
})
}
}
// ---------------------------------------------------------------------------
// Helpers for parity tests
// ---------------------------------------------------------------------------
/// A handler backed by a shared `AtomicU32` counter.
/// Returns Fail on call 0, Success on call >= 1.
struct CounterHandler {
call_count: Arc<std::sync::atomic::AtomicU32>,
}
#[async_trait::async_trait]
impl Handler for CounterHandler {
async fn execute(
&self,
_node: &Node,
_context: &Context,
_graph: &Graph,
_run_dir: &Path,
_services: &fabro_workflow::handler::EngineServices,
) -> Result<Outcome, Error> {
let count = self
.call_count
.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
if count == 0 {
// Use a message that heuristics classify as transient_infra
Ok(Outcome::fail_classify("connection refused"))
} else {
Ok(Outcome::success())
}
}
}
/// A handler that sets a context_update with a large value (>100KB) to trigger
/// artifact offloading.
struct LargeOutputHandler;
#[async_trait::async_trait]
impl Handler for LargeOutputHandler {
async fn execute(
&self,
node: &Node,
_context: &Context,
_graph: &Graph,
_run_dir: &Path,
_services: &fabro_workflow::handler::EngineServices,
) -> Result<Outcome, Error> {
let mut outcome = Outcome::success();
// 150KB string — well above the 100KB artifact threshold
let large_value = "x".repeat(150 * 1024);
outcome.context_updates.insert(
format!("response.{}", node.id),
serde_json::json!(large_value),
);
Ok(outcome)
}
}
#[derive(Clone)]
struct ContextValueCaptureHandler {
values: Arc<std::sync::Mutex<Vec<String>>>,
key: String,
}
#[async_trait::async_trait]
impl Handler for ContextValueCaptureHandler {
async fn execute(
&self,
_node: &Node,
context: &Context,
_graph: &Graph,
_run_dir: &Path,
_services: &fabro_workflow::handler::EngineServices,
) -> Result<Outcome, Error> {
let value = context
.get(&self.key)
.and_then(|value| value.as_str().map(ToOwned::to_owned))
.expect("captured context value should be a string");
self.values.lock().unwrap().push(value);
Ok(Outcome::success())
}
}
/// A handler that sets `context_updates` = {"`my_flag"`: "set"}.
struct ContextSetterHandler;
#[async_trait::async_trait]
impl Handler for ContextSetterHandler {
async fn execute(
&self,
_node: &Node,
_context: &Context,
_graph: &Graph,
_run_dir: &Path,
_services: &fabro_workflow::handler::EngineServices,
) -> Result<Outcome, Error> {
let mut outcome = Outcome::success();
outcome
.context_updates
.insert("my_flag".to_string(), serde_json::json!("set"));
Ok(outcome)
}
}
fn make_full_registry(interviewer: Arc<dyn Interviewer>) -> HandlerRegistry {
let mut registry = HandlerRegistry::new(Box::new(AgentHandler::new(None)));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register("agent", Box::new(AgentHandler::new(None)));
registry.register("conditional", Box::new(ConditionalHandler));
registry.register("command", Box::new(CommandHandler));
registry.register("human", Box::new(HumanHandler::new(interviewer)));
registry.register("wait", Box::new(WaitHandler));
registry.register("stack.manager_loop", Box::new(SubWorkflowHandler));
registry
}
fn make_graph_with_start_exit(name: &str) -> Graph {
let mut graph = Graph::new(name);
let mut start = Node::new("start");
start.attrs.insert(
"shape".to_string(),
AttrValue::String("Mdiamond".to_string()),
);
graph.nodes.insert("start".to_string(), start);
let mut exit = Node::new("exit");
exit.attrs.insert(
"shape".to_string(),
AttrValue::String("Msquare".to_string()),
);
graph.nodes.insert("exit".to_string(), exit);
graph
}
#[tokio::test]
async fn command_schema_validation_failure_does_not_consume_retries() {
let mut graph = make_graph_with_start_exit("CommandSchemaNoRetry");
let mut audit = Node::new("audit");
audit.attrs.insert(
"shape".to_string(),
AttrValue::String("parallelogram".to_string()),
);
audit.attrs.insert(
"script".to_string(),
AttrValue::String(r#"echo '{"passed":"yes"}'"#.to_string()),
);
audit.attrs.insert(
"output_schema".to_string(),
AttrValue::String(
r#"{"type":"object","required":["passed"],"properties":{"passed":{"type":"boolean"}}}"#
.to_string(),
),
);
audit
.attrs
.insert("max_retries".to_string(), AttrValue::Integer(2));
graph.nodes.insert("audit".to_string(), audit);
graph.edges.push(Edge::new("start", "audit"));
let emitter = Emitter::default();
let events = collect_events(&emitter);
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register("command", Box::new(CommandHandler));
let engine = WorkflowRunner::new(registry, Arc::new(emitter), local_env());
let mut run_options = make_run_options(dir.path());
run_options.run_id = test_run_id("command-schema-no-retry");
let (outcome, state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("deterministic command failure should remain a workflow outcome");
assert_eq!(outcome.status, StageOutcome::Failed {
retry_requested: false,
});
assert_eq!(
outcome.failure_category(),
Some(fabro_workflow::outcome::FailureCategory::Deterministic)
);
let checkpoint = state
.current_checkpoint()
.expect("checkpoint should be captured");
let audit_outcome = checkpoint
.node_outcomes
.get("audit")
.expect("audit outcome should be checkpointed");
assert_eq!(audit_outcome.status, StageOutcome::Failed {
retry_requested: false,
});
assert_eq!(
audit_outcome.failure_category(),
Some(fabro_workflow::outcome::FailureCategory::Deterministic)
);
assert_eq!(
checkpoint.node_retries.get("audit").copied().unwrap_or(0),
0,
"schema validation should not consume node retries"
);
let command_starts = events
.lock()
.unwrap()
.iter()
.filter(|event| matches!(event.body, EventBody::CommandStarted(_)))
.count();
assert_eq!(command_starts, 1, "command should execute exactly once");
}
#[tokio::test]
async fn smoke_test_with_mock_codergen_backend() {
// Pipeline:
// start -> plan -> gate (diamond)
// gate -> implement [condition="outcome=succeeded"]
// gate -> fix [condition="outcome!=succeeded"]
// implement -> exit
// fix -> exit
//
// codergen nodes use MockCodergenBackend which returns real Text responses.
// The gate is a conditional node. Since the mock backend returns success,
// we should route through implement.
let mut graph = Graph::new("SmokeTest");
graph.attrs.insert(
"goal".to_string(),
AttrValue::String("Build and validate".to_string()),
);
let mut start = Node::new("start");
start.attrs.insert(
"shape".to_string(),
AttrValue::String("Mdiamond".to_string()),
);
graph.nodes.insert("start".to_string(), start);
let mut exit = Node::new("exit");
exit.attrs.insert(
"shape".to_string(),
AttrValue::String("Msquare".to_string()),
);
graph.nodes.insert("exit".to_string(), exit);
let mut plan = Node::new("plan");
plan.attrs
.insert("shape".to_string(), AttrValue::String("box".to_string()));
plan.attrs.insert(
"prompt".to_string(),
AttrValue::String("Plan to achieve: Build and validate".to_string()),
);
graph.nodes.insert("plan".to_string(), plan);
let mut gate = Node::new("gate");
gate.attrs.insert(
"shape".to_string(),
AttrValue::String("diamond".to_string()),
);
graph.nodes.insert("gate".to_string(), gate);
let mut implement = Node::new("implement");
implement
.attrs
.insert("shape".to_string(), AttrValue::String("box".to_string()));
implement.attrs.insert(
"prompt".to_string(),
AttrValue::String("Implement the plan".to_string()),
);
graph.nodes.insert("implement".to_string(), implement);
let mut fix = Node::new("fix");
fix.attrs
.insert("shape".to_string(), AttrValue::String("box".to_string()));
fix.attrs.insert(
"prompt".to_string(),
AttrValue::String("Fix the issues".to_string()),
);
graph.nodes.insert("fix".to_string(), fix);
graph.edges.push(Edge::new("start", "plan"));
graph.edges.push(Edge::new("plan", "gate"));
let mut gate_impl = Edge::new("gate", "implement");
gate_impl.attrs.insert(
"condition".to_string(),
AttrValue::String("outcome=succeeded".to_string()),
);
graph.edges.push(gate_impl);
let mut gate_fix = Edge::new("gate", "fix");
gate_fix.attrs.insert(
"condition".to_string(),
AttrValue::String("outcome!=succeeded".to_string()),
);
graph.edges.push(gate_fix);
graph.edges.push(Edge::new("implement", "exit"));
graph.edges.push(Edge::new("fix", "exit"));
let dir = tempfile::tempdir().unwrap();
let backend = Box::new(MockCodergenBackend);
let mut registry = HandlerRegistry::new(Box::new(AgentHandler::new(Some(backend))));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"agent",
Box::new(AgentHandler::new(Some(Box::new(MockCodergenBackend)))),
);
registry.register("conditional", Box::new(ConditionalHandler));
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (outcome, state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("smoke test should succeed");
assert_eq!(outcome.status, StageOutcome::Succeeded);
let checkpoint = load_run_checkpoint(dir.path()).unwrap();
assert!(
checkpoint.completed_nodes.contains(&"plan".to_string()),
"plan should have executed"
);
assert!(
checkpoint
.completed_nodes
.contains(&"implement".to_string()),
"should route through implement (success path)"
);
assert!(
!checkpoint.completed_nodes.contains(&"fix".to_string()),
"should NOT have traversed fix path"
);
let plan_state = state.stage(&fabro_types::StageId::new("plan", 1)).unwrap();
let plan_response = plan_state
.response
.as_deref()
.expect("plan response should exist");
assert!(
plan_response.contains("Response for plan"),
"mock backend should have written response, got: {plan_response}"
);
let plan_prompt = plan_state
.prompt
.as_deref()
.expect("plan prompt should exist");
assert!(
plan_prompt.ends_with("Plan to achieve: Build and validate"),
"prompt should end with original prompt, got: {plan_prompt}"
);
}
#[tokio::test]
async fn shared_thread_compaction_before_routing_audit_succeeds() {
use fabro_workflow::steering_hub::SteeringHub;
use httpmock::Method::POST;
use httpmock::MockServer;
fn chat_completion_stream(text: &str, input_tokens: i64, output_tokens: i64) -> String {
let text_chunk = serde_json::json!({
"id": uuid::Uuid::new_v4().to_string(),
"model": "compact-model",
"choices": [{
"delta": {"content": text},
"finish_reason": null
}]
});
let usage_chunk = serde_json::json!({
"id": uuid::Uuid::new_v4().to_string(),
"model": "compact-model",
"choices": [],
"usage": {
"prompt_tokens": input_tokens,
"completion_tokens": output_tokens,
"total_tokens": input_tokens + output_tokens
}
});
format!("data: {text_chunk}\n\ndata: {usage_chunk}\n\ndata: [DONE]\n\n")
}
fn chat_completion_response(text: &str) -> serde_json::Value {
serde_json::json!({
"id": uuid::Uuid::new_v4().to_string(),
"model": "compact-model",
"choices": [{
"message": {"content": text},
"finish_reason": "stop"
}],
"usage": {
"prompt_tokens": 10,
"completion_tokens": 1,
"total_tokens": 11
}
})
}
let server = MockServer::start_async().await;
let warmup_count = 10;
for index in 1..=warmup_count {
let prompt = format!("Warmup {index}");
let next_prompt = if index == warmup_count {
"Audit shared-thread work".to_string()
} else {
format!("Warmup {}", index + 1)
};
let response = chat_completion_stream(r#"{"outcome":"succeeded"}"#, 1, 1);
server
.mock_async(move |when, then| {
when.method(POST)
.path("/chat/completions")
.body_includes(r#""stream":true"#)
.body_includes(prompt)
.body_excludes(next_prompt);
then.status(200)
.header("content-type", "text/event-stream")
.body(response);
})
.await;
}
let audit_stream = chat_completion_stream(
r#"{"outcome":"succeeded","preferred_next_label":"Done"}"#,
1_000_000,
1,
);
let audit_mock = server
.mock_async(|when, then| {
when.method(POST)
.path("/chat/completions")
.body_includes(r#""stream":true"#)
.body_includes("Audit shared-thread work");
then.status(200)
.header("content-type", "text/event-stream")
.body(audit_stream);
})
.await;
let compaction_mock = server
.mock_async(|when, then| {
when.method(POST)
.path("/chat/completions")
.body_excludes(r#""stream":true"#);
then.status(200)
.header("content-type", "application/json")
.json_body(chat_completion_response(
"Previous work completed and the audit can finish.",
));
})
.await;
let settings: LlmCatalogSettings = toml::from_str(&format!(
r#"
[providers.compact]
adapter = "openai_compatible"
agent_profile = "openai"
base_url = "{}"
[providers.compact.auth]
credentials = ["env:COMPACT_API_KEY"]
[models.compact-model]
provider = "compact"
display_name = "Compact Model"
family = "mock"
default = true
[models.compact-model.limits]
context_window = 100000
max_output = 1024
[models.compact-model.features]
tools = true
vision = false
reasoning = false
"#,
server.base_url()
))
.expect("test catalog should parse");
let catalog = Arc::new(Catalog::from_builtin_with_overrides(&settings).unwrap());
let source = auth_test_support::env_credential_source(|name| {
(name == "COMPACT_API_KEY").then(|| "sk-test".to_string())
});
let backend = AgentApiBackend::new_with_catalog(
"compact-model".to_string(),
ProviderId::from("compact"),
ModelFallbackPolicy::default(),
source,
Arc::new(SteeringHub::new(Arc::new(Emitter::default()))),
catalog,
);
let mut graph = make_graph_with_start_exit("SharedThreadCompactionAudit");
graph.attrs.insert(
"default_fidelity".to_string(),
AttrValue::String("full".to_string()),
);
let mut previous = "start".to_string();
for index in 1..=warmup_count {
let node_id = format!("warmup_{index}");
let mut node = Node::new(&node_id);
node.attrs.insert(
"prompt".to_string(),
AttrValue::String(format!("Warmup {index}")),
);
node.attrs.insert(
"thread_id".to_string(),
AttrValue::String("shared-audit-thread".to_string()),
);
graph.nodes.insert(node_id.clone(), node);
graph.edges.push(Edge::new(&previous, &node_id));
previous = node_id;
}
let mut audit = Node::new("audit");
audit.attrs.insert(
"prompt".to_string(),
AttrValue::String("Audit shared-thread work".to_string()),
);
audit.attrs.insert(
"thread_id".to_string(),
AttrValue::String("shared-audit-thread".to_string()),
);
audit.attrs.insert(
"output_schema".to_string(),
AttrValue::String("routing".to_string()),
);
graph.nodes.insert("audit".to_string(), audit);
graph.edges.push(Edge::new(&previous, "audit"));
graph.edges.push(Edge::new("audit", "exit"));
let mut registry = HandlerRegistry::new(Box::new(AgentHandler::new(Some(Box::new(backend)))));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
let dir = tempfile::tempdir().unwrap();
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("shared-thread-compaction-audit"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (_outcome, state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("workflow execution should complete");
assert_eq!(
audit_mock.calls_async().await,
1,
"audit should use the high-usage response that triggers compaction"
);
assert_eq!(
compaction_mock.calls_async().await,
1,
"audit response should trigger context compaction before routing finishes"
);
let checkpoint = state
.current_checkpoint()
.cloned()
.expect("checkpoint should be captured");
let audit_outcome = checkpoint
.node_outcomes
.get("audit")
.expect("audit outcome should be captured");
assert_eq!(
audit_outcome.status,
StageOutcome::Succeeded,
"audit should succeed after compaction, got failure: {:?}",
audit_outcome.failure
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn workflow_persists_authoritative_openrouter_cost_for_agent_stage() {
use fabro_workflow::steering_hub::SteeringHub;
use httpmock::Method::POST;
use httpmock::MockServer;
const AUTHORITATIVE_COST_USD: f64 = 0.125;
const AUTHORITATIVE_COST_USD_MICROS: i64 = 125_000;
let server = MockServer::start_async().await;
let text_chunk = serde_json::json!({
"id": "chatcmpl_authoritative_cost",
"model": "openai/gpt-5.4",
"choices": [{
"delta": {"content": "done"},
"finish_reason": null
}]
});
let usage_chunk = serde_json::json!({
"id": "chatcmpl_authoritative_cost",
"model": "openai/gpt-5.4",
"choices": [],
"usage": {
"prompt_tokens": 11,
"completion_tokens": 7,
"total_tokens": 18,
"cost": AUTHORITATIVE_COST_USD
}
});
let response = format!("data: {text_chunk}\n\ndata: {usage_chunk}\n\ndata: [DONE]\n\n");
let completion_mock = server
.mock_async(|when, then| {
when.method(POST)
.path("/chat/completions")
.body_includes(r#""stream":true"#)
.body_includes("Report completion");
then.status(200)
.header("content-type", "text/event-stream")
.body(response);
})
.await;
let settings: LlmCatalogSettings = toml::from_str(&format!(
r#"
[providers.openrouter]
enabled = true
base_url = "{}"
"#,
server.base_url()
))
.expect("test catalog should parse");
let catalog = Arc::new(Catalog::from_builtin_with_overrides(&settings).unwrap());
let source = auth_test_support::env_credential_source(|name| {
(name == "OPENROUTER_API_KEY").then(|| "sk-test".to_string())
});
let backend = AgentApiBackend::new_with_catalog(
"openai/gpt-5.4".to_string(),
ProviderId::from("openrouter"),
ModelFallbackPolicy::default(),
source,
Arc::new(SteeringHub::new(Arc::new(Emitter::default()))),
catalog,
);
let mut graph = make_graph_with_start_exit("AuthoritativeOpenRouterCost");
let mut work = Node::new("work");
work.attrs.insert(
"prompt".to_string(),
AttrValue::String("Report completion".to_string()),
);
graph.nodes.insert("work".to_string(), work);
graph.edges.push(Edge::new("start", "work"));
graph.edges.push(Edge::new("work", "exit"));
let mut registry = HandlerRegistry::new(Box::new(AgentHandler::new(Some(Box::new(backend)))));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
let events = Arc::new(std::sync::Mutex::new(Vec::new()));
let events_for_listener = Arc::clone(&events);
let emitter = Arc::new(Emitter::default());
emitter.on_event(move |event| {
if event.event_name() == "agent.message" {
std::thread::sleep(Duration::from_millis(50));
}
events_for_listener.lock().unwrap().push(event.clone());
});
let dir = tempfile::tempdir().unwrap();
let engine = WorkflowRunner::new(registry, emitter, local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("authoritative-openrouter-cost"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (outcome, state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("workflow execution should complete");
assert_eq!(outcome.status, StageOutcome::Succeeded);
assert_eq!(completion_mock.calls_async().await, 1);
let work = state
.stage(&fabro_types::StageId::new("work", 1))
.expect("agent stage should be projected");
assert_eq!(work.usage.input_tokens, 11);
assert_eq!(work.usage.output_tokens, 7);
assert_eq!(
work.usage.total_usd_micros,
Some(AUTHORITATIVE_COST_USD_MICROS),
"provider-reported usage.cost should override the catalog estimate"
);
let events = events.lock().unwrap();
let agent_message = events
.iter()
.position(|event| event.event_name() == "agent.message")
.expect("agent message should be emitted");
let stage_completed = events
.iter()
.position(|event| {
event.event_name() == "stage.completed" && event.node_id.as_deref() == Some("work")
})
.expect("work stage completion should be emitted");
assert!(
agent_message < stage_completed,
"agent messages must be forwarded before terminal stage events"
);
}
// ---------------------------------------------------------------------------
// 12. Parallel fan-out / fan-in integration test (Gap #14)
// ---------------------------------------------------------------------------
#[tokio::test]
async fn end_to_end_parallel_fan_out_fan_in() {
use fabro_workflow::handler::fan_in::FanInHandler;
use fabro_workflow::handler::parallel::ParallelHandler;
let input = r#"digraph parallel_test {
start [shape=Mdiamond]
fan_out [shape=component]
branch_a [shape=box, prompt="Branch A work"]
branch_b [shape=box, prompt="Branch B work"]
fan_in_node [shape=tripleoctagon]
done [shape=Msquare]
start -> fan_out
fan_out -> branch_a
fan_out -> branch_b
branch_a -> fan_in_node
branch_b -> fan_in_node
fan_in_node -> done
}"#;
let graph = parse(input).expect("parse should succeed");
validate_or_raise(&graph, &[]).expect("validation should pass");
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(AgentHandler::new(Some(Box::new(
MockCodergenBackend,
)))));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"agent",
Box::new(AgentHandler::new(Some(Box::new(MockCodergenBackend)))),
);
registry.register("parallel", Box::new(ParallelHandler));
registry.register(
"parallel.fan_in",
Box::new(FanInHandler::new(Some(Box::new(MockCodergenBackend)))),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (outcome, state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("parallel pipeline should succeed");
assert_eq!(outcome.status, StageOutcome::Succeeded);
let checkpoint = state
.current_checkpoint()
.cloned()
.expect("checkpoint should be captured");
// The parallel node (fan_out) and fan_in_node should be in completed_nodes.
// Branch nodes run inside the parallel handler, so they are not recorded
// individually by the engine -- but fan_out and fan_in_node are top-level.
assert!(
checkpoint.completed_nodes.contains(&"fan_out".to_string()),
"fan_out should have been executed"
);
assert!(
checkpoint
.completed_nodes
.contains(&"fan_in_node".to_string()),
"fan_in_node should have been executed"
);
// Verify parallel.results was populated (both branches ran)
let parallel_results = checkpoint
.context_values
.get("parallel.results")
.expect("parallel.results should be in context");
let results_arr = parallel_results.as_array().expect("should be an array");
assert_eq!(results_arr.len(), 2, "should have 2 branch results");
}
// ---------------------------------------------------------------------------
// 13. Resume from checkpoint (P1)
// ---------------------------------------------------------------------------
#[tokio::test]
async fn resume_from_checkpoint_completes_pipeline() {
// Build a pipeline: start -> step_a -> step_b -> exit
// Create a checkpoint mid-pipeline (after step_a) and verify
// run_from_checkpoint completes from step_b onward.
let mut graph = Graph::new("ResumeTest");
graph.attrs.insert(
"goal".to_string(),
AttrValue::String("Test resume".to_string()),
);
let mut start = Node::new("start");
start.attrs.insert(
"shape".to_string(),
AttrValue::String("Mdiamond".to_string()),
);
graph.nodes.insert("start".to_string(), start);
let mut exit = Node::new("exit");
exit.attrs.insert(
"shape".to_string(),
AttrValue::String("Msquare".to_string()),
);
graph.nodes.insert("exit".to_string(), exit);
let step_a = Node::new("step_a");
graph.nodes.insert("step_a".to_string(), step_a);
let step_b = Node::new("step_b");
graph.nodes.insert("step_b".to_string(), step_b);
graph.edges.push(Edge::new("start", "step_a"));
graph.edges.push(Edge::new("step_a", "step_b"));
graph.edges.push(Edge::new("step_b", "exit"));
// Simulate a checkpoint saved after step_a completed.
// The checkpoint records step_a as current_node with next_node_id = step_b.
let ctx = Context::new();
ctx.set("graph.goal", serde_json::json!("Test resume"));
ctx.set("outcome", serde_json::json!("success"));
let mut outcomes = std::collections::HashMap::new();
outcomes.insert("start".to_string(), Outcome::success());
outcomes.insert("step_a".to_string(), Outcome::success());
let checkpoint = Checkpoint::from_context(
&ctx,
"step_a",
vec!["start".to_string(), "step_a".to_string()],
std::collections::HashMap::new(),
outcomes,
Some("step_b".to_string()),
std::collections::HashMap::new(),
std::collections::HashMap::new(),
std::collections::HashMap::new(),
);
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (outcome, state) = engine
.run_from_checkpoint_with_state(&graph, &run_options, &checkpoint)
.await
.expect("resume should succeed");
assert_eq!(outcome.status, StageOutcome::Succeeded);
// Verify checkpoint written after resume contains step_b
let final_cp = state
.current_checkpoint()
.cloned()
.expect("checkpoint should be captured");
assert!(
final_cp.completed_nodes.contains(&"step_b".to_string()),
"step_b should have been executed after resume"
);
// step_a should also be present (carried over from the checkpoint)
assert!(
final_cp.completed_nodes.contains(&"step_a".to_string()),
"step_a should be preserved from checkpoint"
);
// start should also be present
assert!(
final_cp.completed_nodes.contains(&"start".to_string()),
"start should be preserved from checkpoint"
);
}
#[tokio::test]
async fn resume_from_checkpoint_preserves_goal_gate_outcomes() {
// Build: start -> gated_work (goal_gate=true) -> step_b -> exit
// Checkpoint after gated_work (success), resume at step_b.
// At exit, goal gate should pass because outcomes are restored.
let mut graph = Graph::new("ResumeGoalGateTest");
let mut start = Node::new("start");
start.attrs.insert(
"shape".to_string(),
AttrValue::String("Mdiamond".to_string()),
);
graph.nodes.insert("start".to_string(), start);
let mut exit = Node::new("exit");
exit.attrs.insert(
"shape".to_string(),
AttrValue::String("Msquare".to_string()),
);
graph.nodes.insert("exit".to_string(), exit);
let mut gated_work = Node::new("gated_work");
gated_work
.attrs
.insert("goal_gate".to_string(), AttrValue::Boolean(true));
graph.nodes.insert("gated_work".to_string(), gated_work);
let step_b = Node::new("step_b");
graph.nodes.insert("step_b".to_string(), step_b);
graph.edges.push(Edge::new("start", "gated_work"));
graph.edges.push(Edge::new("gated_work", "step_b"));
graph.edges.push(Edge::new("step_b", "exit"));
// Checkpoint: gated_work completed with success, next is step_b
let ctx = Context::new();
ctx.set("outcome", serde_json::json!("success"));
let mut outcomes = std::collections::HashMap::new();
outcomes.insert("start".to_string(), Outcome::success());
outcomes.insert("gated_work".to_string(), Outcome::success());
let checkpoint = Checkpoint::from_context(
&ctx,
"gated_work",
vec!["start".to_string(), "gated_work".to_string()],
std::collections::HashMap::new(),
outcomes,
Some("step_b".to_string()),
std::collections::HashMap::new(),
std::collections::HashMap::new(),
std::collections::HashMap::new(),
);
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
// This should succeed because goal gate for gated_work is satisfied
// via restored outcomes
let outcome = engine
.run_from_checkpoint(&graph, &run_options, &checkpoint)
.await
.expect("resume with goal gate should succeed");
assert_eq!(outcome.status, StageOutcome::Succeeded);
}
// ===========================================================================
// Parity tests — P1: Core pipeline behaviors
// ===========================================================================
#[tokio::test]
async fn graph_goal_in_context() {
let input = r#"digraph GoalTest {
graph [goal="Ship the widget"]
start [shape=Mdiamond]
exit [shape=Msquare]
work [shape=box, prompt="Build it"]
start -> work -> exit
}"#;
let graph = parse(input).expect("parse");
let dir = tempfile::tempdir().unwrap();
let engine = WorkflowRunner::new(
make_linear_registry(),
Arc::new(Emitter::default()),
local_env(),
);
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (_outcome, state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("run");
let cp = state
.current_checkpoint()
.cloned()
.expect("checkpoint should exist");
assert_eq!(
cp.context_values.get("graph.goal"),
Some(&serde_json::json!("Ship the widget"))
);
}
#[tokio::test]
async fn event_streaming_lifecycle() {
let input = r#"digraph EventTest {
start [shape=Mdiamond]
exit [shape=Msquare]
task [shape=box, prompt="Do something"]
start -> task -> exit
}"#;
let graph = parse(input).expect("parse");
let dir = tempfile::tempdir().unwrap();
let emitter = Emitter::default();
let events = collect_events(&emitter);
let engine = WorkflowRunner::new(make_linear_registry(), Arc::new(emitter), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
engine.run(&graph, &run_options).await.expect("run");
let collected = events.lock().unwrap();
assert!(collected.iter().any(|e| e.event_name() == "run.started"));
assert!(
collected
.iter()
.any(|e| e.event_name() == "stage.started" && e.node_id.as_deref() == Some("start"))
);
assert!(
collected
.iter()
.any(|e| e.event_name() == "stage.completed" && e.node_id.as_deref() == Some("start"))
);
assert!(
collected
.iter()
.any(|e| e.event_name() == "stage.started" && e.node_id.as_deref() == Some("task"))
);
assert!(
collected
.iter()
.any(|e| e.event_name() == "stage.completed" && e.node_id.as_deref() == Some("task"))
);
assert!(
collected
.iter()
.any(|e| e.event_name() == "checkpoint.completed")
);
assert!(collected.iter().any(|e| e.event_name() == "run.completed"));
// WorkflowRunStarted first, WorkflowRunCompleted last
assert_eq!(collected.first().unwrap().event_name(), "run.started");
assert_eq!(collected.last().unwrap().event_name(), "run.completed");
}
#[tokio::test]
async fn context_flow_between_stages() {
let mut graph = make_graph_with_start_exit("ContextFlowTest");
let mut step_a = Node::new("step_a");
step_a
.attrs
.insert("shape".to_string(), AttrValue::String("box".to_string()));
step_a.attrs.insert(
"prompt".to_string(),
AttrValue::String("Step A work".to_string()),
);
graph.nodes.insert("step_a".to_string(), step_a);
let mut step_b = Node::new("step_b");
step_b
.attrs
.insert("shape".to_string(), AttrValue::String("box".to_string()));
step_b.attrs.insert(
"prompt".to_string(),
AttrValue::String("Step B work".to_string()),
);
graph.nodes.insert("step_b".to_string(), step_b);
graph.edges.push(Edge::new("start", "step_a"));
graph.edges.push(Edge::new("step_a", "step_b"));
graph.edges.push(Edge::new("step_b", "exit"));
let dir = tempfile::tempdir().unwrap();
let engine = WorkflowRunner::new(
make_linear_registry(),
Arc::new(Emitter::default()),
local_env(),
);
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (_outcome, state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("run");
let cp = state
.current_checkpoint()
.cloned()
.expect("checkpoint should exist");
assert_eq!(
cp.context_values.get("last_stage"),
Some(&serde_json::json!("step_b"))
);
let last_response = cp
.context_values
.get("last_response")
.unwrap()
.as_str()
.unwrap();
assert!(last_response.contains("[Simulated]"));
}
#[tokio::test]
async fn tool_handler_e2e() {
let mut graph = make_graph_with_start_exit("ToolTest");
let mut echo_task = Node::new("echo_task");
echo_task.attrs.insert(
"shape".to_string(),
AttrValue::String("parallelogram".to_string()),
);
echo_task.attrs.insert(
"script".to_string(),
AttrValue::String("echo hello-from-script".to_string()),
);
graph.nodes.insert("echo_task".to_string(), echo_task);
graph.edges.push(Edge::new("start", "echo_task"));
graph.edges.push(Edge::new("echo_task", "exit"));
let dir = tempfile::tempdir().unwrap();
let interviewer = Arc::new(AutoApproveInterviewer::engine());
let engine = WorkflowRunner::new(
make_full_registry(interviewer),
Arc::new(Emitter::default()),
local_env(),
);
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (outcome, _state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("run");
assert_eq!(outcome.status, StageOutcome::Succeeded);
let cp = load_run_checkpoint(dir.path()).unwrap();
let command_output = cp
.context_values
.get("command.output")
.expect("command.output should exist");
let command_output = resolve_checkpoint_text(dir.path(), command_output).unwrap();
assert!(command_output.contains("hello-from-script"));
}
#[tokio::test]
async fn auto_approve_interviewer_e2e() {
let mut graph = make_graph_with_start_exit("AutoApproveTest");
let mut gate = Node::new("gate");
gate.attrs.insert(
"shape".to_string(),
AttrValue::String("hexagon".to_string()),
);
gate.attrs
.insert("type".to_string(), AttrValue::String("human".to_string()));
gate.attrs
.insert("label".to_string(), AttrValue::String("Review".to_string()));
graph.nodes.insert("gate".to_string(), gate);
graph
.nodes
.insert("approve".to_string(), Node::new("approve"));
graph
.nodes
.insert("reject".to_string(), Node::new("reject"));
graph.edges.push(Edge::new("start", "gate"));
let mut e_approve = Edge::new("gate", "approve");
e_approve.attrs.insert(
"label".to_string(),
AttrValue::String("[A] Approve".to_string()),
);
graph.edges.push(e_approve);
let mut e_reject = Edge::new("gate", "reject");
e_reject.attrs.insert(
"label".to_string(),
AttrValue::String("[R] Reject".to_string()),
);
graph.edges.push(e_reject);
graph.edges.push(Edge::new("approve", "exit"));
graph.edges.push(Edge::new("reject", "exit"));
let dir = tempfile::tempdir().unwrap();
let interviewer = Arc::new(AutoApproveInterviewer::engine());
let engine = WorkflowRunner::new(
make_full_registry(interviewer),
Arc::new(Emitter::default()),
local_env(),
);
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (outcome, _state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("run");
assert_eq!(outcome.status, StageOutcome::Succeeded);
let cp = load_run_checkpoint(dir.path()).unwrap();
assert!(cp.completed_nodes.contains(&"approve".to_string()));
assert!(!cp.completed_nodes.contains(&"reject".to_string()));
}
#[tokio::test]
async fn codergen_without_backend_simulated() {
let input = r#"digraph SimTest {
start [shape=Mdiamond]
exit [shape=Msquare]
code [shape=box, prompt="Write the code"]
start -> code -> exit
}"#;
let graph = parse(input).expect("parse");
let dir = tempfile::tempdir().unwrap();
let engine = WorkflowRunner::new(
make_linear_registry(),
Arc::new(Emitter::default()),
local_env(),
);
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (_outcome, state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("run");
let cp = state
.current_checkpoint()
.cloned()
.expect("checkpoint should exist");
let last_response = cp
.context_values
.get("last_response")
.unwrap()
.as_str()
.unwrap();
assert!(last_response.contains("[Simulated]"));
assert!(last_response.contains("[Simulated]"));
}
// ===========================================================================
// Parity tests — P2: Complex scenarios
// ===========================================================================
#[tokio::test]
async fn branching_loop_back_on_failure() {
struct FailThenSucceedHandler {
call_count: std::sync::atomic::AtomicU32,
}
#[async_trait::async_trait]
impl Handler for FailThenSucceedHandler {
async fn execute(
&self,
_node: &Node,
_context: &Context,
_graph: &Graph,
_run_dir: &Path,
_services: &fabro_workflow::handler::EngineServices,
) -> Result<Outcome, Error> {
let count = self
.call_count
.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
if count == 0 {
Ok(Outcome::fail_classify("first attempt fails"))
} else {
Ok(Outcome::success())
}
}
}
let mut graph = make_graph_with_start_exit("LoopTest");
let mut implement = Node::new("implement");
implement
.attrs
.insert("shape".to_string(), AttrValue::String("box".to_string()));
implement.attrs.insert(
"prompt".to_string(),
AttrValue::String("Implement".to_string()),
);
graph.nodes.insert("implement".to_string(), implement);
let mut validate_node = Node::new("validate");
validate_node.attrs.insert(
"type".to_string(),
AttrValue::String("fail_then_succeed".to_string()),
);
graph.nodes.insert("validate".to_string(), validate_node);
graph.edges.push(Edge::new("start", "implement"));
graph.edges.push(Edge::new("implement", "validate"));
let mut e_success = Edge::new("validate", "exit");
e_success.attrs.insert(
"condition".to_string(),
AttrValue::String("outcome=succeeded".to_string()),
);
graph.edges.push(e_success);
let mut e_fail = Edge::new("validate", "implement");
e_fail.attrs.insert(
"condition".to_string(),
AttrValue::String("outcome=failed".to_string()),
);
graph.edges.push(e_fail);
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(AgentHandler::new(None)));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register("agent", Box::new(AgentHandler::new(None)));
registry.register(
"fail_then_succeed",
Box::new(FailThenSucceedHandler {
call_count: std::sync::atomic::AtomicU32::new(0),
}),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (outcome, _state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("run");
assert_eq!(outcome.status, StageOutcome::Succeeded);
let cp = load_run_checkpoint(dir.path()).unwrap();
let implement_count = cp
.completed_nodes
.iter()
.filter(|n| *n == "implement")
.count();
assert!(
implement_count >= 2,
"implement should appear at least 2x, got {implement_count}"
);
}
#[tokio::test]
async fn human_gate_loops_back() {
let mut graph = make_graph_with_start_exit("HumanLoopTest");
let mut gate = Node::new("gate");
gate.attrs.insert(
"shape".to_string(),
AttrValue::String("hexagon".to_string()),
);
gate.attrs
.insert("type".to_string(), AttrValue::String("human".to_string()));
gate.attrs
.insert("label".to_string(), AttrValue::String("Review".to_string()));
graph.nodes.insert("gate".to_string(), gate);
graph
.nodes
.insert("approve".to_string(), Node::new("approve"));
graph.nodes.insert("fix".to_string(), Node::new("fix"));
graph.edges.push(Edge::new("start", "gate"));
let mut e_approve = Edge::new("gate", "approve");
e_approve.attrs.insert(
"label".to_string(),
AttrValue::String("[A] Approve".to_string()),
);
graph.edges.push(e_approve);
let mut e_fix = Edge::new("gate", "fix");
e_fix.attrs.insert(
"label".to_string(),
AttrValue::String("[F] Fix".to_string()),
);
graph.edges.push(e_fix);
graph.edges.push(Edge::new("fix", "gate"));
graph.edges.push(Edge::new("approve", "exit"));
let answers = VecDeque::from([
Answer {
value: AnswerValue::Selected("F".to_string()),
selected_option: None,
text: None,
},
Answer {
value: AnswerValue::Selected("A".to_string()),
selected_option: None,
text: None,
},
]);
let interviewer = Arc::new(QueueInterviewer::new(answers));
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register("human", Box::new(HumanHandler::new(interviewer)));
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (outcome, _state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("run");
assert_eq!(outcome.status, StageOutcome::Succeeded);
let cp = load_run_checkpoint(dir.path()).unwrap();
let gate_count = cp.completed_nodes.iter().filter(|n| *n == "gate").count();
assert!(
gate_count >= 2,
"gate should appear at least 2x, got {gate_count}"
);
assert!(cp.completed_nodes.contains(&"approve".to_string()));
}
#[tokio::test]
async fn scenario_ship_a_feature() {
let dot = r#"digraph ShipFeature {
graph [goal="Ship the widget"]
rankdir=LR
start [shape=Mdiamond]
exit [shape=Msquare]
plan [shape=box, prompt="Plan to achieve: {{ goal }}"]
implement [shape=box, prompt="Implement the plan"]
test [shape=parallelogram, script="echo PASS"]
review [shape=hexagon, label="Review Changes"]
start -> plan -> implement -> test -> review
review -> exit [label="[A] Approve"]
review -> implement [label="[F] Fix"]
}"#;
let graph = parse(dot).expect("parse");
validate_or_raise(&graph, &[]).expect("validate");
let graph = TemplateTransform::new(std::collections::HashMap::new())
.apply(graph)
.unwrap();
assert_eq!(
graph.nodes["plan"].prompt().unwrap(),
"Plan to achieve: Ship the widget"
);
let interviewer = Arc::new(AutoApproveInterviewer::engine());
let dir = tempfile::tempdir().unwrap();
let emitter = Emitter::default();
let events = collect_events(&emitter);
let engine = WorkflowRunner::new(
make_full_registry(interviewer),
Arc::new(emitter),
local_env(),
);
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (outcome, _state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("run");
assert_eq!(outcome.status, StageOutcome::Succeeded);
let cp = load_run_checkpoint(dir.path()).unwrap();
let command_output = cp
.context_values
.get("command.output")
.expect("command.output");
let command_output = resolve_checkpoint_text(dir.path(), command_output).unwrap();
assert!(command_output.contains("PASS"));
assert!(cp.completed_nodes.contains(&"plan".to_string()));
assert!(cp.completed_nodes.contains(&"implement".to_string()));
assert!(cp.completed_nodes.contains(&"test".to_string()));
assert!(cp.completed_nodes.contains(&"review".to_string()));
let collected = events.lock().unwrap();
assert!(collected.iter().any(|e| e.event_name() == "run.started"));
assert!(collected.iter().any(|e| e.event_name() == "run.completed"));
}
#[tokio::test]
async fn scenario_parallel_expert_review() {
use fabro_workflow::handler::fan_in::FanInHandler;
use fabro_workflow::handler::parallel::ParallelHandler;
let input = r#"digraph ParallelReview {
start [shape=Mdiamond]
fan_out [shape=component]
expert_a [shape=box, prompt="Expert A review"]
expert_b [shape=box, prompt="Expert B review"]
expert_c [shape=box, prompt="Expert C review"]
fan_in_node [shape=tripleoctagon]
review [shape=hexagon, label="Final Review"]
exit [shape=Msquare]
start -> fan_out
fan_out -> expert_a
fan_out -> expert_b
fan_out -> expert_c
expert_a -> fan_in_node
expert_b -> fan_in_node
expert_c -> fan_in_node
fan_in_node -> review
review -> exit [label="[A] Approve"]
review -> fan_out [label="[F] Redo"]
}"#;
let graph = parse(input).expect("parse");
validate_or_raise(&graph, &[]).expect("validate");
let recorder = Arc::new(RecordingInterviewer::new(Box::new(
AutoApproveInterviewer::engine(),
)));
let dir = tempfile::tempdir().unwrap();
let interviewer: Arc<dyn Interviewer> = recorder.clone();
let mut registry = HandlerRegistry::new(Box::new(AgentHandler::new(Some(Box::new(
MockCodergenBackend,
)))));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"agent",
Box::new(AgentHandler::new(Some(Box::new(MockCodergenBackend)))),
);
registry.register("parallel", Box::new(ParallelHandler));
registry.register(
"parallel.fan_in",
Box::new(FanInHandler::new(Some(Box::new(MockCodergenBackend)))),
);
registry.register("human", Box::new(HumanHandler::new(interviewer)));
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (outcome, state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("run");
assert_eq!(outcome.status, StageOutcome::Succeeded);
let cp = state
.current_checkpoint()
.cloned()
.expect("checkpoint should exist");
let results = cp
.context_values
.get("parallel.results")
.expect("parallel.results");
assert_eq!(results.as_array().unwrap().len(), 3);
let recordings = recorder.recordings();
assert_eq!(recordings.len(), 1, "should have 1 interview recording");
assert!(cp.completed_nodes.contains(&"review".to_string()));
}
#[tokio::test]
async fn scenario_node_retries_on_retry_status() {
struct RetryHandler {
call_count: std::sync::atomic::AtomicU32,
}
#[async_trait::async_trait]
impl Handler for RetryHandler {
async fn execute(
&self,
_node: &Node,
_context: &Context,
_graph: &Graph,
_run_dir: &Path,
_services: &fabro_workflow::handler::EngineServices,
) -> Result<Outcome, Error> {
let count = self
.call_count
.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
if count == 0 {
Ok(Outcome::retry_classify("transient failure"))
} else {
Ok(Outcome::success())
}
}
}
let mut graph = make_graph_with_start_exit("RetryScenarioTest");
let mut flaky = Node::new("flaky");
flaky.attrs.insert(
"type".to_string(),
AttrValue::String("retry_handler".to_string()),
);
flaky
.attrs
.insert("max_retries".to_string(), AttrValue::Integer(2));
flaky.attrs.insert(
"retry_policy".to_string(),
AttrValue::String("linear".to_string()),
);
graph.nodes.insert("flaky".to_string(), flaky);
graph.edges.push(Edge::new("start", "flaky"));
graph.edges.push(Edge::new("flaky", "exit"));
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"retry_handler",
Box::new(RetryHandler {
call_count: std::sync::atomic::AtomicU32::new(0),
}),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (outcome, state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("run");
assert_eq!(outcome.status, StageOutcome::Succeeded);
let cp = state
.current_checkpoint()
.cloned()
.expect("checkpoint should exist");
let retry_count = cp
.node_retries
.get("flaky")
.expect("flaky should have retries");
assert_eq!(*retry_count, 1, "should have retried once");
}
#[tokio::test]
async fn scenario_loop_restart_resets_context() {
let mut graph = make_graph_with_start_exit("LoopRestartTest");
let mut work = Node::new("work");
work.attrs
.insert("type".to_string(), AttrValue::String("counter".to_string()));
graph.nodes.insert("work".to_string(), work);
graph.edges.push(Edge::new("start", "work"));
let mut success_edge = Edge::new("work", "exit");
success_edge.attrs.insert(
"condition".to_string(),
AttrValue::String("outcome=succeeded".to_string()),
);
graph.edges.push(success_edge);
let mut fail_edge = Edge::new("work", "start");
fail_edge.attrs.insert(
"condition".to_string(),
AttrValue::String("outcome=failed".to_string()),
);
fail_edge
.attrs
.insert("loop_restart".to_string(), AttrValue::Boolean(true));
graph.edges.push(fail_edge);
let call_count = Arc::new(std::sync::atomic::AtomicU32::new(0));
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"counter",
Box::new(CounterHandler {
call_count: Arc::clone(&call_count),
}),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let outcome = engine.run(&graph, &run_options).await.expect("run");
assert_eq!(outcome.status, StageOutcome::Succeeded);
assert!(call_count.load(std::sync::atomic::Ordering::SeqCst) >= 2);
}
#[tokio::test]
async fn scenario_bug_triage_router() {
let mut graph = make_graph_with_start_exit("TriageTest");
let mut triage = Node::new("triage");
triage.attrs.insert(
"shape".to_string(),
AttrValue::String("diamond".to_string()),
);
graph.nodes.insert("triage".to_string(), triage);
graph
.nodes
.insert("critical".to_string(), Node::new("critical"));
graph
.nodes
.insert("normal".to_string(), Node::new("normal"));
graph
.nodes
.insert("wontfix".to_string(), Node::new("wontfix"));
graph.edges.push(Edge::new("start", "triage"));
let mut e_critical = Edge::new("triage", "critical");
e_critical.attrs.insert(
"condition".to_string(),
AttrValue::String("outcome=succeeded".to_string()),
);
e_critical
.attrs
.insert("weight".to_string(), AttrValue::Integer(10));
graph.edges.push(e_critical);
let mut e_normal = Edge::new("triage", "normal");
e_normal.attrs.insert(
"condition".to_string(),
AttrValue::String("outcome=succeeded".to_string()),
);
e_normal
.attrs
.insert("weight".to_string(), AttrValue::Integer(5));
graph.edges.push(e_normal);
graph.edges.push(Edge::new("triage", "wontfix"));
graph.edges.push(Edge::new("critical", "exit"));
graph.edges.push(Edge::new("normal", "exit"));
graph.edges.push(Edge::new("wontfix", "exit"));
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register("conditional", Box::new(ConditionalHandler));
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (outcome, state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("run");
assert_eq!(outcome.status, StageOutcome::Succeeded);
let cp = state
.current_checkpoint()
.cloned()
.expect("checkpoint should exist");
assert!(
cp.completed_nodes.contains(&"critical".to_string()),
"critical should be selected (highest weight)"
);
assert!(!cp.completed_nodes.contains(&"normal".to_string()));
assert!(!cp.completed_nodes.contains(&"wontfix".to_string()));
}
#[tokio::test]
async fn scenario_crash_recovery() {
let mut graph = make_graph_with_start_exit("CrashRecoveryTest");
graph.nodes.insert("a".to_string(), Node::new("a"));
graph.nodes.insert("b".to_string(), Node::new("b"));
graph.nodes.insert("c".to_string(), Node::new("c"));
graph.edges.push(Edge::new("start", "a"));
graph.edges.push(Edge::new("a", "b"));
graph.edges.push(Edge::new("b", "c"));
graph.edges.push(Edge::new("c", "exit"));
let ctx = Context::new();
ctx.set("outcome", serde_json::json!("success"));
let mut outcomes = std::collections::HashMap::new();
outcomes.insert("start".to_string(), Outcome::success());
outcomes.insert("a".to_string(), Outcome::success());
let checkpoint = Checkpoint::from_context(
&ctx,
"a",
vec!["start".to_string(), "a".to_string()],
std::collections::HashMap::new(),
outcomes,
Some("b".to_string()),
std::collections::HashMap::new(),
std::collections::HashMap::new(),
std::collections::HashMap::new(),
);
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (outcome, state) = engine
.run_from_checkpoint_with_state(&graph, &run_options, &checkpoint)
.await
.expect("run");
assert_eq!(outcome.status, StageOutcome::Succeeded);
let cp = state
.current_checkpoint()
.cloned()
.expect("checkpoint should be captured");
assert!(cp.completed_nodes.contains(&"b".to_string()));
assert!(cp.completed_nodes.contains(&"c".to_string()));
assert!(cp.completed_nodes.contains(&"a".to_string()));
let a_count = cp.completed_nodes.iter().filter(|n| *n == "a").count();
assert_eq!(a_count, 1, "a should not be re-executed");
}
#[tokio::test]
async fn manager_loop_stop_condition_satisfied_e2e() {
struct DoneSetterHandler;
#[async_trait::async_trait]
impl Handler for DoneSetterHandler {
async fn execute(
&self,
_node: &Node,
_context: &Context,
_graph: &Graph,
_run_dir: &Path,
_services: &fabro_workflow::handler::EngineServices,
) -> Result<Outcome, Error> {
let mut outcome = Outcome::success();
outcome
.context_updates
.insert("done".to_string(), serde_json::json!("true"));
Ok(outcome)
}
}
// A slow handler so the child doesn't finish before the stop condition is
// checked
struct SlowHandler;
#[async_trait::async_trait]
impl Handler for SlowHandler {
async fn execute(
&self,
_node: &Node,
_context: &Context,
_graph: &Graph,
_run_dir: &Path,
_services: &fabro_workflow::handler::EngineServices,
) -> Result<Outcome, Error> {
tokio::time::sleep(std::time::Duration::from_secs(10)).await;
Ok(Outcome::success())
}
}
let mut graph = make_graph_with_start_exit("ManagerStopTest");
let mut setter = Node::new("setter");
setter.attrs.insert(
"type".to_string(),
AttrValue::String("done_setter".to_string()),
);
graph.nodes.insert("setter".to_string(), setter);
let mut manager = Node::new("manager");
manager.attrs.insert(
"type".to_string(),
AttrValue::String("stack.manager_loop".to_string()),
);
manager.attrs.insert(
"stack.child_dot_source".to_string(),
AttrValue::String(
"digraph Child { start [shape=Mdiamond]; slow [shape=box]; exit [shape=Msquare]; start -> slow -> exit }"
.to_string(),
),
);
manager.attrs.insert(
"manager.stop_condition".to_string(),
AttrValue::String("context.done=true".to_string()),
);
manager
.attrs
.insert("manager.max_cycles".to_string(), AttrValue::Integer(10));
manager.attrs.insert(
"manager.poll_interval".to_string(),
AttrValue::Duration(std::time::Duration::from_millis(1)),
);
graph.nodes.insert("manager".to_string(), manager);
graph.edges.push(Edge::new("start", "setter"));
graph.edges.push(Edge::new("setter", "manager"));
graph.edges.push(Edge::new("manager", "exit"));
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(SlowHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register("done_setter", Box::new(DoneSetterHandler));
registry.register("stack.manager_loop", Box::new(SubWorkflowHandler));
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (outcome, state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("run");
let cp = state
.current_checkpoint()
.cloned()
.expect("checkpoint should be captured");
let manager_outcome = cp.node_outcomes.get("manager").expect("manager outcome");
assert_eq!(manager_outcome.status, StageOutcome::Succeeded);
assert!(
manager_outcome
.notes
.as_deref()
.unwrap()
.contains("Stop condition satisfied")
);
// Overall pipeline succeeds because manager succeeded
assert_eq!(outcome.status, StageOutcome::Succeeded);
}
#[tokio::test]
async fn manager_loop_max_cycles_exceeded_e2e() {
// A slow handler so the child doesn't finish before max cycles
struct SlowHandler;
#[async_trait::async_trait]
impl Handler for SlowHandler {
async fn execute(
&self,
_node: &Node,
_context: &Context,
_graph: &Graph,
_run_dir: &Path,
_services: &fabro_workflow::handler::EngineServices,
) -> Result<Outcome, Error> {
tokio::time::sleep(std::time::Duration::from_secs(10)).await;
Ok(Outcome::success())
}
}
let mut graph = make_graph_with_start_exit("ManagerMaxCyclesTest");
let mut manager = Node::new("manager");
manager.attrs.insert(
"type".to_string(),
AttrValue::String("stack.manager_loop".to_string()),
);
manager.attrs.insert(
"stack.child_dot_source".to_string(),
AttrValue::String(
"digraph Child { start [shape=Mdiamond]; slow [shape=box]; exit [shape=Msquare]; start -> slow -> exit }"
.to_string(),
),
);
manager
.attrs
.insert("manager.max_cycles".to_string(), AttrValue::Integer(2));
manager.attrs.insert(
"manager.poll_interval".to_string(),
AttrValue::Duration(std::time::Duration::from_millis(1)),
);
graph.nodes.insert("manager".to_string(), manager);
graph.edges.push(Edge::new("start", "manager"));
graph.edges.push(Edge::new("manager", "exit"));
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(SlowHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register("stack.manager_loop", Box::new(SubWorkflowHandler));
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (outcome, state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("run");
let cp = state
.current_checkpoint()
.cloned()
.expect("checkpoint should be captured");
let manager_outcome = cp.node_outcomes.get("manager").expect("manager outcome");
assert_eq!(manager_outcome.status, StageOutcome::Failed {
retry_requested: false,
});
assert!(
manager_outcome
.failure_reason()
.unwrap()
.contains("Max cycles")
);
// Pipeline reached exit with goal gates satisfied — per spec, SUCCESS.
assert_eq!(outcome.status, StageOutcome::Succeeded);
}
// ===========================================================================
// Parity tests — P3: Validation
// ===========================================================================
#[test]
fn validation_missing_start_node() {
let mut graph = Graph::new("NoStartTest");
let mut exit = Node::new("exit");
exit.attrs.insert(
"shape".to_string(),
AttrValue::String("Msquare".to_string()),
);
graph.nodes.insert("exit".to_string(), exit);
let diagnostics = validate(&graph, &[]);
let start_errors: Vec<_> = diagnostics
.iter()
.filter(|d| d.severity == Severity::Error && d.rule == "start_node")
.collect();
assert!(
!start_errors.is_empty(),
"should have start_node error diagnostic"
);
}
#[test]
fn validation_missing_exit_node() {
let mut graph = Graph::new("NoExitTest");
let mut start = Node::new("start");
start.attrs.insert(
"shape".to_string(),
AttrValue::String("Mdiamond".to_string()),
);
graph.nodes.insert("start".to_string(), start);
graph.nodes.insert("work".to_string(), Node::new("work"));
graph.edges.push(Edge::new("start", "work"));
let diagnostics = validate(&graph, &[]);
let exit_errors: Vec<_> = diagnostics
.iter()
.filter(|d| d.severity == Severity::Error && d.rule == "terminal_node")
.collect();
assert!(
!exit_errors.is_empty(),
"should have terminal_node error diagnostic"
);
}
#[test]
fn validation_orphan_unreachable_node() {
let mut graph = make_graph_with_start_exit("OrphanTest");
graph
.nodes
.insert("orphan".to_string(), Node::new("orphan"));
graph.edges.push(Edge::new("start", "exit"));
let diagnostics = validate(&graph, &[]);
let reachability_errors: Vec<_> = diagnostics
.iter()
.filter(|d| d.rule == "reachability")
.collect();
assert!(
!reachability_errors.is_empty(),
"should have reachability diagnostic for orphan node"
);
}
// ===========================================================================
// Parity tests — P4: Edge selection and cross-feature
// ===========================================================================
#[tokio::test]
async fn conditional_branching_success_fail_paths() {
let mut graph = make_graph_with_start_exit("CondBranchTest");
let mut work = Node::new("work");
work.attrs.insert(
"type".to_string(),
AttrValue::String("always_fail".to_string()),
);
graph.nodes.insert("work".to_string(), work);
graph
.nodes
.insert("success_path".to_string(), Node::new("success_path"));
graph
.nodes
.insert("fail_path".to_string(), Node::new("fail_path"));
graph.edges.push(Edge::new("start", "work"));
let mut e_success = Edge::new("work", "success_path");
e_success.attrs.insert(
"condition".to_string(),
AttrValue::String("outcome=succeeded".to_string()),
);
graph.edges.push(e_success);
let mut e_fail = Edge::new("work", "fail_path");
e_fail.attrs.insert(
"condition".to_string(),
AttrValue::String("outcome=failed".to_string()),
);
graph.edges.push(e_fail);
graph.edges.push(Edge::new("success_path", "exit"));
graph.edges.push(Edge::new("fail_path", "exit"));
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register("always_fail", Box::new(AlwaysFailHandler));
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (outcome, state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("run");
assert_eq!(outcome.status, StageOutcome::Succeeded);
let cp = state
.current_checkpoint()
.cloned()
.expect("checkpoint should exist");
assert!(cp.completed_nodes.contains(&"fail_path".to_string()));
assert!(!cp.completed_nodes.contains(&"success_path".to_string()));
}
#[tokio::test]
async fn edge_selection_condition_match_wins_over_weight() {
let mut graph = make_graph_with_start_exit("CondVsWeightTest");
graph.nodes.insert("a".to_string(), Node::new("a"));
graph
.nodes
.insert("cond_target".to_string(), Node::new("cond_target"));
graph
.nodes
.insert("weighted_target".to_string(), Node::new("weighted_target"));
graph.edges.push(Edge::new("start", "a"));
let mut e_cond = Edge::new("a", "cond_target");
e_cond.attrs.insert(
"condition".to_string(),
AttrValue::String("outcome=succeeded".to_string()),
);
graph.edges.push(e_cond);
let mut e_weight = Edge::new("a", "weighted_target");
e_weight
.attrs
.insert("weight".to_string(), AttrValue::Integer(100));
graph.edges.push(e_weight);
graph.edges.push(Edge::new("cond_target", "exit"));
graph.edges.push(Edge::new("weighted_target", "exit"));
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (_outcome, state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("run");
let cp = state
.current_checkpoint()
.cloned()
.expect("checkpoint should exist");
assert!(cp.completed_nodes.contains(&"cond_target".to_string()));
assert!(!cp.completed_nodes.contains(&"weighted_target".to_string()));
}
#[tokio::test]
async fn edge_selection_weight_breaks_ties() {
let mut graph = make_graph_with_start_exit("WeightTiesTest");
graph.nodes.insert("a".to_string(), Node::new("a"));
graph.nodes.insert("low".to_string(), Node::new("low"));
graph.nodes.insert("high".to_string(), Node::new("high"));
graph.edges.push(Edge::new("start", "a"));
let mut e_low = Edge::new("a", "low");
e_low
.attrs
.insert("weight".to_string(), AttrValue::Integer(1));
graph.edges.push(e_low);
let mut e_high = Edge::new("a", "high");
e_high
.attrs
.insert("weight".to_string(), AttrValue::Integer(10));
graph.edges.push(e_high);
graph.edges.push(Edge::new("low", "exit"));
graph.edges.push(Edge::new("high", "exit"));
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (_outcome, state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("run");
let cp = state
.current_checkpoint()
.cloned()
.expect("checkpoint should exist");
assert!(cp.completed_nodes.contains(&"high".to_string()));
assert!(!cp.completed_nodes.contains(&"low".to_string()));
}
#[tokio::test]
async fn edge_selection_lexical_tiebreak() {
let mut graph = make_graph_with_start_exit("LexicalTieTest");
graph.nodes.insert("a".to_string(), Node::new("a"));
graph.nodes.insert("beta".to_string(), Node::new("beta"));
graph.nodes.insert("alpha".to_string(), Node::new("alpha"));
graph.edges.push(Edge::new("start", "a"));
graph.edges.push(Edge::new("a", "beta"));
graph.edges.push(Edge::new("a", "alpha"));
graph.edges.push(Edge::new("beta", "exit"));
graph.edges.push(Edge::new("alpha", "exit"));
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (_outcome, state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("run");
let cp = state
.current_checkpoint()
.cloned()
.expect("checkpoint should exist");
assert!(cp.completed_nodes.contains(&"alpha".to_string()));
assert!(!cp.completed_nodes.contains(&"beta".to_string()));
}
#[tokio::test]
async fn context_updates_visible_across_nodes() {
let mut graph = make_graph_with_start_exit("ContextVisibilityTest");
let mut setter = Node::new("setter");
setter.attrs.insert(
"type".to_string(),
AttrValue::String("context_setter".to_string()),
);
graph.nodes.insert("setter".to_string(), setter);
let mut gate = Node::new("gate");
gate.attrs.insert(
"shape".to_string(),
AttrValue::String("diamond".to_string()),
);
graph.nodes.insert("gate".to_string(), gate);
graph.nodes.insert("yes".to_string(), Node::new("yes"));
graph.nodes.insert("no".to_string(), Node::new("no"));
graph.edges.push(Edge::new("start", "setter"));
graph.edges.push(Edge::new("setter", "gate"));
let mut e_yes = Edge::new("gate", "yes");
e_yes.attrs.insert(
"condition".to_string(),
AttrValue::String("context.my_flag=set".to_string()),
);
graph.edges.push(e_yes);
graph.edges.push(Edge::new("gate", "no"));
graph.edges.push(Edge::new("yes", "exit"));
graph.edges.push(Edge::new("no", "exit"));
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register("conditional", Box::new(ConditionalHandler));
registry.register("context_setter", Box::new(ContextSetterHandler));
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (_outcome, state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("run");
let cp = state
.current_checkpoint()
.cloned()
.expect("checkpoint should exist");
assert!(cp.completed_nodes.contains(&"yes".to_string()));
assert!(!cp.completed_nodes.contains(&"no".to_string()));
}
#[tokio::test]
async fn stylesheet_applies_model_override() {
let input = r#"digraph StylesheetTest {
graph [
goal="Test stylesheet",
model_stylesheet="* { model: custom-model; }"
]
start [shape=Mdiamond]
exit [shape=Msquare]
work [shape=box, prompt="Do work"]
start -> work -> exit
}"#;
let graph = parse(input).expect("parse");
validate_or_raise(&graph, &[]).expect("validate");
let graph = StylesheetApplicationTransform.apply(graph).unwrap();
assert_eq!(graph.nodes["work"].model(), Some("custom-model"));
let dir = tempfile::tempdir().unwrap();
let engine = WorkflowRunner::new(
make_linear_registry(),
Arc::new(Emitter::default()),
local_env(),
);
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let outcome = engine.run(&graph, &run_options).await.expect("run");
assert_eq!(outcome.status, StageOutcome::Succeeded);
}
#[tokio::test]
async fn custom_handler_registration_and_execution() {
struct CustomHandler;
#[async_trait::async_trait]
impl Handler for CustomHandler {
async fn execute(
&self,
_node: &Node,
_context: &Context,
_graph: &Graph,
_run_dir: &Path,
_services: &fabro_workflow::handler::EngineServices,
) -> Result<Outcome, Error> {
let mut outcome = Outcome::success();
outcome
.context_updates
.insert("custom.ran".to_string(), serde_json::json!("true"));
Ok(outcome)
}
}
let mut graph = make_graph_with_start_exit("CustomHandlerTest");
let mut custom = Node::new("custom");
custom.attrs.insert(
"type".to_string(),
AttrValue::String("my_custom".to_string()),
);
graph.nodes.insert("custom".to_string(), custom);
graph.edges.push(Edge::new("start", "custom"));
graph.edges.push(Edge::new("custom", "exit"));
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register("my_custom", Box::new(CustomHandler));
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (_outcome, state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("run");
let cp = state
.current_checkpoint()
.cloned()
.expect("checkpoint should exist");
assert_eq!(
cp.context_values.get("custom.ran"),
Some(&serde_json::json!("true"))
);
}
#[tokio::test]
async fn integration_smoke_plan_implement_review_done() {
let dot = r#"digraph SmokeIntegration {
graph [
goal="Build the feature",
model_stylesheet="* { model: test-model; }"
]
rankdir=LR
start [shape=Mdiamond]
exit [shape=Msquare]
plan [shape=box, prompt="Plan: {{ goal }}"]
implement [shape=box, prompt="Implement"]
review [shape=hexagon, label="Review"]
start -> plan -> implement -> review
review -> exit [label="[A] Approve"]
review -> implement [label="[F] Fix"]
}"#;
// Parse and validate
let graph = parse(dot).expect("parse");
let diagnostics = validate_or_raise(&graph, &[]).expect("validate");
let errors: Vec<_> = diagnostics
.iter()
.filter(|d| d.severity == Severity::Error)
.collect();
assert!(errors.is_empty());
// Apply transforms
let graph = TemplateTransform::new(std::collections::HashMap::new())
.apply(graph)
.unwrap();
let graph = StylesheetApplicationTransform.apply(graph).unwrap();
// Verify transforms applied
assert_eq!(
graph.nodes["plan"].prompt().unwrap(),
"Plan: Build the feature"
);
assert_eq!(graph.nodes["plan"].model(), Some("test-model"));
// Run pipeline
let interviewer = Arc::new(AutoApproveInterviewer::engine());
let dir = tempfile::tempdir().unwrap();
let emitter = Emitter::default();
let events = collect_events(&emitter);
let engine = WorkflowRunner::new(
make_full_registry(interviewer),
Arc::new(emitter),
local_env(),
);
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (outcome, state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("run");
assert_eq!(outcome.status, StageOutcome::Succeeded);
let cp = load_run_checkpoint(dir.path()).unwrap();
assert!(cp.completed_nodes.contains(&"plan".to_string()));
assert!(cp.completed_nodes.contains(&"implement".to_string()));
assert!(cp.completed_nodes.contains(&"review".to_string()));
let plan_state = state.stage(&fabro_types::StageId::new("plan", 1)).unwrap();
assert!(plan_state.prompt.is_some());
assert!(plan_state.response.is_some());
// Verify events
let collected = events.lock().unwrap();
assert!(collected.iter().any(|e| e.event_name() == "run.started"));
assert!(collected.iter().any(|e| e.event_name() == "run.completed"));
}
// ===========================================================================
// 19b. Manager loop runs child engine E2E
// ===========================================================================
#[tokio::test]
async fn manager_loop_runs_child_engine_e2e() {
let mut graph = Graph::new("ManagerLoopE2E");
graph.attrs.insert(
"goal".to_string(),
AttrValue::String("Test manager loop".to_string()),
);
let mut start = Node::new("start");
start.attrs.insert(
"shape".to_string(),
AttrValue::String("Mdiamond".to_string()),
);
graph.nodes.insert("start".to_string(), start);
let mut exit = Node::new("exit");
exit.attrs.insert(
"shape".to_string(),
AttrValue::String("Msquare".to_string()),
);
graph.nodes.insert("exit".to_string(), exit);
let mut supervisor = Node::new("supervisor");
supervisor.attrs.insert(
"type".to_string(),
AttrValue::String("stack.manager_loop".to_string()),
);
supervisor.attrs.insert(
"stack.child_dot_source".to_string(),
AttrValue::String(
"digraph Child { start [shape=Mdiamond]; exit [shape=Msquare]; start -> exit }"
.to_string(),
),
);
supervisor.attrs.insert(
"manager.poll_interval".to_string(),
AttrValue::Duration(std::time::Duration::from_millis(10)),
);
supervisor
.attrs
.insert("manager.max_cycles".to_string(), AttrValue::Integer(100));
graph.nodes.insert("supervisor".to_string(), supervisor);
graph.edges.push(Edge::new("start", "supervisor"));
graph.edges.push(Edge::new("supervisor", "exit"));
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register("stack.manager_loop", Box::new(SubWorkflowHandler));
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (outcome, state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("manager loop E2E should succeed");
assert_eq!(outcome.status, StageOutcome::Succeeded);
let checkpoint = state
.current_checkpoint()
.cloned()
.expect("checkpoint should be captured");
assert!(
checkpoint
.completed_nodes
.contains(&"supervisor".to_string()),
"supervisor should be in completed_nodes"
);
let supervisor_outcome = checkpoint.node_outcomes.get("supervisor");
assert!(
supervisor_outcome.is_some(),
"supervisor outcome should exist"
);
let notes = supervisor_outcome.unwrap().notes.as_deref().unwrap_or("");
assert!(
notes.contains("Child completed"),
"notes should mention child completion, got: {notes}"
);
}
// ===========================================================================
// 19b-2. Manager loop: context flows parent → child → parent
// ===========================================================================
#[tokio::test]
async fn manager_loop_context_flows_e2e() {
// Handler that reads parent's context value and sets a result
struct ContextEchoHandler;
#[async_trait::async_trait]
impl Handler for ContextEchoHandler {
async fn execute(
&self,
_node: &Node,
context: &Context,
_graph: &Graph,
_run_dir: &Path,
_services: &fabro_workflow::handler::EngineServices,
) -> Result<Outcome, Error> {
let target = context.get_string("review.target", "");
let mut outcome = Outcome::success();
outcome
.context_updates
.insert("review.result".to_string(), serde_json::json!("approved"));
outcome
.context_updates
.insert("review.echo".to_string(), serde_json::json!(target));
Ok(outcome)
}
}
let mut graph = make_graph_with_start_exit("ManagerContextFlowE2E");
// A setter node that puts review.target into context before the manager
struct SetterHandler;
#[async_trait::async_trait]
impl Handler for SetterHandler {
async fn execute(
&self,
_node: &Node,
_context: &Context,
_graph: &Graph,
_run_dir: &Path,
_services: &fabro_workflow::handler::EngineServices,
) -> Result<Outcome, Error> {
let mut outcome = Outcome::success();
outcome.context_updates.insert(
"review.target".to_string(),
serde_json::json!("src/main.rs"),
);
Ok(outcome)
}
}
let mut setter = Node::new("setter");
setter
.attrs
.insert("type".to_string(), AttrValue::String("setter".to_string()));
graph.nodes.insert("setter".to_string(), setter);
let mut supervisor = Node::new("supervisor");
supervisor.attrs.insert(
"type".to_string(),
AttrValue::String("stack.manager_loop".to_string()),
);
supervisor.attrs.insert(
"stack.child_dot_source".to_string(),
AttrValue::String(
"digraph Child { start [shape=Mdiamond]; work [shape=box]; exit [shape=Msquare]; start -> work -> exit }"
.to_string(),
),
);
supervisor.attrs.insert(
"manager.poll_interval".to_string(),
AttrValue::Duration(std::time::Duration::from_millis(10)),
);
supervisor
.attrs
.insert("manager.max_cycles".to_string(), AttrValue::Integer(100));
graph.nodes.insert("supervisor".to_string(), supervisor);
graph.edges.push(Edge::new("start", "setter"));
graph.edges.push(Edge::new("setter", "supervisor"));
graph.edges.push(Edge::new("supervisor", "exit"));
let dir = tempfile::tempdir().unwrap();
// Default handler = ContextEchoHandler (handles the child's "work" node)
let mut registry = HandlerRegistry::new(Box::new(ContextEchoHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register("setter", Box::new(SetterHandler));
registry.register("stack.manager_loop", Box::new(SubWorkflowHandler));
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (outcome, state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("run");
assert_eq!(outcome.status, StageOutcome::Succeeded);
// Check that child's context updates were propagated through the manager
let checkpoint = state
.current_checkpoint()
.cloned()
.expect("checkpoint should be captured");
let sup_outcome = checkpoint.node_outcomes.get("supervisor").unwrap();
assert_eq!(
sup_outcome.context_updates.get("review.result"),
Some(&serde_json::json!("approved")),
"child's review.result should propagate to parent"
);
assert_eq!(
sup_outcome.context_updates.get("review.echo"),
Some(&serde_json::json!("src/main.rs")),
"child should have read parent's review.target"
);
}
// ===========================================================================
// 19b-3. Manager loop with child_workflow E2E
// ===========================================================================
#[tokio::test]
async fn manager_loop_child_workflow_e2e() {
let dir = tempfile::tempdir().unwrap();
let dot_path = dir.path().join("child.dot");
std::fs::write(
&dot_path,
"digraph Child { start [shape=Mdiamond]; exit [shape=Msquare]; start -> exit }",
)
.unwrap();
let mut graph = make_graph_with_start_exit("ManagerDotfileE2E");
let mut supervisor = Node::new("supervisor");
supervisor.attrs.insert(
"type".to_string(),
AttrValue::String("stack.manager_loop".to_string()),
);
supervisor.attrs.insert(
"stack.child_workflow".to_string(),
AttrValue::String(dot_path.to_string_lossy().to_string()),
);
supervisor.attrs.insert(
"manager.poll_interval".to_string(),
AttrValue::Duration(std::time::Duration::from_millis(10)),
);
supervisor
.attrs
.insert("manager.max_cycles".to_string(), AttrValue::Integer(100));
graph.nodes.insert("supervisor".to_string(), supervisor);
graph.edges.push(Edge::new("start", "supervisor"));
graph.edges.push(Edge::new("supervisor", "exit"));
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register("stack.manager_loop", Box::new(SubWorkflowHandler));
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let outcome = engine.run(&graph, &run_options).await.expect("run");
assert_eq!(outcome.status, StageOutcome::Succeeded);
}
// ===========================================================================
// 19c. ImportTransform E2E (TS Scenario 11)
// ===========================================================================
#[tokio::test]
async fn import_e2e_through_engine() {
use fabro_workflow::pipeline::{TransformOptions, transform, validate};
use fabro_workflow::transforms::ModelResolutionTransform;
let dir = tempfile::tempdir().unwrap();
let catalog = std::sync::Arc::new(
fabro_model::Catalog::from_builtin_with_overrides(
&fabro_model::catalog::LlmCatalogSettings::default(),
)
.unwrap(),
);
std::fs::write(
dir.path().join("val.fabro"),
r#"digraph validate {
start [shape=Mdiamond]
lint [prompt="Lint the code"]
test [prompt="Run tests"]
exit [shape=Msquare]
start -> lint -> test -> exit
}"#,
)
.unwrap();
std::fs::write(
dir.path().join("dep.fabro"),
r#"digraph deploy {
start [shape=Mdiamond]
stage [prompt="Stage the release"]
release [prompt="Release it"]
exit [shape=Msquare]
start -> stage -> release -> exit
}"#,
)
.unwrap();
let parsed = fabro_workflow::pipeline::parse(
r#"digraph MergeE2E {
graph [goal="Test file imports"]
start [shape=Mdiamond]
validate [import="./val.fabro"]
deploy [import="./dep.fabro"]
exit [shape=Msquare]
start -> validate -> deploy -> exit
}"#,
)
.expect("parse should succeed");
let transformed = transform(parsed, &TransformOptions {
current_dir: Some(dir.path().to_path_buf()),
file_resolver: Some(std::sync::Arc::new(
fabro_workflow::file_resolver::FilesystemFileResolver::new(None),
)),
template_context: fabro_template::TemplateContext::new(),
source_name: None,
render_mode: fabro_workflow::operations::RenderMode::Strict,
custom_transforms: vec![],
model_resolution: Some(ModelResolutionTransform::new(std::sync::Arc::clone(
&catalog,
))),
})
.unwrap();
let validated = validate(transformed, Some(catalog.as_ref()), &[]);
validated
.raise_on_errors()
.expect("validation should pass after imports expand");
let (graph, _, _) = validated.into_parts();
assert!(graph.nodes.contains_key("validate.lint"));
assert!(graph.nodes.contains_key("validate.test"));
assert!(graph.nodes.contains_key("deploy.stage"));
assert!(graph.nodes.contains_key("deploy.release"));
assert!(
graph
.edges
.iter()
.any(|edge| edge.from == "start" && edge.to == "validate.lint")
);
assert!(
graph
.edges
.iter()
.any(|edge| edge.from == "validate.test" && edge.to == "deploy.stage")
);
assert!(
graph
.edges
.iter()
.any(|edge| edge.from == "deploy.release" && edge.to == "exit")
);
let engine = WorkflowRunner::new(
make_linear_registry(),
Arc::new(Emitter::default()),
local_env(),
);
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (outcome, state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("import E2E should succeed");
assert_eq!(outcome.status, StageOutcome::Succeeded);
let checkpoint = state
.current_checkpoint()
.cloned()
.expect("checkpoint should be captured");
assert!(
checkpoint
.completed_nodes
.contains(&"validate.lint".to_string()),
"validate.lint should be completed"
);
assert!(
checkpoint
.completed_nodes
.contains(&"validate.test".to_string()),
"validate.test should be completed"
);
assert!(
checkpoint
.completed_nodes
.contains(&"deploy.stage".to_string()),
"deploy.stage should be completed"
);
assert!(
checkpoint
.completed_nodes
.contains(&"deploy.release".to_string()),
"deploy.release should be completed"
);
// Verify ordering: validate.test appears before deploy.stage
let val_test_pos = checkpoint
.completed_nodes
.iter()
.position(|n| n == "validate.test")
.expect("validate.test should be in completed_nodes");
let dep_stage_pos = checkpoint
.completed_nodes
.iter()
.position(|n| n == "deploy.stage")
.expect("deploy.stage should be in completed_nodes");
assert!(
val_test_pos < dep_stage_pos,
"validate.test ({val_test_pos}) should execute before deploy.stage ({dep_stage_pos})"
);
}
// ===========================================================================
// Context fidelity integration tests (spec Section 5.4)
// ===========================================================================
type SharedVec<T> = Arc<std::sync::Mutex<Vec<T>>>;
/// Shared capture storage for fidelity tests.
#[derive(Clone)]
struct FidelityCaptures {
fidelities: SharedVec<(String, String)>,
thread_ids: SharedVec<(String, Option<String>)>,
preambles: SharedVec<(String, String)>,
}
impl FidelityCaptures {
fn new() -> Self {
Self {
fidelities: Arc::new(std::sync::Mutex::new(Vec::new())),
thread_ids: Arc::new(std::sync::Mutex::new(Vec::new())),
preambles: Arc::new(std::sync::Mutex::new(Vec::new())),
}
}
}
/// A handler that captures the resolved fidelity and `thread_id` from the
/// context.
struct FidelityCapturingHandler {
captures: FidelityCaptures,
}
struct ParallelFidelitySeedHandler;
#[async_trait::async_trait]
impl Handler for ParallelFidelitySeedHandler {
async fn execute(
&self,
_node: &Node,
_context: &Context,
_graph: &Graph,
_run_dir: &Path,
_services: &fabro_workflow::handler::EngineServices,
) -> Result<Outcome, Error> {
let mut outcome = Outcome::success();
outcome.context_updates.insert(
"parallel_fidelity_marker".to_string(),
serde_json::json!("marker visible to inherited preambles"),
);
Ok(outcome)
}
}
#[async_trait::async_trait]
impl Handler for FidelityCapturingHandler {
async fn execute(
&self,
node: &Node,
context: &Context,
_graph: &Graph,
_run_dir: &Path,
_services: &fabro_workflow::handler::EngineServices,
) -> Result<Outcome, Error> {
let fidelity = context.get_string("internal.fidelity", "none");
self.captures
.fidelities
.lock()
.unwrap()
.push((node.id.clone(), fidelity));
let thread_id = context
.get("internal.thread_id")
.and_then(|v| v.as_str().map(String::from));
self.captures
.thread_ids
.lock()
.unwrap()
.push((node.id.clone(), thread_id));
let preamble = context.get_string("current.preamble", "");
self.captures
.preambles
.lock()
.unwrap()
.push((node.id.clone(), preamble));
Ok(Outcome::success())
}
}
#[tokio::test]
async fn fidelity_default_is_compact() {
let mut graph = make_graph_with_start_exit("FidelityDefaultTest");
let mut work = Node::new("work");
work.attrs.insert(
"type".to_string(),
AttrValue::String("fidelity_capture".to_string()),
);
graph.nodes.insert("work".to_string(), work);
graph.edges.push(Edge::new("start", "work"));
graph.edges.push(Edge::new("work", "exit"));
let captures = FidelityCaptures::new();
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"fidelity_capture",
Box::new(FidelityCapturingHandler {
captures: captures.clone(),
}),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
engine.run(&graph, &run_options).await.expect("run");
let fidelities = captures.fidelities.lock().unwrap();
assert_eq!(fidelities.len(), 1);
assert_eq!(fidelities[0].0, "work");
assert_eq!(fidelities[0].1, "compact");
let preambles = captures.preambles.lock().unwrap();
assert!(
!preambles[0].1.is_empty(),
"compact fidelity should produce a preamble"
);
}
#[tokio::test]
async fn fidelity_graph_default_applied() {
let mut graph = make_graph_with_start_exit("FidelityGraphDefaultTest");
graph.attrs.insert(
"default_fidelity".to_string(),
AttrValue::String("truncate".to_string()),
);
let mut work = Node::new("work");
work.attrs.insert(
"type".to_string(),
AttrValue::String("fidelity_capture".to_string()),
);
graph.nodes.insert("work".to_string(), work);
graph.edges.push(Edge::new("start", "work"));
graph.edges.push(Edge::new("work", "exit"));
let captures = FidelityCaptures::new();
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"fidelity_capture",
Box::new(FidelityCapturingHandler {
captures: captures.clone(),
}),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
engine.run(&graph, &run_options).await.expect("run");
let fidelities = captures.fidelities.lock().unwrap();
assert_eq!(fidelities[0].1, "truncate");
}
#[tokio::test]
async fn fidelity_node_overrides_graph_default() {
let mut graph = make_graph_with_start_exit("FidelityNodeOverrideTest");
graph.attrs.insert(
"default_fidelity".to_string(),
AttrValue::String("truncate".to_string()),
);
let mut work = Node::new("work");
work.attrs.insert(
"type".to_string(),
AttrValue::String("fidelity_capture".to_string()),
);
work.attrs.insert(
"fidelity".to_string(),
AttrValue::String("summary:medium".to_string()),
);
graph.nodes.insert("work".to_string(), work);
graph.edges.push(Edge::new("start", "work"));
graph.edges.push(Edge::new("work", "exit"));
let captures = FidelityCaptures::new();
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"fidelity_capture",
Box::new(FidelityCapturingHandler {
captures: captures.clone(),
}),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
engine.run(&graph, &run_options).await.expect("run");
let fidelities = captures.fidelities.lock().unwrap();
assert_eq!(fidelities[0].1, "summary:medium");
}
#[tokio::test]
async fn fidelity_edge_overrides_node_and_graph() {
let mut graph = make_graph_with_start_exit("FidelityEdgeOverrideTest");
graph.attrs.insert(
"default_fidelity".to_string(),
AttrValue::String("truncate".to_string()),
);
let mut work = Node::new("work");
work.attrs.insert(
"type".to_string(),
AttrValue::String("fidelity_capture".to_string()),
);
work.attrs.insert(
"fidelity".to_string(),
AttrValue::String("compact".to_string()),
);
graph.nodes.insert("work".to_string(), work);
let mut edge_with_fidelity = Edge::new("start", "work");
edge_with_fidelity.attrs.insert(
"fidelity".to_string(),
AttrValue::String("summary:high".to_string()),
);
graph.edges.push(edge_with_fidelity);
graph.edges.push(Edge::new("work", "exit"));
let captures = FidelityCaptures::new();
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"fidelity_capture",
Box::new(FidelityCapturingHandler {
captures: captures.clone(),
}),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
engine.run(&graph, &run_options).await.expect("run");
let fidelities = captures.fidelities.lock().unwrap();
assert_eq!(fidelities[0].1, "summary:high");
}
#[tokio::test]
async fn fidelity_full_produces_empty_preamble() {
let mut graph = make_graph_with_start_exit("FidelityFullPreambleTest");
let mut work = Node::new("work");
work.attrs.insert(
"type".to_string(),
AttrValue::String("fidelity_capture".to_string()),
);
work.attrs.insert(
"fidelity".to_string(),
AttrValue::String("full".to_string()),
);
graph.nodes.insert("work".to_string(), work);
graph.edges.push(Edge::new("start", "work"));
graph.edges.push(Edge::new("work", "exit"));
let captures = FidelityCaptures::new();
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"fidelity_capture",
Box::new(FidelityCapturingHandler {
captures: captures.clone(),
}),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
engine.run(&graph, &run_options).await.expect("run");
let fidelities = captures.fidelities.lock().unwrap();
assert_eq!(fidelities[0].1, "full");
let preambles = captures.preambles.lock().unwrap();
assert_eq!(
preambles[0].1, "",
"full fidelity should produce empty preamble"
);
}
#[tokio::test]
async fn fidelity_truncate_preamble_minimal() {
let mut graph = make_graph_with_start_exit("FidelityTruncateTest");
graph.attrs.insert(
"goal".to_string(),
AttrValue::String("Test truncate mode".to_string()),
);
graph.attrs.insert(
"default_fidelity".to_string(),
AttrValue::String("truncate".to_string()),
);
let mut work = Node::new("work");
work.attrs.insert(
"type".to_string(),
AttrValue::String("fidelity_capture".to_string()),
);
graph.nodes.insert("work".to_string(), work);
graph.edges.push(Edge::new("start", "work"));
graph.edges.push(Edge::new("work", "exit"));
let captures = FidelityCaptures::new();
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"fidelity_capture",
Box::new(FidelityCapturingHandler {
captures: captures.clone(),
}),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
engine.run(&graph, &run_options).await.expect("run");
let preambles = captures.preambles.lock().unwrap();
let preamble = &preambles[0].1;
assert!(
preamble.contains("Goal: Test truncate mode"),
"truncate preamble should contain the goal"
);
assert!(
preamble.contains("Run ID:"),
"truncate preamble should contain run ID"
);
assert!(
!preamble.contains("Completed stages:"),
"truncate should not include stage details"
);
}
#[tokio::test]
async fn fidelity_summary_low_mode() {
let mut graph = make_graph_with_start_exit("SummaryLow");
graph.attrs.insert(
"goal".to_string(),
AttrValue::String("Test summary".to_string()),
);
graph.attrs.insert(
"default_fidelity".to_string(),
AttrValue::String("summary:low".to_string()),
);
let mut step_a = Node::new("step_a");
step_a.attrs.insert(
"type".to_string(),
AttrValue::String("fidelity_capture".to_string()),
);
graph.nodes.insert("step_a".to_string(), step_a);
let mut step_b = Node::new("step_b");
step_b.attrs.insert(
"type".to_string(),
AttrValue::String("fidelity_capture".to_string()),
);
graph.nodes.insert("step_b".to_string(), step_b);
graph.edges.push(Edge::new("start", "step_a"));
graph.edges.push(Edge::new("step_a", "step_b"));
graph.edges.push(Edge::new("step_b", "exit"));
let captures = FidelityCaptures::new();
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"fidelity_capture",
Box::new(FidelityCapturingHandler {
captures: captures.clone(),
}),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
engine.run(&graph, &run_options).await.expect("run");
let fidelities = captures.fidelities.lock().unwrap();
assert_eq!(fidelities[0].1, "summary:low");
assert_eq!(fidelities[1].1, "summary:low");
let preambles = captures.preambles.lock().unwrap();
assert!(
preambles[1].1.contains("Test summary"),
"summary:low preamble should contain goal"
);
}
#[tokio::test]
async fn fidelity_summary_medium_mode() {
let mut graph = make_graph_with_start_exit("SummaryMedium");
graph.attrs.insert(
"goal".to_string(),
AttrValue::String("Test summary".to_string()),
);
graph.attrs.insert(
"default_fidelity".to_string(),
AttrValue::String("summary:medium".to_string()),
);
let mut step_a = Node::new("step_a");
step_a.attrs.insert(
"type".to_string(),
AttrValue::String("fidelity_capture".to_string()),
);
graph.nodes.insert("step_a".to_string(), step_a);
let mut step_b = Node::new("step_b");
step_b.attrs.insert(
"type".to_string(),
AttrValue::String("fidelity_capture".to_string()),
);
graph.nodes.insert("step_b".to_string(), step_b);
graph.edges.push(Edge::new("start", "step_a"));
graph.edges.push(Edge::new("step_a", "step_b"));
graph.edges.push(Edge::new("step_b", "exit"));
let captures = FidelityCaptures::new();
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"fidelity_capture",
Box::new(FidelityCapturingHandler {
captures: captures.clone(),
}),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
engine.run(&graph, &run_options).await.expect("run");
let fidelities = captures.fidelities.lock().unwrap();
assert_eq!(fidelities[0].1, "summary:medium");
assert_eq!(fidelities[1].1, "summary:medium");
let preambles = captures.preambles.lock().unwrap();
assert!(
preambles[1].1.contains("Test summary"),
"summary:medium preamble should contain goal"
);
}
#[tokio::test]
async fn fidelity_summary_high_mode() {
let mut graph = make_graph_with_start_exit("SummaryHigh");
graph.attrs.insert(
"goal".to_string(),
AttrValue::String("Test summary".to_string()),
);
graph.attrs.insert(
"default_fidelity".to_string(),
AttrValue::String("summary:high".to_string()),
);
let mut step_a = Node::new("step_a");
step_a.attrs.insert(
"type".to_string(),
AttrValue::String("fidelity_capture".to_string()),
);
graph.nodes.insert("step_a".to_string(), step_a);
let mut step_b = Node::new("step_b");
step_b.attrs.insert(
"type".to_string(),
AttrValue::String("fidelity_capture".to_string()),
);
graph.nodes.insert("step_b".to_string(), step_b);
graph.edges.push(Edge::new("start", "step_a"));
graph.edges.push(Edge::new("step_a", "step_b"));
graph.edges.push(Edge::new("step_b", "exit"));
let captures = FidelityCaptures::new();
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"fidelity_capture",
Box::new(FidelityCapturingHandler {
captures: captures.clone(),
}),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
engine.run(&graph, &run_options).await.expect("run");
let fidelities = captures.fidelities.lock().unwrap();
assert_eq!(fidelities[0].1, "summary:high");
assert_eq!(fidelities[1].1, "summary:high");
let preambles = captures.preambles.lock().unwrap();
assert!(
preambles[1].1.contains("Test summary"),
"summary:high preamble should contain goal"
);
}
#[tokio::test]
async fn fidelity_full_sets_thread_id_in_context() {
let mut graph = make_graph_with_start_exit("FidelityThreadTest");
let mut work = Node::new("work");
work.attrs.insert(
"type".to_string(),
AttrValue::String("fidelity_capture".to_string()),
);
work.attrs.insert(
"fidelity".to_string(),
AttrValue::String("full".to_string()),
);
work.attrs.insert(
"thread_id".to_string(),
AttrValue::String("my-session".to_string()),
);
graph.nodes.insert("work".to_string(), work);
graph.edges.push(Edge::new("start", "work"));
graph.edges.push(Edge::new("work", "exit"));
let captures = FidelityCaptures::new();
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"fidelity_capture",
Box::new(FidelityCapturingHandler {
captures: captures.clone(),
}),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
engine.run(&graph, &run_options).await.expect("run");
let thread_ids = captures.thread_ids.lock().unwrap();
assert_eq!(thread_ids[0].0, "work");
assert_eq!(thread_ids[0].1, Some("my-session".to_string()));
}
#[tokio::test]
async fn fidelity_full_nodes_share_thread_id() {
let mut graph = make_graph_with_start_exit("FidelitySharedThreadTest");
let mut step_a = Node::new("step_a");
step_a.attrs.insert(
"type".to_string(),
AttrValue::String("fidelity_capture".to_string()),
);
step_a.attrs.insert(
"fidelity".to_string(),
AttrValue::String("full".to_string()),
);
step_a.attrs.insert(
"thread_id".to_string(),
AttrValue::String("shared-session".to_string()),
);
graph.nodes.insert("step_a".to_string(), step_a);
let mut step_b = Node::new("step_b");
step_b.attrs.insert(
"type".to_string(),
AttrValue::String("fidelity_capture".to_string()),
);
step_b.attrs.insert(
"fidelity".to_string(),
AttrValue::String("full".to_string()),
);
step_b.attrs.insert(
"thread_id".to_string(),
AttrValue::String("shared-session".to_string()),
);
graph.nodes.insert("step_b".to_string(), step_b);
graph.edges.push(Edge::new("start", "step_a"));
graph.edges.push(Edge::new("step_a", "step_b"));
graph.edges.push(Edge::new("step_b", "exit"));
let captures = FidelityCaptures::new();
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"fidelity_capture",
Box::new(FidelityCapturingHandler {
captures: captures.clone(),
}),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
engine.run(&graph, &run_options).await.expect("run");
let thread_ids = captures.thread_ids.lock().unwrap();
assert_eq!(thread_ids[0].0, "step_a");
assert_eq!(thread_ids[0].1, Some("shared-session".to_string()));
assert_eq!(thread_ids[1].0, "step_b");
assert_eq!(thread_ids[1].1, Some("shared-session".to_string()));
}
#[tokio::test]
async fn fidelity_resume_degrades_full_to_summary_high() {
let mut graph = make_graph_with_start_exit("FidelityResumeTest");
let mut step_a = Node::new("step_a");
step_a.attrs.insert(
"fidelity".to_string(),
AttrValue::String("full".to_string()),
);
graph.nodes.insert("step_a".to_string(), step_a);
let mut step_b = Node::new("step_b");
step_b.attrs.insert(
"type".to_string(),
AttrValue::String("fidelity_capture".to_string()),
);
step_b.attrs.insert(
"fidelity".to_string(),
AttrValue::String("full".to_string()),
);
graph.nodes.insert("step_b".to_string(), step_b);
graph.edges.push(Edge::new("start", "step_a"));
graph.edges.push(Edge::new("step_a", "step_b"));
graph.edges.push(Edge::new("step_b", "exit"));
let ctx = Context::new();
ctx.set("outcome", serde_json::json!("success"));
ctx.set("internal.fidelity", serde_json::json!("full"));
let mut outcomes = std::collections::HashMap::new();
outcomes.insert("start".to_string(), Outcome::success());
outcomes.insert("step_a".to_string(), Outcome::success());
let checkpoint = Checkpoint::from_context(
&ctx,
"step_a",
vec!["start".to_string(), "step_a".to_string()],
std::collections::HashMap::new(),
outcomes,
Some("step_b".to_string()),
std::collections::HashMap::new(),
std::collections::HashMap::new(),
std::collections::HashMap::new(),
);
let captures = FidelityCaptures::new();
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"fidelity_capture",
Box::new(FidelityCapturingHandler {
captures: captures.clone(),
}),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
engine
.run_from_checkpoint(&graph, &run_options, &checkpoint)
.await
.expect("resume should succeed");
let fidelities = captures.fidelities.lock().unwrap();
assert_eq!(fidelities[0].0, "step_b");
assert_eq!(
fidelities[0].1, "summary:high",
"first node after resume from full fidelity should be degraded to summary:high"
);
}
#[tokio::test]
async fn fidelity_resume_degrade_only_affects_first_hop() {
let mut graph = make_graph_with_start_exit("FidelityResumeSingleHopTest");
let mut step_a = Node::new("step_a");
step_a.attrs.insert(
"fidelity".to_string(),
AttrValue::String("full".to_string()),
);
graph.nodes.insert("step_a".to_string(), step_a);
let mut step_b = Node::new("step_b");
step_b.attrs.insert(
"type".to_string(),
AttrValue::String("fidelity_capture".to_string()),
);
step_b.attrs.insert(
"fidelity".to_string(),
AttrValue::String("full".to_string()),
);
graph.nodes.insert("step_b".to_string(), step_b);
let mut step_c = Node::new("step_c");
step_c.attrs.insert(
"type".to_string(),
AttrValue::String("fidelity_capture".to_string()),
);
step_c.attrs.insert(
"fidelity".to_string(),
AttrValue::String("full".to_string()),
);
graph.nodes.insert("step_c".to_string(), step_c);
graph.edges.push(Edge::new("start", "step_a"));
graph.edges.push(Edge::new("step_a", "step_b"));
graph.edges.push(Edge::new("step_b", "step_c"));
graph.edges.push(Edge::new("step_c", "exit"));
let ctx = Context::new();
ctx.set("outcome", serde_json::json!("success"));
ctx.set("internal.fidelity", serde_json::json!("full"));
let mut outcomes = std::collections::HashMap::new();
outcomes.insert("start".to_string(), Outcome::success());
outcomes.insert("step_a".to_string(), Outcome::success());
let checkpoint = Checkpoint::from_context(
&ctx,
"step_a",
vec!["start".to_string(), "step_a".to_string()],
std::collections::HashMap::new(),
outcomes,
Some("step_b".to_string()),
std::collections::HashMap::new(),
std::collections::HashMap::new(),
std::collections::HashMap::new(),
);
let captures = FidelityCaptures::new();
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"fidelity_capture",
Box::new(FidelityCapturingHandler {
captures: captures.clone(),
}),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
engine
.run_from_checkpoint(&graph, &run_options, &checkpoint)
.await
.expect("resume should succeed");
let fidelities = captures.fidelities.lock().unwrap();
assert_eq!(fidelities[0].0, "step_b");
assert_eq!(fidelities[0].1, "summary:high");
assert_eq!(fidelities[1].0, "step_c");
assert_eq!(fidelities[1].1, "full");
}
#[tokio::test]
async fn fidelity_resume_no_degrade_when_not_full() {
let mut graph = make_graph_with_start_exit("FidelityResumeNoDegrade");
let mut step_a = Node::new("step_a");
step_a.attrs.insert(
"fidelity".to_string(),
AttrValue::String("compact".to_string()),
);
graph.nodes.insert("step_a".to_string(), step_a);
let mut step_b = Node::new("step_b");
step_b.attrs.insert(
"type".to_string(),
AttrValue::String("fidelity_capture".to_string()),
);
step_b.attrs.insert(
"fidelity".to_string(),
AttrValue::String("full".to_string()),
);
graph.nodes.insert("step_b".to_string(), step_b);
graph.edges.push(Edge::new("start", "step_a"));
graph.edges.push(Edge::new("step_a", "step_b"));
graph.edges.push(Edge::new("step_b", "exit"));
let ctx = Context::new();
ctx.set("outcome", serde_json::json!("success"));
ctx.set("internal.fidelity", serde_json::json!("compact"));
let mut outcomes = std::collections::HashMap::new();
outcomes.insert("start".to_string(), Outcome::success());
outcomes.insert("step_a".to_string(), Outcome::success());
let checkpoint = Checkpoint::from_context(
&ctx,
"step_a",
vec!["start".to_string(), "step_a".to_string()],
std::collections::HashMap::new(),
outcomes,
Some("step_b".to_string()),
std::collections::HashMap::new(),
std::collections::HashMap::new(),
std::collections::HashMap::new(),
);
let captures = FidelityCaptures::new();
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"fidelity_capture",
Box::new(FidelityCapturingHandler {
captures: captures.clone(),
}),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
engine
.run_from_checkpoint(&graph, &run_options, &checkpoint)
.await
.expect("resume should succeed");
let fidelities = captures.fidelities.lock().unwrap();
assert_eq!(fidelities[0].0, "step_b");
assert_eq!(fidelities[0].1, "full");
}
#[tokio::test]
async fn fidelity_stored_in_checkpoint_context() {
let mut graph = make_graph_with_start_exit("FidelityCheckpointTest");
graph.attrs.insert(
"default_fidelity".to_string(),
AttrValue::String("summary:low".to_string()),
);
let work = Node::new("work");
graph.nodes.insert("work".to_string(), work);
graph.edges.push(Edge::new("start", "work"));
graph.edges.push(Edge::new("work", "exit"));
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (_outcome, state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("run");
let cp = state
.current_checkpoint()
.cloned()
.expect("checkpoint should be captured");
assert_eq!(
cp.context_values.get("internal.fidelity"),
Some(&serde_json::json!("summary:low")),
"checkpoint should record the resolved fidelity"
);
assert!(
!cp.context_values.contains_key("current.preamble"),
"checkpoint should exclude runtime-only preamble state"
);
}
#[tokio::test]
async fn fidelity_precedence_multi_node_pipeline() {
let mut graph = make_graph_with_start_exit("FidelityPrecedenceTest");
graph.attrs.insert(
"default_fidelity".to_string(),
AttrValue::String("truncate".to_string()),
);
let mut step_a = Node::new("step_a");
step_a.attrs.insert(
"type".to_string(),
AttrValue::String("fidelity_capture".to_string()),
);
graph.nodes.insert("step_a".to_string(), step_a);
let mut step_b = Node::new("step_b");
step_b.attrs.insert(
"type".to_string(),
AttrValue::String("fidelity_capture".to_string()),
);
step_b.attrs.insert(
"fidelity".to_string(),
AttrValue::String("summary:medium".to_string()),
);
graph.nodes.insert("step_b".to_string(), step_b);
let mut step_c = Node::new("step_c");
step_c.attrs.insert(
"type".to_string(),
AttrValue::String("fidelity_capture".to_string()),
);
step_c.attrs.insert(
"fidelity".to_string(),
AttrValue::String("compact".to_string()),
);
graph.nodes.insert("step_c".to_string(), step_c);
graph.edges.push(Edge::new("start", "step_a"));
graph.edges.push(Edge::new("step_a", "step_b"));
let mut edge_b_c = Edge::new("step_b", "step_c");
edge_b_c.attrs.insert(
"fidelity".to_string(),
AttrValue::String("summary:high".to_string()),
);
graph.edges.push(edge_b_c);
graph.edges.push(Edge::new("step_c", "exit"));
let captures = FidelityCaptures::new();
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"fidelity_capture",
Box::new(FidelityCapturingHandler {
captures: captures.clone(),
}),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
engine.run(&graph, &run_options).await.expect("run");
let fidelities = captures.fidelities.lock().unwrap();
assert_eq!(fidelities[0].0, "step_a");
assert_eq!(fidelities[0].1, "truncate");
assert_eq!(fidelities[1].0, "step_b");
assert_eq!(fidelities[1].1, "summary:medium");
assert_eq!(fidelities[2].0, "step_c");
assert_eq!(fidelities[2].1, "summary:high");
}
#[tokio::test]
async fn fidelity_compact_preamble_includes_completed_stages_and_context() {
let mut graph = make_graph_with_start_exit("FidelityCompactContentTest");
graph.attrs.insert(
"goal".to_string(),
AttrValue::String("Build the widget".to_string()),
);
graph.attrs.insert(
"default_fidelity".to_string(),
AttrValue::String("compact".to_string()),
);
let mut step_a = Node::new("step_a");
step_a.attrs.insert(
"type".to_string(),
AttrValue::String("fidelity_capture".to_string()),
);
graph.nodes.insert("step_a".to_string(), step_a);
let mut step_b = Node::new("step_b");
step_b.attrs.insert(
"type".to_string(),
AttrValue::String("fidelity_capture".to_string()),
);
graph.nodes.insert("step_b".to_string(), step_b);
graph.edges.push(Edge::new("start", "step_a"));
graph.edges.push(Edge::new("step_a", "step_b"));
graph.edges.push(Edge::new("step_b", "exit"));
let captures = FidelityCaptures::new();
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"fidelity_capture",
Box::new(FidelityCapturingHandler {
captures: captures.clone(),
}),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
engine.run(&graph, &run_options).await.expect("run");
let preambles = captures.preambles.lock().unwrap();
// step_b's preamble should contain structured summary of completed work
let step_b_preamble = &preambles[1].1;
assert!(
step_b_preamble.contains("Build the widget"),
"compact preamble should contain the goal"
);
assert!(
step_b_preamble.contains("## Completed stages"),
"compact preamble should include completed stages section"
);
assert!(
step_b_preamble.contains("step_a"),
"compact preamble should mention completed node step_a"
);
}
#[tokio::test]
async fn fidelity_summary_low_excludes_context_values_in_pipeline() {
// summary:low should NOT include context values (only goal, run ID, stage
// count, recent stages). summary:medium should include context values.
// This verifies a behavioral difference between detail levels.
let mut graph_low = make_graph_with_start_exit("SummaryLowExcludesContext");
graph_low.attrs.insert(
"goal".to_string(),
AttrValue::String("Context exclusion test".to_string()),
);
graph_low.attrs.insert(
"default_fidelity".to_string(),
AttrValue::String("summary:low".to_string()),
);
let mut step_a_low = Node::new("step_a");
step_a_low.attrs.insert(
"type".to_string(),
AttrValue::String("fidelity_capture".to_string()),
);
graph_low.nodes.insert("step_a".to_string(), step_a_low);
let mut step_b_low = Node::new("step_b");
step_b_low.attrs.insert(
"type".to_string(),
AttrValue::String("fidelity_capture".to_string()),
);
graph_low.nodes.insert("step_b".to_string(), step_b_low);
graph_low.edges.push(Edge::new("start", "step_a"));
graph_low.edges.push(Edge::new("step_a", "step_b"));
graph_low.edges.push(Edge::new("step_b", "exit"));
let captures_low = FidelityCaptures::new();
let dir_low = tempfile::tempdir().unwrap();
let mut registry_low = HandlerRegistry::new(Box::new(StartHandler));
registry_low.register("start", Box::new(StartHandler));
registry_low.register("exit", Box::new(ExitHandler));
registry_low.register(
"fidelity_capture",
Box::new(FidelityCapturingHandler {
captures: captures_low.clone(),
}),
);
let engine_low = WorkflowRunner::new(registry_low, Arc::new(Emitter::default()), local_env());
let run_options_low = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir_low.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
engine_low
.run(&graph_low, &run_options_low)
.await
.expect("run low");
{
let preambles_low = captures_low.preambles.lock().unwrap();
let low_preamble = &preambles_low[1].1;
// summary:low should not include "Context values:" section
assert!(
!low_preamble.contains("Context values:"),
"summary:low preamble should not include context values section"
);
}
// Now run summary:medium and verify it DOES include context values
let mut graph_med = make_graph_with_start_exit("SummaryMedIncludesContext");
graph_med.attrs.insert(
"goal".to_string(),
AttrValue::String("Context exclusion test".to_string()),
);
graph_med.attrs.insert(
"default_fidelity".to_string(),
AttrValue::String("summary:medium".to_string()),
);
let mut step_a_med = Node::new("step_a");
step_a_med.attrs.insert(
"type".to_string(),
AttrValue::String("fidelity_capture".to_string()),
);
graph_med.nodes.insert("step_a".to_string(), step_a_med);
let mut step_b_med = Node::new("step_b");
step_b_med.attrs.insert(
"type".to_string(),
AttrValue::String("fidelity_capture".to_string()),
);
graph_med.nodes.insert("step_b".to_string(), step_b_med);
graph_med.edges.push(Edge::new("start", "step_a"));
graph_med.edges.push(Edge::new("step_a", "step_b"));
graph_med.edges.push(Edge::new("step_b", "exit"));
let captures_med = FidelityCaptures::new();
let dir_med = tempfile::tempdir().unwrap();
let mut registry_med = HandlerRegistry::new(Box::new(StartHandler));
registry_med.register("start", Box::new(StartHandler));
registry_med.register("exit", Box::new(ExitHandler));
registry_med.register(
"fidelity_capture",
Box::new(FidelityCapturingHandler {
captures: captures_med.clone(),
}),
);
let engine_med = WorkflowRunner::new(registry_med, Arc::new(Emitter::default()), local_env());
let run_options_med = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir_med.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
engine_med
.run(&graph_med, &run_options_med)
.await
.expect("run med");
let preambles_med = captures_med.preambles.lock().unwrap();
let med_preamble = &preambles_med[1].1;
// summary:medium should include stage details (unlike summary:low which omits
// them)
assert!(
med_preamble.contains("step_a"),
"summary:medium preamble should include completed stage step_a"
);
// Verify medium and low differ: medium shows more recent stages
let preambles_low = captures_low.preambles.lock().unwrap();
let low_preamble = &preambles_low[1].1;
assert!(
!low_preamble.contains("## Context"),
"summary:low preamble should not include context section"
);
}
#[tokio::test]
async fn fidelity_thread_id_fallback_to_previous_node_in_pipeline() {
// When no thread_id is set on the node, edge, graph, or class,
// the thread ID should fall back to the previous node's ID.
let mut graph = make_graph_with_start_exit("ThreadFallbackTest");
let mut step_a = Node::new("step_a");
step_a.attrs.insert(
"type".to_string(),
AttrValue::String("fidelity_capture".to_string()),
);
graph.nodes.insert("step_a".to_string(), step_a);
let mut step_b = Node::new("step_b");
step_b.attrs.insert(
"type".to_string(),
AttrValue::String("fidelity_capture".to_string()),
);
graph.nodes.insert("step_b".to_string(), step_b);
graph.edges.push(Edge::new("start", "step_a"));
graph.edges.push(Edge::new("step_a", "step_b"));
graph.edges.push(Edge::new("step_b", "exit"));
let captures = FidelityCaptures::new();
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"fidelity_capture",
Box::new(FidelityCapturingHandler {
captures: captures.clone(),
}),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
engine.run(&graph, &run_options).await.expect("run");
let thread_ids = captures.thread_ids.lock().unwrap();
// step_a should have previous node = start
assert_eq!(thread_ids[0].0, "step_a");
assert_eq!(thread_ids[0].1, Some("start".to_string()));
// step_b should have previous node = step_a
assert_eq!(thread_ids[1].0, "step_b");
assert_eq!(thread_ids[1].1, Some("step_a".to_string()));
}
#[tokio::test]
async fn fidelity_thread_id_from_node_class_in_pipeline() {
// When a node has classes (from subgraph derivation), thread_id resolves
// from the first class name per spec step 4.
let mut graph = make_graph_with_start_exit("ThreadClassTest");
let mut work = Node::new("work");
work.attrs.insert(
"type".to_string(),
AttrValue::String("fidelity_capture".to_string()),
);
work.classes = vec!["planning".to_string(), "review".to_string()];
graph.nodes.insert("work".to_string(), work);
graph.edges.push(Edge::new("start", "work"));
graph.edges.push(Edge::new("work", "exit"));
let captures = FidelityCaptures::new();
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"fidelity_capture",
Box::new(FidelityCapturingHandler {
captures: captures.clone(),
}),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
engine.run(&graph, &run_options).await.expect("run");
let thread_ids = captures.thread_ids.lock().unwrap();
assert_eq!(thread_ids[0].0, "work");
assert_eq!(
thread_ids[0].1,
Some("planning".to_string()),
"thread_id should resolve from first class name"
);
}
#[tokio::test]
async fn fidelity_edge_thread_id_override_in_pipeline() {
// Edge thread_id should override the previous-node fallback.
let mut graph = make_graph_with_start_exit("EdgeThreadOverrideTest");
let mut work = Node::new("work");
work.attrs.insert(
"type".to_string(),
AttrValue::String("fidelity_capture".to_string()),
);
graph.nodes.insert("work".to_string(), work);
let mut edge_to_work = Edge::new("start", "work");
edge_to_work.attrs.insert(
"thread_id".to_string(),
AttrValue::String("edge-session".to_string()),
);
graph.edges.push(edge_to_work);
graph.edges.push(Edge::new("work", "exit"));
let captures = FidelityCaptures::new();
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"fidelity_capture",
Box::new(FidelityCapturingHandler {
captures: captures.clone(),
}),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
engine.run(&graph, &run_options).await.expect("run");
let thread_ids = captures.thread_ids.lock().unwrap();
assert_eq!(thread_ids[0].0, "work");
assert_eq!(
thread_ids[0].1,
Some("edge-session".to_string()),
"edge thread_id should override the previous-node fallback"
);
}
#[tokio::test]
async fn fidelity_full_without_explicit_thread_id_uses_previous_node() {
// When fidelity=full but no explicit thread_id is set, thread resolution
// should still fall back to the previous node ID.
let mut graph = make_graph_with_start_exit("FullNoExplicitThreadTest");
let mut work = Node::new("work");
work.attrs.insert(
"type".to_string(),
AttrValue::String("fidelity_capture".to_string()),
);
work.attrs.insert(
"fidelity".to_string(),
AttrValue::String("full".to_string()),
);
// No thread_id set explicitly
graph.nodes.insert("work".to_string(), work);
graph.edges.push(Edge::new("start", "work"));
graph.edges.push(Edge::new("work", "exit"));
let captures = FidelityCaptures::new();
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"fidelity_capture",
Box::new(FidelityCapturingHandler {
captures: captures.clone(),
}),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
engine.run(&graph, &run_options).await.expect("run");
let fidelities = captures.fidelities.lock().unwrap();
assert_eq!(fidelities[0].1, "full");
let thread_ids = captures.thread_ids.lock().unwrap();
assert_eq!(thread_ids[0].0, "work");
assert_eq!(
thread_ids[0].1,
Some("start".to_string()),
"full fidelity without explicit thread_id should fall back to previous node"
);
let preambles = captures.preambles.lock().unwrap();
assert_eq!(
preambles[0].1, "",
"full fidelity should produce empty preamble"
);
}
#[tokio::test]
async fn fidelity_from_parsed_dot_pipeline() {
// Parse a DOT file with fidelity attributes and run the pipeline.
let input = r#"digraph FidelityDotTest {
graph [goal="Test DOT fidelity", default_fidelity="truncate"]
start [shape=Mdiamond]
exit [shape=Msquare]
step_a [type="fidelity_capture"]
step_b [type="fidelity_capture", fidelity="summary:medium"]
step_c [type="fidelity_capture"]
start -> step_a -> step_b
step_b -> step_c [fidelity="summary:high"]
step_c -> exit
}"#;
let graph = parse(input).expect("parsing should succeed");
validate_or_raise(&graph, &[]).expect("validation should pass");
let captures = FidelityCaptures::new();
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"fidelity_capture",
Box::new(FidelityCapturingHandler {
captures: captures.clone(),
}),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
engine.run(&graph, &run_options).await.expect("run");
let fidelities = captures.fidelities.lock().unwrap();
// step_a: no node fidelity, no edge fidelity -> graph default "truncate"
assert_eq!(fidelities[0].0, "step_a");
assert_eq!(fidelities[0].1, "truncate");
// step_b: node fidelity "summary:medium" overrides graph default
assert_eq!(fidelities[1].0, "step_b");
assert_eq!(fidelities[1].1, "summary:medium");
// step_c: node has no fidelity but incoming edge has "summary:high" -> edge
// wins
assert_eq!(fidelities[2].0, "step_c");
assert_eq!(fidelities[2].1, "summary:high");
}
#[tokio::test]
async fn fidelity_checkpoint_roundtrip_preserves_fidelity() {
// Run a pipeline that sets a specific fidelity, save checkpoint,
// load it, and verify the fidelity value survives the roundtrip.
let mut graph = make_graph_with_start_exit("FidelityCheckpointRoundtripTest");
graph.attrs.insert(
"default_fidelity".to_string(),
AttrValue::String("summary:high".to_string()),
);
let work = Node::new("work");
graph.nodes.insert("work".to_string(), work);
graph.edges.push(Edge::new("start", "work"));
graph.edges.push(Edge::new("work", "exit"));
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (_outcome, state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("run");
// Save and load again to verify roundtrip
let cp1 = state
.current_checkpoint()
.cloned()
.expect("checkpoint should be captured");
assert_eq!(
cp1.context_values.get("internal.fidelity"),
Some(&serde_json::json!("summary:high")),
);
let roundtrip_path = dir.path().join("checkpoint_roundtrip.json");
save_checkpoint(&roundtrip_path, &cp1);
let cp2 = load_checkpoint(&roundtrip_path).expect("second load");
assert_eq!(
cp2.context_values.get("internal.fidelity"),
Some(&serde_json::json!("summary:high")),
"fidelity should survive checkpoint save/load roundtrip"
);
}
#[tokio::test]
async fn fidelity_node_thread_id_overrides_edge_thread_id_in_pipeline() {
// When both node and edge have thread_id, the edge's takes precedence (step 1 >
// step 2).
let mut graph = make_graph_with_start_exit("NodeOverridesEdgeThreadTest");
let mut work = Node::new("work");
work.attrs.insert(
"type".to_string(),
AttrValue::String("fidelity_capture".to_string()),
);
work.attrs.insert(
"thread_id".to_string(),
AttrValue::String("node-thread".to_string()),
);
graph.nodes.insert("work".to_string(), work);
let mut edge_to_work = Edge::new("start", "work");
edge_to_work.attrs.insert(
"thread_id".to_string(),
AttrValue::String("edge-thread".to_string()),
);
graph.edges.push(edge_to_work);
graph.edges.push(Edge::new("work", "exit"));
let captures = FidelityCaptures::new();
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"fidelity_capture",
Box::new(FidelityCapturingHandler {
captures: captures.clone(),
}),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
engine.run(&graph, &run_options).await.expect("run");
let thread_ids = captures.thread_ids.lock().unwrap();
assert_eq!(thread_ids[0].0, "work");
assert_eq!(
thread_ids[0].1,
Some("edge-thread".to_string()),
"edge thread_id should take precedence over node thread_id"
);
}
#[tokio::test]
async fn fidelity_resume_preserves_context_values_across_checkpoint() {
// After resuming from a checkpoint, context values from the checkpoint
// should be available to the resumed nodes. This tests that fidelity-related
// context survives the resume path.
let mut graph = make_graph_with_start_exit("FidelityResumeContextTest");
let mut step_a = Node::new("step_a");
step_a.attrs.insert(
"fidelity".to_string(),
AttrValue::String("compact".to_string()),
);
graph.nodes.insert("step_a".to_string(), step_a);
let mut step_b = Node::new("step_b");
step_b.attrs.insert(
"type".to_string(),
AttrValue::String("fidelity_capture".to_string()),
);
step_b.attrs.insert(
"fidelity".to_string(),
AttrValue::String("summary:low".to_string()),
);
graph.nodes.insert("step_b".to_string(), step_b);
graph.edges.push(Edge::new("start", "step_a"));
graph.edges.push(Edge::new("step_a", "step_b"));
graph.edges.push(Edge::new("step_b", "exit"));
let ctx = Context::new();
ctx.set("outcome", serde_json::json!("success"));
ctx.set("internal.fidelity", serde_json::json!("compact"));
ctx.set("context.custom_key", serde_json::json!("custom_value"));
let mut outcomes = std::collections::HashMap::new();
outcomes.insert("start".to_string(), Outcome::success());
outcomes.insert("step_a".to_string(), Outcome::success());
let checkpoint = Checkpoint::from_context(
&ctx,
"step_a",
vec!["start".to_string(), "step_a".to_string()],
std::collections::HashMap::new(),
outcomes,
Some("step_b".to_string()),
std::collections::HashMap::new(),
std::collections::HashMap::new(),
std::collections::HashMap::new(),
);
let captures = FidelityCaptures::new();
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"fidelity_capture",
Box::new(FidelityCapturingHandler {
captures: captures.clone(),
}),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (_outcome, state) = engine
.run_from_checkpoint_with_state(&graph, &run_options, &checkpoint)
.await
.expect("resume should succeed");
let fidelities = captures.fidelities.lock().unwrap();
assert_eq!(fidelities[0].0, "step_b");
assert_eq!(
fidelities[0].1, "summary:low",
"resumed node should use its own fidelity (no degrade since checkpoint was compact, not full)"
);
// Verify the final checkpoint still has the fidelity
let final_cp = state
.current_checkpoint()
.cloned()
.expect("checkpoint should be captured");
assert_eq!(
final_cp.context_values.get("internal.fidelity"),
Some(&serde_json::json!("summary:low")),
);
}
// ===========================================================================
// 20. Real LLM pipeline tests (requires ANTHROPIC_API_KEY)
// ===========================================================================
mod real_llm {
use std::collections::HashMap;
use std::sync::Arc;
use async_trait::async_trait;
use fabro_auth::EnvCredentialSource;
use fabro_graphviz::graph::Node;
use fabro_llm::client::Client;
use fabro_llm::providers::OpenAiAdapter;
use fabro_llm::types::{Message, Request};
use fabro_types::WorkflowSettings;
use fabro_workflow::error::Error;
use fabro_workflow::handler::agent::{
AgentHandler, CodergenBackend, CodergenResult, CodergenRunRequest, OneShotRequest,
};
use tokio_util::sync::CancellationToken;
struct LlmCodergenBackend {
client: Arc<Client>,
model: String,
provider: String,
}
#[async_trait]
impl CodergenBackend for LlmCodergenBackend {
async fn run(&self, request: CodergenRunRequest<'_>) -> Result<CodergenResult, Error> {
self.complete(request.prompt).await
}
async fn one_shot(&self, request: OneShotRequest<'_>) -> Result<CodergenResult, Error> {
self.complete(request.prompt).await
}
}
impl LlmCodergenBackend {
async fn complete(&self, prompt: &str) -> Result<CodergenResult, Error> {
let request = Request {
model: self.model.clone(),
messages: vec![Message::user(prompt)],
provider: Some(self.provider.clone()),
tools: None,
tool_choice: None,
response_format: None,
temperature: Some(0.0),
top_p: None,
max_tokens: Some(200),
stop_sequences: None,
reasoning_effort: None,
speed: None,
metadata: None,
provider_options: None,
};
let response = self
.client
.complete(&request)
.await
.map_err(|e| Error::handler(e.to_string()))?;
Ok(CodergenResult::Text {
text: response.text(),
usage: None,
files_touched: Vec::new(),
last_file_touched: None,
timing: fabro_types::StageTiming::default(),
})
}
}
fn test_llm_model() -> &'static str {
if fabro_test::TestMode::from_env().is_twin() {
"gpt-5.4-mini"
} else {
"claude-haiku-4-5"
}
}
fn test_llm_provider() -> &'static str {
if fabro_test::TestMode::from_env().is_twin() {
"openai"
} else {
"anthropic"
}
}
async fn make_llm_client() -> Option<Arc<Client>> {
if fabro_test::TestMode::from_env().is_twin() {
let (base_url, api_key) = fabro_test::e2e_openai!();
let adapter: Arc<dyn fabro_llm::provider::ProviderAdapter> =
Arc::new(OpenAiAdapter::new(api_key).with_base_url(base_url));
let mut providers: HashMap<String, Arc<dyn fabro_llm::provider::ProviderAdapter>> =
HashMap::new();
providers.insert("openai".to_string(), adapter);
return Some(Arc::new(Client::new(
providers,
Some("openai".to_string()),
Vec::new(),
)));
}
fabro_test::require_env("ANTHROPIC_API_KEY")?;
let source = EnvCredentialSource::new();
Some(Arc::new(
Client::from_source(&source, super::default_catalog())
.await
.expect("unified-llm client should initialize from env source"),
))
}
fn make_llm_backend(client: Arc<Client>) -> Box<LlmCodergenBackend> {
Box::new(LlmCodergenBackend {
client,
model: test_llm_model().to_string(),
provider: test_llm_provider().to_string(),
})
}
use fabro_graphviz::graph::{AttrValue, Edge, Graph};
use fabro_interview::AutoApproveInterviewer;
use fabro_workflow::event::Emitter;
use fabro_workflow::handler::HandlerRegistry;
use fabro_workflow::handler::exit::ExitHandler;
use fabro_workflow::handler::human::HumanHandler;
use fabro_workflow::handler::start::StartHandler;
use fabro_workflow::outcome::StageOutcome;
use fabro_workflow::run_options::RunOptions;
use fabro_workflow::test_support::WorkflowRunner;
use super::{load_run_checkpoint, local_env, test_run_id};
#[fabro_macros::e2e_test(twin, live("ANTHROPIC_API_KEY"))]
async fn real_llm_linear_pipeline() {
let client = make_llm_client().await.unwrap();
let mut graph = Graph::new("RealLLMLinear");
graph.attrs.insert(
"goal".to_string(),
AttrValue::String("Describe a sorting algorithm".to_string()),
);
let mut start = Node::new("start");
start.attrs.insert(
"shape".to_string(),
AttrValue::String("Mdiamond".to_string()),
);
graph.nodes.insert("start".to_string(), start);
let mut exit = Node::new("exit");
exit.attrs.insert(
"shape".to_string(),
AttrValue::String("Msquare".to_string()),
);
graph.nodes.insert("exit".to_string(), exit);
let mut plan = Node::new("plan");
plan.attrs
.insert("shape".to_string(), AttrValue::String("box".to_string()));
plan.attrs.insert(
"prompt".to_string(),
AttrValue::String("Briefly describe quicksort in 2-3 sentences.".to_string()),
);
graph.nodes.insert("plan".to_string(), plan);
let mut review = Node::new("review");
review
.attrs
.insert("shape".to_string(), AttrValue::String("box".to_string()));
review.attrs.insert(
"prompt".to_string(),
AttrValue::String(
"Review the previous description and add one improvement suggestion.".to_string(),
),
);
graph.nodes.insert("review".to_string(), review);
graph.edges.push(Edge::new("start", "plan"));
graph.edges.push(Edge::new("plan", "review"));
graph.edges.push(Edge::new("review", "exit"));
let dir = tempfile::tempdir().unwrap();
let backend = make_llm_backend(client);
let mut registry = HandlerRegistry::new(Box::new(AgentHandler::new(Some(backend))));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"agent",
Box::new(AgentHandler::new(Some(make_llm_backend(
make_llm_client().await.unwrap(),
)))),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (outcome, state) = tokio::time::timeout(
std::time::Duration::from_mins(2),
engine.run_with_state(&graph, &run_options),
)
.await
.expect("should not timeout")
.expect("real LLM pipeline should succeed");
assert_eq!(outcome.status, StageOutcome::Succeeded);
let checkpoint = load_run_checkpoint(dir.path()).unwrap();
assert!(checkpoint.completed_nodes.contains(&"plan".to_string()));
assert!(checkpoint.completed_nodes.contains(&"review".to_string()));
let last_stage = checkpoint
.context_values
.get("last_stage")
.and_then(|v| v.as_str());
assert_eq!(last_stage, Some("review"));
// Verify actual LLM responses were written
let plan_response = state
.stage(&fabro_types::StageId::new("plan", 1))
.and_then(|node| node.response.as_deref())
.unwrap();
assert!(
!plan_response.is_empty(),
"LLM should have generated a response"
);
assert!(
!plan_response.contains("[Simulated]"),
"response should be from real LLM, not simulated"
);
}
#[fabro_macros::e2e_test(twin)]
async fn twin_structured_array_flows_through_for_each_agents_to_fan_in() {
use fabro_test::{TwinScenario, TwinScenarios};
use fabro_workflow::handler::fan_in::FanInHandler;
use fabro_workflow::handler::parallel::ParallelHandler;
use fabro_workflow::handler::prompt::PromptHandler;
let twin = fabro_test::twin_openai().await;
let namespace = format!("{}::for-each", module_path!());
TwinScenarios::new(namespace.clone())
.scenario(TwinScenario::responses("gpt-5.4-mini").text(
r#"{"context_updates":{"candidates":[{"name":"auth","path":"src/auth.rs"},{"label":"api","path":"src/api.rs"}]}}"#,
))
.scenario(
TwinScenario::responses("gpt-5.4-mini")
.text("Reviewed the first security candidate."),
)
.scenario(
TwinScenario::responses("gpt-5.4-mini")
.text("Reviewed the second security candidate."),
)
.scenario(
TwinScenario::responses("gpt-5.4-mini")
.text("Combined both security reviews."),
)
.load(twin)
.await;
let adapter: Arc<dyn fabro_llm::provider::ProviderAdapter> =
Arc::new(OpenAiAdapter::new(namespace.clone()).with_base_url(twin.base_url.clone()));
let providers = HashMap::from([("openai".to_string(), adapter)]);
let client = Arc::new(Client::new(
providers,
Some("openai".to_string()),
Vec::new(),
));
let mut graph = Graph::new("ForEachSecurityReview");
graph.attrs.insert(
"goal".to_string(),
AttrValue::String("Review runtime security candidates".to_string()),
);
let mut start = Node::new("start");
start.attrs.insert(
"shape".to_string(),
AttrValue::String("Mdiamond".to_string()),
);
let mut discover = Node::new("discover");
discover
.attrs
.insert("shape".to_string(), AttrValue::String("tab".to_string()));
discover.attrs.insert(
"prompt".to_string(),
AttrValue::String("Return the candidate array as routing context updates.".to_string()),
);
discover.attrs.insert(
"output_schema".to_string(),
AttrValue::String("routing".to_string()),
);
let mut fanout = Node::new("review_batch");
fanout.attrs.insert(
"shape".to_string(),
AttrValue::String("component".to_string()),
);
fanout.attrs.insert(
"for_each".to_string(),
AttrValue::String("context.candidates".to_string()),
);
fanout
.attrs
.insert("max_parallel".to_string(), AttrValue::Integer(2));
let mut reviewer = Node::new("reviewer");
reviewer.attrs.insert(
"prompt".to_string(),
AttrValue::String("Review this security candidate.".to_string()),
);
let mut aggregate = Node::new("aggregate");
aggregate.attrs.insert(
"shape".to_string(),
AttrValue::String("tripleoctagon".to_string()),
);
aggregate.attrs.insert(
"prompt".to_string(),
AttrValue::String("Synthesize every candidate review.".to_string()),
);
let mut exit = Node::new("exit");
exit.attrs.insert(
"shape".to_string(),
AttrValue::String("Msquare".to_string()),
);
for node in [start, discover, fanout, reviewer, aggregate, exit] {
graph.nodes.insert(node.id.clone(), node);
}
graph.edges.push(Edge::new("start", "discover"));
graph.edges.push(Edge::new("discover", "review_batch"));
graph.edges.push(Edge::new("review_batch", "reviewer"));
graph.edges.push(Edge::new("reviewer", "aggregate"));
graph.edges.push(Edge::new("aggregate", "exit"));
let emitter = Emitter::default();
let events = super::collect_events(&emitter);
let mut registry = HandlerRegistry::new(Box::new(AgentHandler::new(Some(
make_llm_backend(Arc::clone(&client)),
))));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"prompt",
Box::new(PromptHandler::new(Some(make_llm_backend(Arc::clone(
&client,
))))),
);
registry.register(
"agent",
Box::new(AgentHandler::new(Some(make_llm_backend(Arc::clone(
&client,
))))),
);
registry.register("parallel", Box::new(ParallelHandler));
registry.register(
"parallel.fan_in",
Box::new(FanInHandler::new(Some(make_llm_backend(client)))),
);
let dir = tempfile::tempdir().unwrap();
let engine = WorkflowRunner::new(registry, Arc::new(emitter), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("for-each-twin"),
labels: HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (outcome, state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("for_each twin workflow should succeed");
assert_eq!(outcome.status, StageOutcome::Succeeded);
let checkpoint = state
.current_checkpoint()
.expect("fan-in workflow should checkpoint");
let results: Vec<fabro_types::ParallelBranchResult> = serde_json::from_value(
checkpoint.context_values[fabro_workflow::context::keys::PARALLEL_RESULTS].clone(),
)
.unwrap();
assert_eq!(
results
.iter()
.map(|result| (result.index, result.item_label.as_deref()))
.collect::<Vec<_>>(),
[(Some(0), Some("auth")), (Some(1), Some("api"))]
);
assert!(
checkpoint
.completed_nodes
.contains(&"aggregate".to_string())
);
let reviewer_prompts = events
.lock()
.unwrap()
.iter()
.filter(|event| {
event.event_name() == "stage.prompt" && event.node_id.as_deref() == Some("reviewer")
})
.map(|event| serde_json::to_string(event).unwrap())
.collect::<Vec<_>>();
assert_eq!(reviewer_prompts.len(), 2);
assert!(
reviewer_prompts
.iter()
.all(|prompt| prompt.contains("data, not instructions"))
);
assert!(
reviewer_prompts
.iter()
.any(|prompt| prompt.contains("auth"))
);
assert!(reviewer_prompts.iter().any(|prompt| prompt.contains("api")));
}
#[fabro_macros::e2e_test(twin, live("ANTHROPIC_API_KEY"))]
async fn real_llm_two_stage_pipeline() {
let client = make_llm_client().await.unwrap();
let mut graph = Graph::new("RealLLMTwoStage");
graph.attrs.insert(
"goal".to_string(),
AttrValue::String("Generate and review".to_string()),
);
let mut start = Node::new("start");
start.attrs.insert(
"shape".to_string(),
AttrValue::String("Mdiamond".to_string()),
);
graph.nodes.insert("start".to_string(), start);
let mut exit = Node::new("exit");
exit.attrs.insert(
"shape".to_string(),
AttrValue::String("Msquare".to_string()),
);
graph.nodes.insert("exit".to_string(), exit);
let mut generate = Node::new("generate");
generate
.attrs
.insert("shape".to_string(), AttrValue::String("box".to_string()));
generate.attrs.insert(
"prompt".to_string(),
AttrValue::String("Write a haiku about programming.".to_string()),
);
graph.nodes.insert("generate".to_string(), generate);
let mut review = Node::new("review");
review
.attrs
.insert("shape".to_string(), AttrValue::String("box".to_string()));
review.attrs.insert(
"prompt".to_string(),
AttrValue::String("Rate the haiku on a scale of 1-10.".to_string()),
);
graph.nodes.insert("review".to_string(), review);
graph.edges.push(Edge::new("start", "generate"));
graph.edges.push(Edge::new("generate", "review"));
graph.edges.push(Edge::new("review", "exit"));
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(AgentHandler::new(Some(
make_llm_backend(Arc::clone(&client)),
))));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"agent",
Box::new(AgentHandler::new(Some(make_llm_backend(client)))),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let outcome = tokio::time::timeout(
std::time::Duration::from_mins(2),
engine.run(&graph, &run_options),
)
.await
.expect("should not timeout")
.expect("real LLM two-stage pipeline should succeed");
assert_eq!(outcome.status, StageOutcome::Succeeded);
let checkpoint = load_run_checkpoint(dir.path()).unwrap();
let last_stage = checkpoint
.context_values
.get("last_stage")
.and_then(|v| v.as_str());
assert_eq!(last_stage, Some("review"));
}
#[fabro_macros::e2e_test(twin, live("ANTHROPIC_API_KEY"))]
async fn real_llm_human_gate_auto_approve() {
let client = make_llm_client().await.unwrap();
let mut graph = Graph::new("RealLLMGate");
graph.attrs.insert(
"goal".to_string(),
AttrValue::String("Write and approve".to_string()),
);
let mut start = Node::new("start");
start.attrs.insert(
"shape".to_string(),
AttrValue::String("Mdiamond".to_string()),
);
graph.nodes.insert("start".to_string(), start);
let mut exit = Node::new("exit");
exit.attrs.insert(
"shape".to_string(),
AttrValue::String("Msquare".to_string()),
);
graph.nodes.insert("exit".to_string(), exit);
let mut write = Node::new("write");
write
.attrs
.insert("shape".to_string(), AttrValue::String("box".to_string()));
write.attrs.insert(
"prompt".to_string(),
AttrValue::String("Write a one-line greeting.".to_string()),
);
graph.nodes.insert("write".to_string(), write);
let mut gate = Node::new("gate");
gate.attrs.insert(
"shape".to_string(),
AttrValue::String("hexagon".to_string()),
);
gate.attrs
.insert("type".to_string(), AttrValue::String("human".to_string()));
gate.attrs.insert(
"label".to_string(),
AttrValue::String("Approve?".to_string()),
);
graph.nodes.insert("gate".to_string(), gate);
let mut ship = Node::new("ship");
ship.attrs
.insert("shape".to_string(), AttrValue::String("box".to_string()));
ship.attrs.insert(
"prompt".to_string(),
AttrValue::String("Ship the greeting.".to_string()),
);
graph.nodes.insert("ship".to_string(), ship);
let mut revise = Node::new("revise");
revise
.attrs
.insert("shape".to_string(), AttrValue::String("box".to_string()));
revise.attrs.insert(
"prompt".to_string(),
AttrValue::String("Revise the greeting.".to_string()),
);
graph.nodes.insert("revise".to_string(), revise);
graph.edges.push(Edge::new("start", "write"));
graph.edges.push(Edge::new("write", "gate"));
let mut approve_edge = Edge::new("gate", "ship");
approve_edge.attrs.insert(
"label".to_string(),
AttrValue::String("[A] Approve".to_string()),
);
graph.edges.push(approve_edge);
let mut revise_edge = Edge::new("gate", "revise");
revise_edge.attrs.insert(
"label".to_string(),
AttrValue::String("[R] Revise".to_string()),
);
graph.edges.push(revise_edge);
graph.edges.push(Edge::new("ship", "exit"));
graph.edges.push(Edge::new("revise", "gate"));
let dir = tempfile::tempdir().unwrap();
let interviewer = Arc::new(AutoApproveInterviewer::engine());
let mut registry = HandlerRegistry::new(Box::new(AgentHandler::new(Some(
make_llm_backend(Arc::clone(&client)),
))));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"agent",
Box::new(AgentHandler::new(Some(make_llm_backend(client)))),
);
registry.register("human", Box::new(HumanHandler::new(interviewer)));
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let outcome = tokio::time::timeout(
std::time::Duration::from_mins(2),
engine.run(&graph, &run_options),
)
.await
.expect("should not timeout")
.expect("real LLM gate pipeline should succeed");
assert_eq!(outcome.status, StageOutcome::Succeeded);
let checkpoint = load_run_checkpoint(dir.path()).unwrap();
assert!(
checkpoint.completed_nodes.contains(&"write".to_string()),
"write should be completed"
);
assert!(
checkpoint.completed_nodes.contains(&"gate".to_string()),
"gate should be completed"
);
assert!(
checkpoint.completed_nodes.contains(&"ship".to_string()),
"ship should be completed (auto-approve selects first option)"
);
assert!(
!checkpoint.completed_nodes.contains(&"revise".to_string()),
"revise should NOT be traversed with auto-approve"
);
}
#[fabro_macros::e2e_test(twin, live("ANTHROPIC_API_KEY"))]
async fn real_llm_one_shot_pipeline() {
let client = make_llm_client().await.unwrap();
let mut graph = Graph::new("RealLLMOneShot");
graph.attrs.insert(
"goal".to_string(),
AttrValue::String("Classify a fruit".to_string()),
);
let mut start = Node::new("start");
start.attrs.insert(
"shape".to_string(),
AttrValue::String("Mdiamond".to_string()),
);
graph.nodes.insert("start".to_string(), start);
let mut exit = Node::new("exit");
exit.attrs.insert(
"shape".to_string(),
AttrValue::String("Msquare".to_string()),
);
graph.nodes.insert("exit".to_string(), exit);
let mut classify = Node::new("classify");
classify
.attrs
.insert("shape".to_string(), AttrValue::String("tab".to_string()));
classify.attrs.insert(
"prompt".to_string(),
AttrValue::String(
"Reply with exactly one word: is an apple a fruit or vegetable?".to_string(),
),
);
classify.attrs.insert(
"model".to_string(),
AttrValue::String(test_llm_model().to_string()),
);
graph.nodes.insert("classify".to_string(), classify);
graph.edges.push(Edge::new("start", "classify"));
graph.edges.push(Edge::new("classify", "exit"));
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(AgentHandler::new(Some(
make_llm_backend(Arc::clone(&client)),
))));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"prompt",
Box::new(fabro_workflow::handler::prompt::PromptHandler::new(Some(
make_llm_backend(client),
))),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (outcome, state) = tokio::time::timeout(
std::time::Duration::from_secs(30),
engine.run_with_state(&graph, &run_options),
)
.await
.expect("should not timeout")
.expect("one_shot pipeline should succeed");
assert_eq!(outcome.status, StageOutcome::Succeeded);
let response = state
.stage(&fabro_types::StageId::new("classify", 1))
.and_then(|node| node.response.as_deref())
.unwrap();
assert!(!response.is_empty(), "response.md should be non-empty");
}
}
fn openai_responses_payload(text: &str) -> serde_json::Value {
serde_json::json!({
"id": "resp_1",
"model": "gpt-5.4",
"output": [
{
"type": "message",
"role": "assistant",
"content": [
{
"type": "output_text",
"text": text
}
]
}
],
"status": "completed",
"usage": {
"input_tokens": 10,
"output_tokens": 20
}
})
}
// ---------------------------------------------------------------------------
// Wait.human freeform edge integration tests (Section 4.6)
// ---------------------------------------------------------------------------
#[tokio::test]
async fn workflow_run_with_vault_only_openai_codex_builds_pr_body() {
use chrono::Utc;
use fabro_auth::{CredentialSource, VaultCredentialSource};
use fabro_types::Conclusion;
use fabro_vault::{SecretType, Vault};
use httpmock::Method::POST;
use httpmock::MockServer;
use tokio::sync::RwLock as AsyncRwLock;
let server = MockServer::start_async().await;
let response_mock = server
.mock_async(|when, then| {
when.method(POST)
.path("/v1/responses")
.header("authorization", "Bearer vault-openai-key");
then.status(200)
.header("content-type", "application/json")
.json_body(openai_responses_payload(
&serde_json::to_string(&serde_json::json!({
"title": "Vault title",
"body": "Narrative from vault source.",
}))
.unwrap(),
));
})
.await;
let mut graph = Graph::new("VaultOpenAiCodexPrBody");
graph.attrs.insert(
"goal".to_string(),
AttrValue::String("Verify PR body generation uses vault credentials".to_string()),
);
let mut start = Node::new("start");
start.attrs.insert(
"shape".to_string(),
AttrValue::String("Mdiamond".to_string()),
);
graph.nodes.insert("start".to_string(), start);
let mut exit = Node::new("exit");
exit.attrs.insert(
"shape".to_string(),
AttrValue::String("Msquare".to_string()),
);
graph.nodes.insert("exit".to_string(), exit);
graph.edges.push(Edge::new("start", "exit"));
let vault_dir = tempfile::tempdir().unwrap();
let mut vault = Vault::load(vault_dir.path().join("secrets.json")).unwrap();
vault
.set(
"OPENAI_API_KEY",
"vault-openai-key",
SecretType::Token,
None,
)
.unwrap();
let llm_source: Arc<dyn CredentialSource> = Arc::new(VaultCredentialSource::new(Arc::new(
AsyncRwLock::new(vault),
)));
// Use catalog settings to override base_url instead of env var
let catalog = catalog_with_provider_base_url("openai", &server.url("/v1"));
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("vault-only-openai-codex-pr-body"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (outcome, _) = engine
.run_with_state_and_llm_source(&graph, &run_options, Arc::clone(&llm_source))
.await
.expect("workflow run should succeed");
assert_eq!(outcome.status, StageOutcome::Succeeded);
let store_dir = test_store_dir(&run_options.run_dir);
let store = Arc::new(fabro_store::test_support::test_database_at(
Arc::new(LocalFileSystem::new_with_prefix(&store_dir).unwrap()),
"",
Duration::from_millis(1),
None,
&store_dir,
));
let run_store = store.open_run_reader(&run_options.run_id).await.unwrap();
let run_store_handle: fabro_workflow::runtime_store::RunStoreHandle = run_store.into();
let content = fabro_workflow::pull_request::build_pr_content(
"diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n",
"Implement feature",
"gpt-5.4",
&run_store_handle,
llm_source.as_ref(),
catalog,
Some(&Conclusion {
timestamp: Utc::now(),
status: StageOutcome::Succeeded,
timing: fabro_types::RunTiming::wall_only(1),
failure: None,
final_git_commit_sha: None,
stages: Vec::new(),
billing: None,
total_retries: 0,
diff: fabro_types::RunDiff::default(),
}),
None,
)
.await
.expect("PR body should build from vault-only credentials");
assert_eq!(content.title, "Vault title");
assert!(content.body.contains("Narrative from vault source."));
response_mock.assert_async().await;
}
/// Freeform-only human gate: free-text input routes through the freeform edge
/// and stores the text in human.gate.text context variable.
#[tokio::test]
async fn human_gate_freeform_only_routes_text() {
// Graph: start -> gate -> freeform_target -> exit
// gate has only a freeform edge (no fixed choices)
let mut graph = Graph::new("FreeformOnlyTest");
let mut start = Node::new("start");
start.attrs.insert(
"shape".to_string(),
AttrValue::String("Mdiamond".to_string()),
);
graph.nodes.insert("start".to_string(), start);
let mut exit = Node::new("exit");
exit.attrs.insert(
"shape".to_string(),
AttrValue::String("Msquare".to_string()),
);
graph.nodes.insert("exit".to_string(), exit);
let mut gate = Node::new("gate");
gate.attrs.insert(
"shape".to_string(),
AttrValue::String("hexagon".to_string()),
);
gate.attrs
.insert("type".to_string(), AttrValue::String("human".to_string()));
gate.attrs.insert(
"label".to_string(),
AttrValue::String("Enter feedback".to_string()),
);
graph.nodes.insert("gate".to_string(), gate);
graph
.nodes
.insert("freeform_target".to_string(), Node::new("freeform_target"));
graph.edges.push(Edge::new("start", "gate"));
let mut freeform_edge = Edge::new("gate", "freeform_target");
freeform_edge
.attrs
.insert("freeform".to_string(), AttrValue::Boolean(true));
graph.edges.push(freeform_edge);
graph.edges.push(Edge::new("freeform_target", "exit"));
let answers = VecDeque::from([Answer::text("my free text input")]);
let interviewer = Arc::new(QueueInterviewer::new(answers));
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register("human", Box::new(HumanHandler::new(interviewer)));
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (outcome, state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("run should succeed");
assert_eq!(outcome.status, StageOutcome::Succeeded);
let checkpoint = state
.current_checkpoint()
.cloned()
.expect("checkpoint should be captured");
assert!(
checkpoint
.completed_nodes
.contains(&"freeform_target".to_string()),
"should have routed through freeform_target"
);
assert_eq!(
checkpoint.context_values.get("human.gate.text"),
Some(&serde_json::json!("my free text input")),
"human.gate.text should contain the freeform input"
);
assert_eq!(
checkpoint.context_values.get("human.gate.selected"),
Some(&serde_json::json!("freeform")),
"human.gate.selected should be 'freeform'"
);
assert_eq!(
checkpoint.context_values.get("human.gate.label"),
Some(&serde_json::json!("my free text input")),
"human.gate.label should contain the freeform text"
);
}
/// Human gate with both fixed choices and a freeform edge:
/// when the answer matches a fixed choice, it routes to the fixed choice
/// target.
#[tokio::test]
async fn human_gate_freeform_with_fixed_choice_match() {
// Graph: start -> gate -> {approve, reject, freeform_target} -> exit
// gate has fixed choices plus a freeform edge
// Answer selects "A" which matches "Approve" -> routes to approve
let mut graph = Graph::new("FreeformFixedMatchTest");
let mut start = Node::new("start");
start.attrs.insert(
"shape".to_string(),
AttrValue::String("Mdiamond".to_string()),
);
graph.nodes.insert("start".to_string(), start);
let mut exit = Node::new("exit");
exit.attrs.insert(
"shape".to_string(),
AttrValue::String("Msquare".to_string()),
);
graph.nodes.insert("exit".to_string(), exit);
let mut gate = Node::new("gate");
gate.attrs.insert(
"shape".to_string(),
AttrValue::String("hexagon".to_string()),
);
gate.attrs
.insert("type".to_string(), AttrValue::String("human".to_string()));
gate.attrs.insert(
"label".to_string(),
AttrValue::String("Review Changes".to_string()),
);
graph.nodes.insert("gate".to_string(), gate);
graph
.nodes
.insert("approve".to_string(), Node::new("approve"));
graph
.nodes
.insert("reject".to_string(), Node::new("reject"));
graph
.nodes
.insert("freeform_target".to_string(), Node::new("freeform_target"));
graph.edges.push(Edge::new("start", "gate"));
let mut e_approve = Edge::new("gate", "approve");
e_approve.attrs.insert(
"label".to_string(),
AttrValue::String("[A] Approve".to_string()),
);
graph.edges.push(e_approve);
let mut e_reject = Edge::new("gate", "reject");
e_reject.attrs.insert(
"label".to_string(),
AttrValue::String("[R] Reject".to_string()),
);
graph.edges.push(e_reject);
let mut freeform_edge = Edge::new("gate", "freeform_target");
freeform_edge
.attrs
.insert("freeform".to_string(), AttrValue::Boolean(true));
graph.edges.push(freeform_edge);
graph.edges.push(Edge::new("approve", "exit"));
graph.edges.push(Edge::new("reject", "exit"));
graph.edges.push(Edge::new("freeform_target", "exit"));
// Answer selects "A" which matches the Approve choice
let answers = VecDeque::from([Answer {
value: AnswerValue::Selected("A".to_string()),
selected_option: None,
text: None,
}]);
let interviewer = Arc::new(QueueInterviewer::new(answers));
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register("human", Box::new(HumanHandler::new(interviewer)));
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (outcome, state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("run should succeed");
assert_eq!(outcome.status, StageOutcome::Succeeded);
let checkpoint = state
.current_checkpoint()
.cloned()
.expect("checkpoint should be captured");
assert!(
checkpoint.completed_nodes.contains(&"approve".to_string()),
"fixed choice match should route to approve"
);
assert!(
!checkpoint
.completed_nodes
.contains(&"freeform_target".to_string()),
"should NOT route through freeform when fixed choice matches"
);
}
/// Human gate with both fixed choices and a freeform edge:
/// when the answer does NOT match any fixed choice, it falls through to the
/// freeform edge.
#[tokio::test]
async fn human_gate_freeform_fallback_on_unmatched_text() {
// Graph: start -> gate -> {approve, reject, freeform_target} -> exit
// gate has fixed choices plus a freeform edge
// Answer is free text that doesn't match any choice -> routes to
// freeform_target
let mut graph = Graph::new("FreeformFallbackTest");
let mut start = Node::new("start");
start.attrs.insert(
"shape".to_string(),
AttrValue::String("Mdiamond".to_string()),
);
graph.nodes.insert("start".to_string(), start);
let mut exit = Node::new("exit");
exit.attrs.insert(
"shape".to_string(),
AttrValue::String("Msquare".to_string()),
);
graph.nodes.insert("exit".to_string(), exit);
let mut gate = Node::new("gate");
gate.attrs.insert(
"shape".to_string(),
AttrValue::String("hexagon".to_string()),
);
gate.attrs
.insert("type".to_string(), AttrValue::String("human".to_string()));
gate.attrs.insert(
"label".to_string(),
AttrValue::String("Review Changes".to_string()),
);
graph.nodes.insert("gate".to_string(), gate);
graph
.nodes
.insert("approve".to_string(), Node::new("approve"));
graph
.nodes
.insert("reject".to_string(), Node::new("reject"));
graph
.nodes
.insert("freeform_target".to_string(), Node::new("freeform_target"));
graph.edges.push(Edge::new("start", "gate"));
let mut e_approve = Edge::new("gate", "approve");
e_approve.attrs.insert(
"label".to_string(),
AttrValue::String("[A] Approve".to_string()),
);
graph.edges.push(e_approve);
let mut e_reject = Edge::new("gate", "reject");
e_reject.attrs.insert(
"label".to_string(),
AttrValue::String("[R] Reject".to_string()),
);
graph.edges.push(e_reject);
let mut freeform_edge = Edge::new("gate", "freeform_target");
freeform_edge
.attrs
.insert("freeform".to_string(), AttrValue::Boolean(true));
graph.edges.push(freeform_edge);
graph.edges.push(Edge::new("approve", "exit"));
graph.edges.push(Edge::new("reject", "exit"));
graph.edges.push(Edge::new("freeform_target", "exit"));
// Free-text answer that doesn't match any fixed choice
let answers = VecDeque::from([Answer::text("I need more context before deciding")]);
let interviewer = Arc::new(QueueInterviewer::new(answers));
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register("human", Box::new(HumanHandler::new(interviewer)));
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (outcome, state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("run should succeed");
assert_eq!(outcome.status, StageOutcome::Succeeded);
let checkpoint = state
.current_checkpoint()
.cloned()
.expect("checkpoint should be captured");
assert!(
checkpoint
.completed_nodes
.contains(&"freeform_target".to_string()),
"unmatched text should fall through to freeform_target"
);
assert!(
!checkpoint.completed_nodes.contains(&"approve".to_string()),
"should NOT route to approve"
);
assert!(
!checkpoint.completed_nodes.contains(&"reject".to_string()),
"should NOT route to reject"
);
assert_eq!(
checkpoint.context_values.get("human.gate.text"),
Some(&serde_json::json!("I need more context before deciding")),
"human.gate.text should contain the freeform input"
);
assert_eq!(
checkpoint.context_values.get("human.gate.selected"),
Some(&serde_json::json!("freeform")),
"human.gate.selected should be 'freeform' for freeform fallback"
);
assert_eq!(
checkpoint.context_values.get("human.gate.label"),
Some(&serde_json::json!("I need more context before deciding")),
"human.gate.label should contain the freeform text"
);
}
/// Verifies that the Question presented to the interviewer has
/// `allow_freeform=true` when a freeform edge is present on the human gate.
#[tokio::test]
async fn human_gate_freeform_sets_allow_freeform_on_question() {
// Graph: start -> gate -> {approve, freeform_target} -> exit
// gate has a fixed choice plus a freeform edge
// We use RecordingInterviewer to capture the question and verify allow_freeform
let mut graph = Graph::new("AllowFreeformTest");
let mut start = Node::new("start");
start.attrs.insert(
"shape".to_string(),
AttrValue::String("Mdiamond".to_string()),
);
graph.nodes.insert("start".to_string(), start);
let mut exit = Node::new("exit");
exit.attrs.insert(
"shape".to_string(),
AttrValue::String("Msquare".to_string()),
);
graph.nodes.insert("exit".to_string(), exit);
let mut gate = Node::new("gate");
gate.attrs.insert(
"shape".to_string(),
AttrValue::String("hexagon".to_string()),
);
gate.attrs
.insert("type".to_string(), AttrValue::String("human".to_string()));
gate.attrs.insert(
"label".to_string(),
AttrValue::String("Pick or type".to_string()),
);
graph.nodes.insert("gate".to_string(), gate);
graph
.nodes
.insert("approve".to_string(), Node::new("approve"));
graph
.nodes
.insert("freeform_target".to_string(), Node::new("freeform_target"));
graph.edges.push(Edge::new("start", "gate"));
let mut e_approve = Edge::new("gate", "approve");
e_approve.attrs.insert(
"label".to_string(),
AttrValue::String("[A] Approve".to_string()),
);
graph.edges.push(e_approve);
let mut freeform_edge = Edge::new("gate", "freeform_target");
freeform_edge
.attrs
.insert("freeform".to_string(), AttrValue::Boolean(true));
graph.edges.push(freeform_edge);
graph.edges.push(Edge::new("approve", "exit"));
graph.edges.push(Edge::new("freeform_target", "exit"));
let answers = VecDeque::from([Answer {
value: AnswerValue::Selected("A".to_string()),
selected_option: None,
text: None,
}]);
let inner = QueueInterviewer::new(answers);
let recorder = Arc::new(RecordingInterviewer::new(Box::new(inner)));
let interviewer: Arc<dyn Interviewer> = recorder.clone();
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register("human", Box::new(HumanHandler::new(interviewer)));
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let outcome = engine
.run(&graph, &run_options)
.await
.expect("run should succeed");
assert_eq!(outcome.status, StageOutcome::Succeeded);
let recordings = recorder.recordings();
assert_eq!(
recordings.len(),
1,
"should have recorded exactly one question"
);
assert!(
recordings[0].0.allow_freeform,
"Question should have allow_freeform=true when a freeform edge is present"
);
}
/// Verifies that the Question presented to the interviewer has
/// `allow_freeform=false` when no freeform edge is present on the human gate
/// (fixed choices only).
#[tokio::test]
async fn human_gate_without_freeform_sets_allow_freeform_false() {
// Graph: start -> gate -> {approve, reject} -> exit
// gate has only fixed choices, no freeform edge
let mut graph = Graph::new("NoFreeformTest");
let mut start = Node::new("start");
start.attrs.insert(
"shape".to_string(),
AttrValue::String("Mdiamond".to_string()),
);
graph.nodes.insert("start".to_string(), start);
let mut exit = Node::new("exit");
exit.attrs.insert(
"shape".to_string(),
AttrValue::String("Msquare".to_string()),
);
graph.nodes.insert("exit".to_string(), exit);
let mut gate = Node::new("gate");
gate.attrs.insert(
"shape".to_string(),
AttrValue::String("hexagon".to_string()),
);
gate.attrs
.insert("type".to_string(), AttrValue::String("human".to_string()));
gate.attrs.insert(
"label".to_string(),
AttrValue::String("Pick one".to_string()),
);
graph.nodes.insert("gate".to_string(), gate);
graph
.nodes
.insert("approve".to_string(), Node::new("approve"));
graph
.nodes
.insert("reject".to_string(), Node::new("reject"));
graph.edges.push(Edge::new("start", "gate"));
let mut e_approve = Edge::new("gate", "approve");
e_approve.attrs.insert(
"label".to_string(),
AttrValue::String("[A] Approve".to_string()),
);
graph.edges.push(e_approve);
let mut e_reject = Edge::new("gate", "reject");
e_reject.attrs.insert(
"label".to_string(),
AttrValue::String("[R] Reject".to_string()),
);
graph.edges.push(e_reject);
graph.edges.push(Edge::new("approve", "exit"));
graph.edges.push(Edge::new("reject", "exit"));
let answers = VecDeque::from([Answer {
value: AnswerValue::Selected("A".to_string()),
selected_option: None,
text: None,
}]);
let inner = QueueInterviewer::new(answers);
let recorder = Arc::new(RecordingInterviewer::new(Box::new(inner)));
let interviewer: Arc<dyn Interviewer> = recorder.clone();
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register("human", Box::new(HumanHandler::new(interviewer)));
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let outcome = engine
.run(&graph, &run_options)
.await
.expect("run should succeed");
assert_eq!(outcome.status, StageOutcome::Succeeded);
let recordings = recorder.recordings();
assert_eq!(
recordings.len(),
1,
"should have recorded exactly one question"
);
assert!(
!recordings[0].0.allow_freeform,
"Question should have allow_freeform=false when no freeform edge is present"
);
}
// ---------------------------------------------------------------------------
// Subgraph features (Section 2.10)
// ---------------------------------------------------------------------------
#[test]
fn subgraph_node_defaults_scoped_to_subgraph() {
let input = r#"digraph SubgraphDefaults {
graph [goal="Test subgraph defaults"]
start [shape=Mdiamond]
exit [shape=Msquare]
subgraph cluster_loop {
label = "Loop A"
node [thread_id="loop-a", timeout="900s"]
plan [label="Plan next step"]
implement [label="Implement", timeout="1800s"]
}
outside [label="Outside node"]
start -> plan -> implement -> outside -> exit
}"#;
let graph = parse(input).expect("parsing should succeed");
// Plan inherits both thread_id and timeout from subgraph defaults
let plan = &graph.nodes["plan"];
assert_eq!(plan.thread_id(), Some("loop-a"));
assert_eq!(plan.timeout(), Some(std::time::Duration::from_mins(15)));
// Implement inherits thread_id but overrides timeout
let implement = &graph.nodes["implement"];
assert_eq!(implement.thread_id(), Some("loop-a"));
assert_eq!(
implement.timeout(),
Some(std::time::Duration::from_mins(30))
);
// Outside node should NOT have subgraph defaults
let outside = &graph.nodes["outside"];
assert_eq!(outside.thread_id(), None);
assert_eq!(outside.timeout(), None);
}
#[test]
fn subgraph_class_derived_from_label() {
let input = r#"digraph SubgraphClass {
graph [goal="Test class derivation"]
start [shape=Mdiamond]
exit [shape=Msquare]
subgraph cluster_loop {
label = "Loop A"
plan [label="Plan"]
implement [label="Implement"]
}
start -> plan -> implement -> exit
}"#;
let graph = parse(input).expect("parsing should succeed");
// Nodes inside subgraph receive derived class "loop-a"
assert!(graph.nodes["plan"].classes.contains(&"loop-a".to_string()));
assert!(
graph.nodes["implement"]
.classes
.contains(&"loop-a".to_string())
);
// Nodes outside subgraph do not get the class
assert!(!graph.nodes["start"].classes.contains(&"loop-a".to_string()));
assert!(!graph.nodes["exit"].classes.contains(&"loop-a".to_string()));
}
#[test]
fn subgraph_class_derivation_strips_special_chars() {
let input = r#"digraph SubgraphClassStrip {
graph [goal="Test class derivation with special chars"]
subgraph cluster_review {
label = "Code Review!!!"
reviewer [label="Reviewer"]
}
}"#;
let graph = parse(input).expect("parsing should succeed");
// "Code Review!!!" -> lowercase "code review!!!" -> spaces to hyphens
// "code-review!!!" -> strip non-alphanumeric except hyphens ->
// "code-review"
assert!(
graph.nodes["reviewer"]
.classes
.contains(&"code-review".to_string())
);
}
#[test]
fn subgraph_scoping_does_not_leak_to_outer_scope() {
let input = r#"digraph SubgraphScoping {
graph [goal="Test scoping"]
node [timeout="300s"]
subgraph cluster_inner {
label = "Inner"
node [timeout="900s"]
inner_node [label="Inner"]
}
outer_node [label="Outer"]
}"#;
let graph = parse(input).expect("parsing should succeed");
// Inner node gets the subgraph-scoped timeout of 900s
let inner = &graph.nodes["inner_node"];
assert_eq!(inner.timeout(), Some(std::time::Duration::from_mins(15)));
// Outer node gets the graph-level default of 300s, not the subgraph's 900s
let outer = &graph.nodes["outer_node"];
assert_eq!(outer.timeout(), Some(std::time::Duration::from_mins(5)));
}
#[test]
fn subgraph_global_defaults_plus_subgraph_defaults() {
let input = r#"digraph SubgraphMerge {
graph [goal="Test merged defaults"]
node [shape=box, timeout="300s"]
subgraph cluster_loop {
label = "Loop"
node [thread_id="loop-thread"]
step [label="Step"]
}
plain [label="Plain"]
}"#;
let graph = parse(input).expect("parsing should succeed");
// Step should have both the global shape=box + timeout=300s and subgraph
// thread_id
let step = &graph.nodes["step"];
assert_eq!(step.shape(), "box");
assert_eq!(step.thread_id(), Some("loop-thread"));
assert_eq!(step.timeout(), Some(std::time::Duration::from_mins(5)));
// Plain should have the global defaults but no thread_id
let plain = &graph.nodes["plain"];
assert_eq!(plain.shape(), "box");
assert_eq!(plain.thread_id(), None);
assert_eq!(plain.timeout(), Some(std::time::Duration::from_mins(5)));
}
#[test]
fn subgraph_edges_inherit_class() {
let input = r#"digraph SubgraphEdgeClass {
graph [goal="Test edge nodes get class"]
subgraph cluster_loop {
label = "My Loop"
a [label="A"]
b [label="B"]
a -> b
}
}"#;
let graph = parse(input).expect("parsing should succeed");
// Both nodes referenced in edges within the subgraph get the derived class
assert!(graph.nodes["a"].classes.contains(&"my-loop".to_string()));
assert!(graph.nodes["b"].classes.contains(&"my-loop".to_string()));
}
#[test]
fn subgraph_without_label_no_class_derived() {
let input = r#"digraph SubgraphNoLabel {
graph [goal="Test subgraph without label"]
subgraph cluster_unnamed {
node [timeout="600s"]
worker [label="Worker"]
}
}"#;
let graph = parse(input).expect("parsing should succeed");
// No label means no class should be derived
let worker = &graph.nodes["worker"];
assert!(worker.classes.is_empty());
// But the default should still apply
assert_eq!(worker.timeout(), Some(std::time::Duration::from_mins(10)));
}
// ---------------------------------------------------------------------------
// Hook System E2E Tests
// ---------------------------------------------------------------------------
fn hook_runner_from_defs(hooks: Vec<fabro_hooks::HookDefinition>) -> Arc<fabro_hooks::HookRunner> {
Arc::new(fabro_hooks::HookRunner::new(
fabro_hooks::HookSettings { hooks },
auth_test_support::vault_only_credential_source(),
default_catalog(),
))
}
struct HookTestRunner {
emitter: Arc<Emitter>,
hook_runner: Arc<fabro_hooks::HookRunner>,
}
impl HookTestRunner {
async fn run(&self, graph: &Graph, run_options: &RunOptions) -> Result<Outcome, Error> {
run_graph_with_hooks(
make_linear_registry(),
Arc::clone(&self.emitter),
local_env(),
graph,
run_options,
Arc::clone(&self.hook_runner),
None,
)
.await
}
async fn run_with_state(
&self,
graph: &Graph,
run_options: &RunOptions,
) -> Result<(Outcome, fabro_store::RunProjection), Error> {
Box::pin(
fabro_workflow::test_support::run_graph_with_hooks_and_state(
make_linear_registry(),
Arc::clone(&self.emitter),
local_env(),
graph,
run_options,
Arc::clone(&self.hook_runner),
None,
),
)
.await
}
}
fn emitter_with_events() -> (Arc<Emitter>, Arc<std::sync::Mutex<Vec<RunEvent>>>) {
let emitter = Emitter::default();
let events = collect_events(&emitter);
(Arc::new(emitter), events)
}
fn engine_with_hooks(hooks: Vec<fabro_hooks::HookDefinition>) -> HookTestRunner {
HookTestRunner {
emitter: Arc::new(Emitter::default()),
hook_runner: hook_runner_from_defs(hooks),
}
}
fn engine_with_hooks_and_events(
hooks: Vec<fabro_hooks::HookDefinition>,
) -> (HookTestRunner, Arc<std::sync::Mutex<Vec<RunEvent>>>) {
let (emitter, events) = emitter_with_events();
(
HookTestRunner {
emitter,
hook_runner: hook_runner_from_defs(hooks),
},
events,
)
}
fn make_run_options(dir: &std::path::Path) -> RunOptions {
RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("hook-test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
}
}
fn make_hook(event: fabro_hooks::HookEvent, command: &str) -> fabro_hooks::HookDefinition {
fabro_hooks::HookDefinition {
name: None,
event,
command: Some(command.into()),
hook_type: None,
matcher: None,
blocking: None,
timeout_ms: Some(5000),
sandbox: Some(false), // run on host for test reliability
}
}
fn simple_linear_dot() -> &'static str {
r#"digraph HookTest {
graph [goal="Test hooks"]
start [shape=Mdiamond]
exit [shape=Msquare]
work [shape=box, label="Work", prompt="Do work"]
start -> work -> exit
}"#
}
fn two_step_dot() -> &'static str {
r#"digraph HookTest {
graph [goal="Test hooks"]
start [shape=Mdiamond]
exit [shape=Msquare]
step1 [shape=box, label="Step1", prompt="First"]
step2 [shape=box, label="Step2", prompt="Second"]
start -> step1 -> step2 -> exit
}"#
}
fn branching_dot() -> &'static str {
r#"digraph HookTest {
graph [goal="Test routing"]
start [shape=Mdiamond]
exit [shape=Msquare]
plan [shape=box, label="Plan", prompt="Plan it"]
pathA [shape=box, label="PathA", prompt="Path A"]
pathB [shape=box, label="PathB", prompt="Path B"]
start -> plan
plan -> pathA [label="A"]
plan -> pathB [label="B"]
pathA -> exit
pathB -> exit
}"#
}
// --- RunStart hook tests ---
#[tokio::test]
async fn hook_run_start_proceed_allows_run() {
let hooks = vec![make_hook(fabro_hooks::HookEvent::RunStart, "exit 0")];
let engine = engine_with_hooks(hooks);
let graph = parse(simple_linear_dot()).unwrap();
let dir = tempfile::tempdir().unwrap();
let run_options = make_run_options(dir.path());
let (outcome, _state) = Box::pin(engine.run_with_state(&graph, &run_options))
.await
.unwrap();
assert_eq!(outcome.status, StageOutcome::Succeeded);
}
#[tokio::test]
async fn hook_run_start_block_prevents_run() {
let hooks = vec![make_hook(fabro_hooks::HookEvent::RunStart, "exit 1")];
let (engine, events) = engine_with_hooks_and_events(hooks);
let graph = parse(simple_linear_dot()).unwrap();
let dir = tempfile::tempdir().unwrap();
let run_options = make_run_options(dir.path());
let result = engine.run(&graph, &run_options).await;
assert!(result.is_err(), "RunStart block should cause error");
let err = result.unwrap_err();
assert!(
err.to_string().contains("hook"),
"Error should mention hook: {err}"
);
// WorkflowRunStarted should still have been emitted (it fires before the hook)
let captured = events.lock().unwrap();
assert!(
captured.iter().any(|e| e.event_name() == "run.started"),
"WorkflowRunStarted should be emitted before hook blocks"
);
// But no StageStarted — the run never reached node execution
assert!(
!captured.iter().any(|e| e.event_name() == "stage.started"),
"No stage should start when RunStart hook blocks"
);
}
#[tokio::test]
async fn hook_run_start_block_with_json_reason() {
// Hook that outputs JSON with a reason
let hooks = vec![make_hook(
fabro_hooks::HookEvent::RunStart,
r#"echo '{"decision":"block","reason":"policy violation"}'; exit 2"#,
)];
let engine = engine_with_hooks(hooks);
let graph = parse(simple_linear_dot()).unwrap();
let dir = tempfile::tempdir().unwrap();
let run_options = make_run_options(dir.path());
let result = engine.run(&graph, &run_options).await;
assert!(result.is_err());
let err = result.unwrap_err();
assert!(
err.to_string().contains("policy violation"),
"Error should contain JSON reason: {err}"
);
}
// --- StageStart hook tests ---
#[tokio::test]
async fn hook_stage_start_proceed_allows_execution() {
let hooks = vec![make_hook(fabro_hooks::HookEvent::StageStart, "exit 0")];
let engine = engine_with_hooks(hooks);
let graph = parse(simple_linear_dot()).unwrap();
let dir = tempfile::tempdir().unwrap();
let run_options = make_run_options(dir.path());
let (outcome, state) = Box::pin(engine.run_with_state(&graph, &run_options))
.await
.unwrap();
assert_eq!(outcome.status, StageOutcome::Succeeded);
assert!(
state
.stage(&fabro_types::StageId::new("work", 1))
.and_then(|node| node.response.as_ref())
.is_some(),
"response should exist when StageStart hook proceeds"
);
}
#[tokio::test]
async fn hook_stage_start_skip_bypasses_node() {
// Hook that outputs skip decision as JSON
let hooks = vec![make_hook(
fabro_hooks::HookEvent::StageStart,
r#"echo '{"decision":"skip","reason":"not needed"}'; exit 0"#,
)];
let (engine, events) = engine_with_hooks_and_events(hooks);
let graph = parse(simple_linear_dot()).unwrap();
let dir = tempfile::tempdir().unwrap();
let run_options = make_run_options(dir.path());
let (outcome, state) = Box::pin(engine.run_with_state(&graph, &run_options))
.await
.unwrap();
// Pipeline reached exit with goal gates satisfied — per spec, SUCCESS.
assert_eq!(outcome.status, StageOutcome::Succeeded);
assert!(
state
.stage(&fabro_types::StageId::new("work", 1))
.and_then(|node| node.response.as_ref())
.is_none(),
"response should not exist when StageStart hook skips node"
);
// StageStarted should NOT be emitted for hook-skipped stages (the stage never
// started)
let captured = events.lock().unwrap();
let stage_starts: Vec<_> = captured
.iter()
.filter(|e| {
e.event_name() == "stage.started"
&& e.properties().is_ok_and(|properties| {
!matches!(
properties
.get("handler_type")
.and_then(|value| value.as_str()),
Some("start" | "exit")
)
})
})
.collect();
assert!(
stage_starts.is_empty(),
"StageStarted should not be emitted when StageStart hook skips"
);
}
#[tokio::test]
async fn hook_stage_start_block_aborts_run() {
let hooks = vec![make_hook(fabro_hooks::HookEvent::StageStart, "exit 1")];
let engine = engine_with_hooks(hooks);
let graph = parse(simple_linear_dot()).unwrap();
let dir = tempfile::tempdir().unwrap();
let run_options = make_run_options(dir.path());
let result = engine.run(&graph, &run_options).await;
assert!(result.is_err(), "StageStart block should abort the run");
}
#[tokio::test]
async fn hook_stage_start_matcher_filters_by_node_id() {
// Hook that only matches nodes with "step2" in their ID
let mut hook = make_hook(
fabro_hooks::HookEvent::StageStart,
r#"echo '{"decision":"skip","reason":"filtered"}'"#,
);
hook.matcher = Some("step2".into());
let hooks = vec![hook];
let engine = engine_with_hooks(hooks);
let graph = parse(two_step_dot()).unwrap();
let dir = tempfile::tempdir().unwrap();
let run_options = make_run_options(dir.path());
let (outcome, state) = Box::pin(engine.run_with_state(&graph, &run_options))
.await
.unwrap();
// Pipeline reached exit with goal gates satisfied — per spec, SUCCESS.
assert_eq!(outcome.status, StageOutcome::Succeeded);
assert!(
state
.stage(&fabro_types::StageId::new("step1", 1))
.and_then(|node| node.response.as_ref())
.is_some(),
"step1 should execute because matcher doesn't match it"
);
assert!(
state
.stage(&fabro_types::StageId::new("step2", 1))
.and_then(|node| node.response.as_ref())
.is_none(),
"step2 should be skipped because matcher matches it"
);
}
#[tokio::test]
async fn hook_stage_start_matcher_no_match_proceeds() {
// Hook with matcher that matches nothing
let mut hook = make_hook(fabro_hooks::HookEvent::StageStart, "exit 1");
hook.matcher = Some("nonexistent_node".into());
let hooks = vec![hook];
let engine = engine_with_hooks(hooks);
let graph = parse(simple_linear_dot()).unwrap();
let dir = tempfile::tempdir().unwrap();
let run_options = make_run_options(dir.path());
let (outcome, _state) = Box::pin(engine.run_with_state(&graph, &run_options))
.await
.unwrap();
assert_eq!(outcome.status, StageOutcome::Succeeded);
}
// --- StageComplete hook tests ---
#[tokio::test]
async fn hook_stage_complete_fires_after_success() {
let dir = tempfile::tempdir().unwrap();
let marker = dir.path().join("stage_complete_marker.txt");
let hooks = vec![make_hook(
fabro_hooks::HookEvent::StageComplete,
&format!("echo $FABRO_NODE_ID >> {}", marker.display()),
)];
let engine = engine_with_hooks(hooks);
let graph = parse(two_step_dot()).unwrap();
let run_options = make_run_options(dir.path());
let outcome = engine.run(&graph, &run_options).await.unwrap();
assert_eq!(outcome.status, StageOutcome::Succeeded);
// Marker file should exist and contain node IDs
assert!(
marker.exists(),
"StageComplete hook should have written marker file"
);
let content = std::fs::read_to_string(&marker).unwrap();
// start, step1, step2, exit all complete — hook fires for each
assert!(
content.contains("step1"),
"Marker should contain step1: {content}"
);
assert!(
content.contains("step2"),
"Marker should contain step2: {content}"
);
}
#[tokio::test]
async fn hook_stage_complete_failure_does_not_block_pipeline() {
// Non-blocking hook that fails should not affect the pipeline
let hooks = vec![make_hook(fabro_hooks::HookEvent::StageComplete, "exit 1")];
let engine = engine_with_hooks(hooks);
let graph = parse(simple_linear_dot()).unwrap();
let dir = tempfile::tempdir().unwrap();
let run_options = make_run_options(dir.path());
let outcome = engine.run(&graph, &run_options).await.unwrap();
assert_eq!(
outcome.status,
StageOutcome::Succeeded,
"Non-blocking StageComplete hook failure should not block pipeline"
);
}
// --- RunComplete hook tests ---
#[tokio::test]
async fn hook_run_complete_fires_on_success() {
let dir = tempfile::tempdir().unwrap();
let marker = dir.path().join("run_complete_marker.txt");
let hooks = vec![make_hook(
fabro_hooks::HookEvent::RunComplete,
&format!("echo done > {}", marker.display()),
)];
let engine = engine_with_hooks(hooks);
let graph = parse(simple_linear_dot()).unwrap();
let run_options = make_run_options(dir.path());
let outcome = engine.run(&graph, &run_options).await.unwrap();
assert_eq!(outcome.status, StageOutcome::Succeeded);
assert!(
marker.exists(),
"RunComplete hook should have written marker file"
);
let content = std::fs::read_to_string(&marker).unwrap();
assert_eq!(content.trim(), "done");
}
#[tokio::test]
async fn hook_run_complete_does_not_fire_on_blocked_run() {
let dir = tempfile::tempdir().unwrap();
let marker = dir.path().join("run_complete_should_not_exist.txt");
let hooks = vec![
make_hook(
fabro_hooks::HookEvent::RunStart,
"exit 1", // block the run
),
make_hook(
fabro_hooks::HookEvent::RunComplete,
&format!("echo done > {}", marker.display()),
),
];
let engine = engine_with_hooks(hooks);
let graph = parse(simple_linear_dot()).unwrap();
let run_options = make_run_options(dir.path());
let _ = engine.run(&graph, &run_options).await;
assert!(
!marker.exists(),
"RunComplete hook should not fire when run is blocked by RunStart"
);
}
// --- RunFailed hook tests ---
#[tokio::test]
async fn hook_run_failed_fires_on_stage_block() {
let dir = tempfile::tempdir().unwrap();
let marker = dir.path().join("run_failed_marker.txt");
let hooks = vec![
make_hook(
fabro_hooks::HookEvent::StageStart,
"exit 1", // block during stage
),
make_hook(
fabro_hooks::HookEvent::RunFailed,
&format!("echo failed > {}", marker.display()),
),
];
let engine = engine_with_hooks(hooks);
let graph = parse(simple_linear_dot()).unwrap();
let run_options = make_run_options(dir.path());
let _ = engine.run(&graph, &run_options).await;
// RunFailed may or may not fire depending on the error path — a StageStart
// block causes an engine error, which doesn't go through the normal
// WorkflowRunFailed event. Let's just verify no panic occurs.
}
// --- Environment variables ---
#[tokio::test]
async fn hook_receives_env_vars() {
let dir = tempfile::tempdir().unwrap();
let env_file = dir.path().join("hook_env.txt");
let hooks = vec![make_hook(
fabro_hooks::HookEvent::StageComplete,
&format!(
"echo \"event=$FABRO_EVENT run=$FABRO_RUN_ID wf=$FABRO_WORKFLOW node=$FABRO_NODE_ID\" >> {}",
env_file.display()
),
)];
let engine = engine_with_hooks(hooks);
let graph = parse(simple_linear_dot()).unwrap();
let run_options = make_run_options(dir.path());
engine.run(&graph, &run_options).await.unwrap();
assert!(env_file.exists(), "Env file should be written by hook");
let content = std::fs::read_to_string(&env_file).unwrap();
// Should contain lines like: event=stage_complete run=<ulid> wf=HookTest
// node=work
let lines: Vec<&str> = content.lines().collect();
let work_line = lines.iter().find(|l| l.contains("node=work"));
assert!(
work_line.is_some(),
"Should have a line for node=work, got: {content}"
);
let line = work_line.unwrap();
assert!(
line.contains("event=stage_complete"),
"FABRO_EVENT should be set: {line}"
);
assert!(
line.contains(&format!("run={}", test_run_id("hook-test-run"))),
"FABRO_RUN_ID should be set: {line}"
);
assert!(
line.contains("wf=HookTest"),
"FABRO_WORKFLOW should be set: {line}"
);
}
// --- Multiple hooks for same event ---
#[tokio::test]
async fn multiple_hooks_same_event_all_fire() {
let dir = tempfile::tempdir().unwrap();
let marker1 = dir.path().join("hook1.txt");
let marker2 = dir.path().join("hook2.txt");
let hooks = vec![
make_hook(
fabro_hooks::HookEvent::StageComplete,
&format!("echo hook1 > {}", marker1.display()),
),
make_hook(
fabro_hooks::HookEvent::StageComplete,
&format!("echo hook2 > {}", marker2.display()),
),
];
let engine = engine_with_hooks(hooks);
let graph = parse(simple_linear_dot()).unwrap();
let run_options = make_run_options(dir.path());
engine.run(&graph, &run_options).await.unwrap();
assert!(marker1.exists(), "First hook should have fired");
assert!(marker2.exists(), "Second hook should have fired");
}
// --- No hooks configured (baseline) ---
#[tokio::test]
async fn no_hooks_configured_runs_normally() {
let engine = engine_with_hooks(vec![]);
let graph = parse(simple_linear_dot()).unwrap();
let dir = tempfile::tempdir().unwrap();
let run_options = make_run_options(dir.path());
let outcome = engine.run(&graph, &run_options).await.unwrap();
assert_eq!(outcome.status, StageOutcome::Succeeded);
}
// --- EdgeSelected hook tests ---
#[tokio::test]
async fn hook_edge_selected_override_redirects_routing() {
// Hook that overrides edge routing to pathB when it would go to pathA
let mut hook = make_hook(
fabro_hooks::HookEvent::EdgeSelected,
// Override routing to pathB
r#"echo '{"decision":"override","edge_to":"pathB"}'"#,
);
// Only match edges going FROM plan
hook.matcher = Some("^plan$".into());
let hooks = vec![hook];
let (engine, events) = engine_with_hooks_and_events(hooks);
let graph = parse(branching_dot()).unwrap();
let dir = tempfile::tempdir().unwrap();
let run_options = make_run_options(dir.path());
let outcome = engine.run(&graph, &run_options).await.unwrap();
assert_eq!(outcome.status, StageOutcome::Succeeded);
// Verify pathB was executed (override worked)
let captured = events.lock().unwrap();
let completed_nodes: Vec<String> = captured
.iter()
.filter_map(|e| {
(e.event_name() == "stage.completed")
.then(|| e.node_id.clone())
.flatten()
})
.collect();
assert!(
completed_nodes.contains(&"pathB".to_string()),
"pathB should have been executed due to override: {completed_nodes:?}"
);
}
#[tokio::test]
async fn hook_edge_selected_block_aborts_run() {
let mut hook = make_hook(fabro_hooks::HookEvent::EdgeSelected, "exit 1");
hook.matcher = Some("^plan$".into());
let hooks = vec![hook];
let engine = engine_with_hooks(hooks);
let graph = parse(branching_dot()).unwrap();
let dir = tempfile::tempdir().unwrap();
let run_options = make_run_options(dir.path());
let result = engine.run(&graph, &run_options).await;
assert!(result.is_err(), "EdgeSelected block should abort the run");
}
// --- CheckpointSaved hook ---
#[tokio::test]
async fn hook_checkpoint_saved_fires() {
let dir = tempfile::tempdir().unwrap();
let marker = dir.path().join("checkpoint_marker.txt");
let hooks = vec![make_hook(
fabro_hooks::HookEvent::CheckpointSaved,
&format!("echo $FABRO_NODE_ID >> {}", marker.display()),
)];
let engine = engine_with_hooks(hooks);
let graph = parse(simple_linear_dot()).unwrap();
let run_options = make_run_options(dir.path());
let outcome = engine.run(&graph, &run_options).await.unwrap();
assert_eq!(outcome.status, StageOutcome::Succeeded);
// Checkpoint is saved after each node
assert!(marker.exists(), "CheckpointSaved hook should have fired");
let content = std::fs::read_to_string(&marker).unwrap();
assert!(
content.contains("work"),
"Should contain 'work' node checkpoint: {content}"
);
}
// --- StageStart with JSON skip via exit code 2 ---
#[tokio::test]
async fn hook_stage_start_exit_2_blocks() {
let hooks = vec![make_hook(fabro_hooks::HookEvent::StageStart, "exit 2")];
let engine = engine_with_hooks(hooks);
let graph = parse(simple_linear_dot()).unwrap();
let dir = tempfile::tempdir().unwrap();
let run_options = make_run_options(dir.path());
// exit 2 without JSON defaults to Block
let result = engine.run(&graph, &run_options).await;
assert!(result.is_err(), "exit 2 should block");
}
// --- Config merge tests (server + run) ---
#[tokio::test]
async fn hook_config_merge_concatenates() {
use fabro_hooks::{HookDefinition, HookEvent, HookSettings};
let server_hooks = HookSettings {
hooks: vec![HookDefinition {
name: Some("server-hook".into()),
event: HookEvent::RunStart,
command: Some("exit 0".into()),
hook_type: None,
matcher: None,
blocking: None,
timeout_ms: None,
sandbox: Some(false),
}],
};
let run_hooks = HookSettings {
hooks: vec![HookDefinition {
name: Some("run-hook".into()),
event: HookEvent::StageComplete,
command: Some("exit 0".into()),
hook_type: None,
matcher: None,
blocking: None,
timeout_ms: None,
sandbox: Some(false),
}],
};
let merged = server_hooks.merge(run_hooks);
assert_eq!(merged.hooks.len(), 2);
assert_eq!(merged.hooks[0].name.as_deref(), Some("server-hook"));
assert_eq!(merged.hooks[1].name.as_deref(), Some("run-hook"));
}
#[tokio::test]
async fn hook_config_merge_run_overrides_by_name() {
use fabro_hooks::{HookDefinition, HookEvent, HookSettings};
let server_hooks = HookSettings {
hooks: vec![HookDefinition {
name: Some("shared".into()),
event: HookEvent::RunStart,
command: Some("exit 1".into()), // would block
hook_type: None,
matcher: None,
blocking: None,
timeout_ms: None,
sandbox: Some(false),
}],
};
let run_hooks = HookSettings {
hooks: vec![HookDefinition {
name: Some("shared".into()),
event: HookEvent::RunStart,
command: Some("exit 0".into()), // allows
hook_type: None,
matcher: None,
blocking: None,
timeout_ms: None,
sandbox: Some(false),
}],
};
let merged = server_hooks.merge(run_hooks);
assert_eq!(merged.hooks.len(), 1);
// Run config wins — command should be "exit 0"
assert_eq!(
merged.hooks[0]
.command
.as_ref()
.map(fabro_hooks::InterpString::as_source),
Some("exit 0".to_string())
);
// Verify it actually works end-to-end
let engine = engine_with_hooks(merged.hooks);
let graph = parse(simple_linear_dot()).unwrap();
let dir = tempfile::tempdir().unwrap();
let run_options = make_run_options(dir.path());
let outcome = engine.run(&graph, &run_options).await.unwrap();
assert_eq!(outcome.status, StageOutcome::Succeeded);
}
// The legacy `Settings`-based TOML parsing tests were deleted in Stage
// 6.3b. Hook TOML parsing now flows through the v2 config parser path,
// with coverage in fabro-config unit tests and the fabro-cli integration
// tests under `cmd::config`.
// --- Blocking vs non-blocking behavior ---
#[tokio::test]
async fn hook_blocking_override_makes_non_blocking_event_blocking() {
// StageComplete is non-blocking by default, but force it to blocking
let mut hook = make_hook(fabro_hooks::HookEvent::StageComplete, "exit 1");
hook.blocking = Some(true);
let hooks = vec![hook];
let engine = engine_with_hooks(hooks);
let graph = parse(simple_linear_dot()).unwrap();
let dir = tempfile::tempdir().unwrap();
let run_options = make_run_options(dir.path());
// This test verifies that the blocking override is respected
// Note: StageComplete hooks run AFTER execution, so they use the
// non-blocking path in the engine (the engine doesn't check blocking
// for StageComplete since it's always after the fact). This is correct
// behavior — the blocking flag only affects the runner's execution
// strategy (sequential vs parallel), not the engine's decision handling.
let outcome = engine.run(&graph, &run_options).await.unwrap();
assert_eq!(outcome.status, StageOutcome::Succeeded);
}
#[tokio::test]
async fn hook_non_blocking_override_on_blocking_event() {
// RunStart is blocking by default, but force it to non-blocking
let mut hook = make_hook(fabro_hooks::HookEvent::RunStart, "exit 1");
hook.blocking = Some(false);
let hooks = vec![hook];
let engine = engine_with_hooks(hooks);
let graph = parse(simple_linear_dot()).unwrap();
let dir = tempfile::tempdir().unwrap();
let run_options = make_run_options(dir.path());
// With blocking=false, the RunStart hook failure should NOT block the run
// because the runner treats it as non-blocking (doesn't merge decisions)
let outcome = engine.run(&graph, &run_options).await.unwrap();
assert_eq!(outcome.status, StageOutcome::Succeeded);
}
// --- Regex matcher tests ---
#[tokio::test]
async fn hook_matcher_regex_pattern() {
// Hook matches any node starting with "step"
let mut hook = make_hook(
fabro_hooks::HookEvent::StageStart,
r#"echo '{"decision":"skip","reason":"regex match"}'"#,
);
hook.matcher = Some("^step".into());
let hooks = vec![hook];
let engine = engine_with_hooks(hooks);
let graph = parse(two_step_dot()).unwrap();
let dir = tempfile::tempdir().unwrap();
let run_options = make_run_options(dir.path());
let (outcome, state) = Box::pin(engine.run_with_state(&graph, &run_options))
.await
.unwrap();
// Pipeline reached exit with goal gates satisfied — per spec, SUCCESS.
assert_eq!(outcome.status, StageOutcome::Succeeded);
assert!(
state
.stage(&fabro_types::StageId::new("step1", 1))
.and_then(|node| node.response.as_ref())
.is_none(),
"step1 should be skipped by regex ^step"
);
assert!(
state
.stage(&fabro_types::StageId::new("step2", 1))
.and_then(|node| node.response.as_ref())
.is_none(),
"step2 should be skipped by regex ^step"
);
}
// --- JSON decision parsing from hook stdout ---
#[tokio::test]
async fn hook_json_proceed_explicit() {
let hooks = vec![make_hook(
fabro_hooks::HookEvent::RunStart,
r#"echo '{"decision":"proceed"}'"#,
)];
let engine = engine_with_hooks(hooks);
let graph = parse(simple_linear_dot()).unwrap();
let dir = tempfile::tempdir().unwrap();
let run_options = make_run_options(dir.path());
let (outcome, _state) = Box::pin(engine.run_with_state(&graph, &run_options))
.await
.unwrap();
assert_eq!(outcome.status, StageOutcome::Succeeded);
}
#[tokio::test]
async fn hook_json_block_with_reason() {
let hooks = vec![make_hook(
fabro_hooks::HookEvent::RunStart,
r#"echo '{"decision":"block","reason":"forbidden by policy"}'; exit 2"#,
)];
let engine = engine_with_hooks(hooks);
let graph = parse(simple_linear_dot()).unwrap();
let dir = tempfile::tempdir().unwrap();
let run_options = make_run_options(dir.path());
let result = engine.run(&graph, &run_options).await;
assert!(result.is_err());
assert!(
result
.unwrap_err()
.to_string()
.contains("forbidden by policy")
);
}
// --- Sandbox field tests ---
#[tokio::test]
async fn hook_sandbox_false_runs_on_host() {
let dir = tempfile::tempdir().unwrap();
let marker = dir.path().join("host_hook.txt");
let mut hook = make_hook(
fabro_hooks::HookEvent::RunComplete,
&format!("echo host > {}", marker.display()),
);
hook.sandbox = Some(false);
let hooks = vec![hook];
let engine = engine_with_hooks(hooks);
let graph = parse(simple_linear_dot()).unwrap();
let run_options = make_run_options(dir.path());
engine.run(&graph, &run_options).await.unwrap();
assert!(marker.exists(), "Host hook should write marker file");
assert_eq!(std::fs::read_to_string(&marker).unwrap().trim(), "host");
}
// Prompt and Agent hook TOML parsing: the legacy `Settings`-based
// variant of this test was deleted in Stage 6.3b; v2 coverage lives in
// `fabro-types::settings::layer::tests`.
// --- Events emitted correctly alongside hooks ---
#[tokio::test]
async fn hooks_do_not_duplicate_workflow_events() {
let hooks = vec![
make_hook(fabro_hooks::HookEvent::RunStart, "exit 0"),
make_hook(fabro_hooks::HookEvent::StageStart, "exit 0"),
make_hook(fabro_hooks::HookEvent::StageComplete, "exit 0"),
make_hook(fabro_hooks::HookEvent::RunComplete, "exit 0"),
];
let (engine, events) = engine_with_hooks_and_events(hooks);
let graph = parse(simple_linear_dot()).unwrap();
let dir = tempfile::tempdir().unwrap();
let run_options = make_run_options(dir.path());
engine.run(&graph, &run_options).await.unwrap();
let captured = events.lock().unwrap();
// Count WorkflowRunStarted — should be exactly 1
let run_started = captured
.iter()
.filter(|e| e.event_name() == "run.started")
.count();
assert_eq!(run_started, 1, "Should have exactly 1 WorkflowRunStarted");
// Count WorkflowRunCompleted — should be exactly 1
let run_completed = captured
.iter()
.filter(|e| e.event_name() == "run.completed")
.count();
assert_eq!(
run_completed, 1,
"Should have exactly 1 WorkflowRunCompleted"
);
// No WorkflowRunFailed
let run_failed = captured
.iter()
.filter(|e| e.event_name() == "run.failed")
.count();
assert_eq!(run_failed, 0, "Should have 0 WorkflowRunFailed");
}
// ---------------------------------------------------------------------------
// Fidelity preamble injection: verify prompt.md contains preamble + prompt
// for each fidelity mode, using script → codergen pipeline with no live LLM.
// ---------------------------------------------------------------------------
/// Build a `start -> run_tests (script) -> report (codergen) -> exit` pipeline
/// with the given fidelity and goal, then return the contents of
/// `report/prompt.md`.
async fn run_fidelity_prompt_pipeline(fidelity: &str) -> String {
let mut graph = Graph::new("FidelityPromptTest");
graph.attrs.insert(
"goal".to_string(),
AttrValue::String("Validate the build".to_string()),
);
graph.attrs.insert(
"default_fidelity".to_string(),
AttrValue::String(fidelity.to_string()),
);
let mut start = Node::new("start");
start.attrs.insert(
"shape".to_string(),
AttrValue::String("Mdiamond".to_string()),
);
graph.nodes.insert("start".to_string(), start);
let mut exit = Node::new("exit");
exit.attrs.insert(
"shape".to_string(),
AttrValue::String("Msquare".to_string()),
);
graph.nodes.insert("exit".to_string(), exit);
// Script node that produces test output via stdout
let mut run_tests = Node::new("run_tests");
run_tests.attrs.insert(
"shape".to_string(),
AttrValue::String("parallelogram".to_string()),
);
run_tests.attrs.insert(
"script".to_string(),
AttrValue::String("echo '10 passed, 0 failed'".to_string()),
);
graph.nodes.insert("run_tests".to_string(), run_tests);
// Codergen node that should receive the preamble
let mut report = Node::new("report");
report
.attrs
.insert("shape".to_string(), AttrValue::String("box".to_string()));
report.attrs.insert(
"prompt".to_string(),
AttrValue::String("Summarize the test results".to_string()),
);
graph.nodes.insert("report".to_string(), report);
graph.edges.push(Edge::new("start", "run_tests"));
graph.edges.push(Edge::new("run_tests", "report"));
graph.edges.push(Edge::new("report", "exit"));
let dir = tempfile::tempdir().expect("temporary run dir should be created");
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register("command", Box::new(CommandHandler));
registry.register(
"agent",
Box::new(AgentHandler::new(Some(Box::new(MockCodergenBackend)))),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (_outcome, state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("pipeline should succeed");
state
.stage(&fabro_types::StageId::new("report", 1))
.and_then(|node| node.prompt.clone())
.expect("report prompt should exist")
}
async fn run_parallel_fidelity_capture(
fork_fidelity: Option<&str>,
branch_node_fidelity: Option<&str>,
branch_edge_fidelity: Option<&str>,
) -> FidelityCaptures {
use fabro_workflow::handler::fan_in::FanInHandler;
use fabro_workflow::handler::parallel::ParallelHandler;
let mut graph = make_graph_with_start_exit("ParallelFidelityTest");
graph.attrs.insert(
"goal".to_string(),
AttrValue::String("Verify parallel branch context".to_string()),
);
let mut seed = Node::new("seed");
seed.attrs.insert(
"type".to_string(),
AttrValue::String("parallel_fidelity_seed".to_string()),
);
let mut fork = Node::new("fork");
fork.attrs.insert(
"shape".to_string(),
AttrValue::String("component".to_string()),
);
if let Some(fidelity) = fork_fidelity {
fork.attrs.insert(
"fidelity".to_string(),
AttrValue::String(fidelity.to_string()),
);
}
let mut branch_a = Node::new("branch_a");
branch_a.attrs.insert(
"type".to_string(),
AttrValue::String("fidelity_capture".to_string()),
);
if let Some(fidelity) = branch_node_fidelity {
branch_a.attrs.insert(
"fidelity".to_string(),
AttrValue::String(fidelity.to_string()),
);
}
let mut branch_b = Node::new("branch_b");
branch_b.attrs.insert(
"type".to_string(),
AttrValue::String("fidelity_capture".to_string()),
);
let mut fan_in = Node::new("fan_in");
fan_in.attrs.insert(
"shape".to_string(),
AttrValue::String("tripleoctagon".to_string()),
);
graph.nodes.insert(seed.id.clone(), seed);
graph.nodes.insert(fork.id.clone(), fork);
graph.nodes.insert(branch_a.id.clone(), branch_a);
graph.nodes.insert(branch_b.id.clone(), branch_b);
graph.nodes.insert(fan_in.id.clone(), fan_in);
graph.edges.push(Edge::new("start", "seed"));
graph.edges.push(Edge::new("seed", "fork"));
let mut branch_a_edge = Edge::new("fork", "branch_a");
if let Some(fidelity) = branch_edge_fidelity {
branch_a_edge.attrs.insert(
"fidelity".to_string(),
AttrValue::String(fidelity.to_string()),
);
}
graph.edges.push(branch_a_edge);
graph.edges.push(Edge::new("fork", "branch_b"));
graph.edges.push(Edge::new("branch_a", "fan_in"));
graph.edges.push(Edge::new("branch_b", "fan_in"));
graph.edges.push(Edge::new("fan_in", "exit"));
let captures = FidelityCaptures::new();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register("parallel", Box::new(ParallelHandler));
registry.register(
"parallel.fan_in",
Box::new(FanInHandler::new(Some(Box::new(MockCodergenBackend)))),
);
registry.register(
"parallel_fidelity_seed",
Box::new(ParallelFidelitySeedHandler),
);
registry.register(
"fidelity_capture",
Box::new(FidelityCapturingHandler {
captures: captures.clone(),
}),
);
let dir = tempfile::tempdir().expect("parallel fidelity run directory should be created");
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("parallel-fidelity"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (outcome, _state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("parallel fidelity workflow should succeed");
assert_eq!(outcome.status, StageOutcome::Succeeded);
captures
}
fn captured_fidelity_preamble(captures: &FidelityCaptures, node_id: &str) -> (String, String) {
let fidelity = captures
.fidelities
.lock()
.unwrap()
.iter()
.find(|(captured_node_id, _)| captured_node_id == node_id)
.map(|(_, fidelity)| fidelity.clone())
.expect("branch fidelity should be captured");
let preamble = captures
.preambles
.lock()
.unwrap()
.iter()
.find(|(captured_node_id, _)| captured_node_id == node_id)
.map(|(_, preamble)| preamble.clone())
.expect("branch preamble should be captured");
(fidelity, preamble)
}
#[tokio::test]
async fn parallel_branches_get_per_branch_preambles_by_fidelity() {
let captures = run_parallel_fidelity_capture(None, Some("truncate"), None).await;
let (branch_a_fidelity, branch_a_preamble) = captured_fidelity_preamble(&captures, "branch_a");
let (branch_b_fidelity, branch_b_preamble) = captured_fidelity_preamble(&captures, "branch_b");
assert_eq!(branch_a_fidelity, "truncate");
assert!(!branch_a_preamble.contains("parallel_fidelity_marker"));
assert_eq!(branch_b_fidelity, "compact");
assert!(branch_b_preamble.contains("parallel_fidelity_marker"));
}
#[tokio::test]
async fn parallel_fork_fidelity_still_applies_to_all_branches() {
let captures = run_parallel_fidelity_capture(Some("truncate"), None, None).await;
for branch_id in ["branch_a", "branch_b"] {
let (fidelity, preamble) = captured_fidelity_preamble(&captures, branch_id);
assert_eq!(fidelity, "truncate");
assert!(!preamble.contains("parallel_fidelity_marker"));
}
}
#[tokio::test]
async fn parallel_branch_edge_fidelity_overrides_node_fidelity() {
let captures =
run_parallel_fidelity_capture(None, Some("summary:high"), Some("truncate")).await;
let (fidelity, preamble) = captured_fidelity_preamble(&captures, "branch_a");
assert_eq!(fidelity, "truncate");
assert!(!preamble.contains("parallel_fidelity_marker"));
}
#[tokio::test]
async fn fidelity_prompt_compact() {
let prompt = run_fidelity_prompt_pipeline("compact").await;
// Preamble should contain goal, completed stages with handler details, and
// context
assert!(
prompt.contains("Validate the build"),
"compact: should contain goal"
);
assert!(
prompt.contains("## Completed stages"),
"compact: should list completed stages"
);
assert!(
prompt.contains("**run_tests**"),
"compact: should mention run_tests node in bold"
);
assert!(
prompt.contains("Script:"),
"compact: should show script sub-item for run_tests"
);
assert!(
prompt.contains("Output:"),
"compact: should show output sub-item for run_tests"
);
// Original prompt at the end
assert!(
prompt.ends_with("Summarize the test results"),
"compact: should end with original prompt, got:\n{prompt}"
);
}
#[tokio::test]
async fn fidelity_prompt_truncate() {
let prompt = run_fidelity_prompt_pipeline("truncate").await;
// Truncate is minimal: goal + run ID only, no completed stages
assert!(
prompt.contains("Validate the build"),
"truncate: should contain goal"
);
assert!(
!prompt.contains("Completed stages:"),
"truncate: should NOT list completed stages"
);
// Original prompt at the end
assert!(
prompt.ends_with("Summarize the test results"),
"truncate: should end with original prompt, got:\n{prompt}"
);
}
#[tokio::test]
async fn fidelity_prompt_summary_low() {
let prompt = run_fidelity_prompt_pipeline("summary:low").await;
// summary:low includes goal, stage count, recent stages, but NOT context values
assert!(
prompt.contains("Validate the build"),
"summary:low: should contain goal"
);
assert!(
!prompt.contains("Context values:"),
"summary:low: should NOT include context values"
);
// Original prompt at the end
assert!(
prompt.ends_with("Summarize the test results"),
"summary:low: should end with original prompt, got:\n{prompt}"
);
}
#[tokio::test]
async fn fidelity_prompt_summary_medium() {
let prompt = run_fidelity_prompt_pipeline("summary:medium").await;
// summary:medium includes goal, stages, and compact handler details
assert!(
prompt.contains("Validate the build"),
"summary:medium: should contain goal"
);
assert!(
prompt.contains("run_tests"),
"summary:medium: should mention run_tests"
);
assert!(
prompt.contains("Script:"),
"summary:medium: should show script sub-item for run_tests"
);
assert!(
prompt.contains("Output:"),
"summary:medium: should show output sub-item for run_tests"
);
// Original prompt at the end
assert!(
prompt.ends_with("Summarize the test results"),
"summary:medium: should end with original prompt, got:\n{prompt}"
);
}
#[tokio::test]
async fn fidelity_prompt_summary_high() {
let prompt = run_fidelity_prompt_pipeline("summary:high").await;
// summary:high includes goal, all stages as ## Stage headings
assert!(
prompt.contains("Validate the build"),
"summary:high: should contain goal"
);
assert!(
prompt.contains("## Stage: run_tests"),
"summary:high: should have stage heading for run_tests"
);
assert!(
!prompt.contains("## Stage: start"),
"summary:high: should not have stage heading for meta start node"
);
assert!(
prompt.contains("Pipeline progress:"),
"summary:high: should show pipeline progress"
);
// Original prompt at the end
assert!(
prompt.ends_with("Summarize the test results"),
"summary:high: should end with original prompt, got:\n{prompt}"
);
}
#[tokio::test]
async fn fidelity_prompt_full_has_no_preamble() {
let prompt = run_fidelity_prompt_pipeline("full").await;
// Full fidelity produces empty preamble — prompt is just the original
assert_eq!(
prompt, "Summarize the test results",
"full: should be bare prompt with no preamble, got:\n{prompt}"
);
}
// ---------------------------------------------------------------------------
// Artifact offloading integration test
// ---------------------------------------------------------------------------
#[tokio::test]
async fn large_context_values_are_offloaded_to_artifact_store() {
// Pipeline: start -> big_output -> exit
// big_output uses LargeOutputHandler which returns a >100KB context_update.
let mut graph = make_graph_with_start_exit("ArtifactOffload");
graph.attrs.insert(
"goal".to_string(),
AttrValue::String("Test artifact offloading".to_string()),
);
let mut big_output = Node::new("big_output");
big_output.attrs.insert(
"label".to_string(),
AttrValue::String("Big Output".to_string()),
);
graph.nodes.insert("big_output".to_string(), big_output);
graph.edges.push(Edge::new("start", "big_output"));
graph.edges.push(Edge::new("big_output", "exit"));
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(LargeOutputHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
let emitter = Emitter::default();
let events = collect_events(&emitter);
let engine = WorkflowRunner::new(registry, Arc::new(emitter), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (outcome, _state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("pipeline should succeed");
assert_eq!(outcome.status, StageOutcome::Succeeded);
// The checkpoint context should contain a durable blob ref, not the full value.
let checkpoint = load_run_checkpoint(dir.path()).expect("checkpoint should load");
let pointer_value = checkpoint
.context_values
.get("response.big_output")
.expect("context should have response.big_output");
let pointer_str = pointer_value.as_str().expect("pointer should be a string");
assert!(
parse_blob_ref(pointer_str).is_some(),
"value should be a durable blob ref"
);
let resolved = resolve_checkpoint_text(dir.path(), pointer_value)
.expect("offloaded value should resolve through the run store");
assert_eq!(
resolved,
"x".repeat(150 * 1024),
"offloaded value should round-trip through the run store"
);
// WorkflowRunCompleted artifact_count now tracks captured artifacts, not
// offloaded values.
let evts = events.lock().unwrap();
let completed_event = evts
.iter()
.find(|e| e.event_name() == "run.completed")
.expect("should have WorkflowRunCompleted event");
let artifact_count = completed_event.properties().unwrap()["artifact_count"]
.as_u64()
.expect("run.completed should include artifact_count");
assert_eq!(
artifact_count, 0,
"artifact_count should ignore offloaded values"
);
}
// ---------------------------------------------------------------------------
// Artifact sync to remote sandboxs
// ---------------------------------------------------------------------------
/// A mock sandbox where `file_exists` always returns false,
/// simulating a remote container that doesn't have local artifact files.
struct RemoteMockEnv {
working_dir: String,
written: std::sync::Mutex<Vec<(String, String)>>,
existing_paths: std::sync::Mutex<std::collections::HashSet<String>>,
}
impl RemoteMockEnv {
fn new(working_dir: &str) -> Self {
Self {
working_dir: working_dir.to_string(),
written: std::sync::Mutex::new(Vec::new()),
existing_paths: std::sync::Mutex::new(std::collections::HashSet::new()),
}
}
}
#[async_trait::async_trait]
impl fabro_agent::Sandbox for RemoteMockEnv {
async fn read_file_bytes(&self, _path: &str) -> fabro_sandbox::Result<Vec<u8>> {
Err("not implemented".into())
}
async fn write_file(&self, path: &str, content: &str) -> fabro_sandbox::Result<()> {
self.written
.lock()
.unwrap()
.push((path.to_string(), content.to_string()));
self.existing_paths.lock().unwrap().insert(path.to_string());
Ok(())
}
async fn delete_file(&self, _path: &str) -> fabro_sandbox::Result<()> {
Err("not implemented".into())
}
async fn file_exists(&self, path: &str) -> fabro_sandbox::Result<bool> {
Ok(self.existing_paths.lock().unwrap().contains(path))
}
fn runtime_directory(&self) -> Option<&str> {
Some("/tmp/fabro/runtime")
}
async fn list_directory(
&self,
_path: &str,
_depth: Option<usize>,
) -> fabro_sandbox::Result<Vec<fabro_agent::DirEntry>> {
Err("not implemented".into())
}
async fn exec_command(
&self,
_command: &str,
_timeout_ms: u64,
_working_dir: Option<&str>,
_env_vars: Option<&std::collections::HashMap<String, String>>,
_cancel_token: Option<tokio_util::sync::CancellationToken>,
) -> fabro_sandbox::Result<fabro_agent::ExecResult> {
Err("not implemented".into())
}
async fn grep(
&self,
_pattern: &str,
_path: &str,
_options: &fabro_agent::GrepOptions,
) -> fabro_sandbox::Result<Vec<String>> {
Err("not implemented".into())
}
async fn glob(
&self,
_pattern: &str,
_path: Option<&str>,
) -> fabro_sandbox::Result<Vec<String>> {
Err("not implemented".into())
}
async fn initialize(&self) -> fabro_sandbox::Result<()> {
Ok(())
}
async fn cleanup(&self) -> fabro_sandbox::Result<()> {
Ok(())
}
async fn download_file_to_local(
&self,
_: &str,
_: &std::path::Path,
) -> fabro_sandbox::Result<()> {
Err("not implemented".into())
}
async fn upload_file_from_local(
&self,
_: &std::path::Path,
_: &str,
) -> fabro_sandbox::Result<()> {
Err("not implemented".into())
}
fn working_directory(&self) -> &str {
&self.working_dir
}
fn platform(&self) -> &str {
"linux"
}
fn os_version(&self) -> String {
"Linux 5.15".to_string()
}
}
#[tokio::test]
async fn artifact_pointers_rewritten_for_remote_sandbox() {
// Pipeline: start -> big_output -> exit
// big_output uses LargeOutputHandler which returns a >100KB context_update.
// RemoteMockEnv simulates a container where local files don't exist.
let mut graph = make_graph_with_start_exit("ArtifactSync");
graph.attrs.insert(
"goal".to_string(),
AttrValue::String("Test artifact sync to remote env".to_string()),
);
let mut big_output = Node::new("big_output");
big_output.attrs.insert(
"label".to_string(),
AttrValue::String("Big Output".to_string()),
);
graph.nodes.insert("big_output".to_string(), big_output);
graph.edges.push(Edge::new("start", "big_output"));
graph.edges.push(Edge::new("big_output", "exit"));
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(LargeOutputHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
let remote_env = Arc::new(RemoteMockEnv::new("/sandbox"));
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), remote_env.clone());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (outcome, _state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("pipeline should succeed");
assert_eq!(outcome.status, StageOutcome::Succeeded);
// The checkpoint context should contain a durable blob ref.
let checkpoint = load_run_checkpoint(dir.path()).expect("checkpoint should load");
let pointer_value = checkpoint
.context_values
.get("response.big_output")
.expect("context should have response.big_output");
let pointer_str = pointer_value.as_str().expect("pointer should be a string");
assert!(
parse_blob_ref(pointer_str).is_some(),
"checkpoint should persist a blob ref"
);
let resolved = resolve_checkpoint_text(dir.path(), pointer_value)
.expect("offloaded value should resolve through the run store");
assert_eq!(
resolved,
"x".repeat(150 * 1024),
"offloaded value should round-trip through the run store"
);
let written = remote_env.written.lock().unwrap();
assert!(
written.is_empty(),
"blob materialization should not happen until a downstream execution needs it"
);
}
#[tokio::test]
async fn downstream_local_execution_resolves_response_blob_refs_as_text() {
let mut graph = make_graph_with_start_exit("ArtifactMaterializeLocal");
graph.attrs.insert(
"goal".to_string(),
AttrValue::String("Test local blob materialization".to_string()),
);
let mut big_output = Node::new("big_output");
big_output.attrs.insert(
"label".to_string(),
AttrValue::String("Big Output".to_string()),
);
graph.nodes.insert("big_output".to_string(), big_output);
let mut inspect = Node::new("inspect");
inspect.attrs.insert(
"label".to_string(),
AttrValue::String("Inspect".to_string()),
);
inspect.attrs.insert(
"type".to_string(),
AttrValue::String("capture_context".to_string()),
);
graph.nodes.insert("inspect".to_string(), inspect);
graph.edges.push(Edge::new("start", "big_output"));
graph.edges.push(Edge::new("big_output", "inspect"));
graph.edges.push(Edge::new("inspect", "exit"));
let dir = tempfile::tempdir().unwrap();
let captured = Arc::new(std::sync::Mutex::new(Vec::new()));
let mut registry = HandlerRegistry::new(Box::new(LargeOutputHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"capture_context",
Box::new(ContextValueCaptureHandler {
values: Arc::clone(&captured),
key: "response.big_output".to_string(),
}),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (outcome, _state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("pipeline should succeed");
assert_eq!(outcome.status, StageOutcome::Succeeded);
// The downstream handler saw the full inline text, so resolution itself
// did not swap the value for a file reference. Prompt-preamble demotion
// materializes the oversized response for preamble use, confined to the
// run's blob directory.
let captured_value = captured.lock().unwrap().first().cloned().unwrap();
assert_eq!(captured_value, "x".repeat(150 * 1024));
assert!(
RunScratch::new(dir.path())
.runtime_dir()
.join("blobs")
.exists(),
"prompt demotion materializes the oversized response under runtime/blobs"
);
}
#[tokio::test]
async fn downstream_remote_execution_resolves_response_blob_refs_as_text() {
let mut graph = make_graph_with_start_exit("ArtifactMaterializeRemote");
graph.attrs.insert(
"goal".to_string(),
AttrValue::String("Test remote blob materialization".to_string()),
);
let mut big_output = Node::new("big_output");
big_output.attrs.insert(
"label".to_string(),
AttrValue::String("Big Output".to_string()),
);
graph.nodes.insert("big_output".to_string(), big_output);
let mut inspect = Node::new("inspect");
inspect.attrs.insert(
"label".to_string(),
AttrValue::String("Inspect".to_string()),
);
inspect.attrs.insert(
"type".to_string(),
AttrValue::String("capture_context".to_string()),
);
graph.nodes.insert("inspect".to_string(), inspect);
graph.edges.push(Edge::new("start", "big_output"));
graph.edges.push(Edge::new("big_output", "inspect"));
graph.edges.push(Edge::new("inspect", "exit"));
let dir = tempfile::tempdir().unwrap();
let captured = Arc::new(std::sync::Mutex::new(Vec::new()));
let mut registry = HandlerRegistry::new(Box::new(LargeOutputHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"capture_context",
Box::new(ContextValueCaptureHandler {
values: Arc::clone(&captured),
key: "response.big_output".to_string(),
}),
);
let remote_env = Arc::new(RemoteMockEnv::new("/sandbox"));
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), remote_env.clone());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (outcome, _state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("pipeline should succeed");
assert_eq!(outcome.status, StageOutcome::Succeeded);
// The downstream handler saw the full inline text, so resolution itself
// did not swap the value for a file reference. Prompt-preamble demotion
// may still materialize the oversized response into the sandbox blob
// directory, but nowhere else.
let captured_value = captured.lock().unwrap().first().cloned().unwrap();
assert_eq!(captured_value, "x".repeat(150 * 1024));
let written = remote_env.written.lock().unwrap();
assert!(
!written.is_empty(),
"prompt demotion materializes the oversized response into the sandbox"
);
assert!(
written
.iter()
.all(|(path, _)| path.starts_with("/tmp/fabro/runtime/blobs/")),
"nothing is written outside the sandbox runtime blob directory"
);
assert!(
written
.iter()
.all(|(path, _)| !path.starts_with("/sandbox")),
"nothing is written inside the repository checkout"
);
}
// ---------------------------------------------------------------------------
// Node directory visit-count naming
// ---------------------------------------------------------------------------
/// Verify that revisited nodes get distinct stage directories:
/// visit 1 → `stages/{id}@1/`
/// visit 2 → `stages/{id}@2/`
#[tokio::test]
async fn node_dir_uses_visit_count_on_revisit() {
// Handler that fails on first call, succeeds on second.
struct FailOnceHandler {
call_count: std::sync::atomic::AtomicU32,
}
#[async_trait::async_trait]
impl Handler for FailOnceHandler {
async fn execute(
&self,
_node: &Node,
_context: &fabro_workflow::context::Context,
_graph: &Graph,
_run_dir: &Path,
_services: &fabro_workflow::handler::EngineServices,
) -> Result<Outcome, fabro_workflow::error::Error> {
let n = self
.call_count
.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
if n == 0 {
Ok(Outcome::fail_classify("first attempt fails"))
} else {
Ok(Outcome::success())
}
}
}
// Graph: start -> gated_work -> exit
// gated_work has goal_gate=true, retry_target=start
// First visit fails → goal gate unsatisfied → retries from start
// Second visit succeeds → pipeline completes
let mut graph = Graph::new("VisitCountTest");
let mut start = Node::new("start");
start.attrs.insert(
"shape".to_string(),
AttrValue::String("Mdiamond".to_string()),
);
graph.nodes.insert("start".to_string(), start);
let mut exit = Node::new("exit");
exit.attrs.insert(
"shape".to_string(),
AttrValue::String("Msquare".to_string()),
);
graph.nodes.insert("exit".to_string(), exit);
let mut gated_work = Node::new("gated_work");
gated_work
.attrs
.insert("goal_gate".to_string(), AttrValue::Boolean(true));
gated_work
.attrs
.insert("max_retries".to_string(), AttrValue::Integer(0));
gated_work.attrs.insert(
"retry_target".to_string(),
AttrValue::String("start".to_string()),
);
gated_work.attrs.insert(
"type".to_string(),
AttrValue::String("fail_once".to_string()),
);
graph.nodes.insert("gated_work".to_string(), gated_work);
graph.edges.push(Edge::new("start", "gated_work"));
graph.edges.push(Edge::new("gated_work", "exit"));
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"fail_once",
Box::new(FailOnceHandler {
call_count: std::sync::atomic::AtomicU32::new(0),
}),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (outcome, state) = engine
.run_with_state(&graph, &run_options)
.await
.expect("pipeline should succeed");
assert_eq!(outcome.status, StageOutcome::Succeeded);
let first = state
.stage(&fabro_types::StageId::new("gated_work", 1))
.unwrap();
let second = state
.stage(&fabro_types::StageId::new("gated_work", 2))
.unwrap();
assert_eq!(
first.completion.as_ref().unwrap().outcome,
StageOutcome::Failed {
retry_requested: false,
},
"first visit should fail"
);
assert_eq!(
second.completion.as_ref().unwrap().outcome,
StageOutcome::Succeeded,
"second visit should succeed"
);
}
// ---------------------------------------------------------------------------
// Git checkpoint e2e (Local)
// ---------------------------------------------------------------------------
use fabro_auth::test_support as auth_test_support;
use fabro_workflow::handler::fan_in::FanInHandler;
use fabro_workflow::handler::parallel::ParallelHandler;
/// A handler that writes a file named `{node_id}.txt` into the sandbox's
/// working directory. Used to verify shared-checkout writes from parallel
/// branches.
struct FileWriterHandler;
#[async_trait::async_trait]
impl Handler for FileWriterHandler {
async fn execute(
&self,
node: &Node,
_context: &Context,
_graph: &Graph,
_run_dir: &Path,
services: &fabro_workflow::handler::EngineServices,
) -> Result<Outcome, Error> {
let work_dir = services.run.sandbox.working_directory().to_string();
let file_path = format!("{}/{}.txt", work_dir, node.id);
services
.run
.sandbox
.write_file(&file_path, &format!("written by {}", node.id))
.await
.map_err(|e| Error::handler(format!("write_file failed: {e}")))?;
Ok(Outcome::success())
}
}
/// End-to-end test: pipeline with git checkpointing enabled emits
/// `CheckpointCompleted` events with valid commit SHAs and writes `diff.patch`
/// per stage.
#[tokio::test]
async fn git_checkpoint_host_emits_events_and_diff_patch() {
// 1. Create a temporary git repo with an initial commit
let repo = tempfile::tempdir().unwrap();
std::process::Command::new("git")
.args(["init"])
.current_dir(repo.path())
.output()
.unwrap();
std::process::Command::new("git")
.args([
"-c",
"user.name=test",
"-c",
"user.email=test@test",
"commit",
"--allow-empty",
"-m",
"init",
])
.current_dir(repo.path())
.output()
.unwrap();
// 2. Create a branch and worktree (like cli/run.rs setup_worktree)
let base_sha = {
let out = std::process::Command::new("git")
.args(["rev-parse", "HEAD"])
.current_dir(repo.path())
.output()
.unwrap();
String::from_utf8_lossy(&out.stdout).trim().to_string()
};
let run_branch = format!("fabro/run/{}", test_run_id("test-docker"));
std::process::Command::new("git")
.args(["branch", &run_branch, "HEAD"])
.current_dir(repo.path())
.output()
.unwrap();
let worktree_path = repo.path().join("worktree");
std::process::Command::new("git")
.args(["worktree", "add"])
.arg(&worktree_path)
.arg(&run_branch)
.current_dir(repo.path())
.output()
.unwrap();
// Write a file in the worktree so there's something to commit
std::fs::write(worktree_path.join("hello.txt"), "from docker test").unwrap();
// 3. Build a simple pipeline: start -> work -> exit
let mut graph = Graph::new("DockerGitCheckpoint");
graph.attrs.insert(
"goal".to_string(),
AttrValue::String("Test Host git checkpoint".to_string()),
);
let mut start = Node::new("start");
start.attrs.insert(
"shape".to_string(),
AttrValue::String("Mdiamond".to_string()),
);
graph.nodes.insert("start".to_string(), start);
let mut exit = Node::new("exit");
exit.attrs.insert(
"shape".to_string(),
AttrValue::String("Msquare".to_string()),
);
graph.nodes.insert("exit".to_string(), exit);
let mut work = Node::new("work");
work.attrs
.insert("label".to_string(), AttrValue::String("Work".to_string()));
graph.nodes.insert("work".to_string(), work);
graph.edges.push(Edge::new("start", "work"));
graph.edges.push(Edge::new("work", "exit"));
// 4. Set up event collection and engine
let run_dir = tempfile::tempdir().unwrap();
let emitter = Emitter::default();
let events = collect_events(&emitter);
let env: Arc<dyn fabro_agent::Sandbox> =
Arc::new(fabro_agent::LocalSandbox::new(worktree_path.clone()));
let mut registry = HandlerRegistry::new(Box::new(ContextSetterHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
let engine = WorkflowRunner::new(registry, Arc::new(emitter), env);
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: run_dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-docker"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: Some(GitCheckpointOptions {
base_sha: Some(base_sha.clone()),
run_branch: Some(run_branch),
meta_branch: None,
}),
};
// 5. Run pipeline
let outcome = engine
.run(&graph, &run_options)
.await
.expect("pipeline should succeed");
assert_eq!(outcome.status, StageOutcome::Succeeded);
// 6. Assert CheckpointCompleted events with git SHAs were emitted
let events = events.lock().unwrap();
let git_events: Vec<_> = events
.iter()
.filter_map(|e| {
if e.event_name() != "checkpoint.completed" {
return None;
}
let properties = e.properties().ok()?;
Some((
e.node_id.clone()?,
properties.get("git_commit_sha")?.as_str()?.to_string(),
))
})
.collect();
// work node gets a checkpoint commit (start is skipped, exit is terminal)
assert!(
!git_events.is_empty(),
"expected at least 1 CheckpointCompleted event with SHA, got {}",
git_events.len()
);
assert!(
!git_events.iter().any(|(id, _)| id == "start"),
"start node should not have a git checkpoint"
);
// Each SHA should be a valid 40-char hex string
assert!(
git_events
.iter()
.all(|(_, sha)| sha.len() == 40 && sha.chars().all(|c| c.is_ascii_hexdigit())),
"all SHAs should be 40-char hex, got: {git_events:?}"
);
// 7. Verify checkpoint has git_commit_sha
let checkpoint = load_run_checkpoint(run_dir.path()).expect("checkpoint should load");
assert!(
checkpoint.git_commit_sha.is_some(),
"checkpoint should have git_commit_sha"
);
// Cleanup worktree
let _ = std::process::Command::new("git")
.args(["worktree", "remove", "--force"])
.arg(&worktree_path)
.current_dir(repo.path())
.output();
}
/// End-to-end test: pipeline with git checkpointing enabled + `meta_branch`
/// but no worker-side GitHub credentials still writes run-branch checkpoint
/// commits and skips metadata-branch snapshots.
#[tokio::test]
async fn git_checkpoint_host_skips_metadata_branch_without_writer_prereqs() {
// 1. Create a temporary git repo with an initial commit
let repo = tempfile::tempdir().unwrap();
std::process::Command::new("git")
.args(["init"])
.current_dir(repo.path())
.output()
.unwrap();
std::process::Command::new("git")
.args([
"-c",
"user.name=test",
"-c",
"user.email=test@test",
"commit",
"--allow-empty",
"-m",
"init",
])
.current_dir(repo.path())
.output()
.unwrap();
// 2. Create a branch and worktree
let run_id = test_run_id("test-shadow");
let base_sha = {
let out = std::process::Command::new("git")
.args(["rev-parse", "HEAD"])
.current_dir(repo.path())
.output()
.unwrap();
String::from_utf8_lossy(&out.stdout).trim().to_string()
};
std::process::Command::new("git")
.args(["branch", &format!("fabro/run/{run_id}"), "HEAD"])
.current_dir(repo.path())
.output()
.unwrap();
let worktree_path = repo.path().join("worktree");
std::process::Command::new("git")
.args(["worktree", "add"])
.arg(&worktree_path)
.arg(format!("fabro/run/{run_id}"))
.current_dir(repo.path())
.output()
.unwrap();
// Write a file in the worktree so there's something to commit
std::fs::write(worktree_path.join("shadow_test.txt"), "shadow branch test").unwrap();
// 3. Build a simple pipeline: start -> work -> exit
let mut graph = Graph::new("ShadowBranchTest");
graph.attrs.insert(
"goal".to_string(),
AttrValue::String("Test shadow branch".to_string()),
);
let mut start = Node::new("start");
start.attrs.insert(
"shape".to_string(),
AttrValue::String("Mdiamond".to_string()),
);
graph.nodes.insert("start".to_string(), start);
let mut exit = Node::new("exit");
exit.attrs.insert(
"shape".to_string(),
AttrValue::String("Msquare".to_string()),
);
graph.nodes.insert("exit".to_string(), exit);
let mut work = Node::new("work");
work.attrs
.insert("label".to_string(), AttrValue::String("Work".to_string()));
graph.nodes.insert("work".to_string(), work);
graph.edges.push(Edge::new("start", "work"));
graph.edges.push(Edge::new("work", "exit"));
// 4. Set up engine with meta_branch
let run_dir = tempfile::tempdir().unwrap();
// Write graph.fabro so init_run can read it
std::fs::write(run_dir.path().join("graph.fabro"), "digraph {}").unwrap();
let emitter = Emitter::default();
let env: Arc<dyn fabro_agent::Sandbox> =
Arc::new(fabro_agent::LocalSandbox::new(worktree_path.clone()));
let mut registry = HandlerRegistry::new(Box::new(ContextSetterHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
let engine = WorkflowRunner::new(registry, Arc::new(emitter), env);
let meta_branch = format!("fabro/meta/{run_id}");
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: run_dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id,
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: Some(GitCheckpointOptions {
base_sha: Some(base_sha),
run_branch: Some(format!("fabro/run/{run_id}")),
meta_branch: Some(meta_branch.clone()),
}),
};
// 5. Run pipeline
let outcome = engine
.run(&graph, &run_options)
.await
.expect("pipeline should succeed");
assert_eq!(outcome.status, StageOutcome::Succeeded);
// 6. Without pre-run GitHub credentials, metadata snapshots are disabled.
let run_json = std::process::Command::new("git")
.args(["show", &format!("refs/heads/{meta_branch}:run.json")])
.current_dir(repo.path())
.output()
.expect("git show should run");
assert!(
!run_json.status.success(),
"metadata run.json should not exist without writer prerequisites"
);
// 7. Assert run-branch commit still has the run checkpoint trailers.
let output = std::process::Command::new("git")
.args(["log", "--format=%B", "-1"])
.current_dir(&worktree_path)
.output()
.unwrap();
let commit_msg = String::from_utf8_lossy(&output.stdout).trim().to_string();
assert!(
commit_msg.contains("Fabro-Run:"),
"run-branch commit should have Fabro-Run trailer, got:\n{commit_msg}"
);
assert!(
commit_msg.contains("Fabro-Completed:"),
"run-branch commit should have Fabro-Completed trailer, got:\n{commit_msg}"
);
assert!(
!commit_msg.contains("Fabro-Checkpoint:"),
"run-branch commit should not have Fabro-Checkpoint trailer without metadata snapshot, got:\n{commit_msg}"
);
// Cleanup worktree
let _ = std::process::Command::new("git")
.args(["worktree", "remove", "--force"])
.arg(&worktree_path)
.current_dir(repo.path())
.output();
}
// ---------------------------------------------------------------------------
// Host e2e: shared-checkout parallel execution
// ---------------------------------------------------------------------------
/// End-to-end: parallel branches write to one shared checkout and normal
/// run-level checkpointing captures all branch changes after the parallel node.
#[tokio::test]
async fn parallel_shared_checkout_host_e2e() {
// 1. Create a temporary git repo with an initial commit
let repo = tempfile::tempdir().unwrap();
std::process::Command::new("git")
.args(["init"])
.current_dir(repo.path())
.output()
.unwrap();
std::process::Command::new("git")
.args([
"-c",
"user.name=test",
"-c",
"user.email=test@test",
"commit",
"--allow-empty",
"-m",
"init",
])
.current_dir(repo.path())
.output()
.unwrap();
// 2. Set up run branch and worktree (same as cli/run.rs)
let base_sha = {
let out = std::process::Command::new("git")
.args(["rev-parse", "HEAD"])
.current_dir(repo.path())
.output()
.unwrap();
String::from_utf8_lossy(&out.stdout).trim().to_string()
};
let run_id = test_run_id("par-git-test");
let run_branch = format!("fabro/run/{run_id}");
std::process::Command::new("git")
.args(["branch", &run_branch, "HEAD"])
.current_dir(repo.path())
.output()
.unwrap();
let worktree_path = repo.path().join("worktree");
std::process::Command::new("git")
.args(["worktree", "add"])
.arg(&worktree_path)
.arg(&run_branch)
.current_dir(repo.path())
.output()
.unwrap();
// 3. Build pipeline: start -> fan_out -> {branch_a, branch_b} -> fan_in -> exit
let mut graph = Graph::new("ParallelGitBranching");
graph.attrs.insert(
"goal".to_string(),
AttrValue::String("Test parallel git branching".to_string()),
);
let mut start = Node::new("start");
start.attrs.insert(
"shape".to_string(),
AttrValue::String("Mdiamond".to_string()),
);
graph.nodes.insert("start".to_string(), start);
let mut fan_out = Node::new("fan_out");
fan_out.attrs.insert(
"shape".to_string(),
AttrValue::String("component".to_string()),
);
graph.nodes.insert("fan_out".to_string(), fan_out);
let branch_a = Node::new("branch_a");
graph.nodes.insert("branch_a".to_string(), branch_a);
let branch_b = Node::new("branch_b");
graph.nodes.insert("branch_b".to_string(), branch_b);
let mut fan_in = Node::new("fan_in");
fan_in.attrs.insert(
"shape".to_string(),
AttrValue::String("tripleoctagon".to_string()),
);
graph.nodes.insert("fan_in".to_string(), fan_in);
let mut exit = Node::new("exit");
exit.attrs.insert(
"shape".to_string(),
AttrValue::String("Msquare".to_string()),
);
graph.nodes.insert("exit".to_string(), exit);
graph.edges.push(Edge::new("start", "fan_out"));
graph.edges.push(Edge::new("fan_out", "branch_a"));
graph.edges.push(Edge::new("fan_out", "branch_b"));
graph.edges.push(Edge::new("branch_a", "fan_in"));
graph.edges.push(Edge::new("branch_b", "fan_in"));
graph.edges.push(Edge::new("fan_in", "exit"));
// 4. Set up engine with FileWriterHandler for branches
let run_dir = tempfile::tempdir().unwrap();
let emitter = Emitter::default();
let events = collect_events(&emitter);
let env: Arc<dyn fabro_agent::Sandbox> =
Arc::new(fabro_agent::LocalSandbox::new(worktree_path.clone()));
let mut registry = HandlerRegistry::new(Box::new(FileWriterHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register("parallel", Box::new(ParallelHandler));
registry.register("parallel.fan_in", Box::new(FanInHandler::new(None)));
let engine = WorkflowRunner::new(registry, Arc::new(emitter), env);
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: run_dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id,
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: Some(GitCheckpointOptions {
base_sha: Some(base_sha.clone()),
run_branch: Some(run_branch.clone()),
meta_branch: None,
}),
};
// 5. Run pipeline
let outcome = engine
.run(&graph, &run_options)
.await
.expect("parallel pipeline should succeed");
assert_eq!(
outcome.status,
StageOutcome::Succeeded,
"pipeline failed: {:?}",
outcome.failure_reason()
);
// 6. Verify ordered typed results and that no fan-in selection state exists.
let checkpoint = load_run_checkpoint(run_dir.path()).expect("checkpoint should load");
let parallel_results = checkpoint
.context_values
.get("parallel.results")
.expect("parallel.results should be in context");
let results: Vec<fabro_types::ParallelBranchResult> =
serde_json::from_value(parallel_results.clone()).expect("results should be typed");
assert_eq!(
results
.iter()
.map(|result| (result.id.as_str(), result.status))
.collect::<Vec<_>>(),
[
("branch_a", fabro_types::StageOutcome::Succeeded),
("branch_b", fabro_types::StageOutcome::Succeeded),
]
);
assert!(
results
.iter()
.all(|result| result.context_updates.is_empty())
);
assert!(
checkpoint
.context_values
.keys()
.all(|key| !key.starts_with("parallel.fan_in."))
);
// 7. Both branches wrote into the one shared checkout.
for branch in ["branch_a", "branch_b"] {
let file = worktree_path.join(format!("{branch}.txt"));
assert!(
file.exists(),
"{branch} output should remain in the checkout"
);
assert_eq!(
std::fs::read_to_string(file).unwrap(),
format!("written by {branch}")
);
}
// 8. Normal run-level checkpointing captured both files together.
let committed_files = std::process::Command::new("git")
.args(["ls-tree", "-r", "--name-only", "HEAD"])
.current_dir(&worktree_path)
.output()
.unwrap();
assert!(committed_files.status.success());
let committed_files = String::from_utf8_lossy(&committed_files.stdout);
assert!(committed_files.lines().any(|path| path == "branch_a.txt"));
assert!(committed_files.lines().any(|path| path == "branch_b.txt"));
// 9. Fabro created no branch-specific refs, commits, or worktrees.
let parallel_refs = std::process::Command::new("git")
.args([
"for-each-ref",
"--format=%(refname)",
"refs/heads/fabro/run/parallel/",
])
.current_dir(repo.path())
.output()
.unwrap();
assert!(parallel_refs.status.success());
assert!(
parallel_refs.stdout.is_empty(),
"parallel refs must not exist"
);
let worktrees = std::process::Command::new("git")
.args(["worktree", "list", "--porcelain"])
.current_dir(repo.path())
.output()
.unwrap();
assert!(worktrees.status.success());
let worktree_count = String::from_utf8_lossy(&worktrees.stdout)
.lines()
.filter(|line| line.starts_with("worktree "))
.count();
assert_eq!(
worktree_count, 2,
"parallel branches must not add worktrees"
);
let commit_count = std::process::Command::new("git")
.args(["rev-list", "--count", &format!("{base_sha}..HEAD")])
.current_dir(&worktree_path)
.output()
.unwrap();
assert!(commit_count.status.success());
let commit_count: usize = String::from_utf8_lossy(&commit_count.stdout)
.trim()
.parse()
.unwrap();
assert!(
(1..=2).contains(&commit_count),
"only run-level parallel/fan-in checkpoints should be committed, got {commit_count}"
);
// 10. Verify lifecycle events without parallel Git/worktree events.
let events = events.lock().unwrap();
let parallel_started: Vec<_> = events
.iter()
.filter(|e| e.event_name() == "parallel.started")
.collect();
assert_eq!(
parallel_started.len(),
1,
"should have exactly one ParallelStarted event"
);
let parallel_completed: Vec<_> = events
.iter()
.filter(|e| e.event_name() == "parallel.completed")
.collect();
assert_eq!(
parallel_completed.len(),
1,
"should have exactly one ParallelCompleted event"
);
assert!(
events.iter().all(|event| !matches!(
event.event_name(),
"git.branch" | "git.worktree.added" | "git.worktree.removed"
)),
"parallel execution must not emit Git branch or worktree lifecycle events"
);
// Cleanup
let _ = std::process::Command::new("git")
.args(["worktree", "remove", "--force"])
.arg(&worktree_path)
.current_dir(repo.path())
.output();
}
/// When a node produces no file changes, `diff.patch` should NOT be written.
#[tokio::test]
async fn git_checkpoint_host_skips_empty_diff_patch() {
let repo = tempfile::tempdir().unwrap();
std::process::Command::new("git")
.args(["init"])
.current_dir(repo.path())
.output()
.unwrap();
std::process::Command::new("git")
.args([
"-c",
"user.name=test",
"-c",
"user.email=test@test",
"commit",
"--allow-empty",
"-m",
"init",
])
.current_dir(repo.path())
.output()
.unwrap();
let base_sha = {
let out = std::process::Command::new("git")
.args(["rev-parse", "HEAD"])
.current_dir(repo.path())
.output()
.unwrap();
String::from_utf8_lossy(&out.stdout).trim().to_string()
};
let run_branch = format!("fabro/run/{}", test_run_id("empty-diff"));
std::process::Command::new("git")
.args(["branch", &run_branch, "HEAD"])
.current_dir(repo.path())
.output()
.unwrap();
let worktree_path = repo.path().join("worktree");
std::process::Command::new("git")
.args(["worktree", "add"])
.arg(&worktree_path)
.arg(&run_branch)
.current_dir(repo.path())
.output()
.unwrap();
// No files written — handler is a no-op
let mut graph = Graph::new("EmptyDiff");
graph.attrs.insert(
"goal".to_string(),
AttrValue::String("Test empty diff skip".to_string()),
);
let mut start = Node::new("start");
start.attrs.insert(
"shape".to_string(),
AttrValue::String("Mdiamond".to_string()),
);
graph.nodes.insert("start".to_string(), start);
let mut exit = Node::new("exit");
exit.attrs.insert(
"shape".to_string(),
AttrValue::String("Msquare".to_string()),
);
graph.nodes.insert("exit".to_string(), exit);
let mut work = Node::new("work");
work.attrs
.insert("label".to_string(), AttrValue::String("Work".to_string()));
graph.nodes.insert("work".to_string(), work);
graph.edges.push(Edge::new("start", "work"));
graph.edges.push(Edge::new("work", "exit"));
let run_dir = tempfile::tempdir().unwrap();
let emitter = Emitter::default();
let _events = collect_events(&emitter);
let env: Arc<dyn fabro_agent::Sandbox> =
Arc::new(fabro_agent::LocalSandbox::new(worktree_path.clone()));
let mut registry = HandlerRegistry::new(Box::new(ContextSetterHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
let engine = WorkflowRunner::new(registry, Arc::new(emitter), env);
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: run_dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("empty-diff"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: Some(GitCheckpointOptions {
base_sha: Some(base_sha.clone()),
run_branch: Some(run_branch),
meta_branch: None,
}),
};
let outcome = engine
.run(&graph, &run_options)
.await
.expect("pipeline should succeed");
assert_eq!(outcome.status, StageOutcome::Succeeded);
// Cleanup
let _ = std::process::Command::new("git")
.args(["worktree", "remove", "--force"])
.arg(&worktree_path)
.current_dir(repo.path())
.output();
}
// ---------------------------------------------------------------------------
// Failure Signatures & Circuit Breaker E2E Tests
// ---------------------------------------------------------------------------
/// Handler that always fails with a fixed deterministic reason.
struct DeterministicFailHandler {
reason: String,
}
impl DeterministicFailHandler {
fn new(reason: &str) -> Self {
Self {
reason: reason.to_string(),
}
}
}
#[async_trait::async_trait]
impl Handler for DeterministicFailHandler {
async fn execute(
&self,
_node: &Node,
_context: &Context,
_graph: &Graph,
_run_dir: &Path,
_services: &fabro_workflow::handler::EngineServices,
) -> Result<Outcome, Error> {
Ok(Outcome::fail_classify(&self.reason))
}
}
/// Handler that always fails with a transient_infra classification.
struct TransientInfraFailHandler;
#[async_trait::async_trait]
impl Handler for TransientInfraFailHandler {
async fn execute(
&self,
_node: &Node,
_context: &Context,
_graph: &Graph,
_run_dir: &Path,
_services: &fabro_workflow::handler::EngineServices,
) -> Result<Outcome, Error> {
Ok(Outcome::fail_classify("connection refused"))
}
}
/// Handler that provides an explicit `failure_signature` hint via
/// FailureDetail.
struct SignatureHintHandler;
#[async_trait::async_trait]
impl Handler for SignatureHintHandler {
async fn execute(
&self,
_node: &Node,
_context: &Context,
_graph: &Graph,
_run_dir: &Path,
_services: &fabro_workflow::handler::EngineServices,
) -> Result<Outcome, Error> {
Ok(
Outcome::fail_classify("error at line 42 in commit abc123def0")
.with_signature(Some("custom-grouping-key")),
)
}
}
/// Handler that fails with varying reasons each call (truly different after
/// normalization).
struct VaryingReasonFailHandler {
counter: std::sync::atomic::AtomicU32,
}
static E2E_VARYING_REASONS: &[&str] = &[
"syntax error in module alpha",
"type mismatch in module beta",
"missing field in module gamma",
"undefined reference in module delta",
"assertion failed in module epsilon",
"panic in module zeta",
"out of bounds in module eta",
"null pointer in module theta",
"stack overflow in module iota",
"deadlock in module kappa",
];
#[async_trait::async_trait]
impl Handler for VaryingReasonFailHandler {
async fn execute(
&self,
_node: &Node,
_context: &Context,
_graph: &Graph,
_run_dir: &Path,
_services: &fabro_workflow::handler::EngineServices,
) -> Result<Outcome, Error> {
let n = self
.counter
.fetch_add(1, std::sync::atomic::Ordering::SeqCst) as usize;
Ok(Outcome::fail_classify(
E2E_VARYING_REASONS[n % E2E_VARYING_REASONS.len()],
))
}
}
/// Handler that succeeds on the Nth call (0-indexed). Fails deterministically
/// before that.
struct SucceedOnNthHandler {
succeed_on: u32,
counter: std::sync::atomic::AtomicU32,
}
#[async_trait::async_trait]
impl Handler for SucceedOnNthHandler {
async fn execute(
&self,
_node: &Node,
_context: &Context,
_graph: &Graph,
_run_dir: &Path,
_services: &fabro_workflow::handler::EngineServices,
) -> Result<Outcome, Error> {
let n = self
.counter
.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
if n >= self.succeed_on {
Ok(Outcome::success())
} else {
Ok(Outcome::fail_classify("not yet ready"))
}
}
}
/// Build a pipeline: start -> work -> (fail loop back to work, success to exit)
/// This creates a self-loop where work keeps retrying via edge routing.
fn circuit_breaker_self_loop_graph(signature_limit: Option<i64>) -> Graph {
let mut graph = make_graph_with_start_exit("CircuitBreakerSelfLoop");
graph
.attrs
.insert("default_max_retries".to_string(), AttrValue::Integer(0));
// High visit limit so the circuit breaker fires first
graph
.attrs
.insert("max_node_visits".to_string(), AttrValue::Integer(100));
if let Some(limit) = signature_limit {
graph.attrs.insert(
"loop_restart_signature_limit".to_string(),
AttrValue::Integer(limit),
);
}
let mut work = Node::new("work");
work.attrs.insert(
"type".to_string(),
AttrValue::String("test_handler".to_string()),
);
work.attrs
.insert("max_retries".to_string(), AttrValue::Integer(0));
graph.nodes.insert("work".to_string(), work);
graph.edges.push(Edge::new("start", "work"));
let mut fail_edge = Edge::new("work", "work");
fail_edge.attrs.insert(
"condition".to_string(),
AttrValue::String("outcome=failed".to_string()),
);
graph.edges.push(fail_edge);
let mut ok_edge = Edge::new("work", "exit");
ok_edge.attrs.insert(
"condition".to_string(),
AttrValue::String("outcome=succeeded".to_string()),
);
graph.edges.push(ok_edge);
graph
}
/// Build a pipeline: start -> work -> (fail: loop_restart to start, success:
/// exit) This uses loop_restart edges for full pipeline restarts.
fn circuit_breaker_restart_graph(signature_limit: Option<i64>) -> Graph {
let mut graph = make_graph_with_start_exit("CircuitBreakerRestart");
graph
.attrs
.insert("default_max_retries".to_string(), AttrValue::Integer(0));
graph
.attrs
.insert("max_node_visits".to_string(), AttrValue::Integer(100));
if let Some(limit) = signature_limit {
graph.attrs.insert(
"loop_restart_signature_limit".to_string(),
AttrValue::Integer(limit),
);
}
let mut work = Node::new("work");
work.attrs.insert(
"type".to_string(),
AttrValue::String("test_handler".to_string()),
);
work.attrs
.insert("max_retries".to_string(), AttrValue::Integer(0));
graph.nodes.insert("work".to_string(), work);
graph.edges.push(Edge::new("start", "work"));
let mut restart_edge = Edge::new("work", "start");
restart_edge.attrs.insert(
"condition".to_string(),
AttrValue::String("outcome=failed".to_string()),
);
restart_edge
.attrs
.insert("loop_restart".to_string(), AttrValue::Boolean(true));
graph.edges.push(restart_edge);
let mut ok_edge = Edge::new("work", "exit");
ok_edge.attrs.insert(
"condition".to_string(),
AttrValue::String("outcome=succeeded".to_string()),
);
graph.edges.push(ok_edge);
graph
}
// --- E2E Test: normalize_failure_reason produces stable signatures ---
#[test]
fn e2e_normalize_failure_reason_strips_variable_data() {
use fabro_workflow::error::normalize_failure_reason;
// Two error messages that differ only in line numbers and hex hashes
// should normalize to the same string.
let reason_a = "Error at line 42 in commit abc123def0: assertion failed";
let reason_b = "Error at line 999 in commit deadbeef01: assertion failed";
assert_eq!(
normalize_failure_reason(reason_a),
normalize_failure_reason(reason_b),
"errors differing only in line numbers and hashes should normalize identically"
);
// Different semantic errors should NOT normalize to the same string.
let reason_c = "syntax error in module alpha";
let reason_d = "type mismatch in module beta";
assert_ne!(
normalize_failure_reason(reason_c),
normalize_failure_reason(reason_d),
"semantically different errors should produce different normalized forms"
);
}
// --- E2E Test: FailureSignature composite key format ---
#[test]
fn e2e_failure_signature_composite_key() {
use fabro_workflow::error::{FailureCategory, FailureSignature};
let sig = FailureSignature::new(
"verify",
FailureCategory::Deterministic,
None,
Some("assertion failed at line 42"),
);
let sig_str = sig.to_string();
// Verify format: node_id|failure_class|normalized_reason
assert!(sig_str.starts_with("verify|deterministic|"));
// Line number should be normalized away
assert!(
sig_str.contains("<n>"),
"line numbers should be normalized: {sig_str}"
);
assert!(
!sig_str.contains("42"),
"raw digits should be replaced: {sig_str}"
);
}
// --- E2E Test: signature_hint takes priority over failure_reason ---
#[test]
fn e2e_failure_signature_hint_priority() {
use fabro_workflow::error::{FailureCategory, FailureSignature};
let sig = FailureSignature::new(
"build",
FailureCategory::Deterministic,
Some("custom-key-abc"),
Some("raw error with line 123 and hash deadbeef"),
);
// The hint should be used, not the raw reason
assert_eq!(sig.to_string(), "build|deterministic|custom-key-abc");
}
// --- E2E Test: is_signature_tracked only for deterministic + structural ---
#[test]
fn e2e_only_deterministic_and_structural_tracked() {
use fabro_workflow::error::FailureCategory;
// These should be tracked
assert!(FailureCategory::Deterministic.is_signature_tracked());
assert!(FailureCategory::Structural.is_signature_tracked());
// These should NOT be tracked (transient failures retry naturally)
assert!(!FailureCategory::TransientInfra.is_signature_tracked());
assert!(!FailureCategory::BudgetExhausted.is_signature_tracked());
assert!(!FailureCategory::Canceled.is_signature_tracked());
assert!(!FailureCategory::CompilationLoop.is_signature_tracked());
}
// --- E2E Test: loop_restart_signature_limit graph attribute ---
#[test]
fn e2e_loop_restart_signature_limit_from_graph_attr() {
let graph = circuit_breaker_self_loop_graph(Some(5));
assert_eq!(graph.loop_restart_signature_limit(), 5);
let graph_default = circuit_breaker_self_loop_graph(None);
assert_eq!(graph_default.loop_restart_signature_limit(), 3);
}
// --- E2E Test: deterministic failure in self-loop triggers circuit breaker ---
#[tokio::test]
async fn e2e_circuit_breaker_deterministic_self_loop() {
let dir = tempfile::tempdir().unwrap();
let graph = circuit_breaker_self_loop_graph(Some(3));
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"test_handler",
Box::new(DeterministicFailHandler::new(
"assertion failed in foo_test",
)),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("e2e-circuit-breaker"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let result = engine.run(&graph, &run_options).await;
assert!(result.is_err(), "pipeline should abort, not loop forever");
let err = result.unwrap_err().to_string();
assert!(
err.contains("deterministic failure cycle detected"),
"error should mention cycle detection, got: {err}"
);
assert!(
err.contains("repeated 3 times"),
"error should mention the count, got: {err}"
);
assert!(
err.contains("work|deterministic|"),
"error should include the signature, got: {err}"
);
}
// --- E2E Test: custom signature limit (5) ---
#[tokio::test]
async fn e2e_circuit_breaker_custom_limit() {
let dir = tempfile::tempdir().unwrap();
let graph = circuit_breaker_self_loop_graph(Some(5));
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"test_handler",
Box::new(DeterministicFailHandler::new("same error every time")),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("e2e-custom-limit"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let result = engine.run(&graph, &run_options).await;
assert!(result.is_err());
let err = result.unwrap_err().to_string();
assert!(
err.contains("repeated 5 times"),
"should fire at limit=5, got: {err}"
);
}
// --- E2E Test: transient_infra failures do NOT trigger circuit breaker ---
#[tokio::test]
async fn e2e_circuit_breaker_ignores_transient_failures() {
let dir = tempfile::tempdir().unwrap();
let mut graph = circuit_breaker_self_loop_graph(Some(3));
// Lower visit limit so the test terminates quickly via visit limit
graph
.attrs
.insert("max_node_visits".to_string(), AttrValue::Integer(6));
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register("test_handler", Box::new(TransientInfraFailHandler));
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("e2e-transient-no-breaker"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let result = engine.run(&graph, &run_options).await;
assert!(result.is_err());
let err = result.unwrap_err().to_string();
// Should hit visit limit, NOT circuit breaker
assert!(
err.contains("stuck in a cycle"),
"transient failures should not trigger circuit breaker, got: {err}"
);
}
// --- E2E Test: different failure reasons produce different signatures ---
#[tokio::test]
async fn e2e_circuit_breaker_different_reasons_separate_counters() {
let dir = tempfile::tempdir().unwrap();
let mut graph = circuit_breaker_self_loop_graph(Some(3));
// With 10 unique reasons and limit=3, we can do up to 30 iterations before
// any single reason hits 3. But max_node_visits=8 will fire first.
graph
.attrs
.insert("max_node_visits".to_string(), AttrValue::Integer(8));
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"test_handler",
Box::new(VaryingReasonFailHandler {
counter: std::sync::atomic::AtomicU32::new(0),
}),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("e2e-varying-reasons"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let result = engine.run(&graph, &run_options).await;
assert!(result.is_err());
let err = result.unwrap_err().to_string();
// Should hit visit limit because each failure has a unique signature
assert!(
err.contains("stuck in a cycle"),
"varying reasons should not trigger circuit breaker, got: {err}"
);
}
// --- E2E Test: loop_restart edge triggers circuit breaker ---
#[tokio::test]
async fn e2e_circuit_breaker_loop_restart() {
let dir = tempfile::tempdir().unwrap();
let graph = circuit_breaker_restart_graph(Some(3));
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"test_handler",
Box::new(DeterministicFailHandler::new("verify step failed")),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("e2e-restart-breaker"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let result = engine.run(&graph, &run_options).await;
assert!(
result.is_err(),
"pipeline should abort, not restart forever"
);
let err = result.unwrap_err().to_string();
// The loop_restart guard blocks non-transient_infra failures immediately
assert!(
err.contains("loop_restart blocked")
|| err.contains("failure cycle detected")
|| err.contains("circuit breaker"),
"expected loop_restart guard or circuit breaker error, got: {err}"
);
}
// --- E2E Test: failure_signature stored in context (checkpoint verification)
// ---
#[tokio::test]
async fn e2e_failure_signature_persisted_in_context() {
let dir = tempfile::tempdir().unwrap();
// Pipeline: start -> work (fails once) -> exit
// Work fails but the edge routes to exit unconditionally.
let mut graph = make_graph_with_start_exit("SignatureContextTest");
graph
.attrs
.insert("default_max_retries".to_string(), AttrValue::Integer(0));
let mut work = Node::new("work");
work.attrs.insert(
"type".to_string(),
AttrValue::String("test_handler".to_string()),
);
work.attrs
.insert("max_retries".to_string(), AttrValue::Integer(0));
graph.nodes.insert("work".to_string(), work);
graph.edges.push(Edge::new("start", "work"));
graph.edges.push(Edge::new("work", "exit"));
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"test_handler",
Box::new(DeterministicFailHandler::new("test assertion failed")),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("e2e-sig-context"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (outcome, state) = engine.run_with_state(&graph, &run_options).await.unwrap();
// Pipeline reaches exit (terminal) with goal gates satisfied.
// Per spec, reaching exit with satisfied goal gates returns SUCCESS.
assert_eq!(outcome.status, StageOutcome::Succeeded);
// Verify checkpoint has failure_signature in context
let cp = state
.current_checkpoint()
.cloned()
.expect("checkpoint should be captured");
let sig_value = cp
.context_values
.get("failure_signature")
.expect("failure_signature should be in context");
let sig_str = sig_value.as_str().unwrap();
assert!(
sig_str.contains("work|deterministic|"),
"signature should contain node_id|class|, got: {sig_str}"
);
assert!(
sig_str.contains("test assertion failed"),
"signature should contain normalized reason, got: {sig_str}"
);
}
// --- E2E Test: failure_signature hint from handler overrides raw reason ---
#[tokio::test]
async fn e2e_failure_signature_hint_overrides_reason_in_context() {
let dir = tempfile::tempdir().unwrap();
let mut graph = make_graph_with_start_exit("SignatureHintTest");
graph
.attrs
.insert("default_max_retries".to_string(), AttrValue::Integer(0));
let mut work = Node::new("work");
work.attrs.insert(
"type".to_string(),
AttrValue::String("hint_handler".to_string()),
);
work.attrs
.insert("max_retries".to_string(), AttrValue::Integer(0));
graph.nodes.insert("work".to_string(), work);
graph.edges.push(Edge::new("start", "work"));
graph.edges.push(Edge::new("work", "exit"));
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register("hint_handler", Box::new(SignatureHintHandler));
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("e2e-sig-hint"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (_outcome, state) = engine.run_with_state(&graph, &run_options).await.unwrap();
let cp = state
.current_checkpoint()
.cloned()
.expect("checkpoint should be captured");
let sig_str = cp
.context_values
.get("failure_signature")
.and_then(|v| v.as_str())
.expect("failure_signature should be set");
// The hint "custom-grouping-key" should be used, not the raw reason
assert!(
sig_str.contains("custom-grouping-key"),
"hint should override raw reason, got: {sig_str}"
);
// Raw reason contained line numbers and hex — verify they are NOT in the
// signature
assert!(
!sig_str.contains("42"),
"raw reason details should not leak through, got: {sig_str}"
);
}
// --- E2E Test: signature maps persisted in checkpoint and survive save/load
// ---
#[tokio::test]
async fn e2e_signature_maps_persist_in_checkpoint() {
let dir = tempfile::tempdir().unwrap();
// Pipeline where work fails twice then we check the checkpoint
let graph = circuit_breaker_self_loop_graph(Some(5));
// Use a handler that succeeds on the 3rd call (0-indexed), so we get
// exactly 3 failures at the work node before succeeding on the 4th visit.
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"test_handler",
Box::new(SucceedOnNthHandler {
succeed_on: 3,
counter: std::sync::atomic::AtomicU32::new(0),
}),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("e2e-sig-persist"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (outcome, state) = engine.run_with_state(&graph, &run_options).await.unwrap();
assert_eq!(outcome.status, StageOutcome::Succeeded);
// Verify signature maps persisted to the run state checkpoint.
let cp = state
.current_checkpoint()
.cloned()
.expect("checkpoint should be captured");
// The pipeline had 3 deterministic failures at "work" before succeeding.
// loop_failure_signatures should have recorded them.
assert!(
!cp.loop_failure_signatures.is_empty(),
"loop_failure_signatures should have entries after deterministic failures"
);
// Verify the signature key format
let (sig, count) = cp.loop_failure_signatures.iter().next().unwrap();
assert!(
sig.to_string().starts_with("work|deterministic|"),
"signature key should have correct format, got: {sig}"
);
assert_eq!(
*count, 3,
"should have recorded exactly 3 failures before success"
);
}
// --- E2E Test: checkpoint backward compat (old checkpoints without signature
// fields) ---
#[test]
fn e2e_checkpoint_backward_compat_no_signatures() {
// Simulate loading a checkpoint saved before signature fields existed
let json = serde_json::json!({
"timestamp": "2025-06-01T00:00:00Z",
"current_node": "work",
"completed_nodes": ["start", "work"],
"node_retries": {},
"context_values": {"goal": "test"},
"logs": ["some log entry"],
"node_outcomes": {}
});
let cp: Checkpoint = serde_json::from_value(json).expect("should deserialize old checkpoint");
assert!(cp.loop_failure_signatures.is_empty());
assert!(cp.restart_failure_signatures.is_empty());
assert_eq!(cp.current_node, "work");
}
// --- E2E Test: checkpoint with signatures round-trips through save/load ---
#[test]
fn e2e_checkpoint_signatures_roundtrip() {
use fabro_workflow::error::{FailureCategory, FailureSignature};
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("cp.json");
let ctx = Context::new();
ctx.set("goal", serde_json::json!("test roundtrip"));
let mut loop_sigs = std::collections::HashMap::new();
let sig1 = FailureSignature::new(
"verify",
FailureCategory::Deterministic,
None,
Some("assertion failed"),
);
loop_sigs.insert(sig1.clone(), 2usize);
let mut restart_sigs = std::collections::HashMap::new();
let sig2 = FailureSignature::new(
"build",
FailureCategory::Structural,
None,
Some("scope violation"),
);
restart_sigs.insert(sig2.clone(), 1usize);
let cp = Checkpoint::from_context(
&ctx,
"verify",
vec!["start".to_string(), "verify".to_string()],
std::collections::HashMap::new(),
std::collections::HashMap::new(),
None,
loop_sigs,
restart_sigs,
std::collections::HashMap::new(),
);
save_checkpoint(&path, &cp);
let loaded = load_checkpoint(&path).unwrap();
assert_eq!(loaded.loop_failure_signatures.len(), 1);
assert_eq!(loaded.restart_failure_signatures.len(), 1);
assert_eq!(loaded.loop_failure_signatures.get(&sig1), Some(&2));
assert_eq!(loaded.restart_failure_signatures.get(&sig2), Some(&1));
}
// --- E2E Test: pipeline events are emitted before circuit breaker aborts ---
#[tokio::test]
async fn e2e_circuit_breaker_emits_events_before_abort() {
let dir = tempfile::tempdir().unwrap();
let graph = circuit_breaker_self_loop_graph(Some(3));
let emitter = Emitter::default();
let events = collect_events(&emitter);
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"test_handler",
Box::new(DeterministicFailHandler::new("assertion failed")),
);
let engine = WorkflowRunner::new(registry, Arc::new(emitter), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("e2e-events"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let result = engine.run(&graph, &run_options).await;
assert!(result.is_err());
let events = events.lock().unwrap();
// Should have at least WorkflowRunStarted and some StageFailed/StageCompleted
// events
let has_pipeline_started = events.iter().any(|e| e.event_name() == "run.started");
assert!(
has_pipeline_started,
"WorkflowRunStarted event should be emitted"
);
// Verify we got stage events for the failing work node.
// The circuit breaker fires when count reaches the limit (3) *before*
// the stage event for that iteration is emitted, so we see limit-1 events.
let stage_failed_count = events
.iter()
.filter(|e| e.event_name() == "stage.failed" && e.node_id.as_deref() == Some("work"))
.count();
let stage_completed_count = events
.iter()
.filter(|e| e.event_name() == "stage.completed" && e.node_id.as_deref() == Some("work"))
.count();
let total_work_events = stage_completed_count + stage_failed_count;
// With limit=3, the breaker fires on the 3rd failure before its event is
// emitted. So we get 2 events (for failures 1 and 2).
assert!(
total_work_events >= 2,
"should have at least 2 stage events before circuit breaker fires, got: {total_work_events}"
);
}
// --- E2E Test: success resets to success path, but signatures are preserved
// ---
#[tokio::test]
async fn e2e_circuit_breaker_does_not_fire_below_limit() {
let dir = tempfile::tempdir().unwrap();
let graph = circuit_breaker_self_loop_graph(Some(5));
// Handler that fails 4 times (below limit of 5) then succeeds
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"test_handler",
Box::new(SucceedOnNthHandler {
succeed_on: 4,
counter: std::sync::atomic::AtomicU32::new(0),
}),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("e2e-below-limit"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let (outcome, state) = engine.run_with_state(&graph, &run_options).await.unwrap();
assert_eq!(
outcome.status,
StageOutcome::Succeeded,
"pipeline should succeed when failures stay below limit"
);
// Verify signatures were tracked but didn't trigger abort
let cp = state
.current_checkpoint()
.cloned()
.expect("checkpoint should exist");
let total_failures: usize = cp.loop_failure_signatures.values().sum();
assert_eq!(
total_failures, 4,
"should have tracked 4 failures in signatures"
);
}
// --- E2E Test: multi-stage pipeline with impl/verify cycle detection ---
#[tokio::test]
async fn e2e_circuit_breaker_multi_stage_impl_verify_cycle() {
// Pipeline: start -> impl (succeeds) -> verify (fails) -> impl -> verify -> ...
// The verify node always fails with the same deterministic reason.
// Circuit breaker should detect the verify failure cycling.
let dir = tempfile::tempdir().unwrap();
let mut graph = make_graph_with_start_exit("ImplVerifyCycle");
graph
.attrs
.insert("default_max_retries".to_string(), AttrValue::Integer(0));
graph
.attrs
.insert("max_node_visits".to_string(), AttrValue::Integer(100));
graph.attrs.insert(
"loop_restart_signature_limit".to_string(),
AttrValue::Integer(3),
);
let mut impl_node = Node::new("impl");
impl_node.attrs.insert(
"type".to_string(),
AttrValue::String("success_handler".to_string()),
);
graph.nodes.insert("impl".to_string(), impl_node);
let mut verify_node = Node::new("verify");
verify_node.attrs.insert(
"type".to_string(),
AttrValue::String("fail_handler".to_string()),
);
verify_node
.attrs
.insert("max_retries".to_string(), AttrValue::Integer(0));
graph.nodes.insert("verify".to_string(), verify_node);
graph.edges.push(Edge::new("start", "impl"));
graph.edges.push(Edge::new("impl", "verify"));
// verify fail -> back to impl
let mut fail_edge = Edge::new("verify", "impl");
fail_edge.attrs.insert(
"condition".to_string(),
AttrValue::String("outcome=failed".to_string()),
);
graph.edges.push(fail_edge);
// verify success -> exit (never taken)
let mut ok_edge = Edge::new("verify", "exit");
ok_edge.attrs.insert(
"condition".to_string(),
AttrValue::String("outcome=succeeded".to_string()),
);
graph.edges.push(ok_edge);
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register("success_handler", Box::new(StartHandler)); // StartHandler returns success
registry.register(
"fail_handler",
Box::new(DeterministicFailHandler::new(
"test assertion: expected 42, got 0",
)),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("e2e-impl-verify-cycle"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let result = engine.run(&graph, &run_options).await;
assert!(
result.is_err(),
"should detect impl/verify cycle, not loop forever"
);
let err = result.unwrap_err().to_string();
assert!(
err.contains("deterministic failure cycle detected"),
"should identify deterministic failure cycle, got: {err}"
);
assert!(
err.contains("verify|deterministic|"),
"signature should name the verify node, got: {err}"
);
}
// --- E2E Tests: loop_restart guard (only transient_infra may restart) ---
/// Handler that fails with an explicit failure_class hint and succeeds on the
/// Nth call.
struct ClassifiedFailHandler {
failure_class: &'static str,
succeed_on: u32,
counter: std::sync::atomic::AtomicU32,
}
impl ClassifiedFailHandler {
fn always(failure_class: &'static str) -> Self {
Self {
failure_class,
succeed_on: u32::MAX,
counter: std::sync::atomic::AtomicU32::new(0),
}
}
fn succeed_on(failure_class: &'static str, n: u32) -> Self {
Self {
failure_class,
succeed_on: n,
counter: std::sync::atomic::AtomicU32::new(0),
}
}
}
#[async_trait::async_trait]
impl Handler for ClassifiedFailHandler {
async fn execute(
&self,
_node: &Node,
_context: &Context,
_graph: &Graph,
_run_dir: &Path,
_services: &fabro_workflow::handler::EngineServices,
) -> Result<Outcome, Error> {
let n = self
.counter
.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
if n >= self.succeed_on {
return Ok(Outcome::success());
}
let failure_class: fabro_workflow::error::FailureCategory =
self.failure_class.parse().unwrap();
let mut outcome = Outcome::fail_classify("classified failure");
if let Some(ref mut f) = outcome.failure {
f.category = failure_class;
}
Ok(outcome)
}
}
#[tokio::test]
async fn e2e_loop_restart_blocked_for_deterministic_failure() {
let dir = tempfile::tempdir().unwrap();
let graph = circuit_breaker_restart_graph(Some(10));
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"test_handler",
Box::new(ClassifiedFailHandler::always("deterministic")),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("e2e-restart-blocked-det"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let result = engine.run(&graph, &run_options).await;
assert!(
result.is_err(),
"deterministic failure should not loop_restart"
);
let err = result.unwrap_err().to_string();
assert!(
err.contains("loop_restart blocked"),
"expected loop_restart blocked error, got: {err}"
);
}
#[tokio::test]
async fn e2e_loop_restart_blocked_for_structural_failure() {
let dir = tempfile::tempdir().unwrap();
let graph = circuit_breaker_restart_graph(Some(10));
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"test_handler",
Box::new(ClassifiedFailHandler::always("structural")),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("e2e-restart-blocked-struct"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let result = engine.run(&graph, &run_options).await;
assert!(
result.is_err(),
"structural failure should not loop_restart"
);
let err = result.unwrap_err().to_string();
assert!(
err.contains("loop_restart blocked"),
"expected loop_restart blocked error, got: {err}"
);
}
#[tokio::test]
async fn e2e_loop_restart_blocked_for_budget_exhausted_failure() {
let dir = tempfile::tempdir().unwrap();
let graph = circuit_breaker_restart_graph(Some(10));
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"test_handler",
Box::new(ClassifiedFailHandler::always("budget_exhausted")),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("e2e-restart-blocked-budget"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let result = engine.run(&graph, &run_options).await;
assert!(
result.is_err(),
"budget_exhausted failure should not loop_restart"
);
let err = result.unwrap_err().to_string();
assert!(
err.contains("loop_restart blocked"),
"expected loop_restart blocked error, got: {err}"
);
}
#[tokio::test]
async fn e2e_loop_restart_blocked_for_canceled_failure() {
let dir = tempfile::tempdir().unwrap();
let graph = circuit_breaker_restart_graph(Some(10));
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"test_handler",
Box::new(ClassifiedFailHandler::always("canceled")),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("e2e-restart-blocked-canceled"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let result = engine.run(&graph, &run_options).await;
assert!(result.is_err(), "canceled failure should not loop_restart");
let err = result.unwrap_err().to_string();
assert!(
err.contains("loop_restart blocked"),
"expected loop_restart blocked error, got: {err}"
);
}
#[tokio::test]
async fn e2e_loop_restart_blocked_for_compilation_loop_failure() {
let dir = tempfile::tempdir().unwrap();
let graph = circuit_breaker_restart_graph(Some(10));
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"test_handler",
Box::new(ClassifiedFailHandler::always("compilation_loop")),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("e2e-restart-blocked-comploop"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let result = engine.run(&graph, &run_options).await;
assert!(
result.is_err(),
"compilation_loop failure should not loop_restart"
);
let err = result.unwrap_err().to_string();
assert!(
err.contains("loop_restart blocked"),
"expected loop_restart blocked error, got: {err}"
);
}
#[tokio::test]
async fn e2e_loop_restart_allowed_for_transient_infra() {
let dir = tempfile::tempdir().unwrap();
let graph = circuit_breaker_restart_graph(Some(10));
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
// Fails with transient_infra on first call, succeeds on second
registry.register(
"test_handler",
Box::new(ClassifiedFailHandler::succeed_on("transient_infra", 1)),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("e2e-restart-allowed-transient"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let result = engine.run(&graph, &run_options).await;
assert!(
result.is_ok(),
"transient_infra failure should be allowed to loop_restart, got: {:?}",
result.unwrap_err()
);
}
// ---------------------------------------------------------------------------
// Stall watchdog e2e tests
// ---------------------------------------------------------------------------
/// Handler that sleeps forever (for stall watchdog testing).
struct HangingHandler;
#[async_trait::async_trait]
impl Handler for HangingHandler {
async fn execute(
&self,
_node: &Node,
_context: &Context,
_graph: &Graph,
_run_dir: &Path,
_services: &fabro_workflow::handler::EngineServices,
) -> Result<Outcome, Error> {
tokio::time::sleep(std::time::Duration::from_mins(1)).await;
Ok(Outcome::success())
}
}
/// Handler that emits keepalive events periodically, then succeeds.
struct KeepaliveHandler {
interval_ms: u64,
total_ms: u64,
}
#[async_trait::async_trait]
impl Handler for KeepaliveHandler {
async fn execute(
&self,
node: &Node,
_context: &Context,
_graph: &Graph,
_run_dir: &Path,
services: &fabro_workflow::handler::EngineServices,
) -> Result<Outcome, Error> {
let start = std::time::Instant::now();
while start.elapsed() < std::time::Duration::from_millis(self.total_ms) {
tokio::time::sleep(std::time::Duration::from_millis(self.interval_ms)).await;
services.run.emitter.emit(&Event::Prompt {
stage: node.id.clone(),
visit: 1,
text: "keepalive".to_string(),
mode: None,
provider: None,
model: None,
reasoning_effort: None,
speed: None,
});
}
Ok(Outcome::success())
}
}
#[tokio::test]
async fn e2e_stall_watchdog_triggers_from_dot_parsed_pipeline() {
// Parse a DOT graph with stall_timeout set to 200ms
let dot = r#"digraph StallTest {
graph [goal="Test stall watchdog", stall_timeout="50ms", default_max_retries=0]
start [shape=Mdiamond]
work [type="hanging", label="Work"]
exit [shape=Msquare]
start -> work -> exit
}"#;
let graph = parse(dot).expect("parse should succeed");
// Verify the stall_timeout was parsed correctly
assert_eq!(
graph.stall_timeout(),
Some(std::time::Duration::from_millis(50)),
);
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register("hanging", Box::new(HangingHandler));
let events = Arc::new(std::sync::Mutex::new(Vec::new()));
let events_clone = events.clone();
let emitter = Emitter::default();
emitter.on_event(move |event| {
events_clone.lock().unwrap().push(format!("{event:?}"));
});
let engine = WorkflowRunner::new(registry, Arc::new(emitter), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("stall-e2e"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let result = engine.run(&graph, &run_options).await;
assert!(result.is_err(), "expected stall watchdog error");
let err = result.unwrap_err().to_string();
assert!(
err.contains("stall watchdog"),
"expected error to contain 'stall watchdog', got: {err}"
);
// Verify the canonical watchdog timeout envelope was emitted.
let collected = events.lock().unwrap();
assert!(
collected.iter().any(|e| e.contains("StallWatchdogTimeout")),
"expected StallWatchdogTimeout event in: {collected:?}"
);
}
#[tokio::test]
async fn e2e_stall_watchdog_kept_alive_by_handler_events() {
// Parse a DOT graph with stall_timeout 200ms, but the handler emits events
// every 100ms for 500ms total — the watchdog should NOT trigger.
let dot = r#"digraph StallAliveTest {
graph [goal="Test stall keepalive", stall_timeout="100ms", default_max_retries=0]
start [shape=Mdiamond]
work [type="keepalive", label="Work"]
exit [shape=Msquare]
start -> work -> exit
}"#;
let graph = parse(dot).expect("parse should succeed");
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register(
"keepalive",
Box::new(KeepaliveHandler {
interval_ms: 10,
total_ms: 50,
}),
);
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("stall-alive-e2e"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let outcome = engine
.run(&graph, &run_options)
.await
.expect("pipeline should succeed");
assert_eq!(outcome.status, StageOutcome::Succeeded);
}
#[tokio::test]
async fn e2e_stall_watchdog_disabled_with_zero_timeout() {
// Parse a DOT graph with stall_timeout="0s" — watchdog should be disabled,
// and a short sleep handler should complete successfully.
let dot = r#"digraph StallDisabledTest {
graph [goal="Test stall disabled", stall_timeout="0s", default_max_retries=0]
start [shape=Mdiamond]
work [type="slow", label="Work"]
exit [shape=Msquare]
start -> work -> exit
}"#;
let graph = parse(dot).expect("parse should succeed");
assert_eq!(
graph.stall_timeout(),
None,
"zero timeout should disable watchdog"
);
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register("slow", Box::new(SlowTestHandler { sleep_ms: 50 }));
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("stall-disabled-e2e"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let outcome = engine
.run(&graph, &run_options)
.await
.expect("pipeline should succeed");
assert_eq!(outcome.status, StageOutcome::Succeeded);
}
/// Handler that sleeps for a configurable duration, then succeeds (for e2e
/// tests).
struct SlowTestHandler {
sleep_ms: u64,
}
#[async_trait::async_trait]
impl Handler for SlowTestHandler {
async fn execute(
&self,
_node: &Node,
_context: &Context,
_graph: &Graph,
_run_dir: &Path,
_services: &fabro_workflow::handler::EngineServices,
) -> Result<Outcome, Error> {
tokio::time::sleep(std::time::Duration::from_millis(self.sleep_ms)).await;
Ok(Outcome::success())
}
}
#[tokio::test]
async fn e2e_stall_watchdog_with_explicit_timeout_override() {
// A short stall_timeout of 50ms should trigger faster than the default 1800s.
// This tests that the graph attribute is actually respected.
let dot = r#"digraph StallOverrideTest {
graph [goal="Test stall override", stall_timeout="50ms", default_max_retries=0]
start [shape=Mdiamond]
work [type="hanging", label="Work"]
exit [shape=Msquare]
start -> work -> exit
}"#;
let graph = parse(dot).expect("parse should succeed");
assert_eq!(
graph.stall_timeout(),
Some(std::time::Duration::from_millis(50)),
);
let dir = tempfile::tempdir().unwrap();
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register("hanging", Box::new(HangingHandler));
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), local_env());
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("stall-override-e2e"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let start = std::time::Instant::now();
let result = engine.run(&graph, &run_options).await;
let elapsed = start.elapsed();
assert!(result.is_err(), "expected stall watchdog error");
let err = result.unwrap_err().to_string();
assert!(err.contains("stall watchdog"), "got: {err}");
// Should trigger well under 1 second (50ms timeout + check interval overhead)
assert!(
elapsed < std::time::Duration::from_secs(1),
"stall watchdog took too long: {elapsed:?}"
);
}
// Daytona parallel git branching test is in daytona_integration.rs
// ---------------------------------------------------------------------------
// Artifact collection e2e tests
// ---------------------------------------------------------------------------
/// Handler that creates artifact files in the sandbox working directory via
/// exec_command.
struct AssetCreatorHandler {
should_fail: bool,
}
impl AssetCreatorHandler {
fn success() -> Self {
Self { should_fail: false }
}
fn failing() -> Self {
Self { should_fail: true }
}
}
#[async_trait::async_trait]
impl Handler for AssetCreatorHandler {
async fn execute(
&self,
_node: &Node,
_context: &Context,
_graph: &Graph,
_run_dir: &Path,
services: &fabro_workflow::handler::EngineServices,
) -> Result<Outcome, Error> {
// Create artifact files via the sandbox's exec_command
let script = concat!(
"mkdir -p test-results && ",
"echo '<testsuites><testsuite name=\"example\"/></testsuites>' > test-results/report.xml && ",
"echo 'test output' > test-results/output.txt"
);
services
.run
.sandbox
.exec_command(script, 30_000, None, None, None)
.await
.map_err(|e| Error::handler(format!("exec failed: {e}")))?;
if self.should_fail {
Ok(Outcome::fail_classify("intentional failure"))
} else {
Ok(Outcome::success())
}
}
}
/// Local sandbox: artifact collection discovers and downloads files created by
/// a handler.
#[tokio::test]
async fn asset_collection_local_sandbox_success() {
let work_dir = tempfile::tempdir().unwrap();
let run_dir = tempfile::tempdir().unwrap();
let sandbox: Arc<dyn fabro_agent::Sandbox> = Arc::new(fabro_agent::LocalSandbox::new(
work_dir.path().to_path_buf(),
));
sandbox.initialize().await.unwrap();
let mut registry = HandlerRegistry::new(Box::new(AssetCreatorHandler::success()));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
let emitter = Emitter::default();
let events = collect_events(&emitter);
let engine = WorkflowRunner::new(registry, Arc::new(emitter), sandbox.clone());
let mut graph = Graph::new("AssetCollectionTest");
graph.attrs.insert(
"goal".to_string(),
AttrValue::String("Test artifact collection".to_string()),
);
let mut start = Node::new("start");
start.attrs.insert(
"shape".to_string(),
AttrValue::String("Mdiamond".to_string()),
);
graph.nodes.insert("start".to_string(), start);
let mut create_assets = Node::new("create_assets");
create_assets.attrs.insert(
"label".to_string(),
AttrValue::String("Create Assets".to_string()),
);
graph
.nodes
.insert("create_assets".to_string(), create_assets);
let mut exit = Node::new("exit");
exit.attrs.insert(
"shape".to_string(),
AttrValue::String("Msquare".to_string()),
);
graph.nodes.insert("exit".to_string(), exit);
graph.edges.push(Edge::new("start", "create_assets"));
graph.edges.push(Edge::new("create_assets", "exit"));
let run_options = RunOptions {
settings: WorkflowSettings {
run: fabro_types::settings::RunNamespace {
artifacts: fabro_types::settings::run::ArtifactsSettings {
include: vec!["test-results/**".to_string()],
},
..fabro_types::settings::RunNamespace::default()
},
..WorkflowSettings::default()
},
run_dir: run_dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("artifact-test-local"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let outcome = engine
.run(&graph, &run_options)
.await
.expect("run should succeed");
assert_eq!(outcome.status, StageOutcome::Succeeded);
let artifact_store = test_artifact_store(run_dir.path());
let artifacts = artifact_store
.list_for_run(&run_options.run_id)
.await
.unwrap();
assert_eq!(
artifacts.len(),
2,
"expected stored artifacts for both files"
);
assert_eq!(artifacts[0].node, StageId::new("create_assets", 1));
assert_eq!(artifacts[0].filename, "test-results/output.txt");
assert_eq!(artifacts[1].node, StageId::new("create_assets", 1));
assert_eq!(artifacts[1].filename, "test-results/report.xml");
let report_content = String::from_utf8(
artifact_store
.get(
&run_options.run_id,
&ArtifactKey::new(
StageId::new("create_assets", 1),
1,
"test-results/report.xml",
),
)
.await
.unwrap()
.expect("artifact should be stored")
.to_vec(),
)
.unwrap();
assert!(report_content.contains("testsuites"));
assert!(
!run_dir.path().join("cache").join("artifacts").exists(),
"artifact scratch cache should not be created"
);
// Check that ArtifactCaptured events were emitted
let captured_events = events.lock().unwrap();
let asset_events: Vec<&RunEvent> = captured_events
.iter()
.filter(|e| e.event_name() == "artifact.captured")
.collect();
assert!(
!asset_events.is_empty(),
"should emit at least one ArtifactCaptured event"
);
let asset_event = asset_events[0];
let asset_properties = asset_event.properties().unwrap();
assert!(!asset_properties["path"].as_str().unwrap().is_empty());
assert!(!asset_properties["mime"].as_str().unwrap().is_empty());
assert_eq!(asset_properties["content_md5"].as_str().unwrap().len(), 32);
assert_eq!(
asset_properties["content_sha256"].as_str().unwrap().len(),
64
);
assert!(asset_properties["bytes"].as_u64().unwrap() > 0);
assert_eq!(asset_properties["attempt"].as_u64().unwrap(), 1);
}
/// Local sandbox: artifact collection discovers files when the sandbox
/// working directory itself is a symlink.
#[tokio::test]
#[cfg(unix)]
async fn asset_collection_local_sandbox_symlink_working_directory() {
let work_root = tempfile::tempdir().unwrap();
let real_work_dir = work_root.path().join("real-workspace");
let symlink_work_dir = work_root.path().join("workspace-link");
std::fs::create_dir_all(&real_work_dir).expect("real workspace should create");
std::os::unix::fs::symlink(&real_work_dir, &symlink_work_dir)
.expect("workspace symlink should create");
let run_dir = tempfile::tempdir().unwrap();
let sandbox: Arc<dyn fabro_agent::Sandbox> =
Arc::new(fabro_agent::LocalSandbox::new(symlink_work_dir));
sandbox.initialize().await.unwrap();
let mut registry = HandlerRegistry::new(Box::new(AssetCreatorHandler::success()));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
let emitter = Emitter::default();
let events = collect_events(&emitter);
let engine = WorkflowRunner::new(registry, Arc::new(emitter), sandbox.clone());
let mut graph = Graph::new("AssetCollectionSymlinkTest");
graph.attrs.insert(
"goal".to_string(),
AttrValue::String("Test artifact collection from symlinked workdir".to_string()),
);
let mut start = Node::new("start");
start.attrs.insert(
"shape".to_string(),
AttrValue::String("Mdiamond".to_string()),
);
graph.nodes.insert("start".to_string(), start);
let mut create_assets = Node::new("create_assets");
create_assets.attrs.insert(
"label".to_string(),
AttrValue::String("Create Assets".to_string()),
);
graph
.nodes
.insert("create_assets".to_string(), create_assets);
let mut exit = Node::new("exit");
exit.attrs.insert(
"shape".to_string(),
AttrValue::String("Msquare".to_string()),
);
graph.nodes.insert("exit".to_string(), exit);
graph.edges.push(Edge::new("start", "create_assets"));
graph.edges.push(Edge::new("create_assets", "exit"));
let run_options = RunOptions {
settings: WorkflowSettings {
run: fabro_types::settings::RunNamespace {
artifacts: fabro_types::settings::run::ArtifactsSettings {
include: vec!["test-results/**".to_string()],
},
..fabro_types::settings::RunNamespace::default()
},
..WorkflowSettings::default()
},
run_dir: run_dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("artifact-test-symlink-workdir"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let outcome = engine
.run(&graph, &run_options)
.await
.expect("run should succeed");
assert_eq!(outcome.status, StageOutcome::Succeeded);
let artifacts = test_artifact_store(run_dir.path())
.list_for_run(&run_options.run_id)
.await
.unwrap();
assert!(
artifacts
.iter()
.any(|artifact| artifact.filename == "test-results/report.xml"),
"expected artifact created under symlinked working directory: {artifacts:?}"
);
assert!(
events
.lock()
.unwrap()
.iter()
.any(|event| event.event_name() == "artifact.captured"),
"artifact.captured should be emitted for symlinked working directory"
);
}
/// Local sandbox: assets are still collected even when the handler fails.
#[tokio::test]
async fn asset_collection_local_sandbox_on_failure() {
let work_dir = tempfile::tempdir().unwrap();
let run_dir = tempfile::tempdir().unwrap();
let sandbox: Arc<dyn fabro_agent::Sandbox> = Arc::new(fabro_agent::LocalSandbox::new(
work_dir.path().to_path_buf(),
));
sandbox.initialize().await.unwrap();
let mut registry = HandlerRegistry::new(Box::new(AssetCreatorHandler::failing()));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), sandbox.clone());
let mut graph = Graph::new("AssetCollectionFailTest");
graph.attrs.insert(
"goal".to_string(),
AttrValue::String("Test artifact collection on failure".to_string()),
);
let mut start = Node::new("start");
start.attrs.insert(
"shape".to_string(),
AttrValue::String("Mdiamond".to_string()),
);
graph.nodes.insert("start".to_string(), start);
let mut create_assets = Node::new("create_assets");
create_assets.attrs.insert(
"label".to_string(),
AttrValue::String("Create Assets".to_string()),
);
graph
.nodes
.insert("create_assets".to_string(), create_assets);
let mut exit = Node::new("exit");
exit.attrs.insert(
"shape".to_string(),
AttrValue::String("Msquare".to_string()),
);
graph.nodes.insert("exit".to_string(), exit);
graph.edges.push(Edge::new("start", "create_assets"));
graph.edges.push(Edge::new("create_assets", "exit"));
let run_options = RunOptions {
settings: WorkflowSettings {
run: fabro_types::settings::RunNamespace {
artifacts: fabro_types::settings::run::ArtifactsSettings {
include: vec!["test-results/**".to_string()],
},
..fabro_types::settings::RunNamespace::default()
},
..WorkflowSettings::default()
},
run_dir: run_dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("artifact-test-fail"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let outcome = engine
.run(&graph, &run_options)
.await
.expect("run should succeed");
// The pipeline completes with goal gates satisfied — per spec, SUCCESS at exit
// node. Assets should still be collected regardless of intermediate node
// failures.
assert_eq!(outcome.status, StageOutcome::Succeeded);
let report_content = String::from_utf8(
test_artifact_store(run_dir.path())
.get(
&run_options.run_id,
&ArtifactKey::new(
StageId::new("create_assets", 1),
1,
"test-results/report.xml",
),
)
.await
.unwrap()
.expect("artifact should still be stored after handler failure")
.to_vec(),
)
.unwrap();
assert!(report_content.contains("testsuites"));
assert!(
!run_dir.path().join("cache").join("artifacts").exists(),
"artifact scratch cache should not be created"
);
}
/// Docker sandbox: artifact collection works through archive copy.
/// Requires Docker with the default sandbox image available locally.
#[tokio::test]
#[ignore]
async fn asset_collection_docker_sandbox() {
let run_dir = tempfile::tempdir().unwrap();
let config = fabro_agent::DockerSandboxOptions {
auto_pull: false,
skip_clone: true,
..Default::default()
};
let sandbox: Arc<dyn fabro_agent::Sandbox> = Arc::new(
fabro_agent::DockerSandbox::new(config, None, None, None, None, None, None)
.expect("Docker not available"),
);
sandbox.initialize().await.expect("Docker init failed");
let mut registry = HandlerRegistry::new(Box::new(AssetCreatorHandler::success()));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
let engine = WorkflowRunner::new(registry, Arc::new(Emitter::default()), sandbox.clone());
let mut graph = Graph::new("DockerAssetTest");
graph.attrs.insert(
"goal".to_string(),
AttrValue::String("Test artifact collection in Docker".to_string()),
);
let mut start = Node::new("start");
start.attrs.insert(
"shape".to_string(),
AttrValue::String("Mdiamond".to_string()),
);
graph.nodes.insert("start".to_string(), start);
let mut create_assets = Node::new("create_assets");
create_assets.attrs.insert(
"label".to_string(),
AttrValue::String("Create Assets".to_string()),
);
graph
.nodes
.insert("create_assets".to_string(), create_assets);
let mut exit = Node::new("exit");
exit.attrs.insert(
"shape".to_string(),
AttrValue::String("Msquare".to_string()),
);
graph.nodes.insert("exit".to_string(), exit);
graph.edges.push(Edge::new("start", "create_assets"));
graph.edges.push(Edge::new("create_assets", "exit"));
let run_options = RunOptions {
settings: WorkflowSettings {
run: fabro_types::settings::RunNamespace {
artifacts: fabro_types::settings::run::ArtifactsSettings {
include: vec!["test-results/**".to_string()],
},
..fabro_types::settings::RunNamespace::default()
},
..WorkflowSettings::default()
},
run_dir: run_dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("artifact-test-docker"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let outcome = engine
.run(&graph, &run_options)
.await
.expect("pipeline should succeed");
assert_eq!(outcome.status, StageOutcome::Succeeded);
let content = String::from_utf8(
test_artifact_store(run_dir.path())
.get(
&run_options.run_id,
&ArtifactKey::new(
StageId::new("create_assets", 1),
1,
"test-results/report.xml",
),
)
.await
.unwrap()
.expect("artifact should be stored from Docker container")
.to_vec(),
)
.unwrap();
assert!(content.contains("testsuites"));
assert!(
!run_dir.path().join("cache").join("artifacts").exists(),
"artifact scratch cache should not be created"
);
sandbox.cleanup().await.unwrap();
}
#[tokio::test]
async fn wait_timer_e2e() {
let mut graph = make_graph_with_start_exit("WaitTimerTest");
let mut wait_node = Node::new("wait60");
wait_node.attrs.insert(
"shape".to_string(),
AttrValue::String("insulator".to_string()),
);
wait_node.attrs.insert(
"label".to_string(),
AttrValue::String("Wait 1ms".to_string()),
);
wait_node.attrs.insert(
"duration".to_string(),
AttrValue::Duration(std::time::Duration::from_millis(1)),
);
graph.nodes.insert("wait60".to_string(), wait_node);
graph.edges.push(Edge::new("start", "wait60"));
graph.edges.push(Edge::new("wait60", "exit"));
let dir = tempfile::tempdir().unwrap();
let interviewer = Arc::new(AutoApproveInterviewer::engine());
let engine = WorkflowRunner::new(
make_full_registry(interviewer),
Arc::new(Emitter::default()),
local_env(),
);
let run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: dir.path().to_path_buf(),
cancel_token: CancellationToken::new(),
run_id: test_run_id("test-run"),
labels: std::collections::HashMap::new(),
workflow_slug: None,
github_app: None,
base_branch: None,
display_base_sha: None,
pre_run_git: None,
fork_source_ref: None,
git: None,
};
let outcome = engine.run(&graph, &run_options).await.expect("run");
assert_eq!(outcome.status, StageOutcome::Succeeded);
}