fabro/lib/apps/fabro-cli/tests/it/cmd/validate.rs
Bryan Helmkamp c90d195c2f
Merge pull request #806 from fabro-sh/node-on-failure
Add node-level on_failure override
2026-08-26 07:01:49 -04:00

414 lines
14 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

use fabro_test::{fabro_snapshot, test_context};
use crate::support::LightweightCli;
fn fixture(name: &str) -> std::path::PathBuf {
std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
.join(format!("../../../test/{name}"))
.canonicalize()
.expect("fixture path should exist")
}
#[test]
fn help() {
let context = test_context!();
let mut cmd = context.validate();
cmd.arg("--help");
fabro_snapshot!(context.filters(), cmd, @"
success: true
exit_code: 0
----- stdout -----
Validate a workflow
Usage: fabro validate [OPTIONS] <WORKFLOW>
Arguments:
<WORKFLOW> Path to the .fabro workflow file
Options:
--json Output as JSON [env: FABRO_JSON=]
--debug Enable DEBUG-level logging (default is INFO) [env: FABRO_DEBUG=]
--no-upgrade-check Disable automatic upgrade check [env: FABRO_NO_UPGRADE_CHECK=true]
--quiet Suppress non-essential output [env: FABRO_QUIET=]
--verbose Enable verbose output [env: FABRO_VERBOSE=]
-h, --help Print help
----- stderr -----
");
}
#[test]
fn simple() {
let context = test_context!();
let mut cmd = context.validate();
cmd.arg(fixture("simple.fabro"));
fabro_snapshot!(context.filters(), cmd, @"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
Workflow: Simple (4 nodes, 3 edges)
Graph: [FIXTURES]/simple.fabro
Validation: OK
");
}
#[test]
fn simple_does_not_connect_to_configured_server() {
let cli = LightweightCli::new();
let mut cmd = cli.command();
cmd.env("FABRO_SERVER", "http://127.0.0.1:9")
.arg("validate")
.arg(fixture("simple.fabro"));
let output = cmd.output().expect("validate should execute");
assert!(
output.status.success(),
"validate should run locally without connecting to FABRO_SERVER\nstdout:\n{}\nstderr:\n{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr),
);
}
/// Offline validation has no model catalog, so a model or provider the server
/// owns must pass rather than be reported as unknown.
#[test]
fn server_owned_provider_is_not_rejected_by_offline_validation() {
let context = test_context!();
let mut cmd = context.validate();
cmd.arg(fixture("server-model.fabro"));
fabro_snapshot!(context.filters(), cmd, @"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
Workflow: ServerModel (3 nodes, 2 edges)
Graph: [FIXTURES]/server-model.fabro
Validation: OK
");
}
#[test]
fn model_fallback_table_is_not_catalog_checked_by_offline_validation() {
let cli = LightweightCli::new();
let mut cmd = cli.command();
cmd.env("FABRO_SERVER", "http://127.0.0.1:9")
.arg("validate")
.arg(fixture("offline-fallbacks/workflow.fabro"));
let output = cmd.output().expect("validate should execute");
assert!(
output.status.success(),
"offline validation should parse model fallback tables without a server catalog\nstdout:\n{}\nstderr:\n{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr),
);
}
#[test]
fn branching() {
let context = test_context!();
let mut cmd = context.validate();
cmd.arg(fixture("branching.fabro"));
fabro_snapshot!(context.filters(), cmd, @"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
Workflow: Branch (6 nodes, 6 edges)
Graph: [FIXTURES]/branching.fabro
warning [node: implement]: Node 'implement' has goal_gate=true but no retry_target or fallback_retry_target (goal_gate_has_retry)
fix: Add retry_target or fallback_retry_target attribute
Validation: OK
");
}
#[test]
fn conditions() {
let context = test_context!();
let mut cmd = context.validate();
cmd.arg(fixture("conditions.fabro"));
fabro_snapshot!(context.filters(), cmd, @"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
Workflow: Conditions (5 nodes, 5 edges)
Graph: [FIXTURES]/conditions.fabro
Validation: OK
");
}
#[test]
fn parallel() {
let context = test_context!();
let mut cmd = context.validate();
cmd.arg(fixture("parallel.fabro"));
fabro_snapshot!(context.filters(), cmd, @"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
Workflow: Parallel (7 nodes, 7 edges)
Graph: [FIXTURES]/parallel.fabro
Validation: OK
");
}
#[test]
fn styled() {
let context = test_context!();
let mut cmd = context.validate();
cmd.arg(fixture("styled.fabro"));
fabro_snapshot!(context.filters(), cmd, @"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
Workflow: Styled (5 nodes, 4 edges)
Graph: [FIXTURES]/styled.fabro
Validation: OK
");
}
#[test]
fn inferred_command() {
let context = test_context!();
let mut cmd = context.validate();
cmd.arg(fixture("inferred_command.fabro"));
fabro_snapshot!(context.filters(), cmd, @"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
Workflow: InferredCommand (3 nodes, 2 edges)
Graph: [FIXTURES]/inferred_command.fabro
Validation: OK
");
}
#[test]
fn bare_fabro_with_unbound_inputs_validates_structurally_with_warning() {
let context = test_context!();
let mut cmd = context.validate();
cmd.arg(fixture("templated_unbound.fabro"));
fabro_snapshot!(context.filters(), cmd, @"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
Workflow: TemplatedUnbound (3 nodes, 2 edges)
Graph: [FIXTURES]/templated_unbound.fabro
warning: [FIXTURES]/templated_unbound.fabro:2:26: undefined template variable `inputs.app_dir` in graph attribute `goal` (template_undefined_variable)
fix: bind `app_dir` via `[run.inputs]` in workflow.toml, or pass `--input app_dir=<value>`
warning: [FIXTURES]/templated_unbound.fabro:7:44: undefined template variable `inputs.app_dir` in node `work` attribute `prompt` [node: work] (template_undefined_variable)
fix: bind `app_dir` via `[run.inputs]` in workflow.toml, or pass `--input app_dir=<value>`
Validation: OK
");
}
#[test]
fn unbound_model_stylesheet_input_warns_without_css_error() {
let context = test_context!();
let mut cmd = context.validate();
cmd.arg(fixture("model_stylesheet_unbound.fabro"));
fabro_snapshot!(context.filters(), cmd, @"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
Workflow: ModelStylesheetUnbound (3 nodes, 2 edges)
Graph: [FIXTURES]/model_stylesheet_unbound.fabro
warning: [FIXTURES]/model_stylesheet_unbound.fabro:4:38: undefined template variable `inputs.effort` in graph attribute `model_stylesheet` (template_undefined_variable)
fix: bind `effort` via `[run.inputs]` in workflow.toml, or pass `--input effort=<value>`
Validation: OK
");
}
/// Regression: https://github.com/fabro-sh/fabro/issues/286
///
/// Undefined template variables in a prompt loaded via `@file` reference must
/// surface as the same warning diagnostic as an inline prompt — not a hard
/// validation error.
#[test]
fn bare_fabro_with_unbound_inputs_in_imported_prompt_validates_structurally_with_warning() {
let context = test_context!();
let mut cmd = context.validate();
cmd.arg(fixture("templated_unbound_imported/workflow.fabro"));
fabro_snapshot!(context.filters(), cmd, @"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
Workflow: TemplatedUnboundImported (3 nodes, 2 edges)
Graph: [FIXTURES]/templated_unbound_imported/workflow.fabro
warning: [FIXTURES]/templated_unbound_imported/work.md:1:12: undefined template variable `inputs.app_dir` in node `work` attribute `prompt` [node: work] (template_undefined_variable)
fix: bind `app_dir` via `[run.inputs]` in workflow.toml, or pass `--input app_dir=<value>`
Validation: OK
");
}
/// Regression: https://github.com/fabro-sh/fabro/issues/330
///
/// Undefined template variables in partials included by an imported prompt must
/// surface as structural validation warnings, matching direct `@file` prompts.
#[test]
fn bare_fabro_with_unbound_inputs_in_template_partial_validates_structurally_with_warning() {
let context = test_context!();
let mut cmd = context.validate();
cmd.arg(fixture("templated_unbound_partial/workflow.fabro"));
fabro_snapshot!(context.filters(), cmd, @"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
Workflow: TemplatedUnboundPartial (3 nodes, 2 edges)
Graph: [FIXTURES]/templated_unbound_partial/workflow.fabro
warning: [FIXTURES]/templated_unbound_partial/test-include.partial.md:1:4: undefined template variable `inputs.hello` in node `test_imported_include` attribute `prompt` [node: test_imported_include] (template_undefined_variable)
fix: bind `hello` via `[run.inputs]` in workflow.toml, or pass `--input hello=<value>`
Validation: OK
");
}
#[test]
fn bare_fabro_picks_up_sibling_workflow_toml_inputs() {
let context = test_context!();
let mut cmd = context.validate();
cmd.arg(fixture("templated_inputs/workflow.fabro"));
fabro_snapshot!(context.filters(), cmd, @"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
Workflow: TemplatedInputs (3 nodes, 2 edges)
Graph: [FIXTURES]/templated_inputs/workflow.fabro
Validation: OK
");
}
#[test]
fn validate_accepts_static_template_dependencies() {
let context = test_context!();
let mut cmd = context.validate();
cmd.arg(fixture("templates/static_dependencies/workflow.fabro"));
fabro_snapshot!(context.filters(), cmd, @"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
Workflow: TemplateIncludes (4 nodes, 3 edges)
Graph: [FIXTURES]/templates/static_dependencies/workflow.fabro
Validation: OK
");
}
#[test]
fn validate_accepts_template_partial_sibling_under_workflow_root() {
let context = test_context!();
let mut cmd = context.validate();
cmd.arg(fixture("templates/sibling_partial/workflow.fabro"));
fabro_snapshot!(context.filters(), cmd, @"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
Workflow: SiblingPartialTemplateInclude (3 nodes, 2 edges)
Graph: [FIXTURES]/templates/sibling_partial/workflow.fabro
Validation: OK
");
}
#[test]
fn validate_reports_missing_template_dependency() {
let context = test_context!();
let mut cmd = context.validate();
cmd.arg(fixture("templates/missing_dependency/workflow.fabro"));
let mut filters = context.filters();
filters.push((
r"(?:\.\./)*\.\.\[FIXTURES\]/".to_string(),
"[FIXTURES]/".to_string(),
));
fabro_snapshot!(filters, cmd, @"
success: false
exit_code: 1
----- stdout -----
----- stderr -----
× failed to discover template dependencies
╰─▶ missing template dependency `missing.tpl.md` from `[FIXTURES]/templates/missing_dependency/workflow.fabro`
");
}
/// A node named only by an edge is almost always a typo, so validation must
/// fail instead of quietly running it as a default agent stage.
#[test]
fn edge_only_node() {
let context = test_context!();
let mut cmd = context.validate();
cmd.arg(fixture("edge_only_node.fabro"));
fabro_snapshot!(context.filters(), cmd, @"
success: false
exit_code: 1
----- stdout -----
----- stderr -----
Workflow: EdgeOnlyNode (2 nodes, 2 edges)
Graph: [FIXTURES]/edge_only_node.fabro
error [node: misspelled_node]: Node 'misspelled_node' is referenced by edge 'start -> misspelled_node' but has no node declaration (edge_target_exists)
fix: Declare node 'misspelled_node' or correct the edge endpoint
× Validation failed
");
}
#[test]
fn invalid() {
let context = test_context!();
let mut cmd = context.validate();
cmd.arg(fixture("invalid.fabro"));
fabro_snapshot!(context.filters(), cmd, @"
success: false
exit_code: 1
----- stdout -----
----- stderr -----
Workflow: Invalid (2 nodes, 1 edges)
Graph: [FIXTURES]/invalid.fabro
error: Pipeline must have exactly one start node (shape=Mdiamond or id start/Start) (start_node)
fix: Add a node with shape=Mdiamond or id 'start'
error [node: exit]: Exit node 'exit' has 1 outgoing edge(s) but must have none (exit_no_outgoing)
fix: Remove outgoing edges from the exit node
× Validation failed
");
}
#[test]
fn invalid_node_on_failure_is_a_validation_failure() {
let context = test_context!();
let mut cmd = context.validate();
cmd.arg(fixture("on_failure_node_invalid.fabro"));
fabro_snapshot!(context.filters(), cmd, @"
success: false
exit_code: 1
----- stdout -----
----- stderr -----
Workflow: InvalidNodeOnFailure (3 nodes, 2 edges)
Graph: [FIXTURES]/on_failure_node_invalid.fabro
error [node: work]: Node 'work' has invalid on_failure value 'stop' (on_failure_valid)
fix: Use one of: route, exit
× Validation failed
");
}
#[test]
fn invalid_on_failure_is_a_validation_failure() {
let context = test_context!();
let mut cmd = context.validate();
cmd.arg(fixture("on_failure_invalid.fabro"));
fabro_snapshot!(context.filters(), cmd, @"
success: false
exit_code: 1
----- stdout -----
----- stderr -----
Workflow: InvalidOnFailure (2 nodes, 1 edges)
Graph: [FIXTURES]/on_failure_invalid.fabro
error: Graph has invalid on_failure value 'stop' (on_failure_valid)
fix: Use one of: route, exit
× Validation failed
");
}