fabro/lib/crates/fabro-cli/tests/it/cmd/create.rs
Bryan Helmkamp 2b53917759
fix(validate): treat undefined template vars in @file prompts as warnings (#290)
## Summary

`fabro validate` had inconsistent behavior for undefined template
variables depending on whether the prompt was inline or loaded via an
`@file` reference. Inline `{{ inputs.foo }}` produced a warning and
validation passed; the same expression inside a `@file`-imported prompt
produced a hard validation error.

Fixes #286.

## Root cause

Two template-rendering passes with different strictness, applied to
disjoint inputs:

1. **DOT-source pass**
(`lib/crates/fabro-workflow/src/operations/create.rs`) honored
`RenderMode::Structural` for `fabro validate` — undefined variables
downgraded to a `Severity::Warning` diagnostic, then lenient render
finished the job.
2. **Per-attribute pass**
(`lib/crates/fabro-workflow/src/transforms/variable_expansion.rs`)
inside `TemplateTransform` was always strict and had no `RenderMode`
awareness. Because `FileInliningTransform` runs *before*
`TemplateTransform`, expressions inside `@file` content only ever
encountered the strict pass.

## Fix

- Plumb `RenderMode` through `TransformOptions` into
`TemplateTransform`.
- In `RenderMode::Structural`, the transform catches
`TemplateError::UndefinedVariable` per attribute, emits a warning
diagnostic, and falls back to `render_lenient`.
- Diagnostics flow through a new `Transformed.diagnostics` field into
`Validated` alongside lint output.
- Diagnostics now include `node_id` when the undefined variable was
found inside a node attribute, which is more useful than the previous
"at line 1" location.
- `RenderMode` and the shared `template_undefined_variable_diagnostic`
helper moved to `pipeline/types.rs` so the transform layer can reach
them without a circular dep.

Strict mode (`fabro run`, preflight) is unchanged — undefined inputs
still hard-fail before a run is created.

## Behavior

Illustrative output shapes (variable names and line numbers depend on
the fixture):

Inline prompt (unchanged):
```
warning: undefined template variable `inputs.<name>` at line <n> (template_undefined_variable)
Validation: OK
```

`@file`-imported prompt (previously a hard error, now matches inline —
node-attributed instead of line-attributed):
```
warning [node: <id>]: undefined template variable `inputs.<name>` in node `<id>` (template_undefined_variable)
Validation: OK
```

## Test plan

- [x] `cargo nextest run --workspace` — 5773/5773 passing
- [x] `cargo +nightly-2026-04-14 clippy --workspace --all-targets -- -D
warnings` clean
- [x] `cargo +nightly-2026-04-14 fmt --check --all` clean
- [x] New regression test
`bare_fabro_with_unbound_inputs_in_imported_prompt_validates_structurally_with_warning`
in `lib/crates/fabro-cli/tests/it/cmd/validate.rs` against new fixture
`test/templated_unbound_imported/`
- [x] Existing
`bare_fabro_with_unbound_inputs_validates_structurally_with_warning` and
`strict_render_hard_fails_on_unbound_inputs` still pass — verifies
inline structural and run-start strict behavior are both preserved
- [x] Manual reproduction of the exact inputs from the issue now
succeeds with a warning

🤖 Generated with [Claude Code](https://claude.com/claude-code)

---------

Co-authored-by: Aleksi Asikainen <1086393+salieri@users.noreply.github.com>
Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-16 17:31:22 -04:00

522 lines
16 KiB
Rust
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

use fabro_test::{fabro_json_snapshot, fabro_snapshot, test_context};
use httpmock::MockServer;
use insta::assert_snapshot;
use serde_json::json;
use super::support::{
fixture, output_stdout, remote_run_summary_json, resolve_run, run_count_for_test_case,
run_state,
};
use crate::support::unique_run_id;
fn resolved_run(settings: &fabro_types::WorkflowSettings) -> fabro_types::settings::RunNamespace {
settings.run.clone()
}
fn run_status_response(run_id: &str, status: &str) -> serde_json::Value {
let status = match status {
"submitted" => json!({ "kind": "submitted" }),
other => panic!("unsupported test status {other:?}"),
};
remote_run_summary_json(
run_id,
"Test Workflow",
"test-workflow",
"Test run",
&status,
"2026-04-05T12:00:00Z",
)
}
#[test]
fn help() {
let context = test_context!();
let mut cmd = context.command();
cmd.args(["create", "--help"]);
fabro_snapshot!(context.filters(), cmd, @"
success: true
exit_code: 0
----- stdout -----
Create a workflow run (allocate run dir, persist spec)
Usage: fabro create [OPTIONS] <WORKFLOW>
Arguments:
<WORKFLOW> Path to a .fabro workflow file or .toml task config
Options:
--json Output as JSON [env: FABRO_JSON=]
--server <SERVER> Fabro server target: http(s) URL or absolute Unix socket path [env: FABRO_SERVER=]
--debug Enable DEBUG-level logging (default is INFO) [env: FABRO_DEBUG=]
-I, --input <KEY=VALUE> Override a workflow input value (repeatable, format: KEY=VALUE)
--dry-run Execute with simulated LLM backend
--no-upgrade-check Disable automatic upgrade check [env: FABRO_NO_UPGRADE_CHECK=true]
--auto-approve Auto-approve all human gates
--quiet Suppress non-essential output [env: FABRO_QUIET=]
--goal <GOAL> Override the workflow goal (available as {{ goal }} in prompts)
--goal-file <GOAL_FILE> Read the workflow goal from a file
--model <MODEL> Override default LLM model
--provider <PROVIDER> Override default LLM provider
-v, --verbose Enable verbose output
--sandbox <SANDBOX> Sandbox for agent tools [possible values: local, docker, daytona]
--label <KEY=VALUE> Attach a label to this run (repeatable, format: KEY=VALUE)
--parent <RUN> Link this run to an existing orchestration parent run
--preserve-sandbox Keep the sandbox alive after the run finishes (for debugging)
-d, --detach Run the workflow in the background and print the run ID
-h, --help Print help
----- stderr -----
");
}
#[test]
fn create_uses_explicit_server_target_and_prints_remote_run_id() {
let context = test_context!();
let server = MockServer::start();
let run_id = unique_run_id();
let mock = server.mock(|when, then| {
when.method("POST").path("/api/v1/runs");
then.status(201)
.header("Content-Type", "application/json")
.body(run_status_response(run_id.as_str(), "submitted").to_string());
});
let output = context
.create_cmd()
.args([
"--server",
&format!("{}/api/v1", server.base_url()),
"--dry-run",
fixture("simple.fabro").to_str().unwrap(),
])
.output()
.expect("command should execute");
assert!(
output.status.success(),
"command failed:\nstdout:\n{}\nstderr:\n{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
mock.assert();
assert_eq!(output_stdout(&output).trim(), run_id.as_str());
}
#[test]
fn create_uses_configured_server_target_without_server_flag() {
let context = test_context!();
let server = MockServer::start();
let run_id = unique_run_id();
let mock = server.mock(|when, then| {
when.method("POST").path("/api/v1/runs");
then.status(201)
.header("Content-Type", "application/json")
.body(run_status_response(run_id.as_str(), "submitted").to_string());
});
context.set_http_target(&server.base_url());
let output = context
.create_cmd()
.args(["--dry-run", fixture("simple.fabro").to_str().unwrap()])
.output()
.expect("command should execute");
assert!(
output.status.success(),
"command failed:\nstdout:\n{}\nstderr:\n{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
mock.assert();
assert_eq!(output_stdout(&output).trim(), run_id.as_str());
}
#[test]
fn create_parent_resolves_parent_and_sends_parent_id_in_manifest() {
let context = test_context!();
let server = MockServer::start();
let run_id = unique_run_id();
let parent_id = unique_run_id();
let resolve_mock = super::support::mock_resolved_run(&server, "nightly-parent", &parent_id);
let create_mock = server.mock(|when, then| {
when.method("POST")
.path("/api/v1/runs")
.json_body_includes(format!(r#"{{"parent_id":"{parent_id}"}}"#));
then.status(201)
.header("Content-Type", "application/json")
.body(run_status_response(run_id.as_str(), "submitted").to_string());
});
let output = context
.create_cmd()
.args([
"--server",
&format!("{}/api/v1", server.base_url()),
"--dry-run",
"--parent",
"nightly-parent",
fixture("simple.fabro").to_str().unwrap(),
])
.output()
.expect("command should execute");
assert!(
output.status.success(),
"command failed:\nstdout:\n{}\nstderr:\n{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
resolve_mock.assert();
create_mock.assert();
assert_eq!(output_stdout(&output).trim(), run_id.as_str());
}
#[test]
fn create_rejects_storage_dir_flag() {
let context = test_context!();
let output = context
.create_cmd()
.args([
"--storage-dir",
"/tmp/fabro-create",
"--dry-run",
fixture("simple.fabro").to_str().unwrap(),
])
.output()
.expect("command should execute");
assert!(
!output.status.success(),
"command should reject --storage-dir"
);
let stderr = String::from_utf8_lossy(&output.stderr);
assert!(stderr.contains("unexpected argument '--storage-dir'"));
}
#[test]
fn create_cli_server_target_overrides_configured_server_target() {
let context = test_context!();
let config_server = MockServer::start();
let config_mock = config_server.mock(|when, then| {
when.method("POST").path("/api/v1/runs");
then.status(500)
.body("configured-server-should-not-be-used");
});
let cli_server = MockServer::start();
let run_id = unique_run_id();
let cli_mock = cli_server.mock(|when, then| {
when.method("POST").path("/api/v1/runs");
then.status(201)
.header("Content-Type", "application/json")
.body(run_status_response(run_id.as_str(), "submitted").to_string());
});
context.set_http_target(&config_server.base_url());
let output = context
.create_cmd()
.args([
"--server",
&format!("{}/api/v1", cli_server.base_url()),
"--dry-run",
fixture("simple.fabro").to_str().unwrap(),
])
.output()
.expect("command should execute");
assert!(
output.status.success(),
"command failed:\nstdout:\n{}\nstderr:\n{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
cli_mock.assert();
config_mock.assert_calls(0);
assert_eq!(output_stdout(&output).trim(), run_id.as_str());
}
#[test]
fn create_persists_directory_workflow_slug_and_cached_graph() {
let context = test_context!();
context.ensure_home_server_auth_methods();
let run_id = unique_run_id();
let workflow_path = context.temp_dir.join("sluggy/workflow.fabro");
context.write_temp(
"sluggy/workflow.fabro",
"\
digraph BarBaz {
start [shape=Mdiamond, label=\"Start\"]
exit [shape=Msquare, label=\"Exit\"]
start -> exit
}
",
);
context
.command()
.args([
"create",
"--dry-run",
"--auto-approve",
"--run-id",
run_id.as_str(),
workflow_path.to_str().unwrap(),
])
.assert()
.success();
let run_dir = context.find_run_dir(&run_id);
let state = run_state(&run_dir);
let run = &state.spec;
fabro_json_snapshot!(
context,
serde_json::json!({
"workflow_slug": run.workflow_slug,
"graph_name": run.graph.name,
"cached_graph_lines": state.spec.graph_source.as_ref().expect("graph should exist").lines().collect::<Vec<_>>(),
}),
@r#"
{
"workflow_slug": "sluggy",
"graph_name": "BarBaz",
"cached_graph_lines": [
"digraph BarBaz {",
" start [shape=Mdiamond, label=\"Start\"]",
" exit [shape=Msquare, label=\"Exit\"]",
" start -> exit",
"}"
]
}
"#
);
}
#[test]
fn create_persists_file_stem_slug_for_standalone_file() {
let context = test_context!();
context.ensure_home_server_auth_methods();
let run_id = unique_run_id();
let workflow_path = context.temp_dir.join("alpha.fabro");
context.write_temp(
"alpha.fabro",
"\
digraph FooWorkflow {
start [shape=Mdiamond, label=\"Start\"]
exit [shape=Msquare, label=\"Exit\"]
start -> exit
}
",
);
context
.command()
.args([
"create",
"--dry-run",
"--auto-approve",
"--run-id",
run_id.as_str(),
workflow_path.to_str().unwrap(),
])
.assert()
.success();
let run_dir = context.find_run_dir(&run_id);
let state = run_state(&run_dir);
let run = &state.spec;
fabro_json_snapshot!(
context,
serde_json::json!({
"workflow_slug": run.workflow_slug,
"graph_name": run.graph.name,
"cached_graph_lines": state.spec.graph_source.as_ref().expect("graph should exist").lines().collect::<Vec<_>>(),
}),
@r#"
{
"workflow_slug": "alpha",
"graph_name": "FooWorkflow",
"cached_graph_lines": [
"digraph FooWorkflow {",
" start [shape=Mdiamond, label=\"Start\"]",
" exit [shape=Msquare, label=\"Exit\"]",
" start -> exit",
"}"
]
}
"#
);
}
#[test]
fn create_persists_requested_overrides_into_store() {
let context = test_context!();
context.ensure_home_server_auth_methods();
let workflow = fixture("simple.fabro");
let mut cmd = context.command();
cmd.args([
"create",
"--dry-run",
"--auto-approve",
"--goal",
"Ship the release",
"--model",
"gpt-5",
"--provider",
"openai",
"--sandbox",
"local",
"--label",
"env=dev",
"--label",
"team=cli",
"--verbose",
"--preserve-sandbox",
workflow.to_str().unwrap(),
]);
let output = cmd.output().expect("command should execute");
assert!(
output.status.success(),
"command failed:\nstdout:\n{}\nstderr:\n{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
let stdout = output_stdout(&output);
let run_id = stdout
.lines()
.find(|line| !line.trim().is_empty())
.map(str::trim)
.expect("create should print a run ID")
.to_string();
let run = resolve_run(&context, &run_id);
let state = run_state(&run.run_dir);
let run_spec = &state.spec;
let labels = json!({
"env": run_spec.labels.get("env"),
"team": run_spec.labels.get("team"),
});
let settings = &run_spec.settings;
let resolved_run = resolved_run(settings);
let compact = json!({
"workflow_slug": run_spec.workflow_slug,
"settings": {
"goal": match resolved_run.goal.as_ref() {
Some(fabro_types::settings::run::RunGoal::Inline(value)) => Some(value.as_source()),
_ => None,
},
"dry_run": resolved_run.execution.mode == fabro_types::settings::run::RunMode::DryRun,
"auto_approve": resolved_run.execution.approval == fabro_types::settings::run::ApprovalMode::Auto,
"llm": {
"model": resolved_run.model.name.as_ref().map(fabro_types::settings::InterpString::as_source),
"provider": resolved_run.model.provider.as_ref().map(fabro_types::settings::InterpString::as_source),
},
"sandbox": {
"provider": resolved_run.sandbox.provider,
"preserve": resolved_run.sandbox.preserve,
},
},
"labels": labels,
});
assert_snapshot!(serde_json::to_string_pretty(&compact).unwrap(), @r###"
{
"workflow_slug": "simple",
"settings": {
"goal": "Ship the release",
"dry_run": true,
"auto_approve": true,
"llm": {
"model": "gpt-5",
"provider": "openai"
},
"sandbox": {
"provider": "local",
"preserve": true
}
},
"labels": {
"env": "dev",
"team": "cli"
}
}
"###);
}
#[test]
fn create_json_does_not_imply_auto_approve() {
let context = test_context!();
context.ensure_home_server_auth_methods();
let workflow = fixture("simple.fabro");
let output = context
.command()
.args(["--json", "create", "--dry-run", workflow.to_str().unwrap()])
.output()
.expect("command should execute");
assert!(
output.status.success(),
"command failed:\nstdout:\n{}\nstderr:\n{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
let value: serde_json::Value =
serde_json::from_slice(&output.stdout).expect("create JSON should parse");
let run_id = value["run_id"]
.as_str()
.expect("create JSON should include run_id");
let run = resolve_run(&context, run_id);
assert!(
resolved_run(&run_state(&run.run_dir).spec.settings,)
.execution
.approval
!= fabro_types::settings::run::ApprovalMode::Auto
);
}
#[test]
fn create_invalid_workflow_fails_without_creating_run() {
let context = test_context!();
let workflow = fixture("invalid.fabro");
let initial_run_count = run_count_for_test_case(&context);
let mut cmd = context.create_cmd();
cmd.arg(workflow.to_str().unwrap());
fabro_snapshot!(context.filters(), cmd, @"
success: false
exit_code: 1
----- stdout -----
----- stderr -----
× Validation failed
");
let run_count = run_count_for_test_case(&context);
assert_eq!(
run_count, initial_run_count,
"invalid create should not persist a run for this test case"
);
}
#[test]
fn create_rejects_unbound_template_inputs_without_creating_run() {
let context = test_context!();
let workflow = fixture("templated_unbound.fabro");
let initial_run_count = run_count_for_test_case(&context);
let mut cmd = context.create_cmd();
cmd.arg(workflow.to_str().unwrap());
fabro_snapshot!(context.filters(), cmd, @"
success: false
exit_code: 1
----- stdout -----
----- stderr -----
× Validation failed
");
let run_count = run_count_for_test_case(&context);
assert_eq!(
run_count, initial_run_count,
"invalid create should not persist a run for this test case"
);
}