fabro/lib/crates/fabro-cli/tests/it/cmd/create.rs
fabro-sh-0530[bot] 7f84ac5e3f
Replace run-scoped sandbox config with named environments (#360)
## Summary

Replaces the `[run.sandbox]` configuration surface with a named,
provider-explicit environment catalog. Runs now select an environment by
slug (`[run.environment] id = "..."`) rather than configuring a sandbox
inline. Fabro resolves the catalog through normal settings precedence,
applies sparse run-level overrides, and creates a concrete sandbox from
the resolved environment.

This is a clean break — no `[run.sandbox]` compatibility layer.

### Plan Summary

- **New config shape:** Top-level `[environments.<slug>]` catalog valid
in `settings.toml`, `.fabro/project.toml`, and `workflow.toml`. Runs
reference a slug via `[run.environment] id = "..."` with optional sparse
overrides under `[run.environment.*]`.
- **Unified environment fields:** `provider`, `image` (ref +
dockerfile), `resources` (cpu/memory/disk), `network` (mode + allow
CIDRs), `lifecycle` (preserve/stop_on_terminal/auto_stop), `labels`,
`volumes`, `env` — replacing the previous split between `[run.sandbox]`,
`[run.sandbox.docker]`, `[run.sandbox.daytona]`, and
`[run.sandbox.daytona.snapshot]`.
- **OpenAPI schema update:** `RunSandboxSettings`, `DockerSettings`,
`DaytonaSettings`, and `DaytonaNetworkLayer` replaced with
`RunEnvironmentSettings`, `EnvironmentSettings`, `EnvironmentProvider`,
`EnvironmentImageSettings`, `EnvironmentResourcesSettings`,
`EnvironmentNetworkSettings`, `EnvironmentLifecycleSettings`, and
`EnvironmentVolumeSettings`.
- **CLI flag rename:** `--sandbox <provider>` → `--environment <slug>`
on `run`, `create`, `preflight`, and `server start/restart`.
- **Provider capability model:** Hard errors for security properties a
provider cannot enforce (local with blocked/CIDR networking; docker with
CIDR allow-lists). Warnings for unsupported resource limits, volumes,
labels, auto-stop, and Docker Dockerfiles.
- **Docs and internal code updated** throughout: `.fabro/project.toml`,
workflow configs, all public docs, CLI args, manifest builders, and the
runner's GitHub credentials check.

### Provider mapping

| Environment field | Local | Docker | Daytona |
|---|---|---|---|
| `image.ref` | Ignored | Docker image | Snapshot name |
| `image.dockerfile` | Ignored | Warning; ignored | Snapshot Dockerfile
(requires `image.ref`) |
| `resources.cpu/memory/disk` | Warning; ignored | cpu_quota / memory
limit / warning | Snapshot sizing |
| `network.mode = block` | **Error** | `network_mode = none` | Daytona
block |
| `network.mode = cidr_allow_list` | **Error** | **Error** | Daytona
CIDR allow-list |
| `labels` | Warning; ignored | Warning; ignored | Daytona labels |
| `volumes` | Warning; ignored | Warning; ignored | Daytona volume
mounts |
| `lifecycle.auto_stop` | Warning; ignored | Warning; ignored | Daytona
auto-stop interval |
| `env` | Process env overlay | Container env | Sandbox env |


### Fabro Details

<details>
<summary>Ran 11 stages in 217m 39s for $129.86</summary>

| Stage | Duration | Cost | Retries |
|---|---|---|---|
| start | 0s | – | 0 |
| toolchain | 1s | – | 0 |
| preflight_compile | 4m 7s | – | 0 |
| preflight_lint | 4m 9s | – | 0 |
| fix_lints | 3m 46s | $1.06 | 0 |
| implement | 76m 6s | $57.39 | 0 |
| simplify_opus | 71m 50s | $38.17 | 0 |
| simplify_gpt | 8m 27s | $2.24 | 0 |
| verify | 6m 10s | – | 0 |
| fixup | 42m 1s | $31.00 | 0 |
| fmt | 3s | – | 0 |
| **Total** | **217m 39s** | **$129.86** | **0** |

</details>

<details>
<summary>Ran <code>ImplementPlan.fabro</code> (12 nodes and 15
edges)</summary>

```dot
digraph ImplementPlan {
    graph [
        goal="Implement and simplify",
        model_stylesheet="
            * { model: claude-opus-4-7; }
        "
    ]
    rankdir=LR

    start [shape=Mdiamond, label="Start"]
    exit  [shape=Msquare, label="Exit"]

    toolchain         [label="Toolchain", shape=parallelogram, script="command -v cargo >/dev/null || { curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && sudo ln -sf $HOME/.cargo/bin/* /usr/local/bin/; }; cargo --version 2>&1", max_retries=0]
    preflight_compile [label="Preflight Compile", shape=parallelogram, script="cargo check -q --workspace 2>&1", max_retries=0]
    preflight_lint    [label="Preflight Lint", shape=parallelogram, script="cargo +nightly-2026-04-14 clippy -q --workspace --all-targets -- -D warnings 2>&1", max_retries=0]
    fix_lints         [label="Fix Lints", prompt="The preflight lint step failed. Read the build output from context and fix all clippy lint warnings.", max_visits=3]
    implement         [label="Implement", prompt="Read the plan file referenced in the goal and implement every step. Make all the code changes described in the plan. Use red/green TDD.", model="gpt-55", reasoning_effort="xhigh"]
    simplify_opus     [label="Simplify (Opus)", prompt="@prompts/simplify.md"]
    simplify_gpt      [label="Simplify (GPT-55)", prompt="@prompts/simplify.md", model="gpt-55"]
    verify            [label="Verify", shape=parallelogram, script="cargo +nightly-2026-04-14 clippy -q --workspace --all-targets -- -D warnings 2>&1 && cargo nextest run --cargo-quiet --workspace --status-level fail 2>&1 && cargo dev docs refresh 2>&1 && cargo dev docs check 2>&1", goal_gate=true, retry_target="fixup"]
    fixup             [label="Fixup", prompt="The verify step failed. Read the build output from context and fix all clippy lint warnings, test failures, and generated docs errors.", max_visits=3]
    fmt               [label="Format", shape=parallelogram, script="cargo +nightly-2026-04-14 fmt --all 2>&1", max_retries=0]

    start -> toolchain
    toolchain -> preflight_compile [condition="outcome=succeeded"]
    toolchain -> exit
    preflight_compile -> preflight_lint [condition="outcome=succeeded"]
    preflight_compile -> exit
    preflight_lint -> implement [condition="outcome=succeeded"]
    preflight_lint -> fix_lints
    fix_lints -> preflight_lint
    implement -> simplify_opus -> simplify_gpt -> verify
    verify -> fmt   [condition="outcome=succeeded"]
    verify -> fixup
    fixup -> verify
    fmt -> exit
}

```

</details>

⚒️ Generated with [Fabro](https://fabro.sh)

---------

Co-authored-by: Fabro <noreply@fabro.sh>
Co-authored-by: Bryan Helmkamp <bryan@brynary.com>
Co-authored-by: Bryan Helmkamp <bhelmkamp@users.noreply.github.com>
2026-05-23 13:03:21 -04:00

524 lines
16 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

use fabro_test::{fabro_json_snapshot, fabro_snapshot, test_context};
use httpmock::MockServer;
use insta::assert_snapshot;
use serde_json::json;
use super::support::{
fixture, output_stdout, remote_run_summary_json, resolve_run, run_count_for_test_case,
run_state,
};
use crate::support::unique_run_id;
fn resolved_run(settings: &fabro_types::WorkflowSettings) -> fabro_types::settings::RunNamespace {
settings.run.clone()
}
fn run_status_response(run_id: &str, status: &str) -> serde_json::Value {
let status = match status {
"submitted" => json!({ "kind": "submitted" }),
other => panic!("unsupported test status {other:?}"),
};
remote_run_summary_json(
run_id,
"Test Workflow",
"test-workflow",
"Test run",
&status,
"2026-04-05T12:00:00Z",
)
}
#[test]
fn help() {
let context = test_context!();
let mut cmd = context.command();
cmd.args(["create", "--help"]);
fabro_snapshot!(context.filters(), cmd, @"
success: true
exit_code: 0
----- stdout -----
Create a workflow run (allocate run dir, persist spec)
Usage: fabro create [OPTIONS] <WORKFLOW>
Arguments:
<WORKFLOW> Path to a .fabro workflow file or .toml task config
Options:
--json Output as JSON [env: FABRO_JSON=]
--server <SERVER> Fabro server target: http(s) URL or absolute Unix socket path [env: FABRO_SERVER=]
--debug Enable DEBUG-level logging (default is INFO) [env: FABRO_DEBUG=]
-I, --input <KEY=VALUE> Override a workflow input value (repeatable, format: KEY=VALUE)
--dry-run Execute with simulated LLM backend
--no-upgrade-check Disable automatic upgrade check [env: FABRO_NO_UPGRADE_CHECK=true]
--auto-approve Auto-approve all human gates
--quiet Suppress non-essential output [env: FABRO_QUIET=]
--goal <GOAL> Override the workflow goal (available as {{ goal }} in prompts)
--goal-file <GOAL_FILE> Read the workflow goal from a file
--model <MODEL> Override default LLM model
--provider <PROVIDER> Override default LLM provider
-v, --verbose Enable verbose output
--environment <ENVIRONMENT> Named environment for agent tools
--label <KEY=VALUE> Attach a label to this run (repeatable, format: KEY=VALUE)
--parent <RUN> Link this run to an existing orchestration parent run
--preserve-sandbox Keep the sandbox alive after the run finishes (for debugging)
-d, --detach Run the workflow in the background and print the run ID
-h, --help Print help
----- stderr -----
");
}
#[test]
fn create_uses_explicit_server_target_and_prints_remote_run_id() {
let context = test_context!();
let server = MockServer::start();
let run_id = unique_run_id();
let mock = server.mock(|when, then| {
when.method("POST").path("/api/v1/runs");
then.status(201)
.header("Content-Type", "application/json")
.body(run_status_response(run_id.as_str(), "submitted").to_string());
});
let output = context
.create_cmd()
.args([
"--server",
&format!("{}/api/v1", server.base_url()),
"--dry-run",
fixture("simple.fabro").to_str().unwrap(),
])
.output()
.expect("command should execute");
assert!(
output.status.success(),
"command failed:\nstdout:\n{}\nstderr:\n{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
mock.assert();
assert_eq!(output_stdout(&output).trim(), run_id.as_str());
}
#[test]
fn create_uses_configured_server_target_without_server_flag() {
let context = test_context!();
let server = MockServer::start();
let run_id = unique_run_id();
let mock = server.mock(|when, then| {
when.method("POST").path("/api/v1/runs");
then.status(201)
.header("Content-Type", "application/json")
.body(run_status_response(run_id.as_str(), "submitted").to_string());
});
context.set_http_target(&server.base_url());
let output = context
.create_cmd()
.args(["--dry-run", fixture("simple.fabro").to_str().unwrap()])
.output()
.expect("command should execute");
assert!(
output.status.success(),
"command failed:\nstdout:\n{}\nstderr:\n{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
mock.assert();
assert_eq!(output_stdout(&output).trim(), run_id.as_str());
}
#[test]
fn create_parent_resolves_parent_and_sends_parent_id_in_manifest() {
let context = test_context!();
let server = MockServer::start();
let run_id = unique_run_id();
let parent_id = unique_run_id();
let resolve_mock = super::support::mock_resolved_run(&server, "nightly-parent", &parent_id);
let create_mock = server.mock(|when, then| {
when.method("POST")
.path("/api/v1/runs")
.json_body_includes(format!(r#"{{"parent_id":"{parent_id}"}}"#));
then.status(201)
.header("Content-Type", "application/json")
.body(run_status_response(run_id.as_str(), "submitted").to_string());
});
let output = context
.create_cmd()
.args([
"--server",
&format!("{}/api/v1", server.base_url()),
"--dry-run",
"--parent",
"nightly-parent",
fixture("simple.fabro").to_str().unwrap(),
])
.output()
.expect("command should execute");
assert!(
output.status.success(),
"command failed:\nstdout:\n{}\nstderr:\n{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
resolve_mock.assert();
create_mock.assert();
assert_eq!(output_stdout(&output).trim(), run_id.as_str());
}
#[test]
fn create_rejects_storage_dir_flag() {
let context = test_context!();
let output = context
.create_cmd()
.args([
"--storage-dir",
"/tmp/fabro-create",
"--dry-run",
fixture("simple.fabro").to_str().unwrap(),
])
.output()
.expect("command should execute");
assert!(
!output.status.success(),
"command should reject --storage-dir"
);
let stderr = String::from_utf8_lossy(&output.stderr);
assert!(stderr.contains("unexpected argument '--storage-dir'"));
}
#[test]
fn create_cli_server_target_overrides_configured_server_target() {
let context = test_context!();
let config_server = MockServer::start();
let config_mock = config_server.mock(|when, then| {
when.method("POST").path("/api/v1/runs");
then.status(500)
.body("configured-server-should-not-be-used");
});
let cli_server = MockServer::start();
let run_id = unique_run_id();
let cli_mock = cli_server.mock(|when, then| {
when.method("POST").path("/api/v1/runs");
then.status(201)
.header("Content-Type", "application/json")
.body(run_status_response(run_id.as_str(), "submitted").to_string());
});
context.set_http_target(&config_server.base_url());
let output = context
.create_cmd()
.args([
"--server",
&format!("{}/api/v1", cli_server.base_url()),
"--dry-run",
fixture("simple.fabro").to_str().unwrap(),
])
.output()
.expect("command should execute");
assert!(
output.status.success(),
"command failed:\nstdout:\n{}\nstderr:\n{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
cli_mock.assert();
config_mock.assert_calls(0);
assert_eq!(output_stdout(&output).trim(), run_id.as_str());
}
#[test]
fn create_persists_directory_workflow_slug_and_cached_graph() {
let context = test_context!();
context.ensure_home_server_auth_methods();
let run_id = unique_run_id();
let workflow_path = context.temp_dir.join("sluggy/workflow.fabro");
context.write_temp(
"sluggy/workflow.fabro",
"\
digraph BarBaz {
start [shape=Mdiamond, label=\"Start\"]
exit [shape=Msquare, label=\"Exit\"]
start -> exit
}
",
);
context
.command()
.args([
"create",
"--dry-run",
"--auto-approve",
"--run-id",
run_id.as_str(),
workflow_path.to_str().unwrap(),
])
.assert()
.success();
let run_dir = context.find_run_dir(&run_id);
let state = run_state(&run_dir);
let run = &state.spec;
fabro_json_snapshot!(
context,
serde_json::json!({
"workflow_slug": run.workflow_slug,
"graph_name": run.graph.name,
"cached_graph_lines": state.spec.graph_source.as_ref().expect("graph should exist").lines().collect::<Vec<_>>(),
}),
@r#"
{
"workflow_slug": "sluggy",
"graph_name": "BarBaz",
"cached_graph_lines": [
"digraph BarBaz {",
" start [shape=Mdiamond, label=\"Start\"]",
" exit [shape=Msquare, label=\"Exit\"]",
" start -> exit",
"}"
]
}
"#
);
}
#[test]
fn create_persists_file_stem_slug_for_standalone_file() {
let context = test_context!();
context.ensure_home_server_auth_methods();
let run_id = unique_run_id();
let workflow_path = context.temp_dir.join("alpha.fabro");
context.write_temp(
"alpha.fabro",
"\
digraph FooWorkflow {
start [shape=Mdiamond, label=\"Start\"]
exit [shape=Msquare, label=\"Exit\"]
start -> exit
}
",
);
context
.command()
.args([
"create",
"--dry-run",
"--auto-approve",
"--run-id",
run_id.as_str(),
workflow_path.to_str().unwrap(),
])
.assert()
.success();
let run_dir = context.find_run_dir(&run_id);
let state = run_state(&run_dir);
let run = &state.spec;
fabro_json_snapshot!(
context,
serde_json::json!({
"workflow_slug": run.workflow_slug,
"graph_name": run.graph.name,
"cached_graph_lines": state.spec.graph_source.as_ref().expect("graph should exist").lines().collect::<Vec<_>>(),
}),
@r#"
{
"workflow_slug": "alpha",
"graph_name": "FooWorkflow",
"cached_graph_lines": [
"digraph FooWorkflow {",
" start [shape=Mdiamond, label=\"Start\"]",
" exit [shape=Msquare, label=\"Exit\"]",
" start -> exit",
"}"
]
}
"#
);
}
#[test]
fn create_persists_requested_overrides_into_store() {
let context = test_context!();
context.ensure_home_server_auth_methods();
let workflow = fixture("simple.fabro");
let mut cmd = context.command();
cmd.args([
"create",
"--dry-run",
"--auto-approve",
"--goal",
"Ship the release",
"--model",
"gpt-5",
"--provider",
"openai",
"--environment",
"default",
"--label",
"env=dev",
"--label",
"team=cli",
"--verbose",
"--preserve-sandbox",
workflow.to_str().unwrap(),
]);
let output = cmd.output().expect("command should execute");
assert!(
output.status.success(),
"command failed:\nstdout:\n{}\nstderr:\n{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
let stdout = output_stdout(&output);
let run_id = stdout
.lines()
.find(|line| !line.trim().is_empty())
.map(str::trim)
.expect("create should print a run ID")
.to_string();
let run = resolve_run(&context, &run_id);
let state = run_state(&run.run_dir);
let run_spec = &state.spec;
let labels = json!({
"env": run_spec.labels.get("env"),
"team": run_spec.labels.get("team"),
});
let settings = &run_spec.settings;
let resolved_run = resolved_run(settings);
let compact = json!({
"workflow_slug": run_spec.workflow_slug,
"settings": {
"goal": match resolved_run.goal.as_ref() {
Some(fabro_types::settings::run::RunGoal::Inline(value)) => Some(value.as_source()),
_ => None,
},
"dry_run": resolved_run.execution.mode == fabro_types::settings::run::RunMode::DryRun,
"auto_approve": resolved_run.execution.approval == fabro_types::settings::run::ApprovalMode::Auto,
"llm": {
"model": resolved_run.model.name.as_ref().map(fabro_types::settings::InterpString::as_source),
"provider": resolved_run.model.provider.as_ref().map(fabro_types::settings::InterpString::as_source),
},
"environment": {
"id": resolved_run.environment.id,
"provider": resolved_run.environment.provider.to_string(),
"preserve": resolved_run.environment.lifecycle.preserve,
},
},
"labels": labels,
});
assert_snapshot!(serde_json::to_string_pretty(&compact).unwrap(), @r###"
{
"workflow_slug": "simple",
"settings": {
"goal": "Ship the release",
"dry_run": true,
"auto_approve": true,
"llm": {
"model": "gpt-5",
"provider": "openai"
},
"environment": {
"id": "default",
"provider": "docker",
"preserve": true
}
},
"labels": {
"env": "dev",
"team": "cli"
}
}
"###);
}
#[test]
fn create_json_does_not_imply_auto_approve() {
let context = test_context!();
context.ensure_home_server_auth_methods();
let workflow = fixture("simple.fabro");
let output = context
.command()
.args(["--json", "create", "--dry-run", workflow.to_str().unwrap()])
.output()
.expect("command should execute");
assert!(
output.status.success(),
"command failed:\nstdout:\n{}\nstderr:\n{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
let value: serde_json::Value =
serde_json::from_slice(&output.stdout).expect("create JSON should parse");
let run_id = value["run_id"]
.as_str()
.expect("create JSON should include run_id");
let run = resolve_run(&context, run_id);
assert!(
resolved_run(&run_state(&run.run_dir).spec.settings,)
.execution
.approval
!= fabro_types::settings::run::ApprovalMode::Auto
);
}
#[test]
fn create_invalid_workflow_fails_without_creating_run() {
let context = test_context!();
let workflow = fixture("invalid.fabro");
let initial_run_count = run_count_for_test_case(&context);
let mut cmd = context.create_cmd();
cmd.arg(workflow.to_str().unwrap());
fabro_snapshot!(context.filters(), cmd, @"
success: false
exit_code: 1
----- stdout -----
----- stderr -----
× Validation failed
");
let run_count = run_count_for_test_case(&context);
assert_eq!(
run_count, initial_run_count,
"invalid create should not persist a run for this test case"
);
}
#[test]
fn create_rejects_unbound_template_inputs_without_creating_run() {
let context = test_context!();
let workflow = fixture("templated_unbound.fabro");
let initial_run_count = run_count_for_test_case(&context);
let mut cmd = context.create_cmd();
cmd.arg(workflow.to_str().unwrap());
fabro_snapshot!(context.filters(), cmd, @"
success: false
exit_code: 1
----- stdout -----
----- stderr -----
× Validation failed
");
let run_count = run_count_for_test_case(&context);
assert_eq!(
run_count, initial_run_count,
"invalid create should not persist a run for this test case"
);
}