fabro/lib/apps/fabro-cli/tests/it/workflow/mod.rs
Bryan Helmkamp af38d68942
Delete fabro-validate and fabro-acp; validate on Petri's check
Petri judges a workflow at admission, so Fabro's lint rules go:
`fabro-validate` (its 35 rules and the `LintRule` trait) is deleted, and
with it `fabro-acp` (only a rule and two legacy executor tests used it),
the model-resolution transform, the legacy `create`, `compile_create_run`
and `materialize_create_run` stages, and `fabro-graphviz`'s `condition`
and `fidelity` modules. `Diagnostic`, `RelatedDiagnostic` and `Severity`
move to `fabro_types::diagnostic`, the one shape every diagnostic takes.

Validation is now the same question the create handler asks. A new
server module, `petri_check`, builds Petri's check request from a
workflow bundle and the run's settings (every workflow of the bundle at
its bundle-relative path, the inputs, the run variables, the launch),
runs the check, and maps the diagnostics; Fabro adds one rule of its
own, `fabro.model.no_ready_provider`, refusing a model node when no
provider is ready. Admission, the validate and preflight endpoints and
the offline `fabro validate` all go through it:

- `validate_prepared_manifest` runs Fabro's structural pass (parse and
  transform, whose diagnostics stay) and then Petri's check, on the
  blocking pool from the handlers;
- the offline `validate_manifest` checks with no model client and with
  an unbound input as a warning (`CheckRequest.unbound_is_warning`), so
  a workflow validates before its inputs exist; a collected workflow
  before upload checks with unbound inputs as errors, as before;
- preflight resolves each LLM node's selector against the ready
  providers and the catalog for its probe, as the deleted transform did,
  and no longer probes a model Petri refused;
- the graph render endpoint needs only the structural pass;
- a run manifest now carries its `[run.goal] file`, which Petri reads
  from the bundle as it does for a version.

The transforms keep the authored model selector (`sonnet` stays
`sonnet`): Petri pins the catalog model in the admitted graph, not in
the graph Fabro displays or in the settings snapshot. Tests assert that,
and the CLI's validate, preflight and graph snapshots carry Petri's
diagnostics (`attractor.no_start`, `attractor.undeclared_node`,
`attractor.bad_on_failure`, ...) in place of the lint rules' text.

Known gaps, Petri's side: a `workflow.toml` whose `[run.environment]`
names an environment the server catalog defines but the file does not
is refused (`unsupported.workflow_toml.run.environment`), as admission
already refused it; an unbound input inside an included template
partial is a render error rather than the unbound-input warning.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-18 11:32:04 -04:00

232 lines
7.6 KiB
Rust

#![allow(
clippy::absolute_paths,
reason = "This test module prefers explicit type paths over extra imports."
)]
mod agent_linear;
mod artifacts;
mod command_agent_mixed;
mod command_pipeline;
mod command_routing;
mod conditional_branching;
mod dry_run_examples;
mod full_stack;
mod hooks;
mod human_gate;
pub(super) mod plugin;
use std::path::{Path, PathBuf};
use std::time::Duration;
use fabro_store::EventEnvelope;
use fabro_test::{TestContext, expect_reqwest_status};
use serde_json::Value;
use crate::cmd::support::{RunProjection, server_endpoint};
pub(super) fn fixture(name: &str) -> PathBuf {
Path::new(env!("CARGO_MANIFEST_DIR"))
.join("tests/it/workflow/fixtures")
.join(name)
}
pub(super) fn read_conclusion(run_dir: &Path) -> Value {
serde_json::to_value(
run_state(run_dir)
.conclusion
.expect("run store conclusion should exist"),
)
.expect("conclusion should serialize")
}
pub(super) fn read_run_spec(run_dir: &Path) -> Value {
serde_json::to_value(run_state(run_dir).spec).expect("run spec should serialize")
}
pub(super) fn completed_nodes(run_dir: &Path) -> Vec<String> {
let state = run_state(run_dir);
let cp = state
.current_checkpoint()
.expect("run store checkpoint should exist");
cp.completed_nodes.clone()
}
pub(super) fn has_event(run_dir: &Path, event_name: &str) -> bool {
run_events(run_dir)
.into_iter()
.any(|event| event.event.event_name() == event_name)
}
pub(super) fn dump_export(context: &TestContext, run_id: &str) -> PathBuf {
let output_dir = context.temp_dir.join(format!("dump-{run_id}"));
context
.command()
.args([
"dump",
"--output",
output_dir
.to_str()
.expect("dump output path should be valid UTF-8"),
run_id,
])
.assert()
.success();
output_dir
}
#[expect(
clippy::disallowed_methods,
reason = "integration test helpers inspect exported files synchronously"
)]
pub(super) fn stage_dump_dir(export_dir: &Path, stage_id: &str) -> PathBuf {
let stages_dir = export_dir.join("stages");
let mut matches: Vec<_> = std::fs::read_dir(&stages_dir)
.unwrap_or_else(|err| panic!("reading {} should succeed: {err}", stages_dir.display()))
.filter_map(|entry| entry.ok().map(|entry| entry.path()))
.filter(|path| {
path.file_name()
.and_then(|name| name.to_str())
.is_some_and(|name| {
name == stage_id || name.split_once('-').is_some_and(|(_, id)| id == stage_id)
})
})
.collect();
matches.sort();
match matches.as_slice() {
[path] => path.clone(),
[] => panic!(
"stage dump dir for {stage_id} not found in {}",
stages_dir.display()
),
_ => panic!("stage dump dir for {stage_id} was ambiguous: {matches:?}"),
}
}
/// Find the single run directory for this test context.
pub(super) fn find_run_dir(context: &TestContext) -> PathBuf {
context.single_run_dir()
}
pub(super) fn run_id_for(run_dir: &Path) -> String {
infer_run_id(run_dir)
}
fn infer_run_id(run_dir: &Path) -> String {
run_dir
.file_name()
.map(|name| name.to_string_lossy().to_string())
.and_then(|name| name.rsplit('-').next().map(ToOwned::to_owned))
.filter(|value| !value.is_empty())
.expect("run directory name should contain run id suffix")
}
fn block_on<T>(future: impl std::future::Future<Output = T>) -> T {
tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("test runtime should build")
.block_on(future)
}
async fn get_server_json_for_storage<T: serde::de::DeserializeOwned>(
storage_dir: &Path,
path: &str,
) -> T {
let (client, base_url) = server_endpoint(storage_dir).expect("server endpoint should exist");
let response = client
.get(format!("{base_url}{path}"))
.send()
.await
.expect("server request should succeed");
let response =
expect_reqwest_status(response, fabro_http::StatusCode::OK, format!("GET {path}")).await;
response
.json::<T>()
.await
.expect("server response should parse")
}
fn run_state(run_dir: &Path) -> RunProjection {
let run_id = infer_run_id(run_dir);
let runs_dir = run_dir.parent().expect("run dir should have parent");
let storage_dir = runs_dir.parent().expect("runs dir should have parent");
block_on(get_server_json_for_storage(
storage_dir,
&format!("/api/v1/runs/{run_id}/state"),
))
}
fn run_events(run_dir: &Path) -> Vec<EventEnvelope> {
let run_id = infer_run_id(run_dir);
let runs_dir = run_dir.parent().expect("run dir should have parent");
let storage_dir = runs_dir.parent().expect("runs dir should have parent");
let response: serde_json::Value = block_on(get_server_json_for_storage(
storage_dir,
&format!("/api/v1/runs/{run_id}/events"),
));
crate::support::parse_event_envelopes(&response)
}
/// Runs a scenario against every sandbox provider fabro supports:
///
/// - `local`: the bundled Host provider in-process.
/// - `daytona`: the bundled Daytona provider, live credentials required.
/// - `host-plugin`: the driver's Host executable over stdio under the
/// non-bundled `host` kind, a clone-based managed workspace.
/// - `docker-plugin`: the driver's Docker executable over stdio under the
/// non-bundled `docker-plugin` kind.
///
/// The plugin variants need the executables `cargo` builds for
/// `fabro-sandbox`; without them (or without a Docker daemon) they skip,
/// unless `FABRO_REQUIRE_SANDBOX_PLUGINS` is set, as CI sets it.
macro_rules! sandbox_tests {
($name:ident) => {
sandbox_tests!($name, keys = []);
};
($name:ident, keys = [$($key:expr),* $(,)?]) => {
paste::paste! {
#[fabro_macros::e2e_test($(live($key)),*)]
fn [<local_ $name>]() {
[<scenario_ $name>](&fabro_test::test_context!(), "local");
}
#[fabro_macros::e2e_test(live("DAYTONA_API_KEY") $(, live($key))*)]
fn [<daytona_ $name>]() {
[<scenario_ $name>](&fabro_test::test_context!(), "daytona");
}
#[fabro_macros::e2e_test($(live($key)),*)]
fn [<host_plugin_ $name>]() {
let mut context = fabro_test::test_context!();
if let Some(environment) =
$crate::workflow::plugin::configure(&mut context, $crate::workflow::plugin::Plugin::Host)
{
$crate::workflow::plugin::run_with_server_log(&context, || {
[<scenario_ $name>](&context, environment);
});
}
}
#[fabro_macros::e2e_test($(live($key)),*)]
fn [<docker_plugin_ $name>]() {
let mut context = fabro_test::test_context!();
if let Some(environment) =
$crate::workflow::plugin::configure(&mut context, $crate::workflow::plugin::Plugin::Docker)
{
$crate::workflow::plugin::run_with_server_log(&context, || {
[<scenario_ $name>](&context, environment);
});
}
}
}
};
}
pub(super) use sandbox_tests;
pub(super) fn timeout_for(sandbox: &str) -> Duration {
match sandbox {
"daytona" => Duration::from_mins(10),
"docker-plugin" => Duration::from_mins(5),
_ => Duration::from_mins(3),
}
}