fabro/lib/crates/fabro-cli/tests/it/cmd/runner.rs
Bryan Helmkamp c9d1b92725
feat(run): harden server-supervised worker lifecycle
Move subprocess workers fully behind the server-owned run store by
switching worker/server coordination to HTTP-backed run events and
control state. Reconcile stale in-flight runs on boot, terminate live
workers during shutdown, and update process titles to reflect server and
worker lifecycle phases.
2026-04-07 07:59:35 -04:00

343 lines
9.1 KiB
Rust

use fabro_store::EventEnvelope;
use fabro_test::{fabro_snapshot, test_context};
use fabro_types::{EventBody, RunEvent};
use super::support::{run_events, run_state, server_target};
use crate::support::{fabro_json_snapshot, unique_run_id};
const SHARED_DAEMON_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30);
fn stored_worker_events(run_dir: &std::path::Path) -> Vec<RunEvent> {
run_events(run_dir).iter().map(run_event).collect()
}
fn run_event(event: &EventEnvelope) -> RunEvent {
RunEvent::try_from(&event.payload).expect("stored event should parse")
}
fn assert_worker_succeeded(run_dir: &std::path::Path, stdout: &[u8]) {
assert!(
stdout.is_empty(),
"worker should not emit event transport on stdout"
);
let events = stored_worker_events(run_dir);
assert!(events.iter().any(|event| matches!(
&event.body,
EventBody::RunCompleted(props) if props.status == "success"
)));
}
#[test]
fn help() {
let context = test_context!();
let mut cmd = context.command();
cmd.args(["__run-worker", "--help"]);
fabro_snapshot!(context.filters(), cmd, @"
success: true
exit_code: 0
----- stdout -----
Internal: execute a single workflow run locally
Usage: fabro __run-worker [OPTIONS] --server <SERVER> --run-dir <RUN_DIR> --run-id <RUN_ID> --mode <MODE>
Options:
--json Output as JSON [env: FABRO_JSON=]
--server <SERVER> Fabro server target: http(s) URL or absolute Unix socket path
--debug Enable DEBUG-level logging (default is INFO) [env: FABRO_DEBUG=]
--run-dir <RUN_DIR> Run scratch directory
--no-upgrade-check Disable automatic upgrade check [env: FABRO_NO_UPGRADE_CHECK=true]
--run-id <RUN_ID> Run ID
--mode <MODE> Worker mode [possible values: start, resume]
--quiet Suppress non-essential output [env: FABRO_QUIET=]
--verbose Enable verbose output [env: FABRO_VERBOSE=]
-h, --help Print help
----- stderr -----
");
}
#[test]
fn runner_uses_cached_graph_after_source_deleted() {
let context = test_context!();
let run_id = unique_run_id();
let workflow_path = context.temp_dir.join("workflow.fabro");
context.write_temp(
"workflow.fabro",
"\
digraph CachedGraph {
start [shape=Mdiamond, label=\"Start\"]
exit [shape=Msquare, label=\"Exit\"]
start -> exit
}
",
);
context
.command()
.args([
"create",
"--dry-run",
"--auto-approve",
"--run-id",
run_id.as_str(),
workflow_path.to_str().unwrap(),
])
.assert()
.success();
let run_dir = context.find_run_dir(&run_id);
let server = server_target(&context.storage_dir);
std::fs::remove_file(&workflow_path).unwrap();
let output = context
.command()
.args([
"__run-worker",
"--server",
server.as_str(),
"--run-dir",
run_dir.to_str().unwrap(),
"--run-id",
run_id.as_str(),
"--mode",
"start",
])
.timeout(SHARED_DAEMON_TIMEOUT)
.assert()
.success()
.get_output()
.stdout
.clone();
assert_worker_succeeded(&run_dir, &output);
}
#[test]
fn runner_uses_snapshotted_app_id_for_github_credentials() {
let context = test_context!();
let run_id = unique_run_id();
let workflow_path = context.temp_dir.join("workflow.fabro");
context.write_home(
".fabro/settings.toml",
"\
version = 1
[git]
app_id = \"snapshotted-app-id\"
",
);
context.write_temp(
"workflow.fabro",
"\
digraph GitHubApp {
start [shape=Mdiamond, label=\"Start\"]
exit [shape=Msquare, label=\"Exit\"]
start -> exit
}
",
);
context
.command()
.args([
"create",
"--dry-run",
"--auto-approve",
"--run-id",
run_id.as_str(),
workflow_path.to_str().unwrap(),
])
.assert()
.success();
let run_dir = context.find_run_dir(&run_id);
let state = run_state(&run_dir);
let run = state.run.as_ref().expect("run record should exist");
fabro_json_snapshot!(
context,
serde_json::json!({
"app_id": run.settings.git.clone().and_then(|git| git.app_id),
}),
@r#"
{
"app_id": "snapshotted-app-id"
}
"#
);
context.write_home(".fabro/settings.toml", "version = 1\n");
let server = server_target(&context.storage_dir);
let mut cmd = context.command();
cmd.env("GITHUB_APP_PRIVATE_KEY", "%%%not-base64%%%");
cmd.args([
"__run-worker",
"--server",
server.as_str(),
"--run-dir",
run_dir.to_str().unwrap(),
"--run-id",
run_id.as_str(),
"--mode",
"start",
]);
cmd.timeout(SHARED_DAEMON_TIMEOUT);
let assert = cmd.assert().success();
assert_worker_succeeded(&run_dir, &assert.get_output().stdout);
}
#[test]
fn runner_runs_without_run_json_when_run_id_is_explicit() {
let context = test_context!();
let run_id = unique_run_id();
let workflow_path = context.temp_dir.join("workflow.fabro");
context.write_temp(
"workflow.fabro",
"\
digraph DetachedStoreOnly {
start [shape=Mdiamond, label=\"Start\"]
exit [shape=Msquare, label=\"Exit\"]
start -> exit
}
",
);
context
.command()
.args([
"create",
"--dry-run",
"--auto-approve",
"--run-id",
run_id.as_str(),
workflow_path.to_str().unwrap(),
])
.assert()
.success();
let run_dir = context.find_run_dir(&run_id);
let server = server_target(&context.storage_dir);
let output = context
.command()
.args([
"__run-worker",
"--server",
server.as_str(),
"--run-dir",
run_dir.to_str().unwrap(),
"--run-id",
run_id.as_str(),
"--mode",
"start",
])
.timeout(SHARED_DAEMON_TIMEOUT)
.assert()
.success()
.get_output()
.stdout
.clone();
assert_worker_succeeded(&run_dir, &output);
}
#[test]
fn runner_resume_rejects_completed_run_without_mutating_it() {
let context = test_context!();
context.write_temp(
"workflow.fabro",
"\
digraph Test {
start [shape=Mdiamond, label=\"Start\"]
exit [shape=Msquare, label=\"Exit\"]
start -> exit
}
",
);
let run = context
.command()
.args([
"run",
"--dry-run",
"--auto-approve",
"--no-retro",
"--detach",
context.temp_dir.join("workflow.fabro").to_str().unwrap(),
])
.assert()
.success();
let run_id = String::from_utf8(run.get_output().stdout.clone())
.unwrap()
.trim()
.to_string();
let run_dir = context.find_run_dir(&run_id);
let server = server_target(&context.storage_dir);
context
.command()
.args(["wait", &run_id])
.timeout(SHARED_DAEMON_TIMEOUT)
.assert()
.success();
let inspect_before = context
.command()
.args(["inspect", &run_id])
.assert()
.success();
let before: serde_json::Value =
serde_json::from_slice(&inspect_before.get_output().stdout).unwrap();
let before_summary = serde_json::json!({
"run_dir": before[0]["run_dir"],
"start_time": before[0]["start_record"]["start_time"],
"conclusion_timestamp": before[0]["conclusion"]["timestamp"],
"conclusion_status": before[0]["conclusion"]["status"],
});
fabro_json_snapshot!(context, &before_summary, @r#"
{
"run_dir": null,
"start_time": "[TIMESTAMP]",
"conclusion_timestamp": "[TIMESTAMP]",
"conclusion_status": "success"
}
"#);
let mut cmd = context.command();
cmd.args([
"__run-worker",
"--server",
&server,
"--run-dir",
run_dir.to_str().unwrap(),
"--run-id",
&run_id,
"--mode",
"resume",
]);
cmd.timeout(SHARED_DAEMON_TIMEOUT);
fabro_snapshot!(context.filters(), cmd, @"
success: false
exit_code: 1
----- stdout -----
----- stderr -----
error: Precondition failed: run already finished successfully — nothing to resume
");
let inspect_after = context
.command()
.args(["inspect", &run_id])
.assert()
.success();
let after: serde_json::Value =
serde_json::from_slice(&inspect_after.get_output().stdout).unwrap();
let after_summary = serde_json::json!({
"run_dir": after[0]["run_dir"],
"start_time": after[0]["start_record"]["start_time"],
"conclusion_timestamp": after[0]["conclusion"]["timestamp"],
"conclusion_status": after[0]["conclusion"]["status"],
});
assert_eq!(after_summary, before_summary);
}