mirror of
https://github.com/fabro-sh/fabro.git
synced 2026-08-28 05:27:41 +00:00
Unify fabro run foreground to use create + start + attach (#141)
## Summary - **Unify foreground and detach code paths**: Both `fabro run` modes now go through the same `create_run() + start_run()` pipeline, with foreground adding `attach_run()`. Only `--preflight` remains as a special case. - **Fix three bugs in create→start→attach path**: (1) `_run_engine` crashed for `.fabro` workflows by hardcoding `run.toml` — now falls back to `graph.fabro`; (2) `attach_run` couldn't detect crashed engines due to zombie processes — `start_run` now returns the `Child` handle; (3) `create_run` ignored `--run-id`. - **Configure nextest slow-timeout profiles**: Tighten unit test timeout to 2s slow / 4s kill, add `e2e` profile with 10s/30s. Switch CI and docs to `cargo nextest run`. ## Test plan - [ ] `cargo nextest run --workspace` passes with new timeout profiles - [ ] `fabro run <workflow>` works in foreground mode (create + start + attach) - [ ] `fabro run --detach <workflow>` prints run ID and exits - [ ] `fabro attach <run>` works standalone (without child handle) - [ ] `fabro resume <run>` works for both `.toml` and `.fabro` workflows 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Fabro <noreply@fabro.sh> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
165e2495bf
commit
b59c62b33e
35 changed files with 2474 additions and 569 deletions
|
|
@ -17,7 +17,7 @@ Output a report with all the bugs using this format:
|
|||
<bug>
|
||||
<summary>up to 3 sentences</summary>
|
||||
<severity>important OR nit</severity>
|
||||
<pre_exiting>yes OR no</pre_existing>
|
||||
<pre_existing>yes OR no</pre_existing>
|
||||
<location>
|
||||
<file>lib/crates/fabro-cli/src/commands/resume.rs</file>
|
||||
<start_line>115</start_line>
|
||||
|
|
@ -49,7 +49,7 @@ Here is a real-world example:
|
|||
`prepare_from_checkpoint` unconditionally creates a `LocalSandbox` via `local_sandbox_with_callback`, completely ignoring the `--sandbox` flag and TOML config. A user running `fabro resume --checkpoint logs/checkpoint.json --workflow w.fabro --sandbox docker` will silently get a local sandbox instead of Docker; to fix this, call `resolve_sandbox_provider(args.sandbox.map(Into::into), None, run_defaults)` just as `prepare_from_branch` does.
|
||||
</summary>
|
||||
<severity>important</severity>
|
||||
<pre_exiting>no</pre_existing>
|
||||
<pre_existing>no</pre_existing>
|
||||
<location>
|
||||
<file>lib/crates/fabro-cli/src/commands/resume.rs</file>
|
||||
<start_line>208</start_line>
|
||||
|
|
|
|||
|
|
@ -1,3 +1,7 @@
|
|||
[profile.default]
|
||||
# Hard kill after 120s per test
|
||||
slow-timeout = { period = "60s", terminate-after = 2 }
|
||||
# Unit tests: flag SLOW after 2s, hard-kill after 4s
|
||||
slow-timeout = { period = "2s", terminate-after = 2 }
|
||||
|
||||
[profile.e2e]
|
||||
# E2E (ignored) tests: flag SLOW after 10s, hard-kill after 30s
|
||||
slow-timeout = { period = "10s", terminate-after = 3 }
|
||||
|
|
|
|||
6
.github/workflows/rust.yml
vendored
6
.github/workflows/rust.yml
vendored
|
|
@ -66,7 +66,8 @@ jobs:
|
|||
- uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- run: cargo test --workspace
|
||||
- uses: taiki-e/install-action@nextest
|
||||
- run: cargo nextest run --workspace
|
||||
|
||||
test-macos:
|
||||
name: Test (macOS)
|
||||
|
|
@ -80,4 +81,5 @@ jobs:
|
|||
- uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- run: cargo test --workspace
|
||||
- uses: taiki-e/install-action@nextest
|
||||
- run: cargo nextest run --workspace
|
||||
|
|
|
|||
12
AGENTS.md
12
AGENTS.md
|
|
@ -6,11 +6,11 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
|
|||
|
||||
### Rust
|
||||
- `cargo build --workspace` — build all crates
|
||||
- `cargo test --workspace` — run all tests
|
||||
- `cargo test -p fabro-api` — test a single crate
|
||||
- `cargo test -p fabro-workflows -- test_name` — run a single test
|
||||
- `set -a && source .env && set +a && cargo test --workspace -- --ignored` — run all E2E live tests (requires credentials in `.env`, see `.env.example`)
|
||||
- `set -a && source .env && set +a && cargo test -p fabro-llm -- --ignored` — run E2E tests for a single crate
|
||||
- `cargo nextest run --workspace` — run all unit tests
|
||||
- `cargo nextest run -p fabro-api` — test a single crate
|
||||
- `cargo nextest run -p fabro-workflows -- test_name` — run a single test
|
||||
- `set -a && source .env && set +a && cargo nextest run --workspace --profile e2e --run-ignored only` — run all E2E live tests (requires credentials in `.env`, see `.env.example`)
|
||||
- `set -a && source .env && set +a && cargo nextest run -p fabro-llm --profile e2e --run-ignored only` — run E2E tests for a single crate
|
||||
- `cargo fmt --check --all` — check formatting
|
||||
- `cargo clippy --workspace -- -D warnings` — lint
|
||||
|
||||
|
|
@ -42,7 +42,7 @@ The OpenAPI spec at `docs/api-reference/fabro-api.yaml` is the source of truth f
|
|||
1. Edit `docs/api-reference/fabro-api.yaml`
|
||||
2. `cargo build -p fabro-types` — build.rs regenerates Rust types via typify
|
||||
3. Write/update handler in `lib/crates/fabro-api/src/server.rs`, add route to `build_router()`
|
||||
4. `cargo test -p fabro-api` — conformance test catches spec/router drift
|
||||
4. `cargo nextest run -p fabro-api` — conformance test catches spec/router drift
|
||||
5. `cd lib/packages/fabro-api-client && bun run generate` — regenerates TypeScript Axios client
|
||||
|
||||
## Architecture
|
||||
|
|
|
|||
|
|
@ -1,9 +1,11 @@
|
|||
# Fabro Events Strategy
|
||||
|
||||
Fabro emits structured **workflow run events** during execution for observability. Events write to `progress.jsonl` (one JSON object per line) and `live.json` (latest event snapshot) inside the run's log directory. Events are the primary record of what happened during a run — they feed the retro system, CLI verbose output, and live monitoring.
|
||||
Fabro emits structured **workflow run events** during execution for observability. Events write to `progress.jsonl` (one JSON object per line) and `live.json` (latest event snapshot) inside the run's log directory. Events are the primary record of what happened during a run — they feed the retro system, CLI verbose output, `fabro attach`, `fabro logs`, and live monitoring.
|
||||
|
||||
Events are distinct from tracing logs (see `logging-strategy.md`). Tracing is developer diagnostics; events are the structured audit trail consumed by tooling.
|
||||
|
||||
Detached runs rely on this distinction. If a warning or error needs to be visible to users after reattach, emit a `WorkflowRunEvent` rather than only writing to stderr/`detach.log`.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
|
|
@ -37,6 +39,10 @@ Every line in `progress.jsonl` has three envelope fields, then the event's own f
|
|||
|
||||
The envelope is built in `cli/run.rs`. Field names from the event that collide with envelope keys (`ts`, `run_id`, `event`) are dropped — the `run_id` from `WorkflowRunStarted` populates the envelope itself.
|
||||
|
||||
## Run Completion Contract
|
||||
|
||||
`status.json` is the authoritative completion signal for detached runs. Write a terminal `RunStatus` only after all post-run work is complete, including retro generation, finalize-commit work, pull request creation, and sandbox cleanup. `conclusion.json` should be written at that same final point so `attach`, `wait`, and summary rendering all agree on when the run is actually finished.
|
||||
|
||||
## Node Terminology
|
||||
|
||||
- **`node_id`** — programmatic identifier (the id from the DOT graph). Stable, used for matching.
|
||||
|
|
@ -151,6 +157,7 @@ WorkflowRunEvent::MyNewEvent { node_id, duration_ms, .. } => {
|
|||
| `WorkflowRunStarted` | `workflow_name`, `run_id`, `base_sha`?, `run_branch`?, `worktree_dir`? |
|
||||
| `WorkflowRunCompleted` | `duration_ms`, `artifact_count`, `total_cost`?, `final_git_commit_sha`? |
|
||||
| `WorkflowRunFailed` | `error`, `duration_ms`, `git_commit_sha`? |
|
||||
| `RunNotice` | `level`, `code`, `message` |
|
||||
|
||||
### Stage execution
|
||||
|
||||
|
|
|
|||
|
|
@ -22,7 +22,8 @@ Production runs at INFO level. INFO should be low-volume and high-signal — the
|
|||
**Do not log:**
|
||||
|
||||
- Hot loops or per-token streaming events (use DEBUG only if truly needed for diagnosis)
|
||||
- Data that belongs in user-facing output (`eprintln!` for CLI feedback, not tracing)
|
||||
- Data that belongs in user-facing output (`eprintln!` for interactive CLI feedback, not tracing)
|
||||
- Detached user-visible warnings or errors that need to survive `attach`/`logs` (`detach.log` is debug-only; emit a `WorkflowRunEvent` into `progress.jsonl` instead)
|
||||
- Redundant information already captured by a parent event (if you logged "starting X", you don't need to log every sub-step at the same level)
|
||||
- Events that are already traced via `EventEnum::trace()` — the event enums (`AgentEvent`, `PipelineEvent`, `ExecutionEnvEvent`) each have a `trace()` method called automatically at their emit site; do not add manual `info!`/`debug!` calls that duplicate what `trace()` already emits
|
||||
- Wrapper/forwarding variants that re-emit an inner event — `PipelineEvent::Agent`, `PipelineEvent::ExecutionEnv`, and `AgentEvent::SubAgentEvent` are no-ops in `trace()` because the inner event is already traced at its origin
|
||||
|
|
@ -176,5 +177,6 @@ The domain event enums (`AgentEvent`, `PipelineEvent`, `ExecutionEnvEvent`) each
|
|||
|
||||
- **Add tracing for new variants** by adding a match arm in the enum's `trace()` method. Choose the level based on the guidelines above (INFO for lifecycle boundaries, DEBUG for individual steps, WARN/ERROR for failures).
|
||||
- **Do not add manual log calls at emit sites.** The `trace()` call in the emitter handles it. Adding `info!` or `debug!` next to an `emit()` call will double-log.
|
||||
- **Detached UX belongs in events, not stderr.** If an attached user needs to see the message later via `fabro attach` or `fabro logs`, emit a workflow event (for example `RunNotice`) and let tracing capture the developer-oriented copy separately.
|
||||
- **Wrapper variants are no-ops.** When one event enum wraps another (`PipelineEvent::Agent` wraps `AgentEvent`, `AgentEvent::SubAgentEvent` wraps a child `AgentEvent`), the wrapper's `trace()` arm is `{}` because the inner event was already traced at its origin. This prevents double-logging.
|
||||
- **Streaming noise variants are no-ops.** `TextDelta` and `ToolCallOutputDelta` produce no log output — per-token events would flood the logs even at DEBUG level.
|
||||
|
|
|
|||
|
|
@ -25,6 +25,21 @@ implement -> fix [condition="failure_class=transient_infra"]
|
|||
implement -> escalate [condition="failure_class=deterministic"]
|
||||
```
|
||||
|
||||
## Human gates fail closed when unanswered
|
||||
|
||||
Human approval gates are special: lack of an answer is treated as a failure, not as an implicit approval. If a prompt ends because stdin is closed, the user cancels it, a web session disconnects, or a test/replay interviewer has no answer available, the human stage returns a failure outcome instead of selecting an unconditional branch.
|
||||
|
||||
That means an unanswered gate will only continue if you model that path explicitly, for example:
|
||||
|
||||
```dot
|
||||
approve [shape=hexagon, label="Approve release?"]
|
||||
|
||||
approve -> ship [label="[A] Approve"]
|
||||
approve -> manual_review [condition="outcome=fail"]
|
||||
```
|
||||
|
||||
If no `outcome=fail` edge or `retry_target` exists, the run stops rather than advancing past the approval gate.
|
||||
|
||||
## Retry layers
|
||||
|
||||
Fabro retries failures at three levels: **LLM retries** handle transient API errors inside a single model call, **turn-level retries** recover from dropped streams mid-response, and **node retries** re-execute the entire node handler when the first two levels aren't enough. These layers are independent — a node retry re-runs the full handler, which gets its own fresh set of LLM and turn-level retries.
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ Each question carries metadata beyond the prompt text:
|
|||
|
||||
## Answer values
|
||||
|
||||
Answers are one of six variants:
|
||||
Answers are one of seven variants:
|
||||
|
||||
| Value | Meaning |
|
||||
|---|---|
|
||||
|
|
@ -44,7 +44,8 @@ Answers are one of six variants:
|
|||
| `No` | Negative response |
|
||||
| `Selected(key)` | A specific option was chosen (carries the option key) |
|
||||
| `Text(string)` | Free-text input |
|
||||
| `Skipped` | The user dismissed the question without answering |
|
||||
| `Aborted` | No answer was obtained because the prompt ended early (EOF, cancel, disconnected session, exhausted replay/queue) |
|
||||
| `Skipped` | Legacy skip-style answer; not treated as approval by human gates |
|
||||
| `Timeout` | The question's timeout elapsed without a response |
|
||||
|
||||
An answer can also carry a `selected_option` (the full `{key, label}` pair) and a `text` field for freeform input.
|
||||
|
|
@ -76,12 +77,16 @@ fabro run workflow.fabro
|
|||
# Select:
|
||||
```
|
||||
|
||||
If the console prompt is canceled or stdin is already closed, the interviewer returns `Aborted`. That does not count as approval.
|
||||
|
||||
### Web
|
||||
|
||||
The default for API server runs. The web interviewer holds questions in a queue until answers are submitted externally — typically by the web UI or a REST API call. Each question gets a unique ID (e.g. `q-1`), and the `ask()` call blocks on a oneshot channel until `submit_answer(id, answer)` is called.
|
||||
|
||||
This decoupling means the workflow engine and the user interface can run in different processes. The web UI polls for pending questions and posts answers back to the API.
|
||||
|
||||
If the pending session disappears before an answer is submitted, the waiting question resolves as `Aborted`.
|
||||
|
||||
### Slack
|
||||
|
||||
Fabro's Slack integration uses the web interviewer under the hood. When a human gate fires, the pending question is rendered as a Slack message with interactive buttons. When a user clicks a button, the Slack event handler calls `submit_answer()` on the web interviewer, unblocking the workflow.
|
||||
|
|
@ -100,6 +105,8 @@ Enable it with the `--auto-approve` flag:
|
|||
fabro run workflow.fabro --auto-approve
|
||||
```
|
||||
|
||||
`--auto-approve` is intentionally distinct from an unanswered prompt. Auto-approve is an explicit operator choice to advance human gates automatically.
|
||||
|
||||
## Timeouts
|
||||
|
||||
Questions can have a `timeout_seconds` field. When set, Fabro wraps the interviewer call with a timeout:
|
||||
|
|
@ -113,3 +120,5 @@ The human handler then checks the node's `human.default_choice` attribute. If se
|
|||
```dot
|
||||
approve [shape=hexagon, label="Approve?", human.default_choice="deploy"]
|
||||
```
|
||||
|
||||
Outside of timeout defaults, human gates fail closed: `Aborted` and `Skipped` answers do not fall through to ordinary approval edges. To model an explicit unanswered path, add an edge such as `condition="outcome=fail"` or configure a `retry_target`.
|
||||
|
|
|
|||
|
|
@ -57,6 +57,16 @@ approve [shape=hexagon, label="Approve?", human.default_choice="approve"]
|
|||
|
||||
If the timeout elapses without a response, the workflow continues to the default target rather than failing.
|
||||
|
||||
### Unanswered gates fail closed
|
||||
|
||||
Fabro does not treat missing input as approval. A human gate advances only when one of these happens:
|
||||
|
||||
- A person explicitly answers the question
|
||||
- The gate times out and `human.default_choice` is set
|
||||
- The run uses `--auto-approve`
|
||||
|
||||
If stdin is closed, a prompt is canceled, a web session disconnects, or no answer is otherwise captured, the gate is treated as unanswered. Fabro will not fall through to an unconditional approval edge. If you want to handle that case explicitly, add a failure route such as `condition="outcome=fail"` or a `retry_target`.
|
||||
|
||||
### Auto-approve
|
||||
|
||||
For testing or fully automated runs, pass `--auto-approve` to skip all human gates:
|
||||
|
|
|
|||
|
|
@ -1,16 +1,27 @@
|
|||
use std::io::{BufRead, BufReader, IsTerminal};
|
||||
use std::path::Path;
|
||||
use std::io::{BufRead, BufReader, IsTerminal, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::ExitCode;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use anyhow::{bail, Result};
|
||||
|
||||
use fabro_interview::ConsoleInterviewer;
|
||||
use fabro_interview::{AnswerValue, ConsoleInterviewer};
|
||||
use fabro_util::terminal::Styles;
|
||||
use fabro_workflows::event::RunNoticeLevel;
|
||||
use fabro_workflows::run_status::{RunStatus, RunStatusRecord};
|
||||
|
||||
use super::detached_support::append_run_notice;
|
||||
use super::run_progress;
|
||||
|
||||
#[cfg(test)]
|
||||
const ATTACH_STARTUP_GRACE: Duration = Duration::from_millis(200);
|
||||
#[cfg(not(test))]
|
||||
const ATTACH_STARTUP_GRACE: Duration = Duration::from_secs(3);
|
||||
const INTERVIEW_UNANSWERED_MESSAGE: &str =
|
||||
"Interview ended without an answer. The run is still waiting for input; reattach to answer it.";
|
||||
|
||||
/// Attach to a running (or finished) workflow run, rendering progress live.
|
||||
///
|
||||
/// Returns exit code 0 for success/partial_success, 1 otherwise.
|
||||
|
|
@ -18,13 +29,17 @@ pub async fn attach_run(
|
|||
run_dir: &Path,
|
||||
kill_on_detach: bool,
|
||||
styles: &'static Styles,
|
||||
engine_child: Option<std::process::Child>,
|
||||
) -> Result<ExitCode> {
|
||||
let progress_path = run_dir.join("progress.jsonl");
|
||||
let conclusion_path = run_dir.join("conclusion.json");
|
||||
let status_path = run_dir.join("status.json");
|
||||
let interview_request_path = run_dir.join("interview_request.json");
|
||||
let interview_response_path = run_dir.join("interview_response.json");
|
||||
let pid_path = run_dir.join("run.pid");
|
||||
|
||||
let mut engine_guard = engine_child.map(EngineChildGuard::new);
|
||||
|
||||
let is_tty = std::io::stderr().is_terminal();
|
||||
let verbose = fabro_workflows::run_spec::RunSpec::load(run_dir)
|
||||
.map(|spec| spec.verbose)
|
||||
|
|
@ -47,13 +62,21 @@ pub async fn attach_run(
|
|||
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
||||
wait_count += 1;
|
||||
if wait_count > 100 {
|
||||
// Guard's Drop kills+waits on the engine child
|
||||
drop(engine_guard.take());
|
||||
bail!(
|
||||
"Timed out waiting for progress.jsonl to appear in {}",
|
||||
run_dir.display()
|
||||
);
|
||||
}
|
||||
if cancelled.load(Ordering::Relaxed) {
|
||||
return Ok(ExitCode::from(0));
|
||||
if !kill_on_detach {
|
||||
if let Some(guard) = engine_guard.as_mut() {
|
||||
guard.defuse();
|
||||
}
|
||||
}
|
||||
// Guard's Drop kills+waits when kill_on_detach is true
|
||||
return Ok(ExitCode::from(1));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -61,20 +84,28 @@ pub async fn attach_run(
|
|||
let mut reader = BufReader::new(file);
|
||||
let mut line = String::new();
|
||||
let mut cached_pid: Option<u32> = None;
|
||||
let attach_started = Instant::now();
|
||||
|
||||
loop {
|
||||
if cancelled.load(Ordering::Relaxed) {
|
||||
if kill_on_detach {
|
||||
// Kill the engine process
|
||||
kill_engine(&pid_path);
|
||||
// Wait briefly for conclusion
|
||||
// Wait briefly for a terminal status or conclusion
|
||||
for _ in 0..20 {
|
||||
if conclusion_path.exists() {
|
||||
if conclusion_path.exists()
|
||||
|| read_status_record(&status_path)
|
||||
.map(|record| record.status.is_terminal())
|
||||
.unwrap_or(false)
|
||||
{
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
} else {
|
||||
if let Some(guard) = engine_guard.as_mut() {
|
||||
guard.defuse();
|
||||
}
|
||||
eprintln!("Detached from run (engine continues in background)");
|
||||
}
|
||||
break;
|
||||
|
|
@ -94,86 +125,94 @@ pub async fn attach_run(
|
|||
}
|
||||
|
||||
// Check for interview request
|
||||
if interview_request_path.exists() {
|
||||
if let Ok(request_data) = std::fs::read_to_string(&interview_request_path) {
|
||||
// Delete the request file immediately to prevent re-prompting
|
||||
let _ = std::fs::remove_file(&interview_request_path);
|
||||
if interview_request_path.exists() && !interview_response_path.exists() {
|
||||
if let Some(_claim_guard) = InterviewClaimGuard::acquire(run_dir) {
|
||||
if let Ok(request_data) = std::fs::read_to_string(&interview_request_path) {
|
||||
if let Ok(question) =
|
||||
serde_json::from_str::<fabro_interview::Question>(&request_data)
|
||||
{
|
||||
// Hide progress bars during interview
|
||||
progress_ui.hide_bars();
|
||||
|
||||
if let Ok(question) =
|
||||
serde_json::from_str::<fabro_interview::Question>(&request_data)
|
||||
{
|
||||
// Hide progress bars during interview
|
||||
progress_ui.hide_bars();
|
||||
// Prompt user via ConsoleInterviewer
|
||||
let interviewer = ConsoleInterviewer::new(styles);
|
||||
let answer =
|
||||
fabro_interview::Interviewer::ask(&interviewer, question).await;
|
||||
|
||||
// Prompt user via ConsoleInterviewer
|
||||
let interviewer = ConsoleInterviewer::new(styles);
|
||||
let answer = fabro_interview::Interviewer::ask(&interviewer, question).await;
|
||||
// Show progress bars again before any return path.
|
||||
progress_ui.show_bars();
|
||||
|
||||
// Write response
|
||||
if let Ok(response_json) = serde_json::to_string_pretty(&answer) {
|
||||
let _ = std::fs::write(&interview_response_path, response_json);
|
||||
if answer_requires_reattach(&answer) {
|
||||
let _ = append_run_notice(
|
||||
run_dir,
|
||||
RunNoticeLevel::Warn,
|
||||
"interview_unanswered",
|
||||
INTERVIEW_UNANSWERED_MESSAGE,
|
||||
);
|
||||
if let Some(guard) = engine_guard.as_mut() {
|
||||
guard.defuse();
|
||||
}
|
||||
eprintln!("{INTERVIEW_UNANSWERED_MESSAGE}");
|
||||
return Ok(ExitCode::from(1));
|
||||
}
|
||||
|
||||
write_interview_response_atomically(&interview_response_path, &answer)?;
|
||||
}
|
||||
|
||||
// Show progress bars again
|
||||
progress_ui.show_bars();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check if run is complete
|
||||
if conclusion_path.exists() {
|
||||
// Drain any remaining lines
|
||||
drain_remaining(&mut reader, &mut line, &mut progress_ui);
|
||||
break;
|
||||
}
|
||||
let terminal_status = read_status_record(&status_path)
|
||||
.map(|record| record.status)
|
||||
.filter(|status| status.is_terminal());
|
||||
|
||||
// Check if engine process is still alive (cache PID after first read)
|
||||
let engine_alive = match cached_pid {
|
||||
Some(pid) => process_alive(pid),
|
||||
None => {
|
||||
if let Ok(pid_str) = std::fs::read_to_string(&pid_path) {
|
||||
if let Ok(pid) = pid_str.trim().parse::<u32>() {
|
||||
let child_alive_via_handle = engine_guard.as_mut().and_then(|guard| {
|
||||
guard.inner().map(|child| match child.try_wait() {
|
||||
Ok(Some(_)) => false, // child exited
|
||||
Ok(None) => true, // still running
|
||||
Err(_) => false, // error, treat as dead
|
||||
})
|
||||
});
|
||||
|
||||
if let Some(child_alive) = child_alive_via_handle {
|
||||
if !child_alive {
|
||||
drain_remaining(&mut reader, &mut line, &mut progress_ui);
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
if terminal_status.is_some() {
|
||||
drain_remaining(&mut reader, &mut line, &mut progress_ui);
|
||||
break;
|
||||
}
|
||||
|
||||
let engine_alive = match cached_pid {
|
||||
Some(pid) => process_alive(pid),
|
||||
None => {
|
||||
if let Some(pid) = read_pid(&pid_path) {
|
||||
cached_pid = Some(pid);
|
||||
process_alive(pid)
|
||||
} else {
|
||||
true
|
||||
attach_started.elapsed() < ATTACH_STARTUP_GRACE
|
||||
|| !progress_file_is_empty(&progress_path)
|
||||
}
|
||||
} else {
|
||||
true // no PID file yet, assume alive
|
||||
}
|
||||
};
|
||||
if !engine_alive {
|
||||
drain_remaining(&mut reader, &mut line, &mut progress_ui);
|
||||
break;
|
||||
}
|
||||
};
|
||||
if !engine_alive && !conclusion_path.exists() {
|
||||
drain_remaining(&mut reader, &mut line, &mut progress_ui);
|
||||
break;
|
||||
}
|
||||
|
||||
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
|
||||
// Finish progress bars
|
||||
progress_ui.finish();
|
||||
|
||||
// Determine exit code from conclusion
|
||||
if conclusion_path.exists() {
|
||||
match fabro_workflows::conclusion::Conclusion::load(&conclusion_path) {
|
||||
Ok(conclusion) => {
|
||||
let success = matches!(
|
||||
conclusion.status,
|
||||
fabro_workflows::outcome::StageStatus::Success
|
||||
| fabro_workflows::outcome::StageStatus::PartialSuccess
|
||||
);
|
||||
Ok(if success {
|
||||
ExitCode::from(0)
|
||||
} else {
|
||||
ExitCode::from(1)
|
||||
})
|
||||
}
|
||||
Err(_) => Ok(ExitCode::from(1)),
|
||||
}
|
||||
} else {
|
||||
Ok(ExitCode::from(1))
|
||||
}
|
||||
Ok(determine_exit_code(
|
||||
&conclusion_path,
|
||||
read_status_record(&status_path),
|
||||
))
|
||||
}
|
||||
|
||||
fn drain_remaining(
|
||||
|
|
@ -196,6 +235,136 @@ fn drain_remaining(
|
|||
}
|
||||
}
|
||||
|
||||
fn read_status_record(path: &Path) -> Option<RunStatusRecord> {
|
||||
RunStatusRecord::load(path).ok()
|
||||
}
|
||||
|
||||
fn read_pid(pid_path: &Path) -> Option<u32> {
|
||||
std::fs::read_to_string(pid_path)
|
||||
.ok()
|
||||
.and_then(|pid| pid.trim().parse::<u32>().ok())
|
||||
}
|
||||
|
||||
fn progress_file_is_empty(path: &Path) -> bool {
|
||||
std::fs::metadata(path)
|
||||
.map(|meta| meta.len() == 0)
|
||||
.unwrap_or(true)
|
||||
}
|
||||
|
||||
fn interview_claim_path(run_dir: &Path) -> std::path::PathBuf {
|
||||
run_dir.join("interview_request.claim")
|
||||
}
|
||||
|
||||
struct InterviewClaimGuard {
|
||||
claim_path: PathBuf,
|
||||
}
|
||||
|
||||
impl InterviewClaimGuard {
|
||||
fn acquire(run_dir: &Path) -> Option<Self> {
|
||||
if try_claim_interview_request(run_dir) {
|
||||
Some(Self {
|
||||
claim_path: interview_claim_path(run_dir),
|
||||
})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for InterviewClaimGuard {
|
||||
fn drop(&mut self) {
|
||||
let _ = std::fs::remove_file(&self.claim_path);
|
||||
}
|
||||
}
|
||||
|
||||
struct EngineChildGuard {
|
||||
child: Option<std::process::Child>,
|
||||
}
|
||||
|
||||
impl EngineChildGuard {
|
||||
fn new(child: std::process::Child) -> Self {
|
||||
Self { child: Some(child) }
|
||||
}
|
||||
|
||||
fn inner(&mut self) -> Option<&mut std::process::Child> {
|
||||
self.child.as_mut()
|
||||
}
|
||||
|
||||
fn defuse(&mut self) {
|
||||
self.child.take();
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for EngineChildGuard {
|
||||
fn drop(&mut self) {
|
||||
if let Some(mut child) = self.child.take() {
|
||||
let _ = child.kill();
|
||||
let _ = child.wait();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn try_claim_interview_request(run_dir: &Path) -> bool {
|
||||
let claim_path = interview_claim_path(run_dir);
|
||||
if let Ok(existing) = std::fs::read_to_string(&claim_path) {
|
||||
if let Ok(pid) = existing.trim().parse::<u32>() {
|
||||
if process_alive(pid) {
|
||||
return pid == std::process::id();
|
||||
}
|
||||
}
|
||||
let _ = std::fs::remove_file(&claim_path);
|
||||
}
|
||||
|
||||
match std::fs::OpenOptions::new()
|
||||
.create_new(true)
|
||||
.write(true)
|
||||
.open(&claim_path)
|
||||
{
|
||||
Ok(mut file) => {
|
||||
let _ = writeln!(file, "{}", std::process::id());
|
||||
true
|
||||
}
|
||||
Err(_) => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn answer_requires_reattach(answer: &fabro_interview::Answer) -> bool {
|
||||
matches!(answer.value, AnswerValue::Aborted | AnswerValue::Skipped)
|
||||
}
|
||||
|
||||
fn write_interview_response_atomically(
|
||||
response_path: &Path,
|
||||
answer: &fabro_interview::Answer,
|
||||
) -> Result<()> {
|
||||
let response_json = serde_json::to_string_pretty(answer)?;
|
||||
let temp_path = response_path.with_extension("json.tmp");
|
||||
std::fs::write(&temp_path, response_json)?;
|
||||
std::fs::rename(temp_path, response_path)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn determine_exit_code(conclusion_path: &Path, status_record: Option<RunStatusRecord>) -> ExitCode {
|
||||
if conclusion_path.exists() {
|
||||
if let Ok(conclusion) = fabro_workflows::conclusion::Conclusion::load(conclusion_path) {
|
||||
let success = matches!(
|
||||
conclusion.status,
|
||||
fabro_workflows::outcome::StageStatus::Success
|
||||
| fabro_workflows::outcome::StageStatus::PartialSuccess
|
||||
);
|
||||
return if success {
|
||||
ExitCode::from(0)
|
||||
} else {
|
||||
ExitCode::from(1)
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
match status_record.map(|record| record.status) {
|
||||
Some(RunStatus::Succeeded) => ExitCode::from(0),
|
||||
Some(_) | None => ExitCode::from(1),
|
||||
}
|
||||
}
|
||||
|
||||
fn kill_engine(pid_path: &Path) {
|
||||
if let Ok(pid_str) = std::fs::read_to_string(pid_path) {
|
||||
if let Ok(pid) = pid_str.trim().parse::<i32>() {
|
||||
|
|
@ -219,3 +388,188 @@ fn process_alive(pid: u32) -> bool {
|
|||
true
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use chrono::Utc;
|
||||
use fabro_interview::{Answer, AnswerValue};
|
||||
use fabro_util::terminal::Styles;
|
||||
use fabro_workflows::conclusion::Conclusion;
|
||||
use fabro_workflows::outcome::StageStatus;
|
||||
use fabro_workflows::run_status::{write_run_status, StatusReason};
|
||||
|
||||
fn no_color_styles() -> &'static Styles {
|
||||
Box::leak(Box::new(Styles::new(false)))
|
||||
}
|
||||
|
||||
fn sample_conclusion(status: StageStatus) -> Conclusion {
|
||||
Conclusion {
|
||||
timestamp: Utc::now(),
|
||||
status,
|
||||
duration_ms: 0,
|
||||
failure_reason: None,
|
||||
final_git_commit_sha: None,
|
||||
stages: Vec::new(),
|
||||
total_cost: None,
|
||||
total_retries: 0,
|
||||
total_input_tokens: 0,
|
||||
total_output_tokens: 0,
|
||||
total_cache_read_tokens: 0,
|
||||
total_cache_write_tokens: 0,
|
||||
total_reasoning_tokens: 0,
|
||||
has_pricing: false,
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn attach_does_not_return_when_only_conclusion_exists() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
std::fs::write(dir.path().join("progress.jsonl"), "").unwrap();
|
||||
sample_conclusion(StageStatus::Success)
|
||||
.save(&dir.path().join("conclusion.json"))
|
||||
.unwrap();
|
||||
|
||||
let child = std::process::Command::new("sh")
|
||||
.args(["-c", "sleep 0.35"])
|
||||
.spawn()
|
||||
.unwrap();
|
||||
let started = Instant::now();
|
||||
|
||||
let exit = attach_run(dir.path(), false, no_color_styles(), Some(child))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(exit, ExitCode::from(0));
|
||||
assert!(
|
||||
started.elapsed() >= Duration::from_millis(250),
|
||||
"attach returned before the owned child exited"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn attach_missing_pid_and_failed_status_is_not_alive() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
std::fs::write(dir.path().join("progress.jsonl"), "").unwrap();
|
||||
write_run_status(
|
||||
dir.path(),
|
||||
RunStatus::Failed,
|
||||
Some(StatusReason::LaunchFailed),
|
||||
);
|
||||
|
||||
let exit = attach_run(dir.path(), false, no_color_styles(), None)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(exit, ExitCode::from(1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn try_claim_interview_request_reclaims_stale_claim() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
std::fs::write(interview_claim_path(dir.path()), "999999\n").unwrap();
|
||||
|
||||
assert!(try_claim_interview_request(dir.path()));
|
||||
assert_eq!(
|
||||
std::fs::read_to_string(interview_claim_path(dir.path())).unwrap(),
|
||||
format!("{}\n", std::process::id())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn interview_claim_guard_releases_claim_on_drop() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
|
||||
{
|
||||
let _guard = InterviewClaimGuard::acquire(dir.path()).unwrap();
|
||||
assert!(interview_claim_path(dir.path()).exists());
|
||||
}
|
||||
|
||||
assert!(!interview_claim_path(dir.path()).exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn answer_requires_reattach_for_aborted_and_skipped_answers() {
|
||||
let aborted = Answer {
|
||||
value: AnswerValue::Aborted,
|
||||
selected_option: None,
|
||||
selected_options: Vec::new(),
|
||||
text: None,
|
||||
};
|
||||
let skipped = Answer {
|
||||
value: AnswerValue::Skipped,
|
||||
selected_option: None,
|
||||
selected_options: Vec::new(),
|
||||
text: None,
|
||||
};
|
||||
let answered = Answer::yes();
|
||||
|
||||
assert!(answer_requires_reattach(&aborted));
|
||||
assert!(answer_requires_reattach(&skipped));
|
||||
assert!(!answer_requires_reattach(&answered));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn engine_child_guard_kills_on_drop() {
|
||||
let child = std::process::Command::new("sleep")
|
||||
.arg("60")
|
||||
.spawn()
|
||||
.unwrap();
|
||||
let pid = child.id();
|
||||
|
||||
{
|
||||
let _guard = EngineChildGuard::new(child);
|
||||
}
|
||||
|
||||
// Process should be dead after guard is dropped
|
||||
assert!(
|
||||
!process_alive(pid),
|
||||
"process should be dead after guard drop"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn engine_child_guard_defuse_keeps_alive() {
|
||||
let child = std::process::Command::new("sleep")
|
||||
.arg("60")
|
||||
.spawn()
|
||||
.unwrap();
|
||||
let pid = child.id();
|
||||
|
||||
{
|
||||
let mut guard = EngineChildGuard::new(child);
|
||||
guard.defuse();
|
||||
}
|
||||
|
||||
// Process should still be alive after defused guard is dropped
|
||||
assert!(
|
||||
process_alive(pid),
|
||||
"process should still be alive after defused guard drop"
|
||||
);
|
||||
|
||||
// Clean up
|
||||
#[cfg(unix)]
|
||||
unsafe {
|
||||
libc::kill(pid as i32, libc::SIGKILL);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn write_interview_response_atomically_persists_answer() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let response_path = dir.path().join("interview_response.json");
|
||||
let answer = Answer {
|
||||
value: AnswerValue::Text("ship it".to_string()),
|
||||
selected_option: None,
|
||||
selected_options: Vec::new(),
|
||||
text: Some("ship it".to_string()),
|
||||
};
|
||||
|
||||
write_interview_response_atomically(&response_path, &answer).unwrap();
|
||||
|
||||
let saved: Answer =
|
||||
serde_json::from_str(&std::fs::read_to_string(&response_path).unwrap()).unwrap();
|
||||
assert_eq!(saved.text.as_deref(), Some("ship it"));
|
||||
assert!(!response_path.with_extension("json.tmp").exists());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -27,7 +27,10 @@ pub async fn create_run(
|
|||
let goal = prep.graph.goal();
|
||||
|
||||
// Create run directory
|
||||
let run_id = ulid::Ulid::new().to_string();
|
||||
let run_id = args
|
||||
.run_id
|
||||
.clone()
|
||||
.unwrap_or_else(|| ulid::Ulid::new().to_string());
|
||||
let run_dir = args
|
||||
.run_dir
|
||||
.clone()
|
||||
|
|
|
|||
363
lib/crates/fabro-cli/src/commands/detached_support.rs
Normal file
363
lib/crates/fabro-cli/src/commands/detached_support.rs
Normal file
|
|
@ -0,0 +1,363 @@
|
|||
use std::io::Write;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use chrono::{SecondsFormat, Utc};
|
||||
use fabro_workflows::conclusion::Conclusion;
|
||||
use fabro_workflows::event::{RunNoticeLevel, WorkflowRunEvent};
|
||||
use fabro_workflows::outcome::StageStatus;
|
||||
use fabro_workflows::run_status::{self, RunStatus, StatusReason};
|
||||
use serde::Serialize;
|
||||
|
||||
const POSTRUN_ABORTED_MESSAGE: &str = "Run aborted before post-run finalization completed.";
|
||||
|
||||
pub(crate) struct DetachedRunBootstrapGuard {
|
||||
run_dir: PathBuf,
|
||||
active: bool,
|
||||
}
|
||||
|
||||
impl DetachedRunBootstrapGuard {
|
||||
pub(crate) fn arm(run_dir: &Path) -> Result<Self> {
|
||||
std::fs::write(run_dir.join("run.pid"), std::process::id().to_string())
|
||||
.with_context(|| format!("Failed to write {}", run_dir.join("run.pid").display()))?;
|
||||
run_status::write_run_status(
|
||||
run_dir,
|
||||
RunStatus::Starting,
|
||||
Some(StatusReason::SandboxInitializing),
|
||||
);
|
||||
Ok(Self {
|
||||
run_dir: run_dir.to_path_buf(),
|
||||
active: true,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn defuse(&mut self) {
|
||||
self.active = false;
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for DetachedRunBootstrapGuard {
|
||||
fn drop(&mut self) {
|
||||
if self.active {
|
||||
run_status::write_run_status(
|
||||
&self.run_dir,
|
||||
RunStatus::Failed,
|
||||
Some(StatusReason::SandboxInitFailed),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct DetachedRunCompletionGuard {
|
||||
run_dir: PathBuf,
|
||||
active: bool,
|
||||
}
|
||||
|
||||
impl DetachedRunCompletionGuard {
|
||||
pub(crate) fn arm(run_dir: &Path) -> Self {
|
||||
Self {
|
||||
run_dir: run_dir.to_path_buf(),
|
||||
active: true,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn defuse(&mut self) {
|
||||
self.active = false;
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for DetachedRunCompletionGuard {
|
||||
fn drop(&mut self) {
|
||||
if !self.active {
|
||||
return;
|
||||
}
|
||||
|
||||
run_status::write_run_status(
|
||||
&self.run_dir,
|
||||
RunStatus::Failed,
|
||||
Some(StatusReason::WorkflowError),
|
||||
);
|
||||
if !self.run_dir.join("conclusion.json").exists() {
|
||||
let _ = write_failure_conclusion(
|
||||
&self.run_dir,
|
||||
POSTRUN_ABORTED_MESSAGE,
|
||||
Some(StatusReason::WorkflowError),
|
||||
);
|
||||
}
|
||||
if let Some(run_id) = load_run_id(&self.run_dir) {
|
||||
let _ = append_progress_event(
|
||||
&self.run_dir,
|
||||
&run_id,
|
||||
&WorkflowRunEvent::RunNotice {
|
||||
level: RunNoticeLevel::Error,
|
||||
code: "postrun_aborted".to_string(),
|
||||
message: POSTRUN_ABORTED_MESSAGE.to_string(),
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn load_run_id(run_dir: &Path) -> Option<String> {
|
||||
fabro_workflows::run_spec::RunSpec::load(run_dir)
|
||||
.ok()
|
||||
.map(|spec| spec.run_id)
|
||||
.filter(|run_id| !run_id.trim().is_empty())
|
||||
.or_else(|| {
|
||||
std::fs::read_to_string(run_dir.join("id.txt"))
|
||||
.ok()
|
||||
.map(|run_id| run_id.trim().to_string())
|
||||
.filter(|run_id| !run_id.is_empty())
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn build_event_envelope(event: &WorkflowRunEvent, run_id: &str) -> serde_json::Value {
|
||||
let (event_name, event_fields) = fabro_workflows::event::flatten_event(event);
|
||||
let mut envelope = serde_json::Map::new();
|
||||
envelope.insert(
|
||||
"ts".to_string(),
|
||||
serde_json::Value::String(Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true)),
|
||||
);
|
||||
envelope.insert(
|
||||
"run_id".to_string(),
|
||||
serde_json::Value::String(run_id.to_string()),
|
||||
);
|
||||
envelope.insert("event".to_string(), serde_json::Value::String(event_name));
|
||||
for (k, v) in event_fields {
|
||||
if k != "ts" && k != "run_id" && k != "event" {
|
||||
envelope.insert(k, v);
|
||||
}
|
||||
}
|
||||
serde_json::Value::Object(envelope)
|
||||
}
|
||||
|
||||
pub(crate) fn append_progress_event(
|
||||
run_dir: &Path,
|
||||
run_id: &str,
|
||||
event: &WorkflowRunEvent,
|
||||
) -> Result<()> {
|
||||
let envelope = build_event_envelope(event, run_id);
|
||||
let line = serde_json::to_string(&envelope)?;
|
||||
let line = fabro_util::redact::redact_jsonl_line(&line);
|
||||
let mut file = std::fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(run_dir.join("progress.jsonl"))
|
||||
.with_context(|| {
|
||||
format!(
|
||||
"Failed to open {}",
|
||||
run_dir.join("progress.jsonl").display()
|
||||
)
|
||||
})?;
|
||||
writeln!(file, "{line}")?;
|
||||
|
||||
let pretty = serde_json::to_string_pretty(&envelope)?;
|
||||
let pretty = fabro_util::redact::redact_jsonl_line(&pretty);
|
||||
std::fs::write(run_dir.join("live.json"), pretty)
|
||||
.with_context(|| format!("Failed to write {}", run_dir.join("live.json").display()))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn append_run_notice(
|
||||
run_dir: &Path,
|
||||
level: RunNoticeLevel,
|
||||
code: &'static str,
|
||||
message: impl Into<String>,
|
||||
) -> Result<()> {
|
||||
let Some(run_id) = load_run_id(run_dir) else {
|
||||
return Ok(());
|
||||
};
|
||||
append_progress_event(
|
||||
run_dir,
|
||||
&run_id,
|
||||
&WorkflowRunEvent::RunNotice {
|
||||
level,
|
||||
code: code.to_string(),
|
||||
message: message.into(),
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn persist_detached_failure(
|
||||
run_dir: &Path,
|
||||
phase: &'static str,
|
||||
reason: StatusReason,
|
||||
error: &anyhow::Error,
|
||||
) -> Result<()> {
|
||||
#[derive(Serialize)]
|
||||
struct DetachedFailureRecord<'a> {
|
||||
timestamp: chrono::DateTime<Utc>,
|
||||
phase: &'a str,
|
||||
reason: StatusReason,
|
||||
error: String,
|
||||
}
|
||||
|
||||
let message = error.to_string();
|
||||
let record = DetachedFailureRecord {
|
||||
timestamp: Utc::now(),
|
||||
phase,
|
||||
reason,
|
||||
error: message.clone(),
|
||||
};
|
||||
std::fs::write(
|
||||
run_dir.join("detached_failure.json"),
|
||||
serde_json::to_string_pretty(&record)?,
|
||||
)
|
||||
.with_context(|| {
|
||||
format!(
|
||||
"Failed to write {}",
|
||||
run_dir.join("detached_failure.json").display()
|
||||
)
|
||||
})?;
|
||||
|
||||
write_failure_conclusion(run_dir, &message, Some(reason))?;
|
||||
run_status::write_run_status(run_dir, RunStatus::Failed, Some(reason));
|
||||
|
||||
if let Some(run_id) = load_run_id(run_dir) {
|
||||
append_progress_event(
|
||||
run_dir,
|
||||
&run_id,
|
||||
&WorkflowRunEvent::RunNotice {
|
||||
level: RunNoticeLevel::Error,
|
||||
code: format!("{phase}_failed"),
|
||||
message,
|
||||
},
|
||||
)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn write_failure_conclusion(
|
||||
run_dir: &Path,
|
||||
message: &str,
|
||||
_reason: Option<StatusReason>,
|
||||
) -> Result<()> {
|
||||
if run_dir.join("conclusion.json").exists() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let conclusion = Conclusion {
|
||||
timestamp: Utc::now(),
|
||||
status: StageStatus::Fail,
|
||||
duration_ms: 0,
|
||||
failure_reason: Some(message.to_string()),
|
||||
final_git_commit_sha: None,
|
||||
stages: vec![],
|
||||
total_cost: None,
|
||||
total_retries: 0,
|
||||
total_input_tokens: 0,
|
||||
total_output_tokens: 0,
|
||||
total_cache_read_tokens: 0,
|
||||
total_cache_write_tokens: 0,
|
||||
total_reasoning_tokens: 0,
|
||||
has_pricing: false,
|
||||
};
|
||||
conclusion.save(&run_dir.join("conclusion.json"))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use fabro_workflows::run_status::{RunStatusRecord, StatusReason};
|
||||
|
||||
#[test]
|
||||
fn bootstrap_guard_marks_failed_on_drop() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
|
||||
{
|
||||
let _guard = DetachedRunBootstrapGuard::arm(dir.path()).unwrap();
|
||||
let record = RunStatusRecord::load(&dir.path().join("status.json")).unwrap();
|
||||
assert_eq!(record.status, RunStatus::Starting);
|
||||
assert_eq!(record.reason, Some(StatusReason::SandboxInitializing));
|
||||
}
|
||||
|
||||
let record = RunStatusRecord::load(&dir.path().join("status.json")).unwrap();
|
||||
assert_eq!(record.status, RunStatus::Failed);
|
||||
assert_eq!(record.reason, Some(StatusReason::SandboxInitFailed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bootstrap_guard_defuse_leaves_starting_intact() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
|
||||
{
|
||||
let mut guard = DetachedRunBootstrapGuard::arm(dir.path()).unwrap();
|
||||
guard.defuse();
|
||||
}
|
||||
|
||||
let record = RunStatusRecord::load(&dir.path().join("status.json")).unwrap();
|
||||
assert_eq!(record.status, RunStatus::Starting);
|
||||
assert_eq!(record.reason, Some(StatusReason::SandboxInitializing));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completion_guard_marks_failed_on_drop() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
std::fs::write(dir.path().join("id.txt"), "run-123").unwrap();
|
||||
|
||||
{
|
||||
let _guard = DetachedRunCompletionGuard::arm(dir.path());
|
||||
}
|
||||
|
||||
let record = RunStatusRecord::load(&dir.path().join("status.json")).unwrap();
|
||||
assert_eq!(record.status, RunStatus::Failed);
|
||||
assert_eq!(record.reason, Some(StatusReason::WorkflowError));
|
||||
assert!(dir.path().join("conclusion.json").exists());
|
||||
let progress = std::fs::read_to_string(dir.path().join("progress.jsonl")).unwrap();
|
||||
assert!(progress.contains("postrun_aborted"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn load_run_id_falls_back_to_id_txt() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
std::fs::write(dir.path().join("id.txt"), "run-xyz").unwrap();
|
||||
|
||||
assert_eq!(load_run_id(dir.path()).as_deref(), Some("run-xyz"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn persist_detached_failure_writes_status_conclusion_and_progress() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
std::fs::write(dir.path().join("id.txt"), "run-err").unwrap();
|
||||
|
||||
let err = anyhow::anyhow!("bootstrap exploded");
|
||||
persist_detached_failure(dir.path(), "bootstrap", StatusReason::BootstrapFailed, &err)
|
||||
.unwrap();
|
||||
|
||||
let record = RunStatusRecord::load(&dir.path().join("status.json")).unwrap();
|
||||
assert_eq!(record.status, RunStatus::Failed);
|
||||
assert_eq!(record.reason, Some(StatusReason::BootstrapFailed));
|
||||
let conclusion =
|
||||
fabro_workflows::conclusion::Conclusion::load(&dir.path().join("conclusion.json"))
|
||||
.unwrap();
|
||||
assert_eq!(conclusion.status, StageStatus::Fail);
|
||||
assert_eq!(
|
||||
conclusion.failure_reason.as_deref(),
|
||||
Some("bootstrap exploded")
|
||||
);
|
||||
let progress = std::fs::read_to_string(dir.path().join("progress.jsonl")).unwrap();
|
||||
assert!(progress.contains("bootstrap_failed"));
|
||||
assert!(dir.path().join("detached_failure.json").exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn append_run_notice_writes_progress() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
std::fs::write(dir.path().join("id.txt"), "run-notice").unwrap();
|
||||
|
||||
append_run_notice(
|
||||
dir.path(),
|
||||
RunNoticeLevel::Warn,
|
||||
"interview_unanswered",
|
||||
"The run is still waiting for input.",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let progress = std::fs::read_to_string(dir.path().join("progress.jsonl")).unwrap();
|
||||
assert!(progress.contains("\"event\":\"RunNotice\""));
|
||||
assert!(progress.contains("\"code\":\"interview_unanswered\""));
|
||||
}
|
||||
}
|
||||
|
|
@ -299,6 +299,28 @@ pub fn format_event_pretty(line: &str, styles: &Styles) -> Option<String> {
|
|||
styles.red.apply_to(error),
|
||||
))
|
||||
}
|
||||
"RunNotice" => {
|
||||
let level = str_field(&envelope, "level").unwrap_or("info");
|
||||
let code = str_field(&envelope, "code").unwrap_or("");
|
||||
let message = str_field(&envelope, "message").unwrap_or("");
|
||||
let label = match level {
|
||||
"warn" => styles.yellow.apply_to("Warning:").to_string(),
|
||||
"error" => styles.bold_red.apply_to("Error:").to_string(),
|
||||
_ => styles.bold.apply_to("Info:").to_string(),
|
||||
};
|
||||
let code_suffix = if code.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!(" {}", styles.dim.apply_to(format!("[{code}]")))
|
||||
};
|
||||
Some(format!(
|
||||
"{} {} {}{}",
|
||||
styles.dim.apply_to(&ts),
|
||||
label,
|
||||
message,
|
||||
code_suffix,
|
||||
))
|
||||
}
|
||||
"StageStarted" => {
|
||||
let label = str_field(&envelope, "node_label").unwrap_or("?");
|
||||
Some(format!(
|
||||
|
|
@ -886,6 +908,29 @@ mod tests {
|
|||
assert!(result.contains("auth token expired"), "got: {result}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pretty_run_notice_warn() {
|
||||
let styles = no_color_styles();
|
||||
let line = r#"{"ts":"2026-01-01T14:25:00Z","event":"RunNotice","level":"warn","code":"sandbox_cleanup_failed","message":"sandbox cleanup failed: boom"}"#;
|
||||
let result = format_event_pretty(line, &styles).unwrap();
|
||||
assert!(result.contains("Warning:"), "got: {result}");
|
||||
assert!(
|
||||
result.contains("sandbox cleanup failed: boom"),
|
||||
"got: {result}"
|
||||
);
|
||||
assert!(result.contains("[sandbox_cleanup_failed]"), "got: {result}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pretty_run_notice_error() {
|
||||
let styles = no_color_styles();
|
||||
let line = r#"{"ts":"2026-01-01T14:25:00Z","event":"RunNotice","level":"error","code":"launch_failed","message":"failed to start engine"}"#;
|
||||
let result = format_event_pretty(line, &styles).unwrap();
|
||||
assert!(result.contains("Error:"), "got: {result}");
|
||||
assert!(result.contains("failed to start engine"), "got: {result}");
|
||||
assert!(result.contains("[launch_failed]"), "got: {result}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pretty_workflow_run_failed() {
|
||||
let styles = no_color_styles();
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ pub mod asset;
|
|||
pub mod attach;
|
||||
pub mod cp;
|
||||
pub mod create;
|
||||
pub(crate) mod detached_support;
|
||||
pub mod diff;
|
||||
pub mod fork;
|
||||
pub mod graph;
|
||||
|
|
|
|||
|
|
@ -14,16 +14,18 @@ use fabro_util::terminal::Styles;
|
|||
use fabro_workflows::backend::{AgentApiBackend, AgentCliBackend, BackendRouter};
|
||||
use fabro_workflows::checkpoint::Checkpoint;
|
||||
use fabro_workflows::engine::RunConfig;
|
||||
use fabro_workflows::event::EventEmitter;
|
||||
use fabro_workflows::event::{EventEmitter, RunNoticeLevel};
|
||||
use fabro_workflows::manifest::Manifest;
|
||||
use fabro_workflows::outcome::StageStatus;
|
||||
use fabro_workflows::sandbox_provider::SandboxProvider;
|
||||
use indicatif::HumanDuration;
|
||||
|
||||
use super::detached_support::{DetachedRunBootstrapGuard, DetachedRunCompletionGuard};
|
||||
use super::run::{
|
||||
build_event_envelope, default_run_dir, generate_retro, local_sandbox_with_callback,
|
||||
mint_github_token, prepare_workflow_with_project_config, print_assets, print_final_output,
|
||||
resolve_daytona_config, resolve_fallback_chain, resolve_model_provider,
|
||||
build_conclusion, build_event_envelope, classify_engine_result, default_run_dir,
|
||||
emit_run_notice, generate_retro, local_sandbox_with_callback, mint_github_token,
|
||||
persist_terminal_outcome, prepare_workflow_with_project_config, print_assets,
|
||||
print_final_output, resolve_daytona_config, resolve_fallback_chain, resolve_model_provider,
|
||||
resolve_sandbox_provider, resolve_ssh_clone_params, resolve_ssh_config, write_finalize_commit,
|
||||
write_run_config_snapshot, CliSandboxProvider, RunArgs,
|
||||
};
|
||||
|
|
@ -113,44 +115,7 @@ struct ResumeContext {
|
|||
sandbox_provider: SandboxProvider,
|
||||
ssh_data_host: Option<String>,
|
||||
github_app: Option<fabro_github::GitHubAppCredentials>,
|
||||
status_guard: ResumeRunStatusGuard,
|
||||
}
|
||||
|
||||
struct ResumeRunStatusGuard {
|
||||
run_dir: PathBuf,
|
||||
active: bool,
|
||||
}
|
||||
|
||||
impl ResumeRunStatusGuard {
|
||||
fn arm(run_dir: &std::path::Path) -> anyhow::Result<Self> {
|
||||
std::fs::write(run_dir.join("run.pid"), std::process::id().to_string())
|
||||
.with_context(|| format!("Failed to write {}", run_dir.join("run.pid").display()))?;
|
||||
fabro_workflows::run_status::write_run_status(
|
||||
run_dir,
|
||||
fabro_workflows::run_status::RunStatus::Starting,
|
||||
Some(fabro_workflows::run_status::StatusReason::SandboxInitializing),
|
||||
);
|
||||
Ok(Self {
|
||||
run_dir: run_dir.to_path_buf(),
|
||||
active: true,
|
||||
})
|
||||
}
|
||||
|
||||
fn defuse(&mut self) {
|
||||
self.active = false;
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ResumeRunStatusGuard {
|
||||
fn drop(&mut self) {
|
||||
if self.active {
|
||||
fabro_workflows::run_status::write_run_status(
|
||||
&self.run_dir,
|
||||
fabro_workflows::run_status::RunStatus::Failed,
|
||||
Some(fabro_workflows::run_status::StatusReason::SandboxInitFailed),
|
||||
);
|
||||
}
|
||||
}
|
||||
status_guard: DetachedRunBootstrapGuard,
|
||||
}
|
||||
|
||||
fn resume_as_run_args(args: &ResumeArgs, workflow: PathBuf) -> RunArgs {
|
||||
|
|
@ -257,7 +222,7 @@ async fn prepare_from_checkpoint(
|
|||
tokio::fs::create_dir_all(&run_dir).await?;
|
||||
fabro_util::run_log::activate(&run_dir.join("cli.log"))
|
||||
.context("Failed to activate per-run log")?;
|
||||
let status_guard = ResumeRunStatusGuard::arm(&run_dir)?;
|
||||
let status_guard = DetachedRunBootstrapGuard::arm(&run_dir)?;
|
||||
tokio::fs::write(run_dir.join("graph.fabro"), &source).await?;
|
||||
let mut run_cfg = run_cfg;
|
||||
write_run_config_snapshot(&run_dir, run_cfg.as_mut()).await?;
|
||||
|
|
@ -609,7 +574,7 @@ async fn prepare_from_branch(
|
|||
let run_dir = tokio::fs::canonicalize(&run_dir).await.unwrap_or(run_dir);
|
||||
fabro_util::run_log::activate(&run_dir.join("cli.log"))
|
||||
.context("Failed to activate per-run log")?;
|
||||
let status_guard = ResumeRunStatusGuard::arm(&run_dir)?;
|
||||
let status_guard = DetachedRunBootstrapGuard::arm(&run_dir)?;
|
||||
tokio::fs::write(run_dir.join("graph.fabro"), &graph_source).await?;
|
||||
let mut run_cfg = run_cfg;
|
||||
write_run_config_snapshot(&run_dir, run_cfg.as_mut()).await?;
|
||||
|
|
@ -1085,17 +1050,21 @@ async fn run_resumed(
|
|||
} else {
|
||||
match fabro_llm::client::Client::from_env().await {
|
||||
Ok(c) if c.provider_names().is_empty() => {
|
||||
eprintln!(
|
||||
"{} No LLM providers configured. Running in dry-run mode.",
|
||||
styles.yellow.apply_to("Warning:"),
|
||||
emit_run_notice(
|
||||
&emitter,
|
||||
RunNoticeLevel::Warn,
|
||||
"dry_run_no_llm",
|
||||
"No LLM providers configured. Running in dry-run mode.",
|
||||
);
|
||||
true
|
||||
}
|
||||
Ok(_) => false,
|
||||
Err(e) => {
|
||||
eprintln!(
|
||||
"{} Failed to initialize LLM client: {e}. Running in dry-run mode.",
|
||||
styles.yellow.apply_to("Warning:"),
|
||||
emit_run_notice(
|
||||
&emitter,
|
||||
RunNoticeLevel::Warn,
|
||||
"dry_run_llm_init_failed",
|
||||
format!("Failed to initialize LLM client: {e}. Running in dry-run mode."),
|
||||
);
|
||||
true
|
||||
}
|
||||
|
|
@ -1160,9 +1129,11 @@ async fn run_resumed(
|
|||
sandbox_env.insert("GITHUB_TOKEN".to_string(), token);
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!(
|
||||
"{} Failed to mint GitHub token: {e}",
|
||||
styles.yellow.apply_to("Warning:"),
|
||||
emit_run_notice(
|
||||
&emitter,
|
||||
RunNoticeLevel::Warn,
|
||||
"github_token_failed",
|
||||
format!("Failed to mint GitHub token: {e}"),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -1233,7 +1204,7 @@ async fn run_resumed(
|
|||
devcontainer_phases,
|
||||
};
|
||||
|
||||
// Defuse the status guard — engine.run() will write "running" and conclusion handles "concluded"
|
||||
// Defuse the bootstrap guard — engine.run() has taken ownership of lifecycle status.
|
||||
status_guard.defuse();
|
||||
|
||||
// Safety net: if we panic or return early, best-effort cleanup via spawn (mirrors run_command).
|
||||
|
|
@ -1260,93 +1231,22 @@ async fn run_resumed(
|
|||
.run_with_lifecycle(&graph, &mut config, lifecycle, Some(&checkpoint))
|
||||
.await;
|
||||
let run_duration_ms = run_start.elapsed().as_millis() as u64;
|
||||
let mut completion_guard = DetachedRunCompletionGuard::arm(&run_dir);
|
||||
|
||||
// Restore cwd if we changed it (worktree is kept for `fabro cp` access; pruned separately)
|
||||
if let Some(ref cwd) = original_cwd {
|
||||
let _ = std::env::set_current_dir(cwd);
|
||||
}
|
||||
|
||||
// Build and save conclusion.json + final status (mirrors run_command)
|
||||
{
|
||||
let (status, failure_reason) = match &engine_result {
|
||||
Ok(ref o) => (o.status.clone(), o.failure_reason().map(String::from)),
|
||||
Err(e) => (StageStatus::Fail, Some(e.to_string())),
|
||||
};
|
||||
|
||||
let (run_status, status_reason) = match &engine_result {
|
||||
Ok(ref o) => match o.status {
|
||||
StageStatus::Success | StageStatus::Skipped => (
|
||||
fabro_workflows::run_status::RunStatus::Succeeded,
|
||||
Some(fabro_workflows::run_status::StatusReason::Completed),
|
||||
),
|
||||
StageStatus::PartialSuccess => (
|
||||
fabro_workflows::run_status::RunStatus::Succeeded,
|
||||
Some(fabro_workflows::run_status::StatusReason::PartialSuccess),
|
||||
),
|
||||
StageStatus::Fail | StageStatus::Retry => (
|
||||
fabro_workflows::run_status::RunStatus::Failed,
|
||||
Some(fabro_workflows::run_status::StatusReason::WorkflowError),
|
||||
),
|
||||
},
|
||||
Err(fabro_workflows::error::FabroError::Cancelled) => (
|
||||
fabro_workflows::run_status::RunStatus::Failed,
|
||||
Some(fabro_workflows::run_status::StatusReason::Cancelled),
|
||||
),
|
||||
Err(_) => (
|
||||
fabro_workflows::run_status::RunStatus::Failed,
|
||||
Some(fabro_workflows::run_status::StatusReason::WorkflowError),
|
||||
),
|
||||
};
|
||||
|
||||
let checkpoint_loaded = Checkpoint::load(&run_dir.join("checkpoint.json")).ok();
|
||||
let stage_durations = fabro_retro::retro::extract_stage_durations(&run_dir);
|
||||
|
||||
let (stages, total_cost, total_retries) = if let Some(ref cp) = checkpoint_loaded {
|
||||
let mut stages = Vec::new();
|
||||
let mut cost_sum: Option<f64> = None;
|
||||
let mut retries_sum: u32 = 0;
|
||||
|
||||
for node_id in &cp.completed_nodes {
|
||||
let outcome = cp.node_outcomes.get(node_id);
|
||||
let retries = cp
|
||||
.node_retries
|
||||
.get(node_id)
|
||||
.copied()
|
||||
.unwrap_or(1)
|
||||
.saturating_sub(1);
|
||||
retries_sum += retries;
|
||||
|
||||
let cost = outcome.and_then(|o| o.usage.as_ref()).and_then(|u| u.cost);
|
||||
if let Some(c) = cost {
|
||||
*cost_sum.get_or_insert(0.0) += c;
|
||||
}
|
||||
|
||||
stages.push(fabro_workflows::conclusion::StageSummary {
|
||||
stage_id: node_id.clone(),
|
||||
stage_label: node_id.clone(),
|
||||
duration_ms: stage_durations.get(node_id).copied().unwrap_or(0),
|
||||
cost,
|
||||
retries,
|
||||
});
|
||||
}
|
||||
(stages, cost_sum, retries_sum)
|
||||
} else {
|
||||
(vec![], None, 0)
|
||||
};
|
||||
|
||||
let conclusion = fabro_workflows::conclusion::Conclusion {
|
||||
timestamp: chrono::Utc::now(),
|
||||
status,
|
||||
duration_ms: run_duration_ms,
|
||||
failure_reason,
|
||||
final_git_commit_sha: last_git_sha.lock().unwrap().clone(),
|
||||
stages,
|
||||
total_cost,
|
||||
total_retries,
|
||||
};
|
||||
let _ = conclusion.save(&run_dir.join("conclusion.json"));
|
||||
fabro_workflows::run_status::write_run_status(&run_dir, run_status, status_reason);
|
||||
}
|
||||
let (final_status, failure_reason, run_status, status_reason) =
|
||||
classify_engine_result(&engine_result);
|
||||
let conclusion = build_conclusion(
|
||||
&run_dir,
|
||||
final_status.clone(),
|
||||
failure_reason,
|
||||
run_duration_ms,
|
||||
last_git_sha.lock().unwrap().clone(),
|
||||
);
|
||||
|
||||
// Auto-derive retro
|
||||
if !args.no_retro && project_config::is_retro_enabled() {
|
||||
|
|
@ -1460,9 +1360,11 @@ async fn run_resumed(
|
|||
error: e.to_string(),
|
||||
},
|
||||
);
|
||||
eprintln!(
|
||||
"{} PR creation failed: {e}",
|
||||
styles.yellow.apply_to("Warning:")
|
||||
emit_run_notice(
|
||||
&emitter,
|
||||
RunNoticeLevel::Warn,
|
||||
"pull_request_failed",
|
||||
format!("PR creation failed: {e}"),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -1481,12 +1383,19 @@ async fn run_resumed(
|
|||
if preserve {
|
||||
let info = sandbox.sandbox_info();
|
||||
if !info.is_empty() {
|
||||
eprintln!(
|
||||
"\n{} sandbox preserved: {info}",
|
||||
styles.bold.apply_to("Info:")
|
||||
emit_run_notice(
|
||||
&emitter,
|
||||
RunNoticeLevel::Info,
|
||||
"sandbox_preserved",
|
||||
format!("sandbox preserved: {info}"),
|
||||
);
|
||||
} else {
|
||||
eprintln!("\n{} sandbox preserved", styles.bold.apply_to("Info:"));
|
||||
emit_run_notice(
|
||||
&emitter,
|
||||
RunNoticeLevel::Info,
|
||||
"sandbox_preserved",
|
||||
"sandbox preserved",
|
||||
);
|
||||
}
|
||||
}
|
||||
if let Err(e) = engine
|
||||
|
|
@ -1494,18 +1403,21 @@ async fn run_resumed(
|
|||
.await
|
||||
{
|
||||
tracing::warn!(error = %e, "Sandbox cleanup failed");
|
||||
eprintln!(
|
||||
"\n{} sandbox cleanup failed: {e}",
|
||||
styles.yellow.apply_to("Warning:")
|
||||
emit_run_notice(
|
||||
&emitter,
|
||||
RunNoticeLevel::Warn,
|
||||
"sandbox_cleanup_failed",
|
||||
format!("sandbox cleanup failed: {e}"),
|
||||
);
|
||||
}
|
||||
|
||||
let outcome = engine_result?;
|
||||
persist_terminal_outcome(&run_dir, &conclusion, run_status, status_reason);
|
||||
completion_guard.defuse();
|
||||
|
||||
eprintln!("\n{}", styles.bold.apply_to("=== Run Result ==="));
|
||||
eprintln!("{}", styles.dim.apply_to(format!("Run: {run_id}")));
|
||||
let status_str = outcome.status.to_string().to_uppercase();
|
||||
let status_color = match outcome.status {
|
||||
let status_str = final_status.to_string().to_uppercase();
|
||||
let status_color = match final_status {
|
||||
StageStatus::Success | StageStatus::PartialSuccess => &styles.bold_green,
|
||||
_ => &styles.bold_red,
|
||||
};
|
||||
|
|
@ -1567,7 +1479,7 @@ async fn run_resumed(
|
|||
.apply_to(format!("Run: {}", tilde_path(&run_dir)))
|
||||
);
|
||||
|
||||
if let Some(failure) = outcome.failure_reason() {
|
||||
if let Some(failure) = conclusion.failure_reason.as_deref() {
|
||||
eprintln!("Failure: {}", styles.red.apply_to(failure));
|
||||
}
|
||||
|
||||
|
|
@ -1585,7 +1497,7 @@ async fn run_resumed(
|
|||
print_assets(&run_dir, styles);
|
||||
|
||||
fabro_util::run_log::deactivate();
|
||||
match outcome.status {
|
||||
match final_status {
|
||||
StageStatus::Success | StageStatus::PartialSuccess => Ok(()),
|
||||
_ => std::process::exit(1),
|
||||
}
|
||||
|
|
@ -1650,11 +1562,11 @@ mod tests {
|
|||
}
|
||||
|
||||
#[test]
|
||||
fn resume_run_status_guard_marks_failed_on_drop() {
|
||||
fn resume_bootstrap_guard_marks_failed_on_drop() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
|
||||
{
|
||||
let guard = ResumeRunStatusGuard::arm(dir.path()).unwrap();
|
||||
let guard = DetachedRunBootstrapGuard::arm(dir.path()).unwrap();
|
||||
let record = RunStatusRecord::load(&dir.path().join("status.json")).unwrap();
|
||||
assert_eq!(record.status, RunStatus::Starting);
|
||||
assert_eq!(record.reason, Some(StatusReason::SandboxInitializing));
|
||||
|
|
@ -1668,9 +1580,9 @@ mod tests {
|
|||
}
|
||||
|
||||
#[test]
|
||||
fn resume_run_status_guard_does_not_overwrite_after_defuse() {
|
||||
fn resume_bootstrap_guard_does_not_overwrite_after_defuse() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut guard = ResumeRunStatusGuard::arm(dir.path()).unwrap();
|
||||
let mut guard = DetachedRunBootstrapGuard::arm(dir.path()).unwrap();
|
||||
guard.defuse();
|
||||
drop(guard);
|
||||
|
||||
|
|
@ -1678,4 +1590,19 @@ mod tests {
|
|||
assert_eq!(record.status, RunStatus::Starting);
|
||||
assert_eq!(record.reason, Some(StatusReason::SandboxInitializing));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resume_completion_guard_marks_failed_on_drop() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
std::fs::write(dir.path().join("id.txt"), "run-resume").unwrap();
|
||||
|
||||
{
|
||||
let _guard = DetachedRunCompletionGuard::arm(dir.path());
|
||||
}
|
||||
|
||||
let record = RunStatusRecord::load(&dir.path().join("status.json")).unwrap();
|
||||
assert_eq!(record.status, RunStatus::Failed);
|
||||
assert_eq!(record.reason, Some(StatusReason::WorkflowError));
|
||||
assert!(dir.path().join("conclusion.json").exists());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,19 +18,22 @@ use fabro_util::terminal::Styles;
|
|||
use fabro_validate::Severity;
|
||||
use fabro_workflows::backend::{AgentApiBackend, AgentCliBackend, BackendRouter};
|
||||
use fabro_workflows::checkpoint::Checkpoint;
|
||||
use fabro_workflows::conclusion::Conclusion;
|
||||
use fabro_workflows::cost::{compute_stage_cost, format_cost};
|
||||
use fabro_workflows::devcontainer_bridge;
|
||||
use fabro_workflows::engine::{RunConfig, WorkflowRunEngine};
|
||||
use fabro_workflows::event::EventEmitter;
|
||||
use fabro_workflows::event::{EventEmitter, RunNoticeLevel, WorkflowRunEvent};
|
||||
use fabro_workflows::git::GitSyncStatus;
|
||||
use fabro_workflows::handler::default_registry;
|
||||
use fabro_workflows::outcome::StageStatus;
|
||||
use fabro_workflows::outcome::{Outcome, StageStatus};
|
||||
use fabro_workflows::run_status::{RunStatus, StatusReason};
|
||||
use fabro_workflows::sandbox_provider::SandboxProvider;
|
||||
use fabro_workflows::workflow::WorkflowBuilder;
|
||||
use indicatif::HumanDuration;
|
||||
use std::time::Duration;
|
||||
use tracing::debug;
|
||||
|
||||
use super::detached_support::{self, DetachedRunBootstrapGuard, DetachedRunCompletionGuard};
|
||||
use super::run_progress;
|
||||
use crate::commands::shared::{
|
||||
format_tokens_human, print_diagnostics, read_workflow_file, relative_path, tilde_path,
|
||||
|
|
@ -741,22 +744,7 @@ pub async fn run_command(
|
|||
fabro_util::run_log::activate(&run_dir.join("cli.log"))
|
||||
.context("Failed to activate per-run log")?;
|
||||
tokio::fs::write(cached_graph_path(&run_dir), &source).await?;
|
||||
tokio::fs::write(run_dir.join("run.pid"), std::process::id().to_string()).await?;
|
||||
fabro_workflows::run_status::write_run_status(
|
||||
&run_dir,
|
||||
fabro_workflows::run_status::RunStatus::Starting,
|
||||
Some(fabro_workflows::run_status::StatusReason::SandboxInitializing),
|
||||
);
|
||||
|
||||
// Safety net: mark as failed if we exit before engine.run() (e.g. sandbox init failure)
|
||||
let status_run_dir = run_dir.clone();
|
||||
let status_guard = scopeguard::guard((), move |()| {
|
||||
fabro_workflows::run_status::write_run_status(
|
||||
&status_run_dir,
|
||||
fabro_workflows::run_status::RunStatus::Failed,
|
||||
Some(fabro_workflows::run_status::StatusReason::SandboxInitFailed),
|
||||
);
|
||||
});
|
||||
let mut status_guard = DetachedRunBootstrapGuard::arm(&run_dir)?;
|
||||
|
||||
// Serialize the merged run config so the run dir is self-contained.
|
||||
// Skip when the workflow path is already the cached run.toml (i.e. _run_engine
|
||||
|
|
@ -921,9 +909,11 @@ pub async fn run_command(
|
|||
WorkdirStrategy::LocalDirectory => None,
|
||||
};
|
||||
if let Some(env_name) = env_name {
|
||||
eprintln!(
|
||||
"{} Uncommitted changes will not be included in the {env_name}.",
|
||||
styles.yellow.apply_to("Warning:"),
|
||||
emit_run_notice(
|
||||
&emitter,
|
||||
RunNoticeLevel::Warn,
|
||||
"dirty_worktree",
|
||||
format!("Uncommitted changes will not be included in the {env_name}."),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -966,16 +956,20 @@ pub async fn run_command(
|
|||
match result {
|
||||
Ok(()) => {
|
||||
tracing::info!(%branch, "Pushed current branch to origin");
|
||||
eprintln!(
|
||||
"{} {branch} (synced local commits to remote)",
|
||||
styles.bold.apply_to("Pushed branch:")
|
||||
emit_run_notice(
|
||||
&emitter,
|
||||
RunNoticeLevel::Info,
|
||||
"git_push_succeeded",
|
||||
format!("{branch} (synced local commits to remote)"),
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(error = %e, %branch, "Failed to push current branch");
|
||||
eprintln!(
|
||||
"{} Failed to push {branch} to origin: {e}",
|
||||
styles.yellow.apply_to("Warning:")
|
||||
emit_run_notice(
|
||||
&emitter,
|
||||
RunNoticeLevel::Warn,
|
||||
"git_push_failed",
|
||||
format!("Failed to push {branch} to origin: {e}"),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -998,9 +992,11 @@ pub async fn run_command(
|
|||
(Some(wt_path), Some(branch_name), Some(base_sha))
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!(
|
||||
"{} Git worktree setup failed ({e}), running without worktree.",
|
||||
styles.yellow.apply_to("Warning:"),
|
||||
emit_run_notice(
|
||||
&emitter,
|
||||
RunNoticeLevel::Warn,
|
||||
"worktree_setup_failed",
|
||||
format!("Git worktree setup failed ({e}), running without worktree."),
|
||||
);
|
||||
(None, None, None)
|
||||
}
|
||||
|
|
@ -1276,9 +1272,11 @@ pub async fn run_command(
|
|||
Arc::new(wt_sandbox) as Arc<dyn Sandbox>
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!(
|
||||
"{} Git worktree setup failed ({e}), running without worktree.",
|
||||
styles.yellow.apply_to("Warning:"),
|
||||
emit_run_notice(
|
||||
&emitter,
|
||||
RunNoticeLevel::Warn,
|
||||
"worktree_setup_failed",
|
||||
format!("Git worktree setup failed ({e}), running without worktree."),
|
||||
);
|
||||
// Reset so RunConfig does not enable git checkpointing
|
||||
worktree_path = None;
|
||||
|
|
@ -1306,17 +1304,21 @@ pub async fn run_command(
|
|||
} else {
|
||||
match fabro_llm::client::Client::from_env().await {
|
||||
Ok(c) if c.provider_names().is_empty() => {
|
||||
eprintln!(
|
||||
"{} No LLM providers configured. Running in dry-run mode.",
|
||||
styles.yellow.apply_to("Warning:"),
|
||||
emit_run_notice(
|
||||
&emitter,
|
||||
RunNoticeLevel::Warn,
|
||||
"dry_run_no_llm",
|
||||
"No LLM providers configured. Running in dry-run mode.",
|
||||
);
|
||||
(true, None)
|
||||
}
|
||||
Ok(c) => (false, Some(c)),
|
||||
Err(e) => {
|
||||
eprintln!(
|
||||
"{} Failed to initialize LLM client: {e}. Running in dry-run mode.",
|
||||
styles.yellow.apply_to("Warning:"),
|
||||
emit_run_notice(
|
||||
&emitter,
|
||||
RunNoticeLevel::Warn,
|
||||
"dry_run_llm_init_failed",
|
||||
format!("Failed to initialize LLM client: {e}. Running in dry-run mode."),
|
||||
);
|
||||
(true, None)
|
||||
}
|
||||
|
|
@ -1373,9 +1375,11 @@ pub async fn run_command(
|
|||
sandbox_env.insert("GITHUB_TOKEN".to_string(), token);
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!(
|
||||
"{} Failed to mint GitHub token: {e}",
|
||||
styles.yellow.apply_to("Warning:"),
|
||||
emit_run_notice(
|
||||
&emitter,
|
||||
RunNoticeLevel::Warn,
|
||||
"github_token_failed",
|
||||
format!("Failed to mint GitHub token: {e}"),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -1502,8 +1506,8 @@ pub async fn run_command(
|
|||
},
|
||||
};
|
||||
|
||||
// Defuse the status guard — engine.run() will write "running" and conclusion handles "concluded"
|
||||
scopeguard::ScopeGuard::into_inner(status_guard);
|
||||
// Defuse the bootstrap guard — engine.run() has taken ownership of lifecycle status.
|
||||
status_guard.defuse();
|
||||
|
||||
// Safety net: if we panic or return early, best-effort cleanup via spawn.
|
||||
let sandbox_for_cleanup = Arc::clone(&sandbox);
|
||||
|
|
@ -1524,95 +1528,20 @@ pub async fn run_command(
|
|||
.run_with_lifecycle(&graph, &mut config, lifecycle, None)
|
||||
.await;
|
||||
let run_duration_ms = run_start.elapsed().as_millis() as u64;
|
||||
let mut completion_guard = DetachedRunCompletionGuard::arm(&run_dir);
|
||||
|
||||
// Restore cwd (worktree is kept for `fabro cp` access; pruned separately)
|
||||
let _ = std::env::set_current_dir(&original_cwd);
|
||||
|
||||
{
|
||||
let (status, failure_reason) = match &engine_result {
|
||||
Ok(ref o) => (o.status.clone(), o.failure_reason().map(String::from)),
|
||||
Err(e) => (
|
||||
fabro_workflows::outcome::StageStatus::Fail,
|
||||
Some(e.to_string()),
|
||||
),
|
||||
};
|
||||
|
||||
// Map engine result to RunStatus + StatusReason
|
||||
let (run_status, status_reason) = match &engine_result {
|
||||
Ok(ref o) => match o.status {
|
||||
StageStatus::Success | StageStatus::Skipped => (
|
||||
fabro_workflows::run_status::RunStatus::Succeeded,
|
||||
Some(fabro_workflows::run_status::StatusReason::Completed),
|
||||
),
|
||||
StageStatus::PartialSuccess => (
|
||||
fabro_workflows::run_status::RunStatus::Succeeded,
|
||||
Some(fabro_workflows::run_status::StatusReason::PartialSuccess),
|
||||
),
|
||||
StageStatus::Fail | StageStatus::Retry => (
|
||||
fabro_workflows::run_status::RunStatus::Failed,
|
||||
Some(fabro_workflows::run_status::StatusReason::WorkflowError),
|
||||
),
|
||||
},
|
||||
Err(fabro_workflows::error::FabroError::Cancelled) => (
|
||||
fabro_workflows::run_status::RunStatus::Failed,
|
||||
Some(fabro_workflows::run_status::StatusReason::Cancelled),
|
||||
),
|
||||
Err(_) => (
|
||||
fabro_workflows::run_status::RunStatus::Failed,
|
||||
Some(fabro_workflows::run_status::StatusReason::WorkflowError),
|
||||
),
|
||||
};
|
||||
|
||||
// Load checkpoint and stage durations to populate per-stage data
|
||||
let checkpoint = Checkpoint::load(&run_dir.join("checkpoint.json")).ok();
|
||||
let stage_durations = fabro_retro::retro::extract_stage_durations(&run_dir);
|
||||
|
||||
let (stages, total_cost, total_retries) = if let Some(ref cp) = checkpoint {
|
||||
let mut stages = Vec::new();
|
||||
let mut cost_sum: Option<f64> = None;
|
||||
let mut retries_sum: u32 = 0;
|
||||
|
||||
for node_id in &cp.completed_nodes {
|
||||
let outcome = cp.node_outcomes.get(node_id);
|
||||
let retries = cp
|
||||
.node_retries
|
||||
.get(node_id)
|
||||
.copied()
|
||||
.unwrap_or(1)
|
||||
.saturating_sub(1);
|
||||
retries_sum += retries;
|
||||
|
||||
let cost = outcome.and_then(|o| o.usage.as_ref()).and_then(|u| u.cost);
|
||||
if let Some(c) = cost {
|
||||
*cost_sum.get_or_insert(0.0) += c;
|
||||
}
|
||||
|
||||
stages.push(fabro_workflows::conclusion::StageSummary {
|
||||
stage_id: node_id.clone(),
|
||||
stage_label: node_id.clone(),
|
||||
duration_ms: stage_durations.get(node_id).copied().unwrap_or(0),
|
||||
cost,
|
||||
retries,
|
||||
});
|
||||
}
|
||||
(stages, cost_sum, retries_sum)
|
||||
} else {
|
||||
(vec![], None, 0)
|
||||
};
|
||||
|
||||
let conclusion = fabro_workflows::conclusion::Conclusion {
|
||||
timestamp: Utc::now(),
|
||||
status,
|
||||
duration_ms: run_duration_ms,
|
||||
failure_reason,
|
||||
final_git_commit_sha: last_git_sha.lock().unwrap().clone(),
|
||||
stages,
|
||||
total_cost,
|
||||
total_retries,
|
||||
};
|
||||
let _ = conclusion.save(&run_dir.join("conclusion.json"));
|
||||
fabro_workflows::run_status::write_run_status(&run_dir, run_status, status_reason);
|
||||
}
|
||||
let (final_status, failure_reason, run_status, status_reason) =
|
||||
classify_engine_result(&engine_result);
|
||||
let conclusion = build_conclusion(
|
||||
&run_dir,
|
||||
final_status.clone(),
|
||||
failure_reason,
|
||||
run_duration_ms,
|
||||
last_git_sha.lock().unwrap().clone(),
|
||||
);
|
||||
|
||||
// Auto-derive retro (always, cheap) and optionally run retro agent
|
||||
if !args.no_retro && project_config::is_retro_enabled() {
|
||||
|
|
@ -1721,9 +1650,11 @@ pub async fn run_command(
|
|||
error: e.to_string(),
|
||||
},
|
||||
);
|
||||
eprintln!(
|
||||
"{} PR creation failed: {e}",
|
||||
styles.yellow.apply_to("Warning:")
|
||||
emit_run_notice(
|
||||
&emitter,
|
||||
RunNoticeLevel::Warn,
|
||||
"pull_request_failed",
|
||||
format!("PR creation failed: {e}"),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -1734,14 +1665,12 @@ pub async fn run_command(
|
|||
debug!("Skipping PR creation: pull_request not enabled in config");
|
||||
}
|
||||
|
||||
let outcome = engine_result?;
|
||||
|
||||
// 8. Print result
|
||||
eprintln!("\n{}", styles.bold.apply_to("=== Run Result ==="),);
|
||||
|
||||
eprintln!("{}", styles.dim.apply_to(format!("Run: {run_id}")));
|
||||
let status_str = outcome.status.to_string().to_uppercase();
|
||||
let status_color = match outcome.status {
|
||||
let status_str = final_status.to_string().to_uppercase();
|
||||
let status_color = match final_status {
|
||||
StageStatus::Success | StageStatus::PartialSuccess => &styles.bold_green,
|
||||
_ => &styles.bold_red,
|
||||
};
|
||||
|
|
@ -1801,7 +1730,7 @@ pub async fn run_command(
|
|||
.apply_to(format!("Run: {}", tilde_path(&run_dir)))
|
||||
);
|
||||
|
||||
if let Some(failure) = outcome.failure_reason() {
|
||||
if let Some(failure) = conclusion.failure_reason.as_deref() {
|
||||
eprintln!("Failure: {}", styles.red.apply_to(failure));
|
||||
}
|
||||
|
||||
|
|
@ -1823,12 +1752,19 @@ pub async fn run_command(
|
|||
if preserve_sandbox {
|
||||
let info = sandbox.sandbox_info();
|
||||
if !info.is_empty() {
|
||||
eprintln!(
|
||||
"\n{} sandbox preserved: {info}",
|
||||
styles.bold.apply_to("Info:")
|
||||
emit_run_notice(
|
||||
&emitter,
|
||||
RunNoticeLevel::Info,
|
||||
"sandbox_preserved",
|
||||
format!("sandbox preserved: {info}"),
|
||||
);
|
||||
} else {
|
||||
eprintln!("\n{} sandbox preserved", styles.bold.apply_to("Info:"));
|
||||
emit_run_notice(
|
||||
&emitter,
|
||||
RunNoticeLevel::Info,
|
||||
"sandbox_preserved",
|
||||
"sandbox preserved",
|
||||
);
|
||||
}
|
||||
}
|
||||
if let Err(e) = engine
|
||||
|
|
@ -1836,15 +1772,20 @@ pub async fn run_command(
|
|||
.await
|
||||
{
|
||||
tracing::warn!(error = %e, "Sandbox cleanup failed");
|
||||
eprintln!(
|
||||
"\n{} sandbox cleanup failed: {e}",
|
||||
styles.yellow.apply_to("Warning:")
|
||||
emit_run_notice(
|
||||
&emitter,
|
||||
RunNoticeLevel::Warn,
|
||||
"sandbox_cleanup_failed",
|
||||
format!("sandbox cleanup failed: {e}"),
|
||||
);
|
||||
}
|
||||
|
||||
persist_terminal_outcome(&run_dir, &conclusion, run_status, status_reason);
|
||||
completion_guard.defuse();
|
||||
|
||||
// 10. Exit code
|
||||
fabro_util::run_log::deactivate();
|
||||
match outcome.status {
|
||||
match final_status {
|
||||
StageStatus::Success | StageStatus::PartialSuccess => Ok(()),
|
||||
_ => {
|
||||
std::process::exit(1);
|
||||
|
|
@ -1852,6 +1793,238 @@ pub async fn run_command(
|
|||
}
|
||||
}
|
||||
|
||||
pub(crate) fn emit_run_notice(
|
||||
emitter: &EventEmitter,
|
||||
level: RunNoticeLevel,
|
||||
code: impl Into<String>,
|
||||
message: impl Into<String>,
|
||||
) {
|
||||
emitter.emit(&WorkflowRunEvent::RunNotice {
|
||||
level,
|
||||
code: code.into(),
|
||||
message: message.into(),
|
||||
});
|
||||
}
|
||||
|
||||
pub(crate) fn classify_engine_result(
|
||||
engine_result: &Result<Outcome, fabro_workflows::error::FabroError>,
|
||||
) -> (StageStatus, Option<String>, RunStatus, Option<StatusReason>) {
|
||||
match engine_result {
|
||||
Ok(outcome) => {
|
||||
let status = outcome.status.clone();
|
||||
let failure_reason = outcome.failure_reason().map(String::from);
|
||||
let (run_status, status_reason) = match status {
|
||||
StageStatus::Success | StageStatus::Skipped => {
|
||||
(RunStatus::Succeeded, Some(StatusReason::Completed))
|
||||
}
|
||||
StageStatus::PartialSuccess => {
|
||||
(RunStatus::Succeeded, Some(StatusReason::PartialSuccess))
|
||||
}
|
||||
StageStatus::Fail | StageStatus::Retry => {
|
||||
(RunStatus::Failed, Some(StatusReason::WorkflowError))
|
||||
}
|
||||
};
|
||||
(status, failure_reason, run_status, status_reason)
|
||||
}
|
||||
Err(fabro_workflows::error::FabroError::Cancelled) => (
|
||||
StageStatus::Fail,
|
||||
Some("Cancelled".to_string()),
|
||||
RunStatus::Failed,
|
||||
Some(StatusReason::Cancelled),
|
||||
),
|
||||
Err(err) => (
|
||||
StageStatus::Fail,
|
||||
Some(err.to_string()),
|
||||
RunStatus::Failed,
|
||||
Some(StatusReason::WorkflowError),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn build_conclusion(
|
||||
run_dir: &Path,
|
||||
status: StageStatus,
|
||||
failure_reason: Option<String>,
|
||||
run_duration_ms: u64,
|
||||
final_git_commit_sha: Option<String>,
|
||||
) -> Conclusion {
|
||||
let checkpoint = Checkpoint::load(&run_dir.join("checkpoint.json")).ok();
|
||||
let stage_durations = fabro_retro::retro::extract_stage_durations(run_dir);
|
||||
|
||||
let mut total_input_tokens: i64 = 0;
|
||||
let mut total_output_tokens: i64 = 0;
|
||||
let mut total_cache_read_tokens: i64 = 0;
|
||||
let mut total_cache_write_tokens: i64 = 0;
|
||||
let mut total_reasoning_tokens: i64 = 0;
|
||||
let mut has_pricing = false;
|
||||
|
||||
let (stages, total_cost, total_retries) = if let Some(ref cp) = checkpoint {
|
||||
let mut stages = Vec::new();
|
||||
let mut cost_sum: Option<f64> = None;
|
||||
let mut retries_sum: u32 = 0;
|
||||
|
||||
for node_id in &cp.completed_nodes {
|
||||
let outcome = cp.node_outcomes.get(node_id);
|
||||
let retries = cp
|
||||
.node_retries
|
||||
.get(node_id)
|
||||
.copied()
|
||||
.unwrap_or(1)
|
||||
.saturating_sub(1);
|
||||
retries_sum += retries;
|
||||
|
||||
let cost = outcome.and_then(|o| o.usage.as_ref()).and_then(|u| u.cost);
|
||||
if let Some(c) = cost {
|
||||
*cost_sum.get_or_insert(0.0) += c;
|
||||
has_pricing = true;
|
||||
}
|
||||
|
||||
if let Some(usage) = outcome.and_then(|o| o.usage.as_ref()) {
|
||||
total_input_tokens += usage.input_tokens;
|
||||
total_output_tokens += usage.output_tokens;
|
||||
total_cache_read_tokens += usage.cache_read_tokens.unwrap_or(0);
|
||||
total_cache_write_tokens += usage.cache_write_tokens.unwrap_or(0);
|
||||
total_reasoning_tokens += usage.reasoning_tokens.unwrap_or(0);
|
||||
}
|
||||
|
||||
stages.push(fabro_workflows::conclusion::StageSummary {
|
||||
stage_id: node_id.clone(),
|
||||
stage_label: node_id.clone(),
|
||||
duration_ms: stage_durations.get(node_id).copied().unwrap_or(0),
|
||||
cost,
|
||||
retries,
|
||||
});
|
||||
}
|
||||
(stages, cost_sum, retries_sum)
|
||||
} else {
|
||||
(vec![], None, 0)
|
||||
};
|
||||
|
||||
Conclusion {
|
||||
timestamp: Utc::now(),
|
||||
status,
|
||||
duration_ms: run_duration_ms,
|
||||
failure_reason,
|
||||
final_git_commit_sha,
|
||||
stages,
|
||||
total_cost,
|
||||
total_retries,
|
||||
total_input_tokens,
|
||||
total_output_tokens,
|
||||
total_cache_read_tokens,
|
||||
total_cache_write_tokens,
|
||||
total_reasoning_tokens,
|
||||
has_pricing,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn persist_terminal_outcome(
|
||||
run_dir: &Path,
|
||||
conclusion: &Conclusion,
|
||||
run_status: RunStatus,
|
||||
status_reason: Option<StatusReason>,
|
||||
) {
|
||||
let _ = conclusion.save(&run_dir.join("conclusion.json"));
|
||||
fabro_workflows::run_status::write_run_status(run_dir, run_status, status_reason);
|
||||
}
|
||||
|
||||
/// Print a summary of the completed run from `conclusion.json` and `pull_request.json`.
|
||||
///
|
||||
/// Used by the unified create+start+attach path in `main.rs` to display
|
||||
/// the same result block that `run_command` prints in-process.
|
||||
pub fn print_run_summary(run_dir: &Path, run_id: &str, styles: &Styles) {
|
||||
let conclusion_path = run_dir.join("conclusion.json");
|
||||
let Ok(conclusion) = fabro_workflows::conclusion::Conclusion::load(&conclusion_path) else {
|
||||
return;
|
||||
};
|
||||
|
||||
eprintln!("\n{}", styles.bold.apply_to("=== Run Result ==="));
|
||||
eprintln!("{}", styles.dim.apply_to(format!("Run: {run_id}")));
|
||||
|
||||
let status_str = conclusion.status.to_string().to_uppercase();
|
||||
let status_color = match conclusion.status {
|
||||
StageStatus::Success | StageStatus::PartialSuccess => &styles.bold_green,
|
||||
_ => &styles.bold_red,
|
||||
};
|
||||
eprintln!("Status: {}", status_color.apply_to(&status_str));
|
||||
eprintln!(
|
||||
"Duration: {}",
|
||||
HumanDuration(Duration::from_millis(conclusion.duration_ms))
|
||||
);
|
||||
|
||||
let total_tokens = conclusion.total_input_tokens + conclusion.total_output_tokens;
|
||||
if total_tokens > 0 {
|
||||
if conclusion.has_pricing {
|
||||
if let Some(cost) = conclusion.total_cost {
|
||||
if cost > 0.0 {
|
||||
eprintln!(
|
||||
"{}",
|
||||
styles.dim.apply_to(format!(
|
||||
"Cost: {} ({} toks)",
|
||||
format_cost(cost),
|
||||
format_tokens_human(total_tokens)
|
||||
))
|
||||
);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
eprintln!(
|
||||
"{}",
|
||||
styles
|
||||
.dim
|
||||
.apply_to(format!("Toks: {}", format_tokens_human(total_tokens)))
|
||||
);
|
||||
}
|
||||
if conclusion.total_cache_read_tokens > 0 {
|
||||
eprintln!(
|
||||
"{}",
|
||||
styles.dim.apply_to(format!(
|
||||
"Cache: {} read, {} write",
|
||||
format_tokens_human(conclusion.total_cache_read_tokens),
|
||||
format_tokens_human(conclusion.total_cache_write_tokens),
|
||||
)),
|
||||
);
|
||||
}
|
||||
if conclusion.total_reasoning_tokens > 0 {
|
||||
eprintln!(
|
||||
"{}",
|
||||
styles.dim.apply_to(format!(
|
||||
"Reasoning: {} tokens",
|
||||
format_tokens_human(conclusion.total_reasoning_tokens),
|
||||
)),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
eprintln!(
|
||||
"{}",
|
||||
styles
|
||||
.dim
|
||||
.apply_to(format!("Run: {}", tilde_path(run_dir)))
|
||||
);
|
||||
|
||||
if let Some(ref failure) = conclusion.failure_reason {
|
||||
eprintln!("Failure: {}", styles.red.apply_to(failure));
|
||||
}
|
||||
|
||||
// PR info from pull_request.json (saved by _run_engine)
|
||||
if let Ok(content) = std::fs::read_to_string(run_dir.join("pull_request.json")) {
|
||||
if let Ok(record) =
|
||||
serde_json::from_str::<fabro_workflows::pull_request::PullRequestRecord>(&content)
|
||||
{
|
||||
eprintln!();
|
||||
eprintln!(
|
||||
"{} {}",
|
||||
styles.bold.apply_to("Pull request:"),
|
||||
record.html_url
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
print_final_output(run_dir, styles);
|
||||
print_assets(run_dir, styles);
|
||||
}
|
||||
|
||||
/// Print the final stage output from the checkpoint, if available.
|
||||
pub(crate) fn print_final_output(run_dir: &std::path::Path, styles: &Styles) {
|
||||
let Ok(checkpoint) = Checkpoint::load(&run_dir.join("checkpoint.json")) else {
|
||||
|
|
@ -2486,23 +2659,7 @@ pub(crate) fn build_event_envelope(
|
|||
event: &fabro_workflows::event::WorkflowRunEvent,
|
||||
run_id: &str,
|
||||
) -> serde_json::Value {
|
||||
let (event_name, event_fields) = fabro_workflows::event::flatten_event(event);
|
||||
let mut envelope = serde_json::Map::new();
|
||||
envelope.insert(
|
||||
"ts".to_string(),
|
||||
serde_json::Value::String(Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true)),
|
||||
);
|
||||
envelope.insert(
|
||||
"run_id".to_string(),
|
||||
serde_json::Value::String(run_id.to_string()),
|
||||
);
|
||||
envelope.insert("event".to_string(), serde_json::Value::String(event_name));
|
||||
for (k, v) in event_fields {
|
||||
if k != "ts" && k != "run_id" && k != "event" {
|
||||
envelope.insert(k, v);
|
||||
}
|
||||
}
|
||||
serde_json::Value::Object(envelope)
|
||||
detached_support::build_event_envelope(event, run_id)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ use indicatif::{MultiProgress, ProgressBar, ProgressDrawTarget, ProgressStyle};
|
|||
|
||||
use fabro_agent::AgentEvent;
|
||||
use fabro_interview::{Answer, ConsoleInterviewer, Interviewer, Question};
|
||||
use fabro_workflows::event::{EventEmitter, WorkflowRunEvent};
|
||||
use fabro_workflows::event::{EventEmitter, RunNoticeLevel, WorkflowRunEvent};
|
||||
use fabro_workflows::outcome::StageStatus;
|
||||
|
||||
use crate::commands::shared::{format_duration_ms, format_tokens_human, tilde_path};
|
||||
|
|
@ -626,6 +626,19 @@ impl ProgressUI {
|
|||
let dur = format_duration_ms(*duration_ms);
|
||||
self.finish_stage("retro", "Retro", red_cross(), &dur);
|
||||
}
|
||||
WorkflowRunEvent::RunNotice {
|
||||
level,
|
||||
code,
|
||||
message,
|
||||
} => {
|
||||
self.on_run_notice(*level, code, message);
|
||||
}
|
||||
WorkflowRunEvent::PullRequestCreated { pr_url, draft, .. } => {
|
||||
self.on_pull_request_created(pr_url, *draft);
|
||||
}
|
||||
WorkflowRunEvent::PullRequestFailed { error } => {
|
||||
self.on_pull_request_failed(error);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
|
@ -881,6 +894,28 @@ impl ProgressUI {
|
|||
let dur = format_duration_ms(u64_field("duration_ms"));
|
||||
self.finish_stage("retro", "Retro", red_cross(), &dur);
|
||||
}
|
||||
"RunNotice" => {
|
||||
let level = match str_field("level").unwrap_or("info") {
|
||||
"warn" => RunNoticeLevel::Warn,
|
||||
"error" => RunNoticeLevel::Error,
|
||||
_ => RunNoticeLevel::Info,
|
||||
};
|
||||
let code = str_field("code").unwrap_or("");
|
||||
let message = str_field("message").unwrap_or("");
|
||||
self.on_run_notice(level, code, message);
|
||||
}
|
||||
"PullRequestCreated" => {
|
||||
let pr_url = str_field("pr_url").unwrap_or("?");
|
||||
let draft = envelope
|
||||
.get("draft")
|
||||
.and_then(|value| value.as_bool())
|
||||
.unwrap_or(false);
|
||||
self.on_pull_request_created(pr_url, draft);
|
||||
}
|
||||
"PullRequestFailed" => {
|
||||
let error = str_field("error").unwrap_or("unknown error");
|
||||
self.on_pull_request_failed(error);
|
||||
}
|
||||
"DevcontainerResolved" => {
|
||||
let dockerfile_lines = u64_field("dockerfile_lines");
|
||||
let environment_count = u64_field("environment_count");
|
||||
|
|
@ -1585,6 +1620,32 @@ impl ProgressUI {
|
|||
}
|
||||
}
|
||||
|
||||
fn on_run_notice(&mut self, level: RunNoticeLevel, code: &str, message: &str) {
|
||||
let dim = Style::new().dim();
|
||||
let label = match level {
|
||||
RunNoticeLevel::Info => Style::new().bold().apply_to("Info:"),
|
||||
RunNoticeLevel::Warn => Style::new().yellow().apply_to("Warning:"),
|
||||
RunNoticeLevel::Error => Style::new().red().apply_to("Error:"),
|
||||
};
|
||||
let code_suffix = if code.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!(" {}", dim.apply_to(format!("[{code}]")))
|
||||
};
|
||||
self.insert_info_line(&format!("{label} {message}{code_suffix}"));
|
||||
}
|
||||
|
||||
fn on_pull_request_created(&mut self, pr_url: &str, draft: bool) {
|
||||
let label = if draft { "Draft PR:" } else { "PR:" };
|
||||
let bold = Style::new().bold();
|
||||
self.insert_info_line(&format!("{} {pr_url}", bold.apply_to(label)));
|
||||
}
|
||||
|
||||
fn on_pull_request_failed(&mut self, error: &str) {
|
||||
let red = Style::new().red();
|
||||
self.insert_info_line(&format!("{} {error}", red.apply_to("PR failed:")));
|
||||
}
|
||||
|
||||
/// Insert a static info line for a subagent, indented deeper than tool calls.
|
||||
fn insert_subagent_line_for_stage(&mut self, stage_node_id: &str, message: &str) {
|
||||
match &self.renderer {
|
||||
|
|
@ -2045,4 +2106,18 @@ mod tests {
|
|||
"devcontainer_command_count should be set by DevcontainerLifecycleStarted"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn handle_json_line_run_notice_warn() {
|
||||
let mut ui = ProgressUI::new(false, false);
|
||||
let event = r#"{"ts":"2026-01-01T12:00:00Z","event":"RunNotice","level":"warn","code":"sandbox_cleanup_failed","message":"sandbox cleanup failed: boom"}"#;
|
||||
ui.handle_json_line(event);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn handle_json_line_pull_request_failed() {
|
||||
let mut ui = ProgressUI::new(false, false);
|
||||
let event = r#"{"ts":"2026-01-01T12:00:00Z","event":"PullRequestFailed","error":"auth token expired"}"#;
|
||||
ui.handle_json_line(event);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,16 +1,19 @@
|
|||
use std::path::Path;
|
||||
|
||||
use anyhow::{bail, Result};
|
||||
use fabro_workflows::run_status::{RunStatus, StatusReason};
|
||||
|
||||
use super::detached_support::persist_detached_failure;
|
||||
|
||||
/// Spawn a detached engine process for the given run directory.
|
||||
///
|
||||
/// The engine process reads `spec.json` from the run directory and executes the
|
||||
/// workflow. Returns the child process PID.
|
||||
pub fn start_run(run_dir: &Path) -> Result<u32> {
|
||||
/// workflow. Returns the child process handle (use `.id()` for the PID).
|
||||
pub fn start_run(run_dir: &Path) -> Result<std::process::Child> {
|
||||
// Validate status is Submitted
|
||||
let status_path = run_dir.join("status.json");
|
||||
match fabro_workflows::run_status::RunStatusRecord::load(&status_path) {
|
||||
Ok(record) if record.status != fabro_workflows::run_status::RunStatus::Submitted => {
|
||||
Ok(record) if record.status != RunStatus::Submitted => {
|
||||
bail!(
|
||||
"Cannot start run: status is {:?}, expected Submitted",
|
||||
record.status
|
||||
|
|
@ -24,19 +27,38 @@ pub fn start_run(run_dir: &Path) -> Result<u32> {
|
|||
.map_err(|e| anyhow::anyhow!("Cannot start run: failed to load spec.json: {e}"))?;
|
||||
|
||||
// Write Starting status before spawning to prevent duplicate engines
|
||||
fabro_workflows::run_status::write_run_status(
|
||||
run_dir,
|
||||
fabro_workflows::run_status::RunStatus::Starting,
|
||||
None,
|
||||
);
|
||||
fabro_workflows::run_status::write_run_status(run_dir, RunStatus::Starting, None);
|
||||
|
||||
let log_file = std::fs::File::create(run_dir.join("detach.log"))?;
|
||||
let log_file = match std::fs::File::create(run_dir.join("detach.log")) {
|
||||
Ok(file) => file,
|
||||
Err(err) => {
|
||||
let err = err.into();
|
||||
let _ = persist_detached_failure(run_dir, "launch", StatusReason::LaunchFailed, &err);
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
|
||||
let exe = match std::env::current_exe() {
|
||||
Ok(exe) => exe,
|
||||
Err(err) => {
|
||||
let err = err.into();
|
||||
let _ = persist_detached_failure(run_dir, "launch", StatusReason::LaunchFailed, &err);
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
|
||||
let exe = std::env::current_exe()?;
|
||||
let mut cmd = std::process::Command::new(&exe);
|
||||
let stdout_log = match log_file.try_clone() {
|
||||
Ok(file) => file,
|
||||
Err(err) => {
|
||||
let err = err.into();
|
||||
let _ = persist_detached_failure(run_dir, "launch", StatusReason::LaunchFailed, &err);
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
cmd.args(["_run_engine", "--run-dir"])
|
||||
.arg(run_dir)
|
||||
.stdout(log_file.try_clone()?)
|
||||
.stdout(stdout_log)
|
||||
.stderr(log_file)
|
||||
.stdin(std::process::Stdio::null());
|
||||
|
||||
|
|
@ -52,20 +74,36 @@ pub fn start_run(run_dir: &Path) -> Result<u32> {
|
|||
}
|
||||
}
|
||||
|
||||
let child = cmd.spawn()?;
|
||||
let pid = child.id();
|
||||
let mut child = match cmd.spawn() {
|
||||
Ok(child) => child,
|
||||
Err(err) => {
|
||||
let err = err.into();
|
||||
let _ = persist_detached_failure(run_dir, "launch", StatusReason::LaunchFailed, &err);
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
|
||||
// Write PID file
|
||||
std::fs::write(run_dir.join("run.pid"), pid.to_string())?;
|
||||
if let Err(err) = std::fs::write(run_dir.join("run.pid"), child.id().to_string()) {
|
||||
kill_child_best_effort(&mut child);
|
||||
let err = err.into();
|
||||
let _ = persist_detached_failure(run_dir, "launch", StatusReason::LaunchFailed, &err);
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
Ok(pid)
|
||||
Ok(child)
|
||||
}
|
||||
|
||||
fn kill_child_best_effort(child: &mut std::process::Child) {
|
||||
let _ = child.kill();
|
||||
let _ = child.wait();
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use fabro_workflows::run_spec::RunSpec;
|
||||
use fabro_workflows::run_status::{write_run_status, RunStatus, RunStatusRecord};
|
||||
use fabro_workflows::run_status::{write_run_status, RunStatus, RunStatusRecord, StatusReason};
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
|
||||
|
|
@ -88,23 +126,24 @@ mod tests {
|
|||
}
|
||||
}
|
||||
|
||||
// Bug 5: start_run should write Starting status before spawning engine
|
||||
// to prevent duplicate engine processes from concurrent start calls.
|
||||
#[test]
|
||||
fn bug5_start_run_writes_starting_status_before_spawn() {
|
||||
fn start_run_marks_failed_when_spawn_cannot_start_engine() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
write_run_status(dir.path(), RunStatus::Submitted, None);
|
||||
sample_spec().save(dir.path()).unwrap();
|
||||
std::fs::create_dir(dir.path().join("detach.log")).unwrap();
|
||||
|
||||
// start_run may fail on spawn (test binary != fabro), but we only
|
||||
// care about the status file being updated before the spawn attempt.
|
||||
let _ = start_run(dir.path());
|
||||
|
||||
let record = RunStatusRecord::load(&dir.path().join("status.json")).unwrap();
|
||||
assert_eq!(
|
||||
record.status,
|
||||
RunStatus::Starting,
|
||||
"start_run should write Starting status before spawning to prevent duplicate engines"
|
||||
RunStatus::Failed,
|
||||
"start_run should persist a terminal failure on launch errors"
|
||||
);
|
||||
assert_eq!(record.reason, Some(StatusReason::LaunchFailed));
|
||||
assert!(dir.path().join("conclusion.json").exists());
|
||||
let progress = std::fs::read_to_string(dir.path().join("progress.jsonl")).unwrap();
|
||||
assert!(progress.contains("launch_failed"));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -155,6 +155,12 @@ mod tests {
|
|||
stages: vec![],
|
||||
total_cost: Some(0.42),
|
||||
total_retries: 0,
|
||||
total_input_tokens: 0,
|
||||
total_output_tokens: 0,
|
||||
total_cache_read_tokens: 0,
|
||||
total_cache_write_tokens: 0,
|
||||
total_reasoning_tokens: 0,
|
||||
has_pricing: false,
|
||||
};
|
||||
let json = build_json_output(RunStatus::Succeeded, "ABC123", Some(&conclusion));
|
||||
assert_eq!(json["run_id"], "ABC123");
|
||||
|
|
@ -189,6 +195,12 @@ mod tests {
|
|||
stages: vec![],
|
||||
total_cost: None,
|
||||
total_retries: 0,
|
||||
total_input_tokens: 0,
|
||||
total_output_tokens: 0,
|
||||
total_cache_read_tokens: 0,
|
||||
total_cache_write_tokens: 0,
|
||||
total_reasoning_tokens: 0,
|
||||
has_pricing: false,
|
||||
};
|
||||
let json = build_json_output(RunStatus::Failed, "JKL012", Some(&conclusion));
|
||||
assert!(json.get("total_cost").is_none());
|
||||
|
|
@ -207,6 +219,12 @@ mod tests {
|
|||
stages: vec![],
|
||||
total_cost: Some(0.15),
|
||||
total_retries: 0,
|
||||
total_input_tokens: 0,
|
||||
total_output_tokens: 0,
|
||||
total_cache_read_tokens: 0,
|
||||
total_cache_write_tokens: 0,
|
||||
total_reasoning_tokens: 0,
|
||||
has_pricing: false,
|
||||
};
|
||||
// Just verify no panic; actual stderr output is hard to capture
|
||||
print_human_output(RunStatus::Succeeded, "ABC123", Some(&conclusion), &styles);
|
||||
|
|
|
|||
|
|
@ -313,6 +313,103 @@ pub(crate) fn build_github_app_credentials(
|
|||
})
|
||||
}
|
||||
|
||||
async fn run_engine_entrypoint(
|
||||
run_dir: PathBuf,
|
||||
styles: &'static fabro_util::terminal::Styles,
|
||||
) -> Result<()> {
|
||||
let cli_config = cli_config::load_cli_config(None)?;
|
||||
let github_app = build_github_app_credentials(cli_config.app_id());
|
||||
let git_author = fabro_workflows::git::GitAuthor::from_options(
|
||||
cli_config.git_author().and_then(|a| a.name.clone()),
|
||||
cli_config.git_author().and_then(|a| a.email.clone()),
|
||||
);
|
||||
|
||||
let spec = match fabro_workflows::run_spec::RunSpec::load(&run_dir) {
|
||||
Ok(spec) => spec,
|
||||
Err(err) => {
|
||||
let _ = commands::detached_support::persist_detached_failure(
|
||||
&run_dir,
|
||||
"bootstrap",
|
||||
fabro_workflows::run_status::StatusReason::BootstrapFailed,
|
||||
&err,
|
||||
);
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
|
||||
if let Err(err) = std::env::set_current_dir(&spec.working_directory).map_err(|e| {
|
||||
anyhow::anyhow!(
|
||||
"Failed to set working directory to {}: {e}",
|
||||
spec.working_directory.display()
|
||||
)
|
||||
}) {
|
||||
let _ = commands::detached_support::persist_detached_failure(
|
||||
&run_dir,
|
||||
"bootstrap",
|
||||
fabro_workflows::run_status::StatusReason::BootstrapFailed,
|
||||
&err,
|
||||
);
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
let workflow_path = {
|
||||
let config_path = commands::run::cached_run_config_path(&run_dir);
|
||||
if config_path.exists() {
|
||||
config_path
|
||||
} else {
|
||||
commands::run::cached_graph_path(&run_dir)
|
||||
}
|
||||
};
|
||||
|
||||
let run_args = commands::run::RunArgs {
|
||||
workflow: Some(workflow_path),
|
||||
run_dir: Some(run_dir.clone()),
|
||||
dry_run: spec.dry_run,
|
||||
preflight: false,
|
||||
auto_approve: spec.auto_approve,
|
||||
goal: spec.goal,
|
||||
goal_file: None,
|
||||
model: Some(spec.model),
|
||||
provider: Some(spec.provider.unwrap_or_default()).filter(|s| !s.is_empty()),
|
||||
verbose: spec.verbose,
|
||||
sandbox: spec
|
||||
.sandbox_provider
|
||||
.parse::<fabro_workflows::sandbox_provider::SandboxProvider>()
|
||||
.ok()
|
||||
.map(commands::run::CliSandboxProvider::from),
|
||||
label: spec
|
||||
.labels
|
||||
.into_iter()
|
||||
.map(|(k, v)| format!("{k}={v}"))
|
||||
.collect(),
|
||||
no_retro: spec.no_retro,
|
||||
preserve_sandbox: spec.preserve_sandbox,
|
||||
detach: false,
|
||||
run_id: Some(spec.run_id),
|
||||
};
|
||||
|
||||
match commands::run::run_command(
|
||||
run_args,
|
||||
cli_config.run_defaults,
|
||||
styles,
|
||||
github_app,
|
||||
git_author,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(()) => Ok(()),
|
||||
Err(err) => {
|
||||
let _ = commands::detached_support::persist_detached_failure(
|
||||
&run_dir,
|
||||
"bootstrap",
|
||||
fabro_workflows::run_status::StatusReason::SandboxInitFailed,
|
||||
&err,
|
||||
);
|
||||
Err(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
fabro_telemetry::panic::install_panic_hook();
|
||||
|
|
@ -653,24 +750,14 @@ async fn main_inner() -> (String, Result<()>) {
|
|||
let cli_config = cli_config::load_cli_config(None)?;
|
||||
args.verbose = args.verbose || cli_config.verbose;
|
||||
|
||||
if args.detach {
|
||||
// Detach mode: create + start + print run ID
|
||||
let (run_id, run_dir) =
|
||||
commands::create::create_run(&args, cli_config.run_defaults, styles, true)
|
||||
.await?;
|
||||
commands::start::start_run(&run_dir)?;
|
||||
println!("{run_id}");
|
||||
} else {
|
||||
// Foreground mode: use existing run_command
|
||||
if args.preflight {
|
||||
// Preflight validates config without creating a run dir.
|
||||
// Needs github_app for token validation, runs in-process.
|
||||
let github_app = build_github_app_credentials(cli_config.app_id());
|
||||
let git_author = fabro_workflows::git::GitAuthor::from_options(
|
||||
cli_config.git_author().and_then(|a| a.name.clone()),
|
||||
cli_config.git_author().and_then(|a| a.email.clone()),
|
||||
);
|
||||
|
||||
#[cfg(feature = "sleep_inhibitor")]
|
||||
let _sleep_guard = fabro_beastie::guard(cli_config.prevent_idle_sleep);
|
||||
|
||||
commands::run::run_command(
|
||||
args,
|
||||
cli_config.run_defaults,
|
||||
|
|
@ -679,6 +766,29 @@ async fn main_inner() -> (String, Result<()>) {
|
|||
git_author,
|
||||
)
|
||||
.await?;
|
||||
} else {
|
||||
// Unified path: create + start (+ attach for foreground)
|
||||
let quiet = args.detach;
|
||||
let (run_id, run_dir) =
|
||||
commands::create::create_run(&args, cli_config.run_defaults, styles, quiet)
|
||||
.await?;
|
||||
|
||||
#[cfg(feature = "sleep_inhibitor")]
|
||||
let _sleep_guard = fabro_beastie::guard(cli_config.prevent_idle_sleep);
|
||||
|
||||
let child = commands::start::start_run(&run_dir)?;
|
||||
|
||||
if args.detach {
|
||||
println!("{run_id}");
|
||||
} else {
|
||||
let exit_code =
|
||||
commands::attach::attach_run(&run_dir, true, styles, Some(child))
|
||||
.await?;
|
||||
commands::run::print_run_summary(&run_dir, &run_id, styles);
|
||||
if exit_code != std::process::ExitCode::SUCCESS {
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Command::Create(args) => {
|
||||
|
|
@ -693,15 +803,16 @@ async fn main_inner() -> (String, Result<()>) {
|
|||
Command::Start { run } => {
|
||||
let base = fabro_workflows::run_lookup::default_runs_base();
|
||||
let run_info = fabro_workflows::run_lookup::resolve_run(&base, &run)?;
|
||||
let pid = commands::start::start_run(&run_info.path)?;
|
||||
eprintln!("Started engine process (PID {pid})");
|
||||
let child = commands::start::start_run(&run_info.path)?;
|
||||
eprintln!("Started engine process (PID {})", child.id());
|
||||
}
|
||||
Command::Attach { run } => {
|
||||
let styles: &'static fabro_util::terminal::Styles =
|
||||
Box::leak(Box::new(fabro_util::terminal::Styles::detect_stderr()));
|
||||
let base = fabro_workflows::run_lookup::default_runs_base();
|
||||
let run_info = fabro_workflows::run_lookup::resolve_run(&base, &run)?;
|
||||
let exit_code = commands::attach::attach_run(&run_info.path, false, styles).await?;
|
||||
let exit_code =
|
||||
commands::attach::attach_run(&run_info.path, false, styles, None).await?;
|
||||
if exit_code != std::process::ExitCode::SUCCESS {
|
||||
std::process::exit(1);
|
||||
}
|
||||
|
|
@ -709,63 +820,7 @@ async fn main_inner() -> (String, Result<()>) {
|
|||
Command::RunEngine { run_dir } => {
|
||||
let styles: &'static fabro_util::terminal::Styles =
|
||||
Box::leak(Box::new(fabro_util::terminal::Styles::detect_stderr()));
|
||||
let cli_config = cli_config::load_cli_config(None)?;
|
||||
let github_app = build_github_app_credentials(cli_config.app_id());
|
||||
let git_author = fabro_workflows::git::GitAuthor::from_options(
|
||||
cli_config.git_author().and_then(|a| a.name.clone()),
|
||||
cli_config.git_author().and_then(|a| a.email.clone()),
|
||||
);
|
||||
|
||||
// Load spec and reconstruct RunArgs
|
||||
let spec = fabro_workflows::run_spec::RunSpec::load(&run_dir)?;
|
||||
|
||||
// Restore the working directory captured at create time
|
||||
std::env::set_current_dir(&spec.working_directory).map_err(|e| {
|
||||
anyhow::anyhow!(
|
||||
"Failed to set working directory to {}: {e}",
|
||||
spec.working_directory.display()
|
||||
)
|
||||
})?;
|
||||
|
||||
// Prefer the cached run.toml. prepare_workflow() falls back to the
|
||||
// sibling graph snapshot for older detached runs that predate run.toml.
|
||||
let workflow_path = commands::run::cached_run_config_path(&run_dir);
|
||||
|
||||
let run_args = commands::run::RunArgs {
|
||||
workflow: Some(workflow_path),
|
||||
run_dir: Some(run_dir),
|
||||
dry_run: spec.dry_run,
|
||||
preflight: false,
|
||||
auto_approve: spec.auto_approve,
|
||||
goal: spec.goal,
|
||||
goal_file: None,
|
||||
model: Some(spec.model),
|
||||
provider: Some(spec.provider.unwrap_or_default()).filter(|s| !s.is_empty()),
|
||||
verbose: spec.verbose,
|
||||
sandbox: spec
|
||||
.sandbox_provider
|
||||
.parse::<fabro_workflows::sandbox_provider::SandboxProvider>()
|
||||
.ok()
|
||||
.map(commands::run::CliSandboxProvider::from),
|
||||
label: spec
|
||||
.labels
|
||||
.into_iter()
|
||||
.map(|(k, v)| format!("{k}={v}"))
|
||||
.collect(),
|
||||
no_retro: spec.no_retro,
|
||||
preserve_sandbox: spec.preserve_sandbox,
|
||||
detach: false,
|
||||
run_id: Some(spec.run_id),
|
||||
};
|
||||
|
||||
commands::run::run_command(
|
||||
run_args,
|
||||
cli_config.run_defaults,
|
||||
styles,
|
||||
github_app,
|
||||
git_author,
|
||||
)
|
||||
.await?;
|
||||
run_engine_entrypoint(run_dir, styles).await?;
|
||||
}
|
||||
Command::Validate(args) => {
|
||||
let styles = fabro_util::terminal::Styles::detect_stderr();
|
||||
|
|
|
|||
|
|
@ -666,7 +666,7 @@ digraph G {
|
|||
// Bug 3: attach loop must delete interview_request.json after handling it
|
||||
// to prevent re-prompting the user on the next poll iteration.
|
||||
#[test]
|
||||
fn bug3_attach_cleans_up_interview_request_after_handling() {
|
||||
fn bug3_attach_leaves_interview_request_until_engine_consumes_response() {
|
||||
let home = tempfile::tempdir().unwrap();
|
||||
|
||||
let run_dir = setup_run_dir(
|
||||
|
|
@ -714,11 +714,88 @@ fn bug3_attach_cleans_up_interview_request_after_handling() {
|
|||
.timeout(std::time::Duration::from_secs(5))
|
||||
.output();
|
||||
|
||||
// Bug: interview_request.json is never deleted by the attach loop.
|
||||
// After the fix it should be removed immediately after handling.
|
||||
// The attach loop should leave the request durable until the engine consumes
|
||||
// the response, so a crashed attach can be retried safely.
|
||||
assert!(
|
||||
!run_dir.join("interview_request.json").exists(),
|
||||
"bug3: interview_request.json should be deleted after being handled by attach"
|
||||
run_dir.join("interview_request.json").exists(),
|
||||
"bug3: interview_request.json should stay present until the engine consumes the answer"
|
||||
);
|
||||
assert!(
|
||||
run_dir.join("interview_response.json").exists(),
|
||||
"bug3: attach should write interview_response.json after handling the prompt"
|
||||
);
|
||||
let response = std::fs::read_to_string(run_dir.join("interview_response.json")).unwrap();
|
||||
assert!(response.contains("\"value\": \"Yes\""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attach_closed_stdin_keeps_interview_pending() {
|
||||
let home = tempfile::tempdir().unwrap();
|
||||
|
||||
let run_dir = setup_run_dir(
|
||||
home.path(),
|
||||
"attach-closed-stdin",
|
||||
serde_json::json!({}),
|
||||
&[
|
||||
r#"{"ts":"2026-01-01T00:00:01Z","run_id":"attach-closed-stdin","event":"StageStarted","node_id":"gate","name":"Gate","index":0,"attempt":1,"max_attempts":1}"#,
|
||||
],
|
||||
);
|
||||
|
||||
std::fs::write(
|
||||
run_dir.join("status.json"),
|
||||
serde_json::json!({"status": "running", "updated_at": "2026-01-01T00:00:00Z"}).to_string(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let question = serde_json::json!({
|
||||
"text": "Approve?",
|
||||
"question_type": "YesNo",
|
||||
"options": [],
|
||||
"allow_freeform": false,
|
||||
"default": null,
|
||||
"timeout_seconds": null,
|
||||
"stage": "gate",
|
||||
"metadata": {}
|
||||
});
|
||||
std::fs::write(
|
||||
run_dir.join("interview_request.json"),
|
||||
serde_json::to_string(&question).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
std::fs::write(run_dir.join("run.pid"), "99999999").unwrap();
|
||||
|
||||
let assert = arc()
|
||||
.env("HOME", home.path())
|
||||
.env("NO_COLOR", "1")
|
||||
.args(["attach", "attach-closed-stdin"])
|
||||
.timeout(std::time::Duration::from_secs(5))
|
||||
.assert()
|
||||
.failure();
|
||||
|
||||
let stderr = String::from_utf8(assert.get_output().stderr.clone()).unwrap();
|
||||
assert!(
|
||||
stderr.contains("still waiting for input"),
|
||||
"attach should explain that the run is still waiting for a human answer.\nstderr: {stderr}"
|
||||
);
|
||||
assert!(
|
||||
run_dir.join("interview_request.json").exists(),
|
||||
"attach with closed stdin must leave the request pending"
|
||||
);
|
||||
assert!(
|
||||
!run_dir.join("interview_response.json").exists(),
|
||||
"attach with closed stdin must not fabricate a response"
|
||||
);
|
||||
assert!(
|
||||
!run_dir.join("interview_request.claim").exists(),
|
||||
"attach with closed stdin must release the claim so a later attach can answer"
|
||||
);
|
||||
|
||||
let progress = std::fs::read_to_string(run_dir.join("progress.jsonl")).unwrap();
|
||||
assert!(
|
||||
progress.contains("\"event\":\"RunNotice\"")
|
||||
&& progress.contains("\"code\":\"interview_unanswered\""),
|
||||
"attach should emit a structured warning when the interview ends without an answer.\nprogress: {progress}"
|
||||
);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -8,6 +8,12 @@ use tokio::io::{AsyncBufReadExt, BufReader};
|
|||
|
||||
use crate::{Answer, AnswerValue, Interviewer, Question, QuestionOption, QuestionType};
|
||||
|
||||
enum PromptRead {
|
||||
Line(String),
|
||||
Eof,
|
||||
Error,
|
||||
}
|
||||
|
||||
/// Reads from stdin to collect answers. Displays formatted prompts per spec 6.4.
|
||||
pub struct ConsoleInterviewer {
|
||||
styles: &'static Styles,
|
||||
|
|
@ -48,14 +54,55 @@ fn find_matching_option(response: &str, options: &[QuestionOption]) -> Option<An
|
|||
None
|
||||
}
|
||||
|
||||
async fn read_line(prompt: &str) -> std::io::Result<String> {
|
||||
async fn read_line(prompt: &str) -> PromptRead {
|
||||
// Print the prompt to stderr so it doesn't interfere with piped stdout
|
||||
eprint!("{prompt}");
|
||||
let stdin = tokio::io::stdin();
|
||||
let mut reader = BufReader::new(stdin);
|
||||
let mut line = String::new();
|
||||
reader.read_line(&mut line).await?;
|
||||
Ok(line.trim_end().to_string())
|
||||
match reader.read_line(&mut line).await {
|
||||
Ok(0) => PromptRead::Eof,
|
||||
Ok(_) => PromptRead::Line(line.trim_end().to_string()),
|
||||
Err(_) => PromptRead::Error,
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_non_tty_choice_response(question: &Question, prompt_read: PromptRead) -> Answer {
|
||||
let PromptRead::Line(response) = prompt_read else {
|
||||
return Answer::aborted();
|
||||
};
|
||||
if response.trim().is_empty() {
|
||||
return Answer::aborted();
|
||||
}
|
||||
if let Some(answer) = find_matching_option(&response, &question.options) {
|
||||
return answer;
|
||||
}
|
||||
if question.allow_freeform {
|
||||
return Answer::text(response);
|
||||
}
|
||||
find_matching_option(&response, &question.options).unwrap_or_else(Answer::aborted)
|
||||
}
|
||||
|
||||
fn parse_non_tty_confirm_response(prompt_read: PromptRead) -> Answer {
|
||||
let PromptRead::Line(response) = prompt_read else {
|
||||
return Answer::aborted();
|
||||
};
|
||||
match response.trim().to_lowercase().as_str() {
|
||||
"y" | "yes" => Answer::yes(),
|
||||
"n" | "no" => Answer::no(),
|
||||
_ => Answer::aborted(),
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_non_tty_freeform_response(prompt_read: PromptRead) -> Answer {
|
||||
let PromptRead::Line(response) = prompt_read else {
|
||||
return Answer::aborted();
|
||||
};
|
||||
if response.trim().is_empty() {
|
||||
Answer::aborted()
|
||||
} else {
|
||||
Answer::text(response)
|
||||
}
|
||||
}
|
||||
|
||||
/// Ask a multiple-choice question using dialoguer's `Select` widget on a TTY.
|
||||
|
|
@ -84,7 +131,16 @@ fn ask_select_interactive(question: &Question) -> Answer {
|
|||
dialoguer::Input::<String>::with_theme(&ColorfulTheme::default())
|
||||
.with_prompt("Enter your response")
|
||||
.interact_on(&Term::stderr())
|
||||
.map_or_else(|_| Answer::skipped(), Answer::text)
|
||||
.map_or_else(
|
||||
|_| Answer::aborted(),
|
||||
|response| {
|
||||
if response.trim().is_empty() {
|
||||
Answer::aborted()
|
||||
} else {
|
||||
Answer::text(response)
|
||||
}
|
||||
},
|
||||
)
|
||||
}
|
||||
Ok(Some(idx)) if idx < question.options.len() => {
|
||||
let opt = &question.options[idx];
|
||||
|
|
@ -95,7 +151,7 @@ fn ask_select_interactive(question: &Question) -> Answer {
|
|||
text: None,
|
||||
}
|
||||
}
|
||||
_ => Answer::skipped(),
|
||||
_ => Answer::aborted(),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -124,7 +180,7 @@ fn ask_multi_select_interactive(question: &Question) -> Answer {
|
|||
.collect();
|
||||
Answer::multi_selected(keys, options)
|
||||
}
|
||||
_ => Answer::skipped(),
|
||||
_ => Answer::aborted(),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -137,7 +193,8 @@ fn ask_confirm_interactive(question: &Question) -> Answer {
|
|||
|
||||
match confirmed {
|
||||
Ok(Some(true)) => Answer::yes(),
|
||||
_ => Answer::no(),
|
||||
Ok(Some(false)) => Answer::no(),
|
||||
_ => Answer::aborted(),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -146,7 +203,16 @@ fn ask_freeform_interactive(question: &Question) -> Answer {
|
|||
dialoguer::Input::<String>::with_theme(&ColorfulTheme::default())
|
||||
.with_prompt(&question.text)
|
||||
.interact_on(&Term::stderr())
|
||||
.map_or_else(|_| Answer::skipped(), Answer::text)
|
||||
.map_or_else(
|
||||
|_| Answer::aborted(),
|
||||
|response| {
|
||||
if response.trim().is_empty() {
|
||||
Answer::aborted()
|
||||
} else {
|
||||
Answer::text(response)
|
||||
}
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
|
|
@ -167,7 +233,7 @@ impl Interviewer for ConsoleInterviewer {
|
|||
QuestionType::Freeform => ask_freeform_interactive(&q),
|
||||
})
|
||||
.await
|
||||
.unwrap_or_else(|_| Answer::skipped());
|
||||
.unwrap_or_else(|_| Answer::aborted());
|
||||
}
|
||||
|
||||
// Non-TTY fallback: line-based stdin reading
|
||||
|
|
@ -189,29 +255,12 @@ impl Interviewer for ConsoleInterviewer {
|
|||
if question.allow_freeform {
|
||||
eprintln!(" Or type a free-text response");
|
||||
}
|
||||
let response = read_line("Select: ").await.unwrap_or_default();
|
||||
if let Some(answer) = find_matching_option(&response, &question.options) {
|
||||
return answer;
|
||||
}
|
||||
if question.allow_freeform {
|
||||
return Answer::text(response);
|
||||
}
|
||||
// Fallback: try match again (spec says to do this)
|
||||
find_matching_option(&response, &question.options).unwrap_or_else(Answer::skipped)
|
||||
parse_non_tty_choice_response(&question, read_line("Select: ").await)
|
||||
}
|
||||
QuestionType::YesNo | QuestionType::Confirmation => {
|
||||
let response = read_line("[Y/N]: ").await.unwrap_or_default();
|
||||
let trimmed = response.trim().to_lowercase();
|
||||
if trimmed == "y" || trimmed == "yes" {
|
||||
Answer::yes()
|
||||
} else {
|
||||
Answer::no()
|
||||
}
|
||||
}
|
||||
QuestionType::Freeform => {
|
||||
let response = read_line("> ").await.unwrap_or_default();
|
||||
Answer::text(response)
|
||||
parse_non_tty_confirm_response(read_line("[Y/N]: ").await)
|
||||
}
|
||||
QuestionType::Freeform => parse_non_tty_freeform_response(read_line("> ").await),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -290,4 +339,28 @@ mod tests {
|
|||
let result = find_matching_option("5", &options);
|
||||
assert!(result.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_tty_multiple_choice_eof_returns_aborted() {
|
||||
let mut question = Question::new("Approve?", QuestionType::MultipleChoice);
|
||||
question.options = vec![crate::QuestionOption {
|
||||
key: "A".to_string(),
|
||||
label: "Approve".to_string(),
|
||||
}];
|
||||
|
||||
let answer = parse_non_tty_choice_response(&question, PromptRead::Eof);
|
||||
assert_eq!(answer.value, AnswerValue::Aborted);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_tty_confirmation_invalid_response_returns_aborted() {
|
||||
let answer = parse_non_tty_confirm_response(PromptRead::Line(String::new()));
|
||||
assert_eq!(answer.value, AnswerValue::Aborted);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_tty_freeform_blank_response_returns_aborted() {
|
||||
let answer = parse_non_tty_freeform_response(PromptRead::Line(" ".to_string()));
|
||||
assert_eq!(answer.value, AnswerValue::Aborted);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,9 +1,15 @@
|
|||
use std::path::PathBuf;
|
||||
use std::time::Duration;
|
||||
|
||||
use async_trait::async_trait;
|
||||
|
||||
use crate::{Answer, Interviewer, Question};
|
||||
|
||||
#[cfg(test)]
|
||||
const REATTACH_WINDOW: Duration = Duration::from_millis(300);
|
||||
#[cfg(not(test))]
|
||||
const REATTACH_WINDOW: Duration = Duration::from_secs(30);
|
||||
|
||||
/// An interviewer that communicates via JSON files in the run directory.
|
||||
///
|
||||
/// The engine process writes `interview_request.json` and polls for
|
||||
|
|
@ -25,6 +31,24 @@ impl FileInterviewer {
|
|||
fn response_path(&self) -> PathBuf {
|
||||
self.run_dir.join("interview_response.json")
|
||||
}
|
||||
|
||||
fn claim_path(&self) -> PathBuf {
|
||||
self.run_dir.join("interview_request.claim")
|
||||
}
|
||||
|
||||
async fn write_request_atomically(&self, question: &Question) -> std::io::Result<()> {
|
||||
let json = serde_json::to_string_pretty(question).expect("Question serialization failed");
|
||||
let request_path = self.request_path();
|
||||
let temp_path = request_path.with_extension("json.tmp");
|
||||
tokio::fs::write(&temp_path, json).await?;
|
||||
tokio::fs::rename(temp_path, request_path).await
|
||||
}
|
||||
|
||||
async fn cleanup_ipc_files(&self) {
|
||||
let _ = tokio::fs::remove_file(self.request_path()).await;
|
||||
let _ = tokio::fs::remove_file(self.response_path()).await;
|
||||
let _ = tokio::fs::remove_file(self.claim_path()).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
|
|
@ -34,23 +58,23 @@ impl Interviewer for FileInterviewer {
|
|||
let default_answer = question.default.clone();
|
||||
|
||||
// Write the request file
|
||||
let request_path = self.request_path();
|
||||
let json = serde_json::to_string_pretty(&question).expect("Question serialization failed");
|
||||
if let Err(e) = tokio::fs::write(&request_path, json).await {
|
||||
if let Err(e) = self.write_request_atomically(&question).await {
|
||||
tracing::warn!(error = %e, "Failed to write interview request");
|
||||
return default_answer.unwrap_or_else(Answer::timeout);
|
||||
}
|
||||
|
||||
// Poll for response with optional timeout
|
||||
let default_for_claim_timeout = default_answer.clone();
|
||||
let poll = async {
|
||||
let response_path = self.response_path();
|
||||
let claim_path = self.claim_path();
|
||||
let mut claim_was_seen = false;
|
||||
let mut reattach_deadline: Option<tokio::time::Instant> = None;
|
||||
loop {
|
||||
match tokio::fs::read_to_string(&response_path).await {
|
||||
Ok(data) => match serde_json::from_str::<Answer>(&data) {
|
||||
Ok(answer) => {
|
||||
// Clean up both files
|
||||
let _ = tokio::fs::remove_file(&request_path).await;
|
||||
let _ = tokio::fs::remove_file(&response_path).await;
|
||||
self.cleanup_ipc_files().await;
|
||||
return answer;
|
||||
}
|
||||
Err(e) => {
|
||||
|
|
@ -59,13 +83,29 @@ impl Interviewer for FileInterviewer {
|
|||
}
|
||||
},
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
|
||||
// Not written yet, poll again
|
||||
// Not written yet — check claim state below
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(error = %e, "Failed to read interview response, retrying");
|
||||
}
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
||||
|
||||
// Monitor claim file to detect attacher departure
|
||||
if claim_path.exists() {
|
||||
claim_was_seen = true;
|
||||
reattach_deadline = None;
|
||||
} else if claim_was_seen && reattach_deadline.is_none() {
|
||||
reattach_deadline = Some(tokio::time::Instant::now() + REATTACH_WINDOW);
|
||||
}
|
||||
|
||||
if let Some(deadline) = reattach_deadline {
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
self.cleanup_ipc_files().await;
|
||||
return default_for_claim_timeout.unwrap_or_else(Answer::timeout);
|
||||
}
|
||||
}
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
};
|
||||
|
||||
|
|
@ -74,8 +114,7 @@ impl Interviewer for FileInterviewer {
|
|||
match tokio::time::timeout(duration, poll).await {
|
||||
Ok(answer) => answer,
|
||||
Err(_) => {
|
||||
// Clean up request file on timeout
|
||||
let _ = tokio::fs::remove_file(&self.request_path()).await;
|
||||
self.cleanup_ipc_files().await;
|
||||
default_answer.unwrap_or_else(Answer::timeout)
|
||||
}
|
||||
}
|
||||
|
|
@ -135,6 +174,7 @@ mod tests {
|
|||
// Both files should be cleaned up
|
||||
assert!(!request_path.exists());
|
||||
assert!(!response_path.exists());
|
||||
assert!(!run_dir.join("interview_request.claim").exists());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
|
@ -150,6 +190,130 @@ mod tests {
|
|||
assert_eq!(answer.value, AnswerValue::No);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn claim_released_without_response_returns_timeout() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let run_dir = dir.path().to_path_buf();
|
||||
let interviewer = FileInterviewer::new(run_dir.clone());
|
||||
|
||||
let question = Question::new("approve?", QuestionType::YesNo);
|
||||
|
||||
let ask_handle = tokio::spawn(async move { interviewer.ask(question).await });
|
||||
|
||||
// Wait for request file to appear
|
||||
let request_path = run_dir.join("interview_request.json");
|
||||
for _ in 0..50 {
|
||||
if request_path.exists() {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
assert!(request_path.exists());
|
||||
|
||||
// Simulate attacher creating claim file
|
||||
let claim_path = run_dir.join("interview_request.claim");
|
||||
std::fs::write(&claim_path, "12345\n").unwrap();
|
||||
|
||||
// Let the poll loop see the claim
|
||||
tokio::time::sleep(Duration::from_millis(150)).await;
|
||||
|
||||
// Simulate attacher departing (deletes claim without writing response)
|
||||
std::fs::remove_file(&claim_path).unwrap();
|
||||
|
||||
// Should return timeout within REATTACH_WINDOW
|
||||
let started = tokio::time::Instant::now();
|
||||
let answer = tokio::time::timeout(Duration::from_secs(2), ask_handle)
|
||||
.await
|
||||
.expect("should complete within 2s")
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(answer.value, AnswerValue::Timeout);
|
||||
assert!(
|
||||
started.elapsed() <= Duration::from_secs(1),
|
||||
"should resolve well within the reattach window"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn claim_released_without_response_returns_default() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let run_dir = dir.path().to_path_buf();
|
||||
let interviewer = FileInterviewer::new(run_dir.clone());
|
||||
|
||||
let mut question = Question::new("approve?", QuestionType::YesNo);
|
||||
question.default = Some(Answer::no());
|
||||
|
||||
let ask_handle = tokio::spawn(async move { interviewer.ask(question).await });
|
||||
|
||||
// Wait for request file
|
||||
let request_path = run_dir.join("interview_request.json");
|
||||
for _ in 0..50 {
|
||||
if request_path.exists() {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
assert!(request_path.exists());
|
||||
|
||||
// Simulate attacher creating then deleting claim
|
||||
let claim_path = run_dir.join("interview_request.claim");
|
||||
std::fs::write(&claim_path, "12345\n").unwrap();
|
||||
tokio::time::sleep(Duration::from_millis(150)).await;
|
||||
std::fs::remove_file(&claim_path).unwrap();
|
||||
|
||||
let answer = tokio::time::timeout(Duration::from_secs(2), ask_handle)
|
||||
.await
|
||||
.expect("should complete within 2s")
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(answer.value, AnswerValue::No);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn claim_released_then_new_attacher_answers() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let run_dir = dir.path().to_path_buf();
|
||||
let interviewer = FileInterviewer::new(run_dir.clone());
|
||||
|
||||
let question = Question::new("approve?", QuestionType::YesNo);
|
||||
|
||||
let run_dir2 = run_dir.clone();
|
||||
let ask_handle = tokio::spawn(async move { interviewer.ask(question).await });
|
||||
|
||||
// Wait for request file
|
||||
let request_path = run_dir.join("interview_request.json");
|
||||
for _ in 0..50 {
|
||||
if request_path.exists() {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
assert!(request_path.exists());
|
||||
|
||||
// First attacher creates then releases claim
|
||||
let claim_path = run_dir.join("interview_request.claim");
|
||||
std::fs::write(&claim_path, "12345\n").unwrap();
|
||||
tokio::time::sleep(Duration::from_millis(150)).await;
|
||||
std::fs::remove_file(&claim_path).unwrap();
|
||||
|
||||
// Second attacher picks up and answers before reattach window expires
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
std::fs::write(&claim_path, "12346\n").unwrap();
|
||||
|
||||
let answer = Answer::yes();
|
||||
let response_json = serde_json::to_string_pretty(&answer).unwrap();
|
||||
tokio::fs::write(run_dir2.join("interview_response.json"), response_json)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let result = tokio::time::timeout(Duration::from_secs(2), ask_handle)
|
||||
.await
|
||||
.expect("should complete within 2s")
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result.value, AnswerValue::Yes);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn timeout_without_default_returns_timeout() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
|
|
|
|||
|
|
@ -77,6 +77,7 @@ impl Question {
|
|||
pub enum AnswerValue {
|
||||
Yes,
|
||||
No,
|
||||
Aborted,
|
||||
Skipped,
|
||||
Timeout,
|
||||
Selected(String),
|
||||
|
|
@ -115,6 +116,16 @@ impl Answer {
|
|||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn aborted() -> Self {
|
||||
Self {
|
||||
value: AnswerValue::Aborted,
|
||||
selected_option: None,
|
||||
selected_options: Vec::new(),
|
||||
text: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn skipped() -> Self {
|
||||
Self {
|
||||
|
|
@ -256,6 +267,12 @@ mod tests {
|
|||
assert_eq!(a.value, AnswerValue::Skipped);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn answer_aborted() {
|
||||
let a = Answer::aborted();
|
||||
assert_eq!(a.value, AnswerValue::Aborted);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn answer_timeout() {
|
||||
let a = Answer::timeout();
|
||||
|
|
@ -297,6 +314,7 @@ mod tests {
|
|||
fn answer_value_variants() {
|
||||
assert_ne!(AnswerValue::Yes, AnswerValue::No);
|
||||
assert_ne!(AnswerValue::Skipped, AnswerValue::Timeout);
|
||||
assert_ne!(AnswerValue::Aborted, AnswerValue::Timeout);
|
||||
assert_eq!(
|
||||
AnswerValue::Selected("x".to_string()),
|
||||
AnswerValue::Selected("x".to_string())
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ use async_trait::async_trait;
|
|||
|
||||
use crate::{Answer, Interviewer, Question};
|
||||
|
||||
/// Reads answers from a pre-filled queue. Returns Skipped when empty.
|
||||
/// Reads answers from a pre-filled queue. Returns Aborted when empty.
|
||||
pub struct QueueInterviewer {
|
||||
answers: Mutex<VecDeque<Answer>>,
|
||||
}
|
||||
|
|
@ -23,7 +23,7 @@ impl QueueInterviewer {
|
|||
impl Interviewer for QueueInterviewer {
|
||||
async fn ask(&self, _question: Question) -> Answer {
|
||||
let mut queue = self.answers.lock().expect("queue lock poisoned");
|
||||
queue.pop_front().unwrap_or_else(Answer::skipped)
|
||||
queue.pop_front().unwrap_or_else(Answer::aborted)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -46,21 +46,21 @@ mod tests {
|
|||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn returns_skipped_when_empty() {
|
||||
async fn returns_aborted_when_empty() {
|
||||
let interviewer = QueueInterviewer::new(VecDeque::new());
|
||||
let q = Question::new("q", QuestionType::YesNo);
|
||||
let answer = interviewer.ask(q).await;
|
||||
assert_eq!(answer.value, AnswerValue::Skipped);
|
||||
assert_eq!(answer.value, AnswerValue::Aborted);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn returns_skipped_after_exhausted() {
|
||||
async fn returns_aborted_after_exhausted() {
|
||||
let answers = VecDeque::from([Answer::yes()]);
|
||||
let interviewer = QueueInterviewer::new(answers);
|
||||
let q = Question::new("q", QuestionType::YesNo);
|
||||
|
||||
let _ = interviewer.ask(q.clone()).await;
|
||||
let answer = interviewer.ask(q).await;
|
||||
assert_eq!(answer.value, AnswerValue::Skipped);
|
||||
assert_eq!(answer.value, AnswerValue::Aborted);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ use async_trait::async_trait;
|
|||
use crate::{Answer, Interviewer, Question};
|
||||
|
||||
/// Replays recorded answers in sequence. When recordings are exhausted,
|
||||
/// returns `Answer::skipped()`.
|
||||
/// returns `Answer::aborted()`.
|
||||
pub struct ReplayInterviewer {
|
||||
answers: Mutex<Vec<Answer>>,
|
||||
}
|
||||
|
|
@ -27,7 +27,7 @@ impl Interviewer for ReplayInterviewer {
|
|||
async fn ask(&self, _question: Question) -> Answer {
|
||||
let mut answers = self.answers.lock().expect("answers lock poisoned");
|
||||
if answers.is_empty() {
|
||||
Answer::skipped()
|
||||
Answer::aborted()
|
||||
} else {
|
||||
answers.remove(0)
|
||||
}
|
||||
|
|
@ -66,7 +66,7 @@ mod tests {
|
|||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn returns_skipped_when_exhausted() {
|
||||
async fn returns_aborted_when_exhausted() {
|
||||
let recordings = vec![(
|
||||
Question::new("approve?", QuestionType::YesNo),
|
||||
Answer::yes(),
|
||||
|
|
@ -82,11 +82,11 @@ mod tests {
|
|||
let a2 = replayer
|
||||
.ask(Question::new("second", QuestionType::YesNo))
|
||||
.await;
|
||||
assert_eq!(a2.value, AnswerValue::Skipped);
|
||||
assert_eq!(a2.value, AnswerValue::Aborted);
|
||||
|
||||
let a3 = replayer
|
||||
.ask(Question::new("third", QuestionType::YesNo))
|
||||
.await;
|
||||
assert_eq!(a3.value, AnswerValue::Skipped);
|
||||
assert_eq!(a3.value, AnswerValue::Aborted);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -102,7 +102,7 @@ impl Interviewer for WebInterviewer {
|
|||
}
|
||||
|
||||
// Block until answer arrives or sender is dropped
|
||||
rx.await.unwrap_or_else(|_| Answer::skipped())
|
||||
rx.await.unwrap_or_else(|_| Answer::aborted())
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -253,4 +253,35 @@ mod tests {
|
|||
|
||||
assert!(interviewer.pending_questions().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ask_returns_aborted_when_pending_sender_is_dropped() {
|
||||
let interviewer = Arc::new(WebInterviewer::new());
|
||||
let interviewer_clone = Arc::clone(&interviewer);
|
||||
|
||||
let ask_handle = tokio::spawn(async move {
|
||||
let q = Question::new("approve?", QuestionType::YesNo);
|
||||
interviewer_clone.ask(q).await
|
||||
});
|
||||
|
||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||
|
||||
{
|
||||
let mut inner = interviewer
|
||||
.inner
|
||||
.lock()
|
||||
.expect("web interviewer lock poisoned");
|
||||
let pending_id = inner
|
||||
.questions
|
||||
.first()
|
||||
.expect("question should be pending")
|
||||
.id
|
||||
.clone();
|
||||
inner.pending.remove(&pending_id);
|
||||
inner.questions.retain(|pq| pq.id != pending_id);
|
||||
}
|
||||
|
||||
let answer = ask_handle.await.expect("task should complete");
|
||||
assert_eq!(answer.value, AnswerValue::Aborted);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -77,6 +77,7 @@ fn format_answer(answer: &Answer) -> String {
|
|||
match &answer.value {
|
||||
AnswerValue::Yes => "Yes".to_string(),
|
||||
AnswerValue::No => "No".to_string(),
|
||||
AnswerValue::Aborted => "Aborted".to_string(),
|
||||
AnswerValue::Text(t) => t.clone(),
|
||||
AnswerValue::Selected(k) => {
|
||||
if let Some(opt) = &answer.selected_option {
|
||||
|
|
|
|||
|
|
@ -31,6 +31,18 @@ pub struct Conclusion {
|
|||
pub total_cost: Option<f64>,
|
||||
#[serde(default)]
|
||||
pub total_retries: u32,
|
||||
#[serde(default)]
|
||||
pub total_input_tokens: i64,
|
||||
#[serde(default)]
|
||||
pub total_output_tokens: i64,
|
||||
#[serde(default)]
|
||||
pub total_cache_read_tokens: i64,
|
||||
#[serde(default)]
|
||||
pub total_cache_write_tokens: i64,
|
||||
#[serde(default)]
|
||||
pub total_reasoning_tokens: i64,
|
||||
#[serde(default)]
|
||||
pub has_pricing: bool,
|
||||
}
|
||||
|
||||
impl Conclusion {
|
||||
|
|
@ -72,6 +84,12 @@ mod tests {
|
|||
],
|
||||
total_cost: Some(0.15),
|
||||
total_retries: 1,
|
||||
total_input_tokens: 5000,
|
||||
total_output_tokens: 1500,
|
||||
total_cache_read_tokens: 2000,
|
||||
total_cache_write_tokens: 500,
|
||||
total_reasoning_tokens: 300,
|
||||
has_pricing: true,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -127,6 +145,12 @@ mod tests {
|
|||
stages: vec![],
|
||||
total_cost: None,
|
||||
total_retries: 0,
|
||||
total_input_tokens: 0,
|
||||
total_output_tokens: 0,
|
||||
total_cache_read_tokens: 0,
|
||||
total_cache_write_tokens: 0,
|
||||
total_reasoning_tokens: 0,
|
||||
has_pricing: false,
|
||||
};
|
||||
conclusion.save(&path).unwrap();
|
||||
|
||||
|
|
@ -152,6 +176,12 @@ mod tests {
|
|||
stages: vec![],
|
||||
total_cost: None,
|
||||
total_retries: 0,
|
||||
total_input_tokens: 0,
|
||||
total_output_tokens: 0,
|
||||
total_cache_read_tokens: 0,
|
||||
total_cache_write_tokens: 0,
|
||||
total_reasoning_tokens: 0,
|
||||
has_pricing: false,
|
||||
};
|
||||
conclusion.save(&path).unwrap();
|
||||
let loaded = Conclusion::load(&path).unwrap();
|
||||
|
|
|
|||
|
|
@ -446,13 +446,21 @@ pub struct EdgeSelection<'a> {
|
|||
pub reason: &'static str,
|
||||
}
|
||||
|
||||
fn blocks_unconditional_failure_fallthrough(node: &Node, outcome: &Outcome) -> bool {
|
||||
node.handler_type() == Some("human")
|
||||
&& outcome.status == StageStatus::Fail
|
||||
&& outcome.preferred_label.is_none()
|
||||
&& outcome.suggested_next_ids.is_empty()
|
||||
}
|
||||
|
||||
pub fn select_edge<'a>(
|
||||
node_id: &str,
|
||||
node: &Node,
|
||||
outcome: &Outcome,
|
||||
context: &Context,
|
||||
graph: &'a Graph,
|
||||
selection: &str,
|
||||
) -> Option<EdgeSelection<'a>> {
|
||||
let node_id = &node.id;
|
||||
let edges = graph.outgoing_edges(node_id);
|
||||
if edges.is_empty() {
|
||||
return None;
|
||||
|
|
@ -501,6 +509,10 @@ pub fn select_edge<'a>(
|
|||
}
|
||||
}
|
||||
|
||||
if blocks_unconditional_failure_fallthrough(node, outcome) {
|
||||
return None;
|
||||
}
|
||||
|
||||
// Step 4 & 5: Weight with lexical tiebreak (unconditional edges only)
|
||||
let unconditional: Vec<&Edge> = edges
|
||||
.iter()
|
||||
|
|
@ -2026,7 +2038,7 @@ impl WorkflowRunEngine {
|
|||
});
|
||||
(None, Some(target.clone()))
|
||||
} else {
|
||||
let selection = select_edge(&node.id, &outcome, &context, graph, node.selection());
|
||||
let selection = select_edge(node, &outcome, &context, graph, node.selection());
|
||||
if let Some(sel) = &selection {
|
||||
self.services.emitter.emit(&WorkflowRunEvent::EdgeSelected {
|
||||
from_node: node.id.clone(),
|
||||
|
|
@ -2840,17 +2852,19 @@ mod tests {
|
|||
#[test]
|
||||
fn select_edge_no_edges() {
|
||||
let g = Graph::new("test");
|
||||
let node = Node::new("a");
|
||||
let outcome = Outcome::success();
|
||||
let context = Context::new();
|
||||
assert!(select_edge("a", &outcome, &context, &g, "deterministic").is_none());
|
||||
assert!(select_edge(&node, &outcome, &context, &g, "deterministic").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn select_edge_single_unconditional() {
|
||||
let g = make_graph_with_edges(vec![Edge::new("a", "b")]);
|
||||
let node = g.nodes.get("a").unwrap();
|
||||
let outcome = Outcome::success();
|
||||
let context = Context::new();
|
||||
let sel = select_edge("a", &outcome, &context, &g, "deterministic").unwrap();
|
||||
let sel = select_edge(node, &outcome, &context, &g, "deterministic").unwrap();
|
||||
assert_eq!(sel.edge.to, "b");
|
||||
assert_eq!(sel.reason, "unconditional");
|
||||
}
|
||||
|
|
@ -2868,9 +2882,10 @@ mod tests {
|
|||
AttrValue::String("outcome=success".to_string()),
|
||||
);
|
||||
let g = make_graph_with_edges(vec![e1, e2]);
|
||||
let node = g.nodes.get("a").unwrap();
|
||||
let outcome = Outcome::success();
|
||||
let context = Context::new();
|
||||
let sel = select_edge("a", &outcome, &context, &g, "deterministic").unwrap();
|
||||
let sel = select_edge(node, &outcome, &context, &g, "deterministic").unwrap();
|
||||
assert_eq!(sel.edge.to, "success_path");
|
||||
assert_eq!(sel.reason, "condition");
|
||||
}
|
||||
|
|
@ -2888,10 +2903,11 @@ mod tests {
|
|||
AttrValue::String("[F] Fix".to_string()),
|
||||
);
|
||||
let g = make_graph_with_edges(vec![e1, e2]);
|
||||
let node = g.nodes.get("a").unwrap();
|
||||
let mut outcome = Outcome::success();
|
||||
outcome.preferred_label = Some("Fix".to_string());
|
||||
let context = Context::new();
|
||||
let sel = select_edge("a", &outcome, &context, &g, "deterministic").unwrap();
|
||||
let sel = select_edge(node, &outcome, &context, &g, "deterministic").unwrap();
|
||||
assert_eq!(sel.edge.to, "fix");
|
||||
assert_eq!(sel.reason, "preferred_label");
|
||||
}
|
||||
|
|
@ -2901,10 +2917,11 @@ mod tests {
|
|||
let e1 = Edge::new("a", "path1");
|
||||
let e2 = Edge::new("a", "path2");
|
||||
let g = make_graph_with_edges(vec![e1, e2]);
|
||||
let node = g.nodes.get("a").unwrap();
|
||||
let mut outcome = Outcome::success();
|
||||
outcome.suggested_next_ids = vec!["path2".to_string()];
|
||||
let context = Context::new();
|
||||
let sel = select_edge("a", &outcome, &context, &g, "deterministic").unwrap();
|
||||
let sel = select_edge(node, &outcome, &context, &g, "deterministic").unwrap();
|
||||
assert_eq!(sel.edge.to, "path2");
|
||||
assert_eq!(sel.reason, "suggested_next");
|
||||
}
|
||||
|
|
@ -2917,9 +2934,10 @@ mod tests {
|
|||
e2.attrs
|
||||
.insert("weight".to_string(), AttrValue::Integer(10));
|
||||
let g = make_graph_with_edges(vec![e1, e2]);
|
||||
let node = g.nodes.get("a").unwrap();
|
||||
let outcome = Outcome::success();
|
||||
let context = Context::new();
|
||||
let sel = select_edge("a", &outcome, &context, &g, "deterministic").unwrap();
|
||||
let sel = select_edge(node, &outcome, &context, &g, "deterministic").unwrap();
|
||||
assert_eq!(sel.edge.to, "high");
|
||||
assert_eq!(sel.reason, "unconditional");
|
||||
}
|
||||
|
|
@ -2929,9 +2947,10 @@ mod tests {
|
|||
let e1 = Edge::new("a", "charlie");
|
||||
let e2 = Edge::new("a", "alpha");
|
||||
let g = make_graph_with_edges(vec![e1, e2]);
|
||||
let node = g.nodes.get("a").unwrap();
|
||||
let outcome = Outcome::success();
|
||||
let context = Context::new();
|
||||
let sel = select_edge("a", &outcome, &context, &g, "deterministic").unwrap();
|
||||
let sel = select_edge(node, &outcome, &context, &g, "deterministic").unwrap();
|
||||
assert_eq!(sel.edge.to, "alpha");
|
||||
assert_eq!(sel.reason, "unconditional");
|
||||
}
|
||||
|
|
@ -2945,9 +2964,10 @@ mod tests {
|
|||
);
|
||||
let e_uncond = Edge::new("a", "uncond_path");
|
||||
let g = make_graph_with_edges(vec![e_cond, e_uncond]);
|
||||
let node = g.nodes.get("a").unwrap();
|
||||
let outcome = Outcome::success();
|
||||
let context = Context::new();
|
||||
let sel = select_edge("a", &outcome, &context, &g, "deterministic").unwrap();
|
||||
let sel = select_edge(node, &outcome, &context, &g, "deterministic").unwrap();
|
||||
assert_eq!(sel.edge.to, "cond_path");
|
||||
assert_eq!(sel.reason, "condition");
|
||||
}
|
||||
|
|
@ -2957,9 +2977,10 @@ mod tests {
|
|||
let e1 = Edge::new("a", "b");
|
||||
let e2 = Edge::new("a", "c");
|
||||
let g = make_graph_with_edges(vec![e1, e2]);
|
||||
let node = g.nodes.get("a").unwrap();
|
||||
let outcome = Outcome::success();
|
||||
let context = Context::new();
|
||||
let sel = select_edge("a", &outcome, &context, &g, "random").unwrap();
|
||||
let sel = select_edge(node, &outcome, &context, &g, "random").unwrap();
|
||||
assert!(sel.edge.to == "b" || sel.edge.to == "c");
|
||||
assert_eq!(sel.reason, "unconditional");
|
||||
}
|
||||
|
|
@ -2973,14 +2994,56 @@ mod tests {
|
|||
);
|
||||
let e2 = Edge::new("a", "other");
|
||||
let g = make_graph_with_edges(vec![e1, e2]);
|
||||
let node = g.nodes.get("a").unwrap();
|
||||
let mut outcome = Outcome::success();
|
||||
outcome.preferred_label = Some("Approve".to_string());
|
||||
let context = Context::new();
|
||||
let sel = select_edge("a", &outcome, &context, &g, "random").unwrap();
|
||||
let sel = select_edge(node, &outcome, &context, &g, "random").unwrap();
|
||||
assert_eq!(sel.edge.to, "approve");
|
||||
assert_eq!(sel.reason, "preferred_label");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn select_edge_failed_human_gate_does_not_fall_through_to_unconditional() {
|
||||
let g = make_graph_with_edges(vec![
|
||||
Edge::new("gate", "approve"),
|
||||
Edge::new("gate", "skip"),
|
||||
]);
|
||||
let mut node = g.nodes.get("gate").unwrap().clone();
|
||||
node.attrs.insert(
|
||||
"shape".to_string(),
|
||||
AttrValue::String("hexagon".to_string()),
|
||||
);
|
||||
let outcome =
|
||||
Outcome::fail_deterministic("human interaction aborted before an answer was provided");
|
||||
let context = Context::new();
|
||||
|
||||
assert!(select_edge(&node, &outcome, &context, &g, "deterministic").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn select_edge_failed_human_gate_routes_via_fail_condition() {
|
||||
let mut fail = Edge::new("gate", "retry");
|
||||
fail.attrs.insert(
|
||||
"condition".to_string(),
|
||||
AttrValue::String("outcome=fail".to_string()),
|
||||
);
|
||||
let approve = Edge::new("gate", "approve");
|
||||
let g = make_graph_with_edges(vec![fail, approve]);
|
||||
let mut node = g.nodes.get("gate").unwrap().clone();
|
||||
node.attrs.insert(
|
||||
"shape".to_string(),
|
||||
AttrValue::String("hexagon".to_string()),
|
||||
);
|
||||
let outcome =
|
||||
Outcome::fail_deterministic("human interaction aborted before an answer was provided");
|
||||
let context = Context::new();
|
||||
|
||||
let sel = select_edge(&node, &outcome, &context, &g, "deterministic").unwrap();
|
||||
assert_eq!(sel.edge.to, "retry");
|
||||
assert_eq!(sel.reason, "condition");
|
||||
}
|
||||
|
||||
// --- check_goal_gates tests ---
|
||||
|
||||
#[test]
|
||||
|
|
@ -5740,9 +5803,13 @@ mod tests {
|
|||
"type".to_string(),
|
||||
AttrValue::String("fail_once".to_string()),
|
||||
);
|
||||
// Allow 1 retry → 2 attempts total
|
||||
// Allow 1 retry → 2 attempts total, use aggressive backoff (500ms) for fast tests
|
||||
work.attrs
|
||||
.insert("max_retries".to_string(), AttrValue::Integer(1));
|
||||
work.attrs.insert(
|
||||
"retry_policy".to_string(),
|
||||
AttrValue::String("aggressive".to_string()),
|
||||
);
|
||||
g.nodes.insert("work".to_string(), work);
|
||||
|
||||
let mut exit = Node::new("exit");
|
||||
|
|
|
|||
|
|
@ -6,6 +6,14 @@ use serde::{Deserialize, Serialize};
|
|||
use crate::outcome::StageUsage;
|
||||
use fabro_agent::{AgentEvent, SandboxEvent, WorktreeEvent, WorktreeEventCallback};
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum RunNoticeLevel {
|
||||
Info,
|
||||
Warn,
|
||||
Error,
|
||||
}
|
||||
|
||||
/// Events emitted during workflow run execution for observability.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub enum WorkflowRunEvent {
|
||||
|
|
@ -39,6 +47,11 @@ pub enum WorkflowRunEvent {
|
|||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
git_commit_sha: Option<String>,
|
||||
},
|
||||
RunNotice {
|
||||
level: RunNoticeLevel,
|
||||
code: String,
|
||||
message: String,
|
||||
},
|
||||
StageStarted {
|
||||
node_id: String,
|
||||
name: String,
|
||||
|
|
@ -340,6 +353,21 @@ impl WorkflowRunEvent {
|
|||
} => {
|
||||
error!(error = %error, duration_ms, "Workflow run failed");
|
||||
}
|
||||
Self::RunNotice {
|
||||
level,
|
||||
code,
|
||||
message,
|
||||
} => match level {
|
||||
RunNoticeLevel::Info => {
|
||||
info!(code, message, "Run notice");
|
||||
}
|
||||
RunNoticeLevel::Warn => {
|
||||
warn!(code, message, "Run notice");
|
||||
}
|
||||
RunNoticeLevel::Error => {
|
||||
error!(code, message, "Run notice");
|
||||
}
|
||||
},
|
||||
Self::StageStarted {
|
||||
node_id,
|
||||
name,
|
||||
|
|
@ -2284,6 +2312,44 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn run_notice_event_serialization() {
|
||||
let event = WorkflowRunEvent::RunNotice {
|
||||
level: RunNoticeLevel::Warn,
|
||||
code: "sandbox_cleanup_failed".to_string(),
|
||||
message: "sandbox cleanup failed: boom".to_string(),
|
||||
};
|
||||
let json = serde_json::to_string(&event).unwrap();
|
||||
assert!(json.contains("RunNotice"));
|
||||
assert!(json.contains("\"level\":\"warn\""));
|
||||
|
||||
let deserialized: WorkflowRunEvent = serde_json::from_str(&json).unwrap();
|
||||
assert!(matches!(
|
||||
deserialized,
|
||||
WorkflowRunEvent::RunNotice {
|
||||
level: RunNoticeLevel::Warn,
|
||||
code,
|
||||
..
|
||||
} if code == "sandbox_cleanup_failed"
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flatten_event_run_notice() {
|
||||
let event = WorkflowRunEvent::RunNotice {
|
||||
level: RunNoticeLevel::Error,
|
||||
code: "bootstrap_failed".to_string(),
|
||||
message: "working directory missing".to_string(),
|
||||
};
|
||||
let (name, fields) = flatten_event(&event);
|
||||
assert_eq!(name, "RunNotice");
|
||||
assert_eq!(fields.get("level").and_then(|v| v.as_str()), Some("error"));
|
||||
assert_eq!(
|
||||
fields.get("code").and_then(|v| v.as_str()),
|
||||
Some("bootstrap_failed")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn workflow_run_completed_serialization_with_status_and_usage() {
|
||||
let event = WorkflowRunEvent::WorkflowRunCompleted {
|
||||
|
|
|
|||
|
|
@ -232,9 +232,14 @@ impl Handler for HumanHandler {
|
|||
return Ok(Outcome::retry_classify("human gate timeout, no default"));
|
||||
}
|
||||
|
||||
// 5. Handle skipped
|
||||
// 5. Handle unanswered / aborted interview sessions.
|
||||
if answer.value == AnswerValue::Aborted {
|
||||
return Ok(unanswered_human_gate(
|
||||
"human interaction aborted before an answer was provided",
|
||||
));
|
||||
}
|
||||
if answer.value == AnswerValue::Skipped {
|
||||
return Ok(Outcome::fail_deterministic("human skipped interaction"));
|
||||
return Ok(unanswered_human_gate("human skipped interaction"));
|
||||
}
|
||||
|
||||
// Emit interview completed for successful interactions
|
||||
|
|
@ -294,6 +299,10 @@ fn make_choice_outcome(key: &str, label: &str, to: &str) -> Outcome {
|
|||
outcome
|
||||
}
|
||||
|
||||
fn unanswered_human_gate(reason: impl Into<String>) -> Outcome {
|
||||
Outcome::fail_deterministic(reason)
|
||||
}
|
||||
|
||||
fn find_choice_match<'a>(answer: &Answer, choices: &'a [Choice]) -> Option<&'a Choice> {
|
||||
match &answer.value {
|
||||
AnswerValue::Selected(key) => choices.iter().find(|c| c.key == *key),
|
||||
|
|
@ -317,6 +326,7 @@ fn answer_text(answer: &Answer) -> String {
|
|||
AnswerValue::MultiSelected(keys) => keys.join(", "),
|
||||
AnswerValue::Yes => "yes".to_string(),
|
||||
AnswerValue::No => "no".to_string(),
|
||||
AnswerValue::Aborted => "aborted".to_string(),
|
||||
AnswerValue::Skipped => "skipped".to_string(),
|
||||
AnswerValue::Timeout => "timeout".to_string(),
|
||||
}
|
||||
|
|
@ -326,7 +336,7 @@ fn answer_text(answer: &Answer) -> String {
|
|||
mod tests {
|
||||
use super::*;
|
||||
use fabro_graphviz::graph::{AttrValue, Edge};
|
||||
use fabro_interview::{AutoApproveInterviewer, RecordingInterviewer};
|
||||
use fabro_interview::{AutoApproveInterviewer, CallbackInterviewer, RecordingInterviewer};
|
||||
|
||||
fn make_services() -> EngineServices {
|
||||
EngineServices::test_default()
|
||||
|
|
@ -432,6 +442,49 @@ mod tests {
|
|||
assert_eq!(outcome.status, crate::outcome::StageStatus::Fail);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn wait_human_aborted_returns_fail_without_routing_hints() {
|
||||
let interviewer = Arc::new(CallbackInterviewer::new(|_| Answer::aborted()));
|
||||
let handler = HumanHandler::new(interviewer);
|
||||
let graph = build_graph_with_human_gate();
|
||||
let node = graph.nodes.get("gate").unwrap();
|
||||
let context = Context::new();
|
||||
let run_dir = Path::new("/tmp/test");
|
||||
|
||||
let outcome = handler
|
||||
.execute(node, &context, &graph, run_dir, &make_services())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(outcome.status, crate::outcome::StageStatus::Fail);
|
||||
assert!(outcome.preferred_label.is_none());
|
||||
assert!(outcome.suggested_next_ids.is_empty());
|
||||
assert_eq!(
|
||||
outcome.failure_reason(),
|
||||
Some("human interaction aborted before an answer was provided")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn wait_human_skipped_returns_fail_without_routing_hints() {
|
||||
let interviewer = Arc::new(CallbackInterviewer::new(|_| Answer::skipped()));
|
||||
let handler = HumanHandler::new(interviewer);
|
||||
let graph = build_graph_with_human_gate();
|
||||
let node = graph.nodes.get("gate").unwrap();
|
||||
let context = Context::new();
|
||||
let run_dir = Path::new("/tmp/test");
|
||||
|
||||
let outcome = handler
|
||||
.execute(node, &context, &graph, run_dir, &make_services())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(outcome.status, crate::outcome::StageStatus::Fail);
|
||||
assert!(outcome.preferred_label.is_none());
|
||||
assert!(outcome.suggested_next_ids.is_empty());
|
||||
assert_eq!(outcome.failure_reason(), Some("human skipped interaction"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn wait_human_with_freeform_edge() {
|
||||
let interviewer = Arc::new(fabro_interview::CallbackInterviewer::new(|_| {
|
||||
|
|
|
|||
|
|
@ -458,6 +458,12 @@ mod tests {
|
|||
],
|
||||
total_cost: Some(0.42),
|
||||
total_retries: 0,
|
||||
total_input_tokens: 0,
|
||||
total_output_tokens: 0,
|
||||
total_cache_read_tokens: 0,
|
||||
total_cache_write_tokens: 0,
|
||||
total_reasoning_tokens: 0,
|
||||
has_pricing: true,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -141,6 +141,8 @@ pub enum StatusReason {
|
|||
Terminated,
|
||||
TransientInfra,
|
||||
BudgetExhausted,
|
||||
LaunchFailed,
|
||||
BootstrapFailed,
|
||||
SandboxInitFailed,
|
||||
// Non-terminal reasons
|
||||
SandboxInitializing,
|
||||
|
|
@ -309,6 +311,22 @@ mod tests {
|
|||
assert_eq!(loaded.reason, Some(StatusReason::Completed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn launch_failed_reason_roundtrip() {
|
||||
let record = RunStatusRecord::new(RunStatus::Failed, Some(StatusReason::LaunchFailed));
|
||||
let json = serde_json::to_string(&record).unwrap();
|
||||
let parsed: RunStatusRecord = serde_json::from_str(&json).unwrap();
|
||||
assert_eq!(parsed.reason, Some(StatusReason::LaunchFailed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bootstrap_failed_reason_roundtrip() {
|
||||
let record = RunStatusRecord::new(RunStatus::Failed, Some(StatusReason::BootstrapFailed));
|
||||
let json = serde_json::to_string(&record).unwrap();
|
||||
let parsed: RunStatusRecord = serde_json::from_str(&json).unwrap();
|
||||
assert_eq!(parsed.reason, Some(StatusReason::BootstrapFailed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn load_missing_file() {
|
||||
let result = RunStatusRecord::load(Path::new("/nonexistent/status.json"));
|
||||
|
|
|
|||
|
|
@ -7,8 +7,8 @@ use fabro_config::run::WorkflowRunConfig;
|
|||
use fabro_graphviz::graph::{AttrValue, Edge, Graph, Node};
|
||||
use fabro_graphviz::parser::parse;
|
||||
use fabro_interview::{
|
||||
Answer, AnswerValue, AutoApproveInterviewer, Interviewer, QueueInterviewer,
|
||||
RecordingInterviewer,
|
||||
Answer, AnswerValue, AutoApproveInterviewer, CallbackInterviewer, Interviewer,
|
||||
QueueInterviewer, RecordingInterviewer,
|
||||
};
|
||||
use fabro_llm::provider::Provider;
|
||||
use fabro_validate::{validate, validate_or_raise, Severity};
|
||||
|
|
@ -501,6 +501,221 @@ async fn end_to_end_human_gate_pipeline() {
|
|||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn human_gate_aborted_input_fails_closed_without_fail_route() {
|
||||
let mut graph = Graph::new("HumanGateAbortedClosed");
|
||||
|
||||
let mut start = Node::new("start");
|
||||
start.attrs.insert(
|
||||
"shape".to_string(),
|
||||
AttrValue::String("Mdiamond".to_string()),
|
||||
);
|
||||
graph.nodes.insert("start".to_string(), start);
|
||||
|
||||
let mut exit = Node::new("exit");
|
||||
exit.attrs.insert(
|
||||
"shape".to_string(),
|
||||
AttrValue::String("Msquare".to_string()),
|
||||
);
|
||||
graph.nodes.insert("exit".to_string(), exit);
|
||||
|
||||
let mut gate = Node::new("gate");
|
||||
gate.attrs.insert(
|
||||
"shape".to_string(),
|
||||
AttrValue::String("hexagon".to_string()),
|
||||
);
|
||||
gate.attrs
|
||||
.insert("type".to_string(), AttrValue::String("human".to_string()));
|
||||
gate.attrs.insert(
|
||||
"label".to_string(),
|
||||
AttrValue::String("Approve release?".to_string()),
|
||||
);
|
||||
graph.nodes.insert("gate".to_string(), gate);
|
||||
graph
|
||||
.nodes
|
||||
.insert("approve".to_string(), Node::new("approve"));
|
||||
graph
|
||||
.nodes
|
||||
.insert("revise".to_string(), Node::new("revise"));
|
||||
|
||||
graph.edges.push(Edge::new("start", "gate"));
|
||||
|
||||
let mut approve_edge = Edge::new("gate", "approve");
|
||||
approve_edge.attrs.insert(
|
||||
"label".to_string(),
|
||||
AttrValue::String("[A] Approve".to_string()),
|
||||
);
|
||||
graph.edges.push(approve_edge);
|
||||
|
||||
let mut revise_edge = Edge::new("gate", "revise");
|
||||
revise_edge.attrs.insert(
|
||||
"label".to_string(),
|
||||
AttrValue::String("[R] Revise".to_string()),
|
||||
);
|
||||
graph.edges.push(revise_edge);
|
||||
|
||||
graph.edges.push(Edge::new("approve", "exit"));
|
||||
graph.edges.push(Edge::new("revise", "exit"));
|
||||
|
||||
let interviewer = Arc::new(CallbackInterviewer::new(|_| Answer::aborted()));
|
||||
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
|
||||
registry.register("start", Box::new(StartHandler));
|
||||
registry.register("exit", Box::new(ExitHandler));
|
||||
registry.register("human", Box::new(HumanHandler::new(interviewer)));
|
||||
|
||||
let engine = WorkflowRunEngine::new(registry, Arc::new(EventEmitter::new()), local_env());
|
||||
let config = RunConfig {
|
||||
run_dir: dir.path().to_path_buf(),
|
||||
cancel_token: None,
|
||||
dry_run: false,
|
||||
run_id: "test-run".into(),
|
||||
git_checkpoint_enabled: false,
|
||||
host_repo_path: None,
|
||||
base_sha: None,
|
||||
run_branch: None,
|
||||
meta_branch: None,
|
||||
labels: std::collections::HashMap::new(),
|
||||
checkpoint_exclude_globs: Vec::new(),
|
||||
github_app: None,
|
||||
git_author: fabro_workflows::git::GitAuthor::default(),
|
||||
base_branch: None,
|
||||
pull_request: None,
|
||||
asset_globs: Vec::new(),
|
||||
workflow_slug: None,
|
||||
};
|
||||
|
||||
let error = engine
|
||||
.run(&graph, &config)
|
||||
.await
|
||||
.expect_err("aborted human gate should fail closed");
|
||||
assert!(
|
||||
error
|
||||
.to_string()
|
||||
.contains("stage gate failed with no outgoing fail edge"),
|
||||
"unexpected error: {error}"
|
||||
);
|
||||
|
||||
let checkpoint = Checkpoint::load(&dir.path().join("checkpoint.json")).unwrap();
|
||||
assert!(
|
||||
checkpoint.node_outcomes.contains_key("gate"),
|
||||
"gate outcome should be checkpointed before termination"
|
||||
);
|
||||
assert!(
|
||||
!checkpoint.completed_nodes.contains(&"approve".to_string()),
|
||||
"approval path must not execute on aborted input"
|
||||
);
|
||||
assert!(
|
||||
!checkpoint.completed_nodes.contains(&"revise".to_string()),
|
||||
"other unconditional choice edges must not execute on aborted input"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn human_gate_aborted_input_routes_via_outcome_fail_condition() {
|
||||
let mut graph = Graph::new("HumanGateAbortedFailRoute");
|
||||
|
||||
let mut start = Node::new("start");
|
||||
start.attrs.insert(
|
||||
"shape".to_string(),
|
||||
AttrValue::String("Mdiamond".to_string()),
|
||||
);
|
||||
graph.nodes.insert("start".to_string(), start);
|
||||
|
||||
let mut exit = Node::new("exit");
|
||||
exit.attrs.insert(
|
||||
"shape".to_string(),
|
||||
AttrValue::String("Msquare".to_string()),
|
||||
);
|
||||
graph.nodes.insert("exit".to_string(), exit);
|
||||
|
||||
let mut gate = Node::new("gate");
|
||||
gate.attrs.insert(
|
||||
"shape".to_string(),
|
||||
AttrValue::String("hexagon".to_string()),
|
||||
);
|
||||
gate.attrs
|
||||
.insert("type".to_string(), AttrValue::String("human".to_string()));
|
||||
gate.attrs.insert(
|
||||
"label".to_string(),
|
||||
AttrValue::String("Approve release?".to_string()),
|
||||
);
|
||||
graph.nodes.insert("gate".to_string(), gate);
|
||||
graph
|
||||
.nodes
|
||||
.insert("approve".to_string(), Node::new("approve"));
|
||||
graph
|
||||
.nodes
|
||||
.insert("manual_review".to_string(), Node::new("manual_review"));
|
||||
|
||||
graph.edges.push(Edge::new("start", "gate"));
|
||||
|
||||
let mut approve_edge = Edge::new("gate", "approve");
|
||||
approve_edge.attrs.insert(
|
||||
"label".to_string(),
|
||||
AttrValue::String("[A] Approve".to_string()),
|
||||
);
|
||||
graph.edges.push(approve_edge);
|
||||
|
||||
let mut fail_edge = Edge::new("gate", "manual_review");
|
||||
fail_edge.attrs.insert(
|
||||
"condition".to_string(),
|
||||
AttrValue::String("outcome=fail".to_string()),
|
||||
);
|
||||
graph.edges.push(fail_edge);
|
||||
|
||||
graph.edges.push(Edge::new("approve", "exit"));
|
||||
graph.edges.push(Edge::new("manual_review", "exit"));
|
||||
|
||||
let interviewer = Arc::new(CallbackInterviewer::new(|_| Answer::aborted()));
|
||||
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut registry = HandlerRegistry::new(Box::new(StartHandler));
|
||||
registry.register("start", Box::new(StartHandler));
|
||||
registry.register("exit", Box::new(ExitHandler));
|
||||
registry.register("human", Box::new(HumanHandler::new(interviewer)));
|
||||
|
||||
let engine = WorkflowRunEngine::new(registry, Arc::new(EventEmitter::new()), local_env());
|
||||
let config = RunConfig {
|
||||
run_dir: dir.path().to_path_buf(),
|
||||
cancel_token: None,
|
||||
dry_run: false,
|
||||
run_id: "test-run".into(),
|
||||
git_checkpoint_enabled: false,
|
||||
host_repo_path: None,
|
||||
base_sha: None,
|
||||
run_branch: None,
|
||||
meta_branch: None,
|
||||
labels: std::collections::HashMap::new(),
|
||||
checkpoint_exclude_globs: Vec::new(),
|
||||
github_app: None,
|
||||
git_author: fabro_workflows::git::GitAuthor::default(),
|
||||
base_branch: None,
|
||||
pull_request: None,
|
||||
asset_globs: Vec::new(),
|
||||
workflow_slug: None,
|
||||
};
|
||||
|
||||
let outcome = engine
|
||||
.run(&graph, &config)
|
||||
.await
|
||||
.expect("aborted human gate should follow explicit fail route");
|
||||
assert_eq!(outcome.status, StageStatus::Success);
|
||||
|
||||
let checkpoint = Checkpoint::load(&dir.path().join("checkpoint.json")).unwrap();
|
||||
assert!(
|
||||
checkpoint
|
||||
.completed_nodes
|
||||
.contains(&"manual_review".to_string()),
|
||||
"explicit fail route should handle unanswered human gates"
|
||||
);
|
||||
assert!(
|
||||
!checkpoint.completed_nodes.contains(&"approve".to_string()),
|
||||
"approval path must not execute on aborted input"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 5. Goal gate enforcement
|
||||
// ---------------------------------------------------------------------------
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue