scriptoriumadmin/fabro
1
0
Fork 0
mirror of https://github.com/fabro-sh/fabro.git synced 2026-10-01 02:04:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Delete fabro-core and the engine half of fabro-workflow

Browse source 
Every run executes on Petri, so the in-process legacy executor goes:
`fabro-core` and, in `fabro-workflow`, the handlers, lifecycle, pipeline
execution, routing, retry, conditions, node handlers, steering, agent
memory, artifacts, checkpoints, command log, and the `start`, `resume`,
`retry`, `fork`, `rewind` and `timeline` operations. The two are deleted
together because the engine half of `fabro-workflow` was the only user of
`fabro-core` and `fabro-core` the only runtime of that half; neither
compiles without the other.

Kept in `fabro-workflow`, narrowed: the parse/transform/validate/persist
pipeline and `create`, `archive`, `validate` (workflow definitions still
come from DOT and settings); the run tools (`run_tools`, moved from
`handler/llm/fabro_tools.rs`) for Ask Fabro, `fabro exec` and Petri's
host tools; the pull request pipeline (`pull_request`, moved from
`pipeline/`, for the step 0 port); Run Files' diff helpers in
`sandbox_git`; `git_identity`, `usage_rollup`, `run_status`,
`run_materialization`, `web_search` and `workflow_bundle`.

Server: `RegistryFactoryOverride` becomes `execute_in_process`;
`RunAnswerTransport::InProcess` carries only the interviewer; the
interrupt endpoint answers 501 `interrupt_unsupported` and every pair
endpoint 501 `pair_unsupported` (status lists none); rewind, fork, retry
and timeline handlers and routes are removed; the command log is served
from the stage output blob; usage rollups accumulate from the settled
projection after an in-process run as after a worker exit.

Ported while here:

- `materialize_admitted_run` materializes the goal and drops a disabled
  pull request block, as the legacy materializer did.
- A run whose admitted graph has an agent or prompt node is refused at
  create when no LLM provider is ready (`fabro.model.no_ready_provider`);
  a workflow of commands and gates needs no model and is admitted.
- The projection's question type falls back on the options, as the
  interview adapter does, so a gate with edge-label options answers as
  multiple choice.

Tests: the server scenarios (lifecycle, run completion, SSE, helpers)
run in process on Petri and assert Petri's stage labels and stream
names; the reconcile tests assert Petri's relaunch semantics; legacy
unit tests of the deleted executor are removed; three server unit tests
the removal took with it are restored; the pair fixtures go with the
pair feature. Petri test fixtures no longer name `[workflow] engine`.

Still red after this commit, all legacy consumers the next steps
delete or port: fabro-store's Slate/reducer fixtures and fabro-types
legacy JSON tests (step 4); server unit tests over legacy run events
(retry endpoints, list_run_events, artifacts, per-event pause/unpause,
run history activation, legacy sandbox fixtures) (steps 3-4); CLI tests
that parse legacy event envelopes, the legacy `events`/`attach`/`diff`/
`dump`/`inspect` snapshots, `run rewind`/`run fork`, the ACP and
git-identity workflow tests, and the runner tests that drive the legacy
worker by hand (steps 3-4); the web app's Petri fixtures still carry
`engine` (regenerate with `FABRO_CAPTURE_PETRI_FIXTURES` in step 4).

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Bryan Helmkamp 2026-09-18 10:44:41 -04:00
parent a36bea15d2
commit d90a5d9cbb
No known key found for this signature in database
131 changed files with 2605 additions and 72572 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
Cargo.lock generated
View file

@ -2534,22 +2534,6 @@ dependencies = [
"ulid",
]
[[package]]
name = "fabro-core"
version = "0.357.0-nightly.0"
dependencies = [
"async-trait",
"fabro-types",
"fabro-util",
"serde",
"serde_json",
"strum 0.28.0",
"thiserror 2.0.18",
"tokio",
"tokio-util",
"tracing",
]
[[package]]
name = "fabro-db"
version = "0.357.0-nightly.0"
@ -3102,7 +3086,6 @@ dependencies = [
"fabro-interview",
"fabro-static",
"fabro-types",
"fabro-workflow",
"futures-util",
"rustls",
"serde",
@ -3367,29 +3350,22 @@ name = "fabro-workflow"
version = "0.357.0-nightly.0"
dependencies = [
"anyhow",
"assert_cmd",
"async-trait",
"base64",
"bytes",
"chrono",
"dirs",
"fabro-acp",
"fabro-api",
"fabro-auth",
"fabro-checkpoint",
"fabro-client",
"fabro-config",
"fabro-core",
"fabro-dump",
"fabro-environment",
"fabro-github",
"fabro-graphviz",
"fabro-hooks",
"fabro-http",
"fabro-interview",
"fabro-llm",
"fabro-macros",
"fabro-mcp",
"fabro-redact",
"fabro-sandbox",
"fabro-static",
@ -3415,7 +3391,6 @@ dependencies = [
"object_store",
"pebble-agent",
"pebble-coding-agent",
"predicates",
"rand 0.9.4",
"regex",
"sandbox-driver",
@ -3423,7 +3398,6 @@ dependencies = [
"serde",
"serde_json",
"sha2 0.10.9",
"shlex",
"strum 0.28.0",
"tempfile",
"thiserror 2.0.18",

4
lib/apps/fabro-cli/tests/it/scenario/petri_tools.rs
View file

@ -59,7 +59,7 @@ const CHILD_DOT: &str = r#"digraph Child {
start -> say -> exit
}"#;
const CHILD_SETTINGS: &str =
"_version = 1\n\n[workflow]\ngraph = \"workflow.fabro\"\nengine = \"petri\"\n";
"_version = 1\n\n[workflow]\ngraph = \"workflow.fabro\"\n";
/// A `[[run.hooks]]` entry that blocks every `fabro_run_search` call.
const BLOCKING_HOOK: &str = r#"
@ -95,7 +95,7 @@ fn write_agent_workspace(context: &fabro_test::TestContext, extra_settings: &str
std::fs::write(
workspace.join("workflow.toml"),
format!(
"_version = 1\n\n[workflow]\ngraph = \"workflow.fabro\"\nengine = \"petri\"\n\n[run]\n\
"_version = 1\n\n[workflow]\ngraph = \"workflow.fabro\"\n\n[run]\n\
goal = \"Use the run tools\"\n\n[run.agent]\nfabro_tools = true\n{extra_settings}"
),
)

14
lib/apps/fabro-server/src/demo/mod.rs
View file

@ -1101,9 +1101,9 @@ mod runs {
};
use fabro_types::settings::{InterpString, ProjectNamespace, WorkflowNamespace};
use fabro_types::{
AuthMethod, IdpIdentity, PendingReason, PetriAdmission, Principal, RepositoryRef, RunId,
RunLifecycle, RunLinks, RunOrigin, RunSize, RunTimestamps, StageId, WorkflowRef,
WorkflowSettings,
AuthMethod, BlobHash, IdpIdentity, PendingReason, PetriAdmission, PetriGraphRef, Principal,
RepositoryRef, RunId, RunLifecycle, RunLinks, RunOrigin, RunSize, RunTimestamps, StageId,
WorkflowRef, WorkflowSettings,
};
use lithos_llm::catalog::ProviderId;
use lithos_llm::types::{Cost, CostSource, TokenCounts, Usage};
@ -1747,7 +1747,13 @@ mod runs {
spec_blob: None,
git: None,
fork_source_ref: None,
admission: PetriAdmission::default(),
admission: PetriAdmission {
graph: PetriGraphRef {
blob: BlobHash::new(b"demo-run"),
digest: "demo".to_string(),
},
children: Vec::new(),
},
};
let mut projection = RunProjection::new(
"Detect and fix environment drift".to_string(),

2
lib/apps/fabro-server/src/petri_runs.rs
View file

@ -190,7 +190,7 @@ mod tests {
}"#;
const PETRI_SETTINGS: &str =
"_version = 1\n\n[workflow]\ngraph = \"workflow.fabro\"\nengine = \"petri\"\n";
"_version = 1\n\n[workflow]\ngraph = \"workflow.fabro\"\n";
/// A worker runtime whose one worker runs until the test ends it, so
/// the test can act while the server waits on the worker. It keeps the

176
lib/apps/fabro-server/src/run_manifest.rs
View file

@ -1,4 +1,4 @@
use std::collections::{BTreeMap, HashMap};
use std::collections::HashMap;
use std::future::Future;
use std::path::{Path, PathBuf};
use std::sync::Arc;
@ -22,18 +22,15 @@ use fabro_sandbox::{
CloneRequest, ProviderAccess, RunSandbox, SandboxSpec, sandbox_spec_for_environment,
};
use fabro_static::EnvVars;
use fabro_types::settings::ModelRef;
use fabro_types::settings::cli::OutputVerbosity;
use fabro_types::settings::interp::InterpString;
use fabro_types::settings::run::{McpServerSettings, RunGoal, RunNamespace};
use fabro_types::{
BundledProvider, ManifestPath, RunId, RunNoticeLevel, SandboxProviderKind, ServerSettings,
WorkflowSettings,
BundledProvider, ManifestPath, RunId, SandboxProviderKind, ServerSettings, WorkflowSettings,
};
use fabro_util::check_report::{CheckDetail, CheckReport, CheckResult, CheckSection, CheckStatus};
use fabro_validate::Severity;
use fabro_workflow::Error as WorkflowError;
use fabro_workflow::model_fallback::resolve_model_fallbacks;
use fabro_workflow::operations::{
ValidateInput, WorkflowInput, validate, validate_with_catalog, validate_with_ready_providers,
};
@ -459,12 +456,6 @@ async fn build_preflight_report(
));
}
run_environment_capability_check(&mut checks, &resolved_run);
let model_fallbacks_ok = run_model_fallback_check(
&mut checks,
catalog.as_ref(),
&ready_providers,
&resolved_run.model.fallbacks,
);
let needs_github_credentials = sandbox_provider.clones_workspace()
|| resolved_run.integrations.github.is_token_requested();
let github_app = if needs_github_credentials {
@ -513,8 +504,7 @@ async fn build_preflight_report(
let github_token_ok =
run_github_token_check(&mut checks, prepared, &resolved_run, github_app).await;
let checks_ok =
model_fallbacks_ok && sandbox_ok && repository_access_ok && llm_ok && github_token_ok;
let checks_ok = sandbox_ok && repository_access_ok && llm_ok && github_token_ok;
Ok((
CheckReport {
@ -528,72 +518,6 @@ async fn build_preflight_report(
))
}
fn run_model_fallback_check(
checks: &mut Vec<CheckResult>,
catalog: &Catalog,
ready_providers: &[ProviderId],
configured: &BTreeMap<String, Vec<ModelRef>>,
) -> bool {
if configured.is_empty() {
return true;
}
let resolved = match resolve_model_fallbacks(catalog, ready_providers, configured) {
Ok(resolved) => resolved,
Err(error) => {
checks.push(CheckResult {
name: "Model Fallbacks".into(),
status: CheckStatus::Error,
summary: "invalid".into(),
details: configured
.keys()
.map(|model| CheckDetail::new(format!("Requested model: {model}")))
.collect(),
remediation: Some(error.to_string()),
});
return false;
}
};
let has_warning = resolved
.notices
.iter()
.any(|notice| notice.level() != RunNoticeLevel::Info);
let mut details = resolved
.policy
.iter()
.map(|(model, targets)| {
let chain = if targets.is_empty() {
"(none)".to_string()
} else {
targets
.iter()
.map(ToString::to_string)
.collect::<Vec<_>>()
.join(" -> ")
};
CheckDetail::new(format!("{model}: {chain}"))
})
.collect::<Vec<_>>();
details.extend(resolved.notices.iter().map(|notice| CheckDetail {
text: notice.message(),
warn: notice.level() != RunNoticeLevel::Info,
}));
checks.push(CheckResult {
name: "Model Fallbacks".into(),
status: if has_warning {
CheckStatus::Warning
} else {
CheckStatus::Pass
},
summary: format!("{} requested model chain(s)", resolved.policy.len()),
details,
remediation: None,
});
true
}
fn base_preflight_checks(prepared: &PreparedManifest, graph: &Graph) -> Vec<CheckResult> {
let setup_command_count = prepared.settings.run.prepare.steps.len();
let repo_summary = prepared.git.as_ref().map_or_else(
@ -1786,100 +1710,6 @@ mod tests {
Arc::new(fabro_llm::test_support::test_catalog())
}
fn openrouter_catalog() -> Catalog {
fabro_llm::test_support::test_catalog_with_overlay(
"[providers.openrouter]\nenabled = true\n",
)
}
fn model_refs(values: &[&str]) -> Vec<fabro_types::settings::ModelRef> {
values
.iter()
.map(|value| value.parse().expect("fallback reference should parse"))
.collect()
}
#[test]
fn model_fallback_preflight_resolves_each_requested_model_chain() {
let mut checks = Vec::new();
let configured = std::collections::BTreeMap::from([
("gpt-sol".to_string(), model_refs(&["claude-opus"])),
(
"claude-fable".to_string(),
model_refs(&["gpt-sol", "claude-opus"]),
),
]);
assert!(run_model_fallback_check(
&mut checks,
&openrouter_catalog(),
&[ProviderId::new("openrouter")],
&configured,
));
let check = checks.last().expect("fallback check should be present");
assert_eq!(check.status, CheckStatus::Pass);
assert!(
check
.details
.iter()
.any(|detail| detail.text == "gpt-5.6-sol: openrouter:claude-opus-5")
);
assert!(check.details.iter().any(|detail| {
detail.text == "claude-fable-5: openrouter:gpt-5.6-sol -> openrouter:claude-opus-5"
}));
}
#[test]
fn model_fallback_preflight_warns_when_a_provider_is_not_ready() {
let mut checks = Vec::new();
let configured = std::collections::BTreeMap::from([(
"kimi-k3".to_string(),
model_refs(&["moonshot:kimi-k3", "openrouter:kimi-k3"]),
)]);
assert!(run_model_fallback_check(
&mut checks,
&openrouter_catalog(),
&[ProviderId::new("openrouter")],
&configured,
));
let check = checks.last().expect("fallback check should be present");
assert_eq!(check.status, CheckStatus::Warning);
assert!(check.details.iter().any(|detail| {
detail.warn
&& detail
.text
.contains("provider `moonshot` is not configured")
}));
}
#[test]
fn model_fallback_preflight_rejects_duplicate_canonical_keys() {
let mut checks = Vec::new();
let configured = std::collections::BTreeMap::from([
("gpt-sol".to_string(), model_refs(&["claude-opus"])),
("gpt-5.6-sol".to_string(), model_refs(&["claude-fable"])),
]);
assert!(!run_model_fallback_check(
&mut checks,
&openrouter_catalog(),
&[ProviderId::new("openrouter")],
&configured,
));
let check = checks.last().expect("fallback check should be present");
assert_eq!(check.status, CheckStatus::Error);
assert!(
check
.remediation
.as_deref()
.is_some_and(|message| message.contains("both resolve to requested model"))
);
}
fn openai_compatible_completion(model: &str) -> serde_json::Value {
serde_json::json!({
"id": "chatcmpl_preflight",

2
lib/apps/fabro-server/src/serve.rs
View file

@ -788,7 +788,7 @@ where
let shutdown = CancellationToken::new();
let state = build_app_state(AppStateConfig {
resolved_settings: resolved_app_settings,
registry_factory_override: None,
execute_in_process: false,
max_concurrent_runs,
store,
artifact_store,

209
lib/apps/fabro-server/src/server.rs
View file

@ -55,7 +55,7 @@ use fabro_config::{LlmLayer, RunLayer, Storage, WorkflowSettingsBuilder};
use fabro_db::DbPool;
use fabro_environment::EnvironmentStore;
use fabro_interview::{
Answer, AnswerSubmission, ControlInterviewer, Interviewer, Question, WorkerControlEnvelope,
Answer, AnswerSubmission, ControlInterviewer, Question, WorkerControlEnvelope,
};
use fabro_llm::credentials::CredentialProvider;
use fabro_llm::lithos_catalog::Catalog;
@ -89,10 +89,10 @@ use fabro_types::settings::server::{
GithubIntegrationSettings, GithubIntegrationStrategy, LogDestination,
};
use fabro_types::{
AgentBackend, AskFabro, AskFabroUnavailableReason, BlobHash, EventBody,
InterviewQuestionRecord, ModelRef, ModelTestMode, PairId, PairMessageId, PairTarget,
PendingReason, Principal, PullRequestLink, QuestionType, RunControlAction, RunEvent, RunId,
RunRunnableSource, RunStatusKind, SandboxProviderKind, ServerSettings, SessionCapability,
AskFabro, AskFabroUnavailableReason, BlobHash, EventBody, InterviewQuestionRecord, ModelRef,
ModelTestMode, PendingReason, Principal, PullRequestLink, QuestionType, RunControlAction,
RunEvent, RunId, RunRunnableSource, RunStatusKind, SandboxProviderKind, ServerSettings,
SessionCapability,
};
use fabro_util::error::{
SharedError, collect_causes, render_compact_with_causes, render_with_causes,
@ -100,10 +100,7 @@ use fabro_util::error::{
use fabro_util::version::FABRO_VERSION;
use fabro_variable::{Error as VariableError, VariableStore};
use fabro_vault::{SecretStore, SecretStoreError, SecretType, Vault};
#[cfg(test)]
use fabro_workflow::command_log::command_log_path;
use fabro_workflow::event::{self as workflow_event};
use fabro_workflow::handler::HandlerRegistry;
use fabro_workflow::records::Checkpoint;
use fabro_workflow::run_lookup::{
RunInfo, StatusFilter, filter_runs, scan_runs_with_summaries, scratch_base,
@ -265,7 +262,6 @@ struct ManagedRun {
active_steerable_stages: HashMap<StageId, String>,
/// API-mode session targets eligible for live pair control. ACP sessions
/// can be steerable but are intentionally excluded from pairing.
active_api_targets: HashMap<StageId, PairTarget>,
/// Stage IDs of currently running agent sessions that have no live
/// steering capability, keyed to the session id that owns the marker.
active_non_steerable_stages: HashMap<StageId, String>,
@ -327,9 +323,6 @@ pub(crate) struct UsageAccumulator {
pub(crate) by_model: HashMap<ModelRef, ModelUsageTotals>,
}
pub(crate) type RegistryFactoryOverride =
dyn Fn(Arc<dyn Interviewer>) -> HandlerRegistry + Send + Sync;
#[derive(Clone)]
enum RunAnswerTransport {
Worker {
@ -337,8 +330,7 @@ enum RunAnswerTransport {
bus: Arc<dyn WorkerControlBus>,
},
InProcess {
interviewer: Arc<ControlInterviewer>,
steering_hub: Arc<fabro_workflow::SteeringHub>,
interviewer: Arc<ControlInterviewer>,
},
}
@ -348,13 +340,6 @@ enum AnswerTransportError {
Timeout,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum PairTransportError {
Closed,
Timeout,
Control(fabro_workflow::PairControlError),
}
impl RunAnswerTransport {
async fn publish_worker_control(
run_id: RunId,
@ -376,15 +361,6 @@ impl RunAnswerTransport {
}
}
fn pair_error_from_bus(error: &WorkerControlBusError) -> PairTransportError {
match error {
WorkerControlBusError::PublishTimeout => PairTransportError::Timeout,
WorkerControlBusError::Closed
| WorkerControlBusError::Unavailable
| WorkerControlBusError::InvalidCursor { .. } => PairTransportError::Closed,
}
}
async fn submit(
&self,
qid: &str,
@ -419,8 +395,8 @@ impl RunAnswerTransport {
}
}
/// Forward a steer to the worker (subprocess) or directly into the
/// in-process steering hub.
/// Forward a steer to the worker. The in-process test path drives no
/// steer: its run has no live agent session to steer.
async fn steer(&self, text: String, actor: Principal) -> Result<(), AnswerTransportError> {
match self {
Self::Worker { run_id, bus } => {
@ -429,111 +405,7 @@ impl RunAnswerTransport {
.await
.map_err(|err| Self::answer_error_from_bus(&err))
}
Self::InProcess { steering_hub, .. } => {
steering_hub.deliver_steer(text, Some(actor));
Ok(())
}
}
}
async fn interrupt(&self, actor: Principal) -> Result<(), AnswerTransportError> {
match self {
Self::Worker { run_id, bus } => {
let message = WorkerControlEnvelope::interrupt(actor);
Self::publish_worker_control(*run_id, bus, message)
.await
.map_err(|err| Self::answer_error_from_bus(&err))
}
Self::InProcess { steering_hub, .. } => {
steering_hub.interrupt(Some(&actor));
Ok(())
}
}
}
async fn interrupt_then_steer(
&self,
text: String,
actor: Principal,
) -> Result<(), AnswerTransportError> {
match self {
Self::Worker { run_id, bus } => {
let message = WorkerControlEnvelope::interrupt_then_steer(text, actor);
Self::publish_worker_control(*run_id, bus, message)
.await
.map_err(|err| Self::answer_error_from_bus(&err))
}
Self::InProcess { steering_hub, .. } => {
steering_hub.interrupt_then_steer(&text, Some(&actor));
Ok(())
}
}
}
async fn start_pair(
&self,
run_id: RunId,
pair_id: PairId,
target: PairTarget,
actor: Principal,
) -> Result<(), PairTransportError> {
match self {
Self::Worker {
run_id: worker_run_id,
bus,
} => {
let message = WorkerControlEnvelope::start_pair(run_id, pair_id, target, actor);
Self::publish_worker_control(*worker_run_id, bus, message)
.await
.map_err(|err| Self::pair_error_from_bus(&err))
}
Self::InProcess { steering_hub, .. } => steering_hub
.start_pair(run_id, pair_id, target, Some(actor))
.map(|_| ())
.map_err(PairTransportError::Control),
}
}
async fn send_pair_message(
&self,
pair_id: PairId,
message_id: PairMessageId,
text: String,
client_message_id: Option<String>,
actor: Principal,
) -> Result<(), PairTransportError> {
match self {
Self::Worker { run_id, bus } => {
let message = WorkerControlEnvelope::pair_message(
pair_id,
message_id,
text.clone(),
client_message_id.clone(),
actor,
);
Self::publish_worker_control(*run_id, bus, message)
.await
.map_err(|err| Self::pair_error_from_bus(&err))
}
Self::InProcess { steering_hub, .. } => steering_hub
.send_pair_message(pair_id, message_id, text, client_message_id, Some(actor))
.map(|_| ())
.map_err(PairTransportError::Control),
}
}
async fn end_pair(&self, pair_id: PairId, actor: Principal) -> Result<(), PairTransportError> {
match self {
Self::Worker { run_id, bus } => {
let message = WorkerControlEnvelope::end_pair(pair_id, actor);
Self::publish_worker_control(*run_id, bus, message)
.await
.map_err(|err| Self::pair_error_from_bus(&err))
}
Self::InProcess { steering_hub, .. } => steering_hub
.end_pair(pair_id, Some(actor))
.map(|_| ())
.map_err(PairTransportError::Control),
Self::InProcess { .. } => Err(AnswerTransportError::Closed),
}
}
@ -1139,7 +1011,8 @@ pub struct AppState {
sandbox_inventory: SandboxInventory,
shutdown: CancellationToken,
shutting_down: AtomicBool,
registry_factory_override: Option<Box<RegistryFactoryOverride>>,
/// Test switch: execute runs in this process instead of a worker.
execute_in_process: bool,
slack_service: Option<Arc<SlackService>>,
slack_started: AtomicBool,
github_webhook_secret: Option<String>,
@ -1297,7 +1170,8 @@ impl AskFabroReadiness {
pub(crate) struct AppStateConfig {
pub(crate) resolved_settings: ResolvedAppStateSettings,
pub(crate) registry_factory_override: Option<Box<RegistryFactoryOverride>>,
/// Execute runs in this process instead of a worker (tests only).
pub(crate) execute_in_process: bool,
pub(crate) max_concurrent_runs: usize,
pub(crate) store: Arc<Database>,
pub(crate) artifact_store: ArtifactStore,
@ -1325,6 +1199,25 @@ pub(crate) struct ResolvedAppStateSettings {
pub(crate) llm_overlay: LlmLayer,
}
/// Add a concluded run's usage to the server's aggregate; a run that
/// recorded no conclusion adds nothing.
pub(crate) fn accumulate_concluded_run_usage(
state: &AppState,
final_state: &fabro_store::RunProjection,
) {
if final_state.conclusion.is_none() {
return;
}
let mut agg = state
.aggregate_usage
.lock()
.expect("aggregate_usage lock poisoned");
accumulate_usage_rollup(
&mut agg,
&fabro_workflow::usage_rollup_from_projection(final_state),
);
}
fn accumulate_usage_rollup(
accumulator: &mut UsageAccumulator,
rollup: &fabro_workflow::ProjectionUsageRollup,
@ -2437,7 +2330,7 @@ where
pub(crate) fn build_app_state(config: AppStateConfig) -> anyhow::Result<Arc<AppState>> {
let AppStateConfig {
resolved_settings,
registry_factory_override,
execute_in_process,
max_concurrent_runs,
store,
artifact_store,
@ -2648,7 +2541,7 @@ pub(crate) fn build_app_state(config: AppStateConfig) -> anyhow::Result<Arc<AppS
sandbox_inventory,
shutdown,
shutting_down: AtomicBool::new(false),
registry_factory_override,
execute_in_process,
slack_service,
slack_started: AtomicBool::new(false),
// Startup snapshot for the sync router build; rotating the webhook
@ -3051,7 +2944,6 @@ fn octet_stream_response(bytes: Bytes) -> Response {
fn clear_live_run_state(run: &mut ManagedRun) {
run.answer_transport = None;
run.accepted_questions.clear();
run.active_api_targets.clear();
run.active_steerable_stages.clear();
run.active_non_steerable_stages.clear();
run.event_tx = None;
@ -3431,7 +3323,6 @@ fn managed_run(
created_at,
answer_transport: None,
accepted_questions: HashSet::new(),
active_api_targets: HashMap::new(),
active_steerable_stages: HashMap::new(),
active_non_steerable_stages: HashMap::new(),
event_tx: None,
@ -3553,7 +3444,6 @@ fn update_live_run_from_event(state: &AppState, run_id: RunId, event: &RunEvent)
reason: props.reason,
};
managed_run.error = None;
managed_run.active_api_targets.clear();
managed_run.active_steerable_stages.clear();
managed_run.active_non_steerable_stages.clear();
cleanup_worker_control_bus_for_run(state, run_id);
@ -3566,7 +3456,6 @@ fn update_live_run_from_event(state: &AppState, run_id: RunId, event: &RunEvent)
&props.failure.detail.message,
&props.failure.detail.causes,
));
managed_run.active_api_targets.clear();
managed_run.active_steerable_stages.clear();
managed_run.active_non_steerable_stages.clear();
cleanup_worker_control_bus_for_run(state, run_id);
@ -3583,26 +3472,11 @@ fn update_live_run_from_event(state: &AppState, run_id: RunId, event: &RunEvent)
.active_steerable_stages
.insert(stage_id.clone(), session_id.clone());
managed_run.active_non_steerable_stages.remove(stage_id);
let acp_provider: &'static str = AgentBackend::Acp.into();
if props.provider.as_deref() == Some(acp_provider) {
managed_run.active_api_targets.remove(stage_id);
} else {
managed_run
.active_api_targets
.insert(stage_id.clone(), PairTarget {
stage_id: stage_id.clone(),
node_label: event
.node_label
.clone()
.unwrap_or_else(|| stage_id.node_id().to_string()),
});
}
} else {
managed_run
.active_non_steerable_stages
.insert(stage_id.clone(), session_id.clone());
managed_run.active_steerable_stages.remove(stage_id);
managed_run.active_api_targets.remove(stage_id);
}
}
}
@ -3616,7 +3490,6 @@ fn update_live_run_from_event(state: &AppState, run_id: RunId, event: &RunEvent)
.is_some_and(|current| current == session_id)
{
managed_run.active_steerable_stages.remove(stage_id);
managed_run.active_api_targets.remove(stage_id);
}
if managed_run
.active_non_steerable_stages
@ -3635,7 +3508,6 @@ fn update_live_run_from_event(state: &AppState, run_id: RunId, event: &RunEvent)
| EventBody::StageCompleted(_)
| EventBody::StageFailed(_) => {
if let Some(stage_id) = &event.stage_id {
managed_run.active_api_targets.remove(stage_id);
managed_run.active_steerable_stages.remove(stage_id);
managed_run.active_non_steerable_stages.remove(stage_id);
}
@ -4016,7 +3888,7 @@ async fn execute_run(state: Arc<AppState>, run_id: RunId) {
// A run executes in its worker process. Under the test override it
// executes in this process instead, so the scenario tests need no worker
// binary.
if state.registry_factory_override.is_some() {
if state.execute_in_process {
Box::pin(petri_runs::execute(state, run_id)).await;
return;
}
@ -4226,16 +4098,7 @@ async fn execute_run_subprocess(state: Arc<AppState>, run_id: RunId) {
}
};
if final_state.current_checkpoint().is_some() {
let mut agg = state
.aggregate_usage
.lock()
.expect("aggregate_usage lock poisoned");
accumulate_usage_rollup(
&mut agg,
&fabro_workflow::usage_rollup_from_projection(&final_state),
);
}
accumulate_concluded_run_usage(&state, &final_state);
let mut runs = state.runs.lock().expect("runs lock poisoned");
if let Some(managed_run) = runs.get_mut(&run_id) {

202
lib/apps/fabro-server/src/server/handler/lifecycle.rs
View file

@ -9,16 +9,14 @@ use super::super::{
BatchDeleteRunsResult, BatchDeleteRunsResultOutcome, BatchDeleteRunsSummary,
BatchRunLifecycleRequest, BatchRunLifecycleResponse, BatchRunLifecycleResult,
BatchRunLifecycleResultOutcome, BatchRunLifecycleSummary, DeleteRunOutcome, DeleteRunSandbox,
DenyRunRequest, FailureReason, ForkRequest, ForkResponse, HeaderMap, IntoResponse, Json, Path,
PendingReason, Principal, RequireRunManagementTarget, RequiredUser, Response, RewindRequest,
RewindResponse, Router, RunAnswerTransport, RunControlAction, RunExecutionMode, RunId,
RunRunnableSource, RunStatus, StartRunRequest, State, StatusCode, Storage,
TimelineEntryResponse, WORKER_CANCEL_GRACE, WorkflowError, append_control_request,
clear_live_run_state, delete_run_internal, durable_run_status, get, load_pending_control,
DenyRunRequest, FailureReason, IntoResponse, Json, Path, PendingReason, Principal,
RequireRunManagementTarget, RequiredUser, Response, Router, RunAnswerTransport,
RunControlAction, RunExecutionMode, RunId, RunRunnableSource, RunStatus, StartRunRequest,
State, StatusCode, Storage, WORKER_CANCEL_GRACE, WorkflowError, append_control_request,
clear_live_run_state, delete_run_internal, durable_run_status, load_pending_control,
managed_run, operations, parse_run_id_path, persist_cancelled_run_status, post,
reject_if_archived, update_live_run_from_event, workflow_event,
};
use super::runs::run_provenance;
use crate::worker_runtime::WorkerRef;
pub(super) fn routes() -> Router<Arc<AppState>> {
@ -33,10 +31,6 @@ pub(super) fn routes() -> Router<Arc<AppState>> {
.route("/runs/delete", post(batch_delete_runs))
.route("/runs/unarchive", post(batch_unarchive_runs))
.route("/runs/{id}/archive", post(archive_run))
.route("/runs/{id}/rewind", post(rewind_run))
.route("/runs/{id}/retry", post(retry_run))
.route("/runs/{id}/fork", post(fork_run))
.route("/runs/{id}/timeline", get(run_timeline))
.route("/runs/{id}/unarchive", post(unarchive_run))
}
@ -864,192 +858,6 @@ async fn batch_delete_runs(
.into_response()
}
async fn rewind_run(
subject: RequiredUser,
State(state): State<Arc<AppState>>,
Path(id): Path<String>,
body: Option<Json<RewindRequest>>,
) -> Response {
let id = match parse_run_id_path(&id) {
Ok(id) => id,
Err(response) => return response,
};
if let Some(response) = reject_if_archived(state.as_ref(), &id).await {
return response;
}
let request = body.map(|Json(body)| body).unwrap_or_default();
let target = match parse_fork_target(request.target) {
Ok(target) => target,
Err(err) => return err.into_response(),
};
let input = operations::RewindInput { run_id: id, target };
match Box::pin(operations::rewind(
&state.stores.runs,
&input,
Some(Principal::User(subject.0.clone())),
))
.await
{
Ok(operations::RewindOutcome::Full {
source_run_id,
new_run_id,
target,
}) => (
StatusCode::OK,
Json(RewindResponse {
source_run_id: source_run_id.to_string(),
new_run_id: new_run_id.to_string(),
target: target.response_target(),
archived: true,
archive_error: None,
}),
)
.into_response(),
Ok(operations::RewindOutcome::Partial {
source_run_id,
new_run_id,
target,
archive_error,
}) => (
StatusCode::MULTI_STATUS,
Json(RewindResponse {
source_run_id: source_run_id.to_string(),
new_run_id: new_run_id.to_string(),
target: target.response_target(),
archived: false,
archive_error: Some(archive_error),
}),
)
.into_response(),
Err(err) => workflow_operation_error_response(err),
}
}
async fn fork_run(
_subject: RequiredUser,
State(state): State<Arc<AppState>>,
Path(id): Path<String>,
body: Option<Json<ForkRequest>>,
) -> Response {
let id = match parse_run_id_path(&id) {
Ok(id) => id,
Err(response) => return response,
};
if let Some(response) = reject_if_archived(state.as_ref(), &id).await {
return response;
}
let request = body.map(|Json(body)| body).unwrap_or_default();
let target = match parse_fork_target(request.target) {
Ok(target) => target,
Err(err) => return err.into_response(),
};
let input = operations::ForkRunInput {
source_run_id: id,
target,
};
match Box::pin(operations::fork_run(&state.stores.runs, &input)).await {
Ok(outcome) => (
StatusCode::OK,
Json(ForkResponse {
source_run_id: outcome.source_run_id.to_string(),
new_run_id: outcome.new_run_id.to_string(),
target: outcome.target.response_target(),
}),
)
.into_response(),
Err(err) => workflow_operation_error_response(err),
}
}
async fn retry_run(
RequiredUser(user): RequiredUser,
State(state): State<Arc<AppState>>,
headers: HeaderMap,
Path(id): Path<String>,
) -> Response {
let id = match parse_run_id_path(&id) {
Ok(id) => id,
Err(response) => return response,
};
let actor = Principal::User(user);
let new_run_id = RunId::new();
let input = operations::RetryRunInput {
source_run_id: id,
new_run_id,
provenance: run_provenance(&headers, &actor),
web_url: state.run_web_url(&new_run_id),
};
match Box::pin(operations::retry_run(&state.stores.runs, &input)).await {
Ok(outcome) => {
let new_run_id = outcome.new_run_id;
if let Err(err) = queue_run_start(state.as_ref(), new_run_id, false, actor).await {
return err.into_response();
}
run_response(state.as_ref(), new_run_id, StatusCode::CREATED).await
}
Err(err) => workflow_operation_error_response(err),
}
}
async fn run_timeline(
_auth: RequiredUser,
State(state): State<Arc<AppState>>,
Path(id): Path<String>,
) -> Response {
let id = match parse_run_id_path(&id) {
Ok(id) => id,
Err(response) => return response,
};
match operations::timeline(&state.stores.runs, &id).await {
Ok(entries) => Json(
entries
.into_iter()
.map(|entry| TimelineEntryResponse {
ordinal: std::num::NonZeroU64::new(entry.ordinal as u64)
.expect("timeline ordinals start at 1"),
node_name: entry.node_name,
visit: std::num::NonZeroU64::new(entry.visit as u64)
.expect("timeline visits start at 1"),
checkpoint_seq: std::num::NonZeroU64::new(u64::from(entry.checkpoint_seq))
.expect("checkpoint event sequence starts at 1"),
run_commit_sha: entry.run_commit_sha,
})
.collect::<Vec<_>>(),
)
.into_response(),
Err(err) => workflow_operation_error_response(err),
}
}
fn parse_fork_target(target: Option<String>) -> Result<Option<operations::ForkTarget>, ApiError> {
target
.map(|target| {
target
.parse::<operations::ForkTarget>()
.map_err(|err| ApiError::bad_request(err.to_string()))
})
.transpose()
}
fn workflow_operation_error_response(err: WorkflowError) -> Response {
match err {
WorkflowError::Parse(message) | WorkflowError::Validation(message) => {
ApiError::bad_request(message).into_response()
}
WorkflowError::ValidationFailed { .. } => {
ApiError::bad_request("Validation failed").into_response()
}
WorkflowError::Precondition(message) => {
ApiError::new(StatusCode::CONFLICT, message).into_response()
}
WorkflowError::RunNotFound(_) => ApiError::not_found("Run not found.").into_response(),
WorkflowError::Unsupported(message) => {
ApiError::new(StatusCode::NOT_IMPLEMENTED, message).into_response()
}
err => ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response(),
}
}
#[derive(Clone, Copy)]
enum ArchiveAction {
Archive,

1115
lib/apps/fabro-server/src/server/handler/pair.rs
View file

File diff suppressed because it is too large Load diff

80
lib/apps/fabro-server/src/server/handler/runs.rs
View file

@ -35,7 +35,6 @@ use fabro_types::{
};
use fabro_util::error as error_util;
use fabro_util::version::FABRO_VERSION;
use fabro_workflow::command_log::{command_log_path, read_json_string_blob, read_log_slice};
use fabro_workflow::run_status::RunStatus;
use fabro_workflow::{Error as WorkflowError, operations};
use lithos_llm::catalog::ProviderId;
@ -1497,40 +1496,19 @@ async fn get_run_stage_command_log(
let live_streaming = node
.live_streaming
.unwrap_or_else(|| cas_ref.is_none() && node.completion.is_none());
let run_dir = Storage::new(state.server_storage_dir())
.run_scratch(&id)
.root()
.to_path_buf();
let scratch_path = command_log_path(&run_dir, &stage_id);
match read_log_slice(&scratch_path, query.offset, limit).await {
Ok((bytes, total_bytes)) => {
return build_command_log_response(
query.offset,
limit,
LogSource::Sliced { bytes, total_bytes },
cas_ref.is_some(),
cas_ref,
live_streaming,
);
}
Err(err) if err.kind() == ErrorKind::NotFound => {}
Err(err) => {
return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string())
.into_response();
}
}
// A stage's output is on its record: inline, or in the blob table when
// Petri offloaded it. The blob holds the output value as JSON (a string
// for a command's output), so a string decodes and anything else is
// served as written.
if let Some(cas_ref) = cas_ref {
let run_store = match state.stores.runs.open_run_reader(&id).await {
Ok(run_store) => run_store,
Err(err) => {
return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string())
.into_response();
}
let Some(hash) = parse_blob_ref(&cas_ref) else {
return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, "invalid output blob ref")
.into_response();
};
let text = match read_json_string_blob(&run_store.into(), &cas_ref).await {
Ok(Some(text)) => text,
let text = match state.store_ref().blobs().read(&hash).await {
Ok(Some(bytes)) => serde_json::from_slice::<String>(&bytes)
.unwrap_or_else(|_| String::from_utf8_lossy(&bytes).into_owned()),
Ok(None) => String::new(),
Err(err) => {
return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string())
@ -1540,7 +1518,7 @@ async fn get_run_stage_command_log(
return build_command_log_response(
query.offset,
limit,
LogSource::Full(text.as_bytes()),
text.as_bytes(),
true,
Some(cas_ref),
live_streaming,
@ -1551,7 +1529,7 @@ async fn get_run_stage_command_log(
return build_command_log_response(
query.offset,
limit,
LogSource::Full(inline_text.as_bytes()),
inline_text.as_bytes(),
true,
None,
live_streaming,
@ -1561,44 +1539,28 @@ async fn get_run_stage_command_log(
build_command_log_response(
query.offset,
limit,
LogSource::Full(&[]),
&[],
node.completion.is_some(),
None,
live_streaming,
)
}
enum LogSource<'a> {
Sliced {
bytes: Vec<u8>,
total_bytes: u64,
},
Full(&'a [u8]),
}
fn build_command_log_response(
requested_offset: u64,
limit: u64,
source: LogSource<'_>,
bytes: &[u8],
eof: bool,
cas_ref: Option<String>,
live_streaming: bool,
) -> Response {
let (body_bytes, total_bytes, offset) = match source {
LogSource::Sliced { bytes, total_bytes } => {
let offset = requested_offset.min(total_bytes);
(bytes, total_bytes, offset)
}
LogSource::Full(bytes) => {
let total_bytes = u64::try_from(bytes.len()).unwrap_or(u64::MAX);
let offset = requested_offset.min(total_bytes);
let start = usize::try_from(offset).unwrap_or(bytes.len());
let end = start
.saturating_add(usize::try_from(limit).unwrap_or(usize::MAX))
.min(bytes.len());
(bytes[start..end].to_vec(), total_bytes, offset)
}
};
let total_bytes = u64::try_from(bytes.len()).unwrap_or(u64::MAX);
let offset = requested_offset.min(total_bytes);
let start = usize::try_from(offset).unwrap_or(bytes.len());
let end = start
.saturating_add(usize::try_from(limit).unwrap_or(usize::MAX))
.min(bytes.len());
let body_bytes = bytes[start..end].to_vec();
Json(CommandLogResponseBody {
offset,
next_offset: offset + u64::try_from(body_bytes.len()).unwrap_or(u64::MAX),

6
lib/apps/fabro-server/src/server/handler/sessions.rs
View file

@ -29,7 +29,7 @@ use fabro_types::run_event::{
};
use fabro_types::settings::ModelRef as SettingsModelRef;
use fabro_types::{EventBody, EventEnvelope, RunEvent, RunId, SessionDetail, SessionId, TurnId};
use fabro_workflow::handler::llm::register_named_fabro_run_tools;
use fabro_workflow::run_tools::register_named_fabro_run_tools;
use fabro_workflow::services::FabroRunToolServices;
use lithos_llm::catalog::ProviderId;
use pebble_coding_agent::environment::Environment;
@ -2040,9 +2040,7 @@ mod resume_tests {
.max_concurrent_runs(2)
// A registry factory runs the dry run in this process, so no
// worker executable is needed.
.registry_factory(|interviewer| {
fabro_workflow::handler::default_registry(interviewer, || None)
})
.in_process_execution()
.llm_overlay(llm_overlay_with_provider_base_url("openai", base_url))
.vault_entries([(EnvVars::OPENAI_API_KEY, namespace.to_string())])
.env_lookup(move |name| (name == EnvVars::OPENAI_API_KEY).then(|| api_key.clone()))

74
lib/apps/fabro-server/src/server/handler/steer.rs
View file

@ -21,14 +21,6 @@ pub(super) fn routes() -> axum::Router<Arc<AppState>> {
enum RunControlRequest {
Steer { text: String },
Interrupt,
InterruptThenSteer { text: String },
}
impl RunControlRequest {
const fn requires_active_steerable_session(&self) -> bool {
matches!(self, Self::Interrupt | Self::InterruptThenSteer { .. })
}
}
async fn steer_run(
@ -43,20 +35,31 @@ async fn steer_run(
if text.trim().is_empty() {
return ApiError::bad_request("Steer text must not be empty.").into_response();
}
let control = if interrupt {
RunControlRequest::InterruptThenSteer { text }
} else {
RunControlRequest::Steer { text }
};
control_run(actor, state, id, control).await
if interrupt {
return interrupt_unsupported();
}
control_run(actor, state, id, RunControlRequest::Steer { text }).await
}
/// Interrupting a live agent turn has no adapter over Petri's control
/// service yet, which delivers a steer to a live stage and cancels a whole
/// run but does not interrupt one stage's turn; the request is refused
/// with that reason rather than accepted and dropped.
async fn interrupt_run(
RequireRunManagementTarget(id, actor): RequireRunManagementTarget,
State(state): State<Arc<AppState>>,
RequireRunManagementTarget(_id, _actor): RequireRunManagementTarget,
State(_state): State<Arc<AppState>>,
) -> Response {
control_run(actor, state, id, RunControlRequest::Interrupt).await
interrupt_unsupported()
}
fn interrupt_unsupported() -> Response {
ApiError::with_code(
StatusCode::NOT_IMPLEMENTED,
"Interrupting a run's agent turn is not supported: Petri's control service has no \
per-stage interrupt yet. Steer the run without `interrupt`, or cancel it.",
"interrupt_unsupported",
)
.into_response()
}
async fn control_run(
@ -118,18 +121,6 @@ async fn control_run(
)
.into_response();
}
// Interrupts need a live session because there's nothing to
// cancel otherwise.
if managed_run.active_steerable_stages.is_empty()
&& control.requires_active_steerable_session()
{
return ApiError::with_code(
StatusCode::CONFLICT,
"Run has no active steerable agent session.",
"no_active_steerable_session",
)
.into_response();
}
Some(managed_run.answer_transport.clone())
}
None => None,
@ -148,13 +139,8 @@ async fn control_run(
.into_response();
};
let result = match control {
RunControlRequest::Steer { text } => answer_transport.steer(text, actor).await,
RunControlRequest::Interrupt => answer_transport.interrupt(actor).await,
RunControlRequest::InterruptThenSteer { text } => {
answer_transport.interrupt_then_steer(text, actor).await
}
};
let RunControlRequest::Steer { text } = control;
let result = answer_transport.steer(text, actor).await;
match result {
Ok(()) => StatusCode::ACCEPTED.into_response(),
@ -173,13 +159,13 @@ async fn control_run(
}
}
fn terminal_control_response(control: &RunControlRequest) -> Response {
let code = if matches!(control, RunControlRequest::Interrupt) {
"run_not_interruptible"
} else {
"run_not_steerable"
};
ApiError::with_code(StatusCode::CONFLICT, "Run is no longer steerable.", code).into_response()
fn terminal_control_response(_control: &RunControlRequest) -> Response {
ApiError::with_code(
StatusCode::CONFLICT,
"Run is no longer steerable.",
"run_not_steerable",
)
.into_response()
}
async fn unmanaged_control_response(

36
lib/apps/fabro-server/src/server/petri_runs.rs
View file

@ -51,7 +51,6 @@ use fabro_types::{PetriAdmission, RunId, RunRunnableSource, RunTarget, RunTiming
use fabro_util::error as error_util;
use fabro_validate::{Diagnostic as FabroDiagnostic, Severity};
use fabro_workflow::Error as WorkflowError;
use fabro_workflow::event::Emitter;
use fabro_workflow::run_status::{FailureReason, RunStatus, SuccessReason};
use lithos_llm::catalog::ProviderId;
use tokio::task;
@ -204,6 +203,28 @@ pub(crate) async fn admit(
for warning in &admitted.warnings {
info!(code = %warning.code, message = %warning.message, "Petri warned at admission");
}
// Without a ready provider there is no model client, so Petri admitted
// the model nodes unchecked: refuse a run they would fail at once, as
// the legacy compiler refused every run without a default model.
if eligible.is_empty() && admitted.needs_model() {
return Err(RunCompilerError::Workflow(
WorkflowError::ValidationFailed {
diagnostics: vec![FabroDiagnostic {
rule: "fabro.model.no_ready_provider".to_string(),
severity: Severity::Error,
message: "no default model is available: no LLM provider is ready, and the \
workflow has a node that runs a model"
.to_string(),
fix: Some(
"configure a provider credential (for example `OPENAI_API_KEY`) or a \
`[run.model]`"
.to_string(),
),
..FabroDiagnostic::default()
}],
},
));
}
admission::persist(&state.store_ref().blobs(), &admitted)
.await
.map_err(|err| {
@ -350,9 +371,6 @@ pub(crate) async fn execute(state: Arc<AppState>, run_id: RunId) {
// The answer endpoint reaches this interviewer directly, as it does
// for a legacy run in this process.
let interviewer = Arc::new(ControlInterviewer::new());
let steering_hub = Arc::new(fabro_workflow::SteeringHub::new(Arc::new(Emitter::new(
run_id,
))));
{
let mut runs = state.runs.lock().expect("runs lock poisoned");
if let Some(managed_run) = runs.get_mut(&run_id) {
@ -360,7 +378,6 @@ pub(crate) async fn execute(state: Arc<AppState>, run_id: RunId) {
managed_run.status = RunStatus::Running;
managed_run.answer_transport = Some(RunAnswerTransport::InProcess {
interviewer: Arc::clone(&interviewer),
steering_hub,
});
}
}
@ -433,6 +450,15 @@ pub(crate) async fn execute(state: Arc<AppState>, run_id: RunId) {
if let Err(err) = workflow_event::append_event(&run_store, &run_id, &event).await {
error!(run_id = %run_id, error = %err, "Failed to persist run outcome");
}
// The view trails the terminal record; the aggregate reads the settled
// projection, as the worker path reads the final state at worker exit.
state.petri_projector.settle(run_id).await;
match state.load_run_projection(&run_id).await {
Ok(final_state) => super::accumulate_concluded_run_usage(&state, &final_state),
Err(err) => {
warn!(run_id = %run_id, error = ?err, "the run's final state could not be read for the usage aggregate");
}
}
finish(&state, run_id, status, error);
}

1181
lib/apps/fabro-server/src/server/tests.rs
View file

File diff suppressed because it is too large Load diff

50
lib/apps/fabro-server/src/test_support.rs
View file

@ -18,7 +18,6 @@ use chrono::Duration as ChronoDuration;
use fabro_config::user::default_storage_dir;
use fabro_config::{LlmLayer, RunLayer, ServerSettingsBuilder, Storage, envfile};
use fabro_db::DbPool;
use fabro_interview::Interviewer;
use fabro_llm::lithos_catalog::Catalog;
use fabro_sandbox::SandboxInventory;
use fabro_static::EnvVars;
@ -29,7 +28,6 @@ use fabro_types::{
WorkflowVersionId,
};
use fabro_vault::{SecretType, Vault};
use fabro_workflow::handler::HandlerRegistry;
use lithos_llm::catalog::ProviderId;
use object_store::memory::InMemory as MemoryObjectStore;
use tokio::runtime::Builder as TokioRuntimeBuilder;
@ -43,8 +41,8 @@ use crate::jwt_auth::{AuthMode, ConfiguredAuth};
#[cfg(test)]
use crate::principal_middleware::{AuthContextSlot, RequestAuthContext};
use crate::server::{
self, AppState, AppStateConfig, EnvLookup, RegistryFactoryOverride, ResolvedAppStateSettings,
RouterOptions, build_app_state,
self, AppState, AppStateConfig, EnvLookup, ResolvedAppStateSettings, RouterOptions,
build_app_state,
};
use crate::server_secrets::ServerSecrets;
#[cfg(test)]
@ -93,7 +91,7 @@ pub struct TestAppStateBuilder {
server_settings: ServerSettings,
manifest_run_defaults: RunLayer,
max_concurrent_runs: usize,
registry_factory_override: Option<Box<RegistryFactoryOverride>>,
execute_in_process: bool,
sandbox_inventory: Option<SandboxInventory>,
store_bundle: Option<(Arc<Database>, ArtifactStore)>,
vault_path: Option<PathBuf>,
@ -115,7 +113,7 @@ impl Default for TestAppStateBuilder {
server_settings: default_test_server_settings(),
manifest_run_defaults: RunLayer::default(),
max_concurrent_runs: 5,
registry_factory_override: None,
execute_in_process: false,
sandbox_inventory: None,
store_bundle: None,
vault_path: None,
@ -153,14 +151,10 @@ impl TestAppStateBuilder {
self
}
pub fn registry_factory(
mut self,
registry_factory_override: impl Fn(Arc<dyn Interviewer>) -> HandlerRegistry
+ Send
+ Sync
+ 'static,
) -> Self {
self.registry_factory_override = Some(Box::new(registry_factory_override));
/// Execute runs in the server process instead of a worker, so a
/// scenario needs no worker binary.
pub fn in_process_execution(mut self) -> Self {
self.execute_in_process = true;
self
}
@ -300,7 +294,7 @@ impl TestAppStateBuilder {
self.manifest_run_defaults,
self.llm_overlay,
),
registry_factory_override: self.registry_factory_override,
execute_in_process: self.execute_in_process,
max_concurrent_runs: self.max_concurrent_runs,
store,
artifact_store,
@ -387,35 +381,31 @@ pub fn test_app_state() -> Arc<AppState> {
ready_test_app_state_builder().build()
}
pub fn test_app_state_with_registry_factory(
registry_factory_override: impl Fn(Arc<dyn Interviewer>) -> HandlerRegistry + Send + Sync + 'static,
) -> Arc<AppState> {
pub fn test_app_state_in_process() -> Arc<AppState> {
ready_test_app_state_builder()
.registry_factory(registry_factory_override)
.in_process_execution()
.build()
}
pub fn test_app_state_with_settings_and_registry_factory(
pub fn test_app_state_with_settings_in_process(
server_settings: ServerSettings,
manifest_run_defaults: RunLayer,
registry_factory_override: impl Fn(Arc<dyn Interviewer>) -> HandlerRegistry + Send + Sync + 'static,
) -> Arc<AppState> {
ready_test_app_state_builder()
.runtime_settings(server_settings, manifest_run_defaults)
.registry_factory(registry_factory_override)
.in_process_execution()
.build()
}
pub fn test_app_state_with_options_and_registry_factory(
pub fn test_app_state_with_options_in_process(
server_settings: ServerSettings,
manifest_run_defaults: RunLayer,
max_concurrent_runs: usize,
registry_factory_override: impl Fn(Arc<dyn Interviewer>) -> HandlerRegistry + Send + Sync + 'static,
) -> Arc<AppState> {
ready_test_app_state_builder()
.runtime_settings(server_settings, manifest_run_defaults)
.max_concurrent_runs(max_concurrent_runs)
.registry_factory(registry_factory_override)
.in_process_execution()
.build()
}
@ -446,27 +436,25 @@ pub(crate) fn resolved_runtime_settings_for_tests(
}
}
pub fn test_app_state_with_runtime_settings_and_registry_factory(
pub fn test_app_state_with_runtime_settings_in_process(
server_settings: ServerSettings,
manifest_run_defaults: RunLayer,
registry_factory_override: impl Fn(Arc<dyn Interviewer>) -> HandlerRegistry + Send + Sync + 'static,
) -> Arc<AppState> {
ready_test_app_state_builder()
.runtime_settings(server_settings, manifest_run_defaults)
.registry_factory(registry_factory_override)
.in_process_execution()
.build()
}
pub fn test_app_state_with_runtime_settings_and_options_and_registry_factory(
pub fn test_app_state_with_runtime_settings_and_options_in_process(
server_settings: ServerSettings,
manifest_run_defaults: RunLayer,
max_concurrent_runs: usize,
registry_factory_override: impl Fn(Arc<dyn Interviewer>) -> HandlerRegistry + Send + Sync + 'static,
) -> Arc<AppState> {
ready_test_app_state_builder()
.runtime_settings(server_settings, manifest_run_defaults)
.max_concurrent_runs(max_concurrent_runs)
.registry_factory(registry_factory_override)
.in_process_execution()
.build()
}

5
lib/apps/fabro-server/tests/it/helpers.rs
View file

@ -9,7 +9,7 @@ use fabro_server::server::{AppState, spawn_scheduler};
use fabro_server::test_support::{
TestAppStateBuilder, build_test_router, llm_overlay_with_provider_base_url,
test_app_state as server_test_app_state, test_app_state_with_runtime_settings_and_env_lookup,
test_app_state_with_runtime_settings_and_options_and_registry_factory,
test_app_state_with_runtime_settings_and_options_in_process,
};
use fabro_test::{
assert_axum_status, assert_reqwest_status, expect_axum_json, expect_axum_status,
@ -83,11 +83,10 @@ pub(crate) fn test_app_state_with_options(
settings: TestAppSettings,
max_concurrent_runs: usize,
) -> Arc<AppState> {
test_app_state_with_runtime_settings_and_options_and_registry_factory(
test_app_state_with_runtime_settings_and_options_in_process(
settings.server_settings,
settings.manifest_run_defaults,
max_concurrent_runs,
|interviewer| fabro_workflow::handler::default_registry(interviewer, || None),
)
}

64
lib/apps/fabro-server/tests/it/scenario/lifecycle.rs
View file

@ -2,14 +2,8 @@ use std::sync::Arc;
use axum::body::Body;
use axum::http::{Request, StatusCode};
use fabro_interview::Interviewer;
use fabro_server::server::spawn_scheduler;
use fabro_server::test_support::test_app_state_with_runtime_settings_and_registry_factory;
use fabro_workflow::handler::HandlerRegistry;
use fabro_workflow::handler::agent::AgentHandler;
use fabro_workflow::handler::exit::ExitHandler;
use fabro_workflow::handler::human::HumanHandler;
use fabro_workflow::handler::start::StartHandler;
use fabro_server::test_support::test_app_state_with_runtime_settings_in_process;
use tokio::time::sleep;
use tower::ServiceExt;
@ -18,15 +12,6 @@ use crate::helpers::{
run_json, test_settings, wait_for_run_status,
};
fn gate_registry(interviewer: Arc<dyn Interviewer>) -> HandlerRegistry {
let mut registry = HandlerRegistry::new(Box::new(AgentHandler::new(None)));
registry.register("start", Box::new(StartHandler));
registry.register("exit", Box::new(ExitHandler));
registry.register("agent", Box::new(AgentHandler::new(None)));
registry.register("human", Box::new(HumanHandler::new(interviewer)));
registry
}
async fn wait_for_question_id(app: &axum::Router, run_id: &str) -> String {
for _ in 0..POLL_ATTEMPTS {
let req = Request::builder()
@ -119,10 +104,9 @@ const GATE_DOT: &str = r#"digraph GateTest {
async fn full_http_lifecycle_approve_and_complete() {
let workspace = tempfile::tempdir().unwrap();
let settings = test_settings();
let state = test_app_state_with_runtime_settings_and_registry_factory(
let state = test_app_state_with_runtime_settings_in_process(
settings.server_settings,
settings.manifest_run_defaults,
gate_registry,
);
spawn_scheduler(Arc::clone(&state));
let app = fabro_server::test_support::build_test_router(Arc::clone(&state));
@ -159,15 +143,19 @@ async fn full_http_lifecycle_approve_and_complete() {
// 2. Poll for question to appear (run goes start -> work -> gate, then blocks)
let question = wait_for_question(&app, &run_id).await;
let question_id = question["id"].as_str().unwrap().to_string();
assert_eq!(question["stage"], "gate");
assert_eq!(question["stage"], "gate@1");
assert!(question["timeout_seconds"].is_null());
assert!(question["context_display"].is_null() || question["context_display"].is_string());
// 3. Submit answer selecting first option (Approve)
// 3. Submit answer selecting first option (Approve). Petri's id
// (`gate#3`) travels as one percent-encoded path segment.
let encoded_id =
percent_encoding::utf8_percent_encode(&question_id, percent_encoding::NON_ALPHANUMERIC)
.to_string();
let req = Request::builder()
.method("POST")
.uri(api(&format!(
"/runs/{run_id}/questions/{question_id}/answer"
"/runs/{run_id}/questions/{encoded_id}/answer"
)))
.header("content-type", "application/json")
.body(Body::from(
@ -213,10 +201,9 @@ async fn full_http_lifecycle_approve_and_complete() {
async fn full_http_lifecycle_cancel() {
let workspace = tempfile::tempdir().unwrap();
let settings = test_settings();
let state = test_app_state_with_runtime_settings_and_registry_factory(
let state = test_app_state_with_runtime_settings_in_process(
settings.server_settings,
settings.manifest_run_defaults,
gate_registry,
);
spawn_scheduler(Arc::clone(&state));
let app = fabro_server::test_support::build_test_router(Arc::clone(&state));
@ -297,10 +284,9 @@ async fn full_http_lifecycle_cancel() {
async fn cancel_at_human_gate_persists_cancelled_terminal_event() {
let workspace = tempfile::tempdir().unwrap();
let settings = test_settings();
let state = test_app_state_with_runtime_settings_and_registry_factory(
let state = test_app_state_with_runtime_settings_in_process(
settings.server_settings,
settings.manifest_run_defaults,
gate_registry,
);
spawn_scheduler(Arc::clone(&state));
let app = fabro_server::test_support::build_test_router(Arc::clone(&state));
@ -348,37 +334,19 @@ async fn cancel_at_human_gate_persists_cancelled_terminal_event() {
let status = wait_for_run_status(&app, &run_id, &["failed"]).await;
assert_eq!(status, "failed");
// The run's record says it was cancelled: Petri's finish, and the
// terminal lifecycle record Fabro wrote after it, both name the reason.
let req = Request::builder()
.method("GET")
.uri(api(&format!("/runs/{run_id}/events")))
.uri(api(&format!("/runs/{run_id}")))
.body(Body::empty())
.unwrap();
let response = app.oneshot(req).await.unwrap();
let body = response_json(
response,
StatusCode::OK,
format!("GET /api/v1/runs/{run_id}/events"),
format!("GET /api/v1/runs/{run_id}"),
)
.await;
let failed_reasons = body["data"]
.as_array()
.unwrap()
.iter()
.filter(|&event| event["event"] == "run.failed")
.map(|event| {
(
event["properties"]["failure"]["reason"]
.as_str()
.map(ToOwned::to_owned),
event["properties"]["failure"]["detail"]["message"]
.as_str()
.map(ToOwned::to_owned),
)
})
.collect::<Vec<_>>();
assert_eq!(failed_reasons, vec![(
Some("cancelled".to_string()),
Some("Pipeline cancelled".to_string())
)]);
assert_eq!(body["lifecycle"]["status"]["reason"], "cancelled", "{body}");
}

4
lib/apps/fabro-server/tests/it/scenario/petri.rs
View file

@ -270,9 +270,7 @@ async fn the_hello_bundle_runs_on_petri() {
let state = TestAppStateBuilder::new()
.runtime_settings(settings.server_settings, settings.manifest_run_defaults)
.max_concurrent_runs(5)
.registry_factory(|interviewer| {
fabro_workflow::handler::default_registry(interviewer, || None)
})
.in_process_execution()
.llm_overlay(llm_overlay_with_provider_base_url(
"openai",
twin.base_url.clone(),

112
lib/apps/fabro-server/tests/it/scenario/run_completion.rs
View file

@ -1,11 +1,7 @@
use std::sync::Arc;
use axum::body::Body;
use axum::http::{Request, StatusCode};
use fabro_auth::test_support;
use fabro_static::EnvVars;
use fabro_test::{TwinScenario, TwinScenarios, twin_openai};
use fabro_types::RunId;
use tokio::time::sleep;
use tower::ServiceExt;
@ -24,48 +20,24 @@ const PROJECT_SKILL_AGENT_DOT: &str = r#"digraph ProjectSkillAgent {
start [shape=Mdiamond, label="Start"]
exit [shape=Msquare, label="Exit"]
work [shape=box, label="Work", prompt="Respond with done."]
work [shape=box, label="Work", prompt="Respond with done.", model="gpt-5.4"]
start -> work -> exit
}"#;
/// A server whose agent stages reach the OpenAI twin through Petri's model
/// client, executing runs in this process.
fn test_app_with_openai_agent_backend(openai_base_url: String, api_key: String) -> axum::Router {
let settings = test_settings();
let llm_overlay =
fabro_server::test_support::llm_overlay_with_provider_base_url("openai", openai_base_url);
let catalog = Arc::new(fabro_server::test_support::test_catalog_with_overlay(
&llm_overlay,
));
let source_api_key = api_key.clone();
let env_api_key = api_key.clone();
let llm_source: Arc<dyn fabro_llm::credentials::CredentialProvider> =
test_support::env_credential_source(move |name| match name {
"OPENAI_API_KEY" => Some(source_api_key.clone()),
_ => None,
});
let state = fabro_server::test_support::TestAppStateBuilder::new()
.runtime_settings(settings.server_settings, settings.manifest_run_defaults)
.max_concurrent_runs(5)
.llm_overlay(llm_overlay)
.vault_entries([(EnvVars::OPENAI_API_KEY, api_key)])
.registry_factory(move |interviewer| {
let catalog = Arc::clone(&catalog);
let llm_source = Arc::clone(&llm_source);
let emitter = Arc::new(fabro_workflow::event::Emitter::new(RunId::new()));
let steering_hub = Arc::new(fabro_workflow::SteeringHub::new(emitter));
fabro_workflow::handler::default_registry(interviewer, move || {
Some(Box::new(
fabro_workflow::handler::llm::PebbleBackend::new_with_catalog(
OPENAI_AGENT_MODEL.to_string(),
lithos_llm::catalog::builtin::openai(),
fabro_workflow::model_fallback::ModelFallbackPolicy::default(),
Arc::clone(&llm_source),
Arc::clone(&steering_hub),
Arc::clone(&catalog),
),
))
})
})
.in_process_execution()
.env_lookup(move |name| match name {
"OPENAI_API_KEY" => Some(env_api_key.clone()),
_ => None,
@ -108,6 +80,9 @@ async fn agent_run_includes_project_skills_from_local_sandbox_working_directory(
)
.await
.expect("project skill should write");
// The run's workspace is a clone of the project, so the skill has to be
// committed there.
commit_all(project.path());
let twin = twin_openai().await;
let namespace = format!("{}::{}", module_path!(), line!());
@ -200,9 +175,11 @@ async fn attach_run_events_replays_terminal_event_after_completion() {
let status = wait_for_run_status(&app, &run_id, &["succeeded", "failed"]).await;
assert_eq!(status, "succeeded");
// The stream replays from its first item and ends with the terminal
// lifecycle record Fabro wrote after Petri's own finish.
let req = Request::builder()
.method("GET")
.uri(api(&format!("/runs/{run_id}/attach?since_seq=1")))
.uri(api(&format!("/runs/{run_id}/attach?after=0")))
.body(Body::empty())
.unwrap();
@ -210,22 +187,71 @@ async fn attach_run_events_replays_terminal_event_after_completion() {
let body = response_text(
response,
StatusCode::OK,
format!("GET /api/v1/runs/{run_id}/attach?since_seq=1"),
format!("GET /api/v1/runs/{run_id}/attach?after=0"),
)
.await;
let event_names = body
let items = body
.lines()
.filter_map(|line| line.strip_prefix("data:"))
.filter_map(|line| serde_json::from_str::<serde_json::Value>(line.trim()).ok())
.filter_map(|event| event["event"].as_str().map(ToString::to_string))
.collect::<Vec<_>>();
let names = items
.iter()
.map(|item| {
if item["kind"] == "platform" {
item["item"]["record"]["kind"]
.as_str()
.unwrap_or_default()
.to_string()
} else {
item["item"]["record"]["body"]["event"]
.as_str()
.or_else(|| item["item"]["derived"]["event"].as_str())
.unwrap_or_default()
.to_string()
}
})
.collect::<Vec<_>>();
assert!(
event_names.iter().any(|event| event == "run.completed"),
"expected a replayed terminal event, got {event_names:?}"
);
assert_eq!(
event_names.last().map(String::as_str),
Some("run.completed")
names.iter().any(|name| name == "run.finished"),
"expected Petri's finish in the replay, got {names:?}"
);
let last = items.last().expect("the replay has items");
assert_eq!(last["kind"], "platform", "{last}");
assert_eq!(last["item"]["record"]["kind"], "run.lifecycle", "{last}");
assert_eq!(last["item"]["record"]["transition"], "succeeded", "{last}");
}
/// Make `path` a git repository with every file committed, so a run whose
/// target is the folder starts from a clone that holds them.
#[expect(
clippy::disallowed_methods,
reason = "the fixture commits with the real git CLI, synchronously"
)]
fn commit_all(path: &std::path::Path) {
for args in [
vec!["init", "--quiet", "--initial-branch=main"],
vec!["add", "--all"],
vec![
"-c",
"user.name=Fabro Test",
"-c",
"user.email=test@fabro.sh",
"commit",
"--quiet",
"--message",
"project",
],
] {
let output = std::process::Command::new("git")
.args(&args)
.current_dir(path)
.output()
.expect("git should run");
assert!(
output.status.success(),
"git {args:?} failed: {}",
String::from_utf8_lossy(&output.stderr)
);
}
}

65
lib/apps/fabro-server/tests/it/scenario/sse.rs
View file

@ -3,12 +3,11 @@ use std::time::Duration;
use axum::body::Body;
use axum::http::{Request, StatusCode};
use http_body_util::BodyExt;
use tokio::time::{sleep, timeout};
use tokio::time::timeout;
use tower::ServiceExt;
use crate::helpers::{
POLL_ATTEMPTS, POLL_INTERVAL, api, checked_response, checked_response_in,
create_and_start_run_from_intent, minimal_intent_json_with_dry_run, response_json,
api, checked_response, create_and_start_run_from_intent, minimal_intent_json_with_dry_run,
test_app_state_with_options, test_app_with_scheduler, test_settings,
wait_for_run_status_not_in,
};
@ -21,34 +20,6 @@ const SIMPLE_DOT: &str = r#"digraph SSETest {
start -> work -> exit
}"#;
async fn wait_for_checkpoint(app: &axum::Router, run_id: &str) -> serde_json::Value {
for _ in 0..POLL_ATTEMPTS {
let req = Request::builder()
.method("GET")
.uri(api(&format!("/runs/{run_id}/checkpoint")))
.body(Body::empty())
.expect("checkpoint request should build");
let response = app.clone().oneshot(req).await.unwrap();
let status = response.status();
if status == StatusCode::OK {
return response_json(
response,
StatusCode::OK,
format!("GET /api/v1/runs/{run_id}/checkpoint"),
)
.await;
}
checked_response_in(
response,
&[StatusCode::OK, StatusCode::NOT_FOUND],
format!("GET /api/v1/runs/{run_id}/checkpoint"),
)
.await;
sleep(POLL_INTERVAL).await;
}
panic!("checkpoint did not become available for {run_id}");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn sse_stream_contains_expected_event_types() {
let workspace = tempfile::tempdir().unwrap();
@ -67,13 +38,13 @@ async fn sse_stream_contains_expected_event_types() {
// the run advances before the attach request is handled.
let req = Request::builder()
.method("GET")
.uri(api(&format!("/runs/{run_id}/attach?since_seq=1")))
.uri(api(&format!("/runs/{run_id}/attach?after=0")))
.body(Body::empty())
.unwrap();
let response = checked_response(
app.clone().oneshot(req).await.unwrap(),
StatusCode::OK,
format!("GET /api/v1/runs/{run_id}/attach?since_seq=1"),
format!("GET /api/v1/runs/{run_id}/attach?after=0"),
)
.await;
@ -94,34 +65,24 @@ async fn sse_stream_contains_expected_event_types() {
}
}
// Parse SSE data lines and extract event types
// Every frame is one stream item: Petri's events name the stage they
// belong to, so the run's stages show up as `visit.started`.
let mut event_types: Vec<String> = Vec::new();
for line in sse_data.lines() {
if let Some(json_str) = line.strip_prefix("data:") {
let json_str = json_str.trim();
if let Ok(event) = serde_json::from_str::<serde_json::Value>(json_str) {
if let Some(event_name) = event["event"].as_str() {
event_types.push(event_name.to_string());
if let Ok(item) = serde_json::from_str::<serde_json::Value>(json_str.trim()) {
let name = item["item"]["record"]["body"]["event"]
.as_str()
.or_else(|| item["item"]["derived"]["event"].as_str());
if let Some(name) = name {
event_types.push(name.to_string());
}
}
}
}
assert!(
event_types
.iter()
.any(|t| t == "stage.started" || t == "stage.completed"),
event_types.iter().any(|t| t == "visit.started"),
"should contain stage events, got: {event_types:?}"
);
// Pipeline is complete (SSE stream ended), verify checkpoint
let cp_body = wait_for_checkpoint(&app, &run_id).await;
// If run completed, checkpoint should have completed_nodes
if !cp_body.is_null() {
let completed = cp_body["completed_nodes"].as_array();
if let Some(nodes) = completed {
let names: Vec<&str> = nodes.iter().filter_map(|v| v.as_str()).collect();
assert!(names.contains(&"work"), "work should be in completed_nodes");
}
}
}

13
lib/components/fabro-petri/src/check.rs
View file

@ -20,6 +20,7 @@
use std::collections::BTreeMap;
use std::path::PathBuf;
use petri_frontend_attractor::kinds::{AGENT_KIND, PROMPT_KIND};
use petri_runtime::LoadError;
use petri_runtime::frontend::{
self, CompileInputs, LAUNCH_MODEL_VAR, LAUNCH_PROVIDER_VAR, MapFiles, REPOSITORY_VAR, Severity,
@ -116,6 +117,18 @@ pub struct Admitted {
pub warnings: Vec<Diagnostic>,
}
impl Admitted {
/// Whether any admitted graph has a node that runs a model: an agent
/// or a prompt node. A workflow of commands and gates needs none.
#[must_use]
pub fn needs_model(&self) -> bool {
std::iter::once(&self.graph)
.chain(&self.children)
.flat_map(|graph| &graph.body.nodes)
.any(|node| node.step.kind == AGENT_KIND || node.step.kind == PROMPT_KIND)
}
}
/// Why a check produced no graph.
#[derive(Debug, thiserror::Error)]
pub enum CheckError {

2
lib/components/fabro-petri/src/host_tools.rs
View file

@ -36,7 +36,7 @@
//! logged as an error and the session gets no run tools rather than the
//! wrong ones.
use fabro_workflow::handler::llm::register_fabro_run_tools;
use fabro_workflow::run_tools::register_fabro_run_tools;
use fabro_workflow::services::FabroRunToolServices;
use pebble_coding_agent::tools::RegisteredTool;
use petri_attractor_steps::host_tools::{HostToolContext, HostTools};

2
lib/components/fabro-petri/src/interview.rs
View file

@ -658,7 +658,7 @@ fn asked_question(request: &InterviewRequest, stage: String) -> AskedQuestion {
/// Fabro's question type: the one the gate names, else what the shape
/// implies.
fn question_type(question: &Question) -> QuestionType {
pub(crate) fn question_type(question: &Question) -> QuestionType {
question
.kind
.as_deref()

18
lib/components/fabro-petri/src/projection.rs
View file

@ -36,11 +36,11 @@ use fabro_types::{
BlockedReason, CheckpointRecord as ViewCheckpoint, CodingAgentEvent, CodingEvent, Conclusion,
FailureCategory, FailureDetail, FailureReason, InterviewOption, InterviewQuestionRecord,
ModelRef, ModelUsage, ParallelBranchId, ParallelBranchResult, PendingInterviewRecord,
PullRequestLink, QuestionType, RunApproval, RunApprovalState, RunControlAction, RunDiff,
RunFailure, RunId, RunProjection, RunSandbox, RunSandboxPlan, RunStatus, RunTiming,
SandboxProviderKind, StageCompletion, StageHandler, StageId, StageInferenceProjection,
StageModelUsage, StageOutcome, StageProjection, StageState, StageTiming, StartRecord,
SuccessReason, first_event_seq, timing, usage_rollup,
PullRequestLink, RunApproval, RunApprovalState, RunControlAction, RunDiff, RunFailure, RunId,
RunProjection, RunSandbox, RunSandboxPlan, RunStatus, RunTiming, SandboxProviderKind,
StageCompletion, StageHandler, StageId, StageInferenceProjection, StageModelUsage,
StageOutcome, StageProjection, StageState, StageTiming, StartRecord, SuccessReason,
first_event_seq, timing, usage_rollup,
};
use lithos_llm::catalog::{ModelId, ProviderId};
use lithos_llm::types::Usage;
@ -52,6 +52,8 @@ use serde::{Deserialize, Serialize};
use serde_json::Value;
use tracing::debug;
use crate::interview::question_type;
/// One item the projector hands the fold, with its delivery sequence.
pub enum Item<'a> {
Petri(&'a RunEvent),
@ -710,11 +712,7 @@ impl RunView {
id: question.id.clone(),
text: question.text.clone(),
stage: label,
question_type: question
.kind
.as_deref()
.and_then(|kind| kind.parse::<QuestionType>().ok())
.unwrap_or_default(),
question_type: question_type(question),
options: question
.options
.iter()

2
lib/components/fabro-petri/tests/host_tools.rs
View file

@ -26,7 +26,7 @@ use fabro_petri::host_tools::recorded::{self, ExecutionId, InvocationId};
use fabro_petri::runtime::RuntimeSpec;
use fabro_tool::fabro_client::ClientBackend;
use fabro_types::{BlobHash, RunId, WorkflowVersionId};
use fabro_workflow::handler::llm::register_fabro_run_tools;
use fabro_workflow::run_tools::register_fabro_run_tools;
use fabro_workflow::services::FabroRunToolServices;
use httpmock::{Method, MockServer};
use lithos_llm::types::Request;

1
lib/components/fabro-slack/Cargo.toml
View file

@ -15,7 +15,6 @@ workspace = true
[dependencies]
fabro-interview = { path = "../fabro-interview" }
fabro-types = { path = "../../foundation/fabro-types" }
fabro-workflow = { path = "../fabro-workflow" }
fabro-http.workspace = true
fabro-static.workspace = true
futures-util.workspace = true

14
lib/components/fabro-workflow/Cargo.toml
View file

@ -22,14 +22,11 @@ workspace = true
[dependencies]
anyhow.workspace = true
fabro-auth = { path = "../../foundation/fabro-auth" }
fabro-acp = { path = "../fabro-acp" }
fabro-config = { path = "../../foundation/fabro-config" }
fabro-graphviz = { path = "../fabro-graphviz" }
fabro-hooks = { path = "../fabro-hooks" }
fabro-validate = { path = "../fabro-validate" }
fabro-sandbox = { path = "../fabro-sandbox" }
sandbox-driver.workspace = true
fabro-mcp = { path = "../fabro-mcp" }
pebble-agent.workspace = true
pebble-coding-agent.workspace = true
fabro-github = { path = "../fabro-github" }
@ -40,7 +37,6 @@ fabro-util = { path = "../../foundation/fabro-util" }
fabro-redact.workspace = true
fabro-checkpoint = { path = "../fabro-checkpoint" }
fabro-llm = { path = "../fabro-llm" }
fabro-core = { path = "../../foundation/fabro-core" }
fabro-store = { path = "../fabro-store" }
fabro-static.workspace = true
fabro-types = { path = "../../foundation/fabro-types" }
@ -79,24 +75,16 @@ fabro-vault = { path = "../../foundation/fabro-vault" }
fabro-dump = { path = "../fabro-dump" }
fabro-client = { path = "../../foundation/fabro-client" }
fabro-workflow-version = { path = "../fabro-workflow-version" }
fabro-environment = { path = "../fabro-environment" }
fabro-llm = { path = "../fabro-llm", features = ["test-support"] }
fabro-store = { path = "../fabro-store", features = ["test-support"] }
fabro-auth = { path = "../../foundation/fabro-auth", features = ["test-support"] }
fabro-github = { path = "../fabro-github", features = ["test-support"] }
base64.workspace = true
fabro-acp = { path = "../fabro-acp", features = ["test-support"] }
fabro-workflow = { path = ".", features = ["test-support"] }
fabro-api = { path = "../../foundation/fabro-api" }
fabro-environment = { path = "../fabro-environment" }
fabro-sandbox = { path = "../fabro-sandbox", features = ["test-support"] }
fabro-mcp = { path = "../fabro-mcp" }
tokio = { workspace = true, features = ["test-util", "macros"] }
pebble-coding-agent = { workspace = true, features = ["test-util"] }
object_store.workspace = true
assert_cmd = "2"
predicates = "3"
httpmock = "0.8"
fabro-macros = { path = "../../foundation/fabro-macros" }
fabro-test = { workspace = true }
fabro-types = { path = "../../foundation/fabro-types", features = ["test-support"] }
shlex = "1"

86
lib/components/fabro-workflow/src/agent_memory.rs
View file

@ -1,86 +0,0 @@
//! Project memory for prompt stages.
//!
//! Agent stages ask pebble to discover the profile's instruction files from
//! the repository root down (`MemoryDiscovery::from_git_root`). A prompt
//! stage reads the working directory alone, as it always has, through the
//! same discovery and the same loader, so the two agree on which files a
//! harness reads and how much of them fits.
use fabro_sandbox::RunSandbox;
use fabro_types::AgentProfileKind;
use pebble_coding_agent::environment::Environment;
use pebble_coding_agent::{InterruptReason, MemoryDiscovery, ProjectMemory};
use tokio_util::sync::CancellationToken;
use crate::error::Error;
/// The memory text a prompt stage inlines into its system prompt: the
/// profile's instruction files in the sandbox working directory, loaded by
/// pebble's [`ProjectMemory`] rules.
///
/// # Errors
///
/// Returns [`Error::Cancelled`] when `cancel` fires around a read.
pub async fn load_memory_text(
sandbox: &RunSandbox,
profile_kind: AgentProfileKind,
cancel: &CancellationToken,
) -> Result<Option<String>, Error> {
let environment: &dyn Environment = sandbox;
let paths = MemoryDiscovery::working_directory()
.resolve(environment, profile_kind, cancel)
.await
.map_err(cancelled_or_handler)?;
let memory = ProjectMemory::load(environment, &paths, cancel)
.await
.map_err(cancelled_or_handler)?;
Ok((!memory.is_empty()).then(|| memory.text()))
}
fn cancelled_or_handler(error: pebble_coding_agent::Error) -> Error {
match error {
pebble_coding_agent::Error::Interrupted(InterruptReason::Cancelled) => Error::Cancelled,
other => Error::handler_with_source("Failed to load project memory", other),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn memory_text_dedupes_and_skips_missing_files() {
let dir = tempfile::tempdir().unwrap();
tokio::fs::write(dir.path().join("AGENTS.md"), "shared")
.await
.unwrap();
tokio::fs::write(dir.path().join("CLAUDE.md"), "shared")
.await
.unwrap();
let sandbox = fabro_sandbox::local_sandbox(dir.path().to_path_buf())
.await
.unwrap();
let text = load_memory_text(
&sandbox,
AgentProfileKind::Anthropic,
&CancellationToken::new(),
)
.await
.unwrap();
assert_eq!(text.as_deref(), Some("shared"));
let gemini = load_memory_text(
&sandbox,
AgentProfileKind::Gemini,
&CancellationToken::new(),
)
.await
.unwrap();
assert_eq!(
gemini.as_deref(),
Some("shared"),
"AGENTS.md is every harness's"
);
}
}

1645
lib/components/fabro-workflow/src/artifact.rs
View file

File diff suppressed because it is too large Load diff

447
lib/components/fabro-workflow/src/artifact_snapshot.rs
View file

@ -1,447 +0,0 @@
use std::path::Path;
use fabro_sandbox::{RunSandbox, SandboxFile, WalkOptions};
use fabro_types::ArtifactUpload;
use fabro_util::workspace_glob::WorkspaceGlobSet;
use futures::{StreamExt as _, TryStreamExt as _, stream};
use sha2::{Digest, Sha256};
use tokio::fs;
use tokio::io::AsyncReadExt as _;
use tracing::warn;
/// Summary of an artifact collection run.
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct ArtifactCollectionSummary {
pub files_copied: usize,
pub total_bytes: u64,
pub files_skipped: usize,
pub download_errors: usize,
pub hash_errors: usize,
pub captured_assets: Vec<ArtifactUpload>,
}
/// Directories to exclude from artifact traversal and checkpoint commits.
pub const EXCLUDE_DIRS: &[&str] = &[
".git",
"node_modules",
".pnpm-store",
".npm",
"target",
".next",
"__pycache__",
".venv",
"venv",
".cache",
".tox",
".pytest_cache",
".mypy_cache",
"dist",
];
/// Maximum number of files to collect.
const MAX_FILE_COUNT: usize = 100;
/// Maximum size for a single file (10 MB).
const MAX_FILE_SIZE: u64 = 10 * 1024 * 1024;
/// Maximum total size for all collected files (50 MB).
const MAX_TOTAL_SIZE: u64 = 50 * 1024 * 1024;
/// Independent traversal roots may run concurrently, but remote providers
/// should not receive an unbounded burst of file-walk operations.
const MAX_CONCURRENT_ARTIFACT_WALKS: usize = 4;
/// Select which files should be collected based on size budgets.
pub fn select_files_to_collect(discovered: Vec<SandboxFile>) -> Vec<SandboxFile> {
let mut candidates: Vec<SandboxFile> = discovered
.into_iter()
.filter(|file| file.size <= MAX_FILE_SIZE)
.collect();
candidates.sort_by(|left, right| {
left.size
.cmp(&right.size)
.then_with(|| left.relative_path.cmp(&right.relative_path))
});
let mut total = 0;
let mut selected = Vec::new();
for file in candidates {
if selected.len() >= MAX_FILE_COUNT || total + file.size > MAX_TOTAL_SIZE {
break;
}
total += file.size;
selected.push(file);
}
selected
}
async fn compute_artifact_info(
relative_path: &str,
local_path: &Path,
) -> std::result::Result<Option<ArtifactUpload>, String> {
let mime = mime_guess::from_path(relative_path)
.first_or_octet_stream()
.to_string();
let file = fs::File::open(local_path)
.await
.map_err(|error| format!("failed to open {}: {error}", local_path.display()))?;
let mut data = Vec::new();
file.take(MAX_FILE_SIZE + 1)
.read_to_end(&mut data)
.await
.map_err(|error| format!("failed to read {}: {error}", local_path.display()))?;
let bytes = u64::try_from(data.len()).unwrap_or(u64::MAX);
if bytes > MAX_FILE_SIZE {
return Ok(None);
}
let content_md5 = format!("{:x}", md5::compute(&data));
let content_sha256 = hex::encode(Sha256::digest(&data));
Ok(Some(ArtifactUpload {
path: relative_path.to_string(),
mime,
content_md5,
content_sha256,
bytes,
}))
}
/// Collect artifact files matching the configured workspace globs.
pub async fn collect_artifacts(
sandbox: &RunSandbox,
artifact_capture_dir: &Path,
globs: &WorkspaceGlobSet,
) -> Result<ArtifactCollectionSummary, String> {
let mut walk_options = WalkOptions::default();
walk_options.exclude_dirs = EXCLUDE_DIRS
.iter()
.map(|directory| (*directory).to_string())
.collect();
let walk_options = &walk_options;
let traversal_roots = globs
.traversal_roots()
.into_iter()
.map(str::to_string)
.collect::<Vec<_>>();
let walks = stream::iter(traversal_roots)
.map(|traversal_root| async move {
sandbox
.walk_files(sandbox.working_directory(), &traversal_root, walk_options)
.await
.map_err(|error| {
format!(
"artifact file traversal failed below {traversal_root:?}: {}",
error.display_with_causes()
)
})
})
.buffer_unordered(MAX_CONCURRENT_ARTIFACT_WALKS)
.try_collect::<Vec<_>>()
.await?;
let discovered = walks
.into_iter()
.flatten()
.filter(|file| globs.is_match(&file.relative_path))
.collect::<Vec<_>>();
let total_discovered = discovered.len();
let to_collect = select_files_to_collect(discovered);
let mut files_skipped = total_discovered - to_collect.len();
let mut files_copied = 0;
let mut total_bytes: u64 = 0;
let mut download_errors = 0;
let mut hash_errors = 0;
let mut captured_assets = Vec::new();
for file in &to_collect {
let dest = artifact_capture_dir.join(&file.relative_path);
match sandbox.download_file_to_local(&file.path, &dest).await {
Ok(()) => match compute_artifact_info(&file.relative_path, &dest).await {
Ok(Some(info)) if total_bytes.saturating_add(info.bytes) <= MAX_TOTAL_SIZE => {
files_copied += 1;
total_bytes += info.bytes;
captured_assets.push(info);
}
Ok(Some(_) | None) => {
let _ = fs::remove_file(&dest).await;
files_skipped += 1;
}
Err(error) => {
warn!(
path = file.relative_path.as_str(),
error = error.as_str(),
"Asset hash failed"
);
let _ = fs::remove_file(&dest).await;
hash_errors += 1;
}
},
Err(error) => {
let rendered = error.display_with_causes();
warn!(
path = file.relative_path.as_str(),
error = rendered.as_str(),
"Asset download failed"
);
download_errors += 1;
}
}
}
Ok(ArtifactCollectionSummary {
files_copied,
total_bytes,
files_skipped,
download_errors,
hash_errors,
captured_assets,
})
}
#[cfg(test)]
#[expect(clippy::disallowed_methods, reason = "tests write fixtures to disk")]
mod tests {
use std::collections::{BTreeSet, HashMap};
use fabro_sandbox::test_support::MockSandbox;
use super::*;
fn sandbox_file(relative_path: &str, size: u64) -> SandboxFile {
SandboxFile {
path: format!("/home/test/{relative_path}"),
relative_path: relative_path.to_string(),
size,
}
}
fn asset_sandbox(contents: HashMap<String, String>) -> MockSandbox {
let mut files = HashMap::new();
let mut discovered = Vec::new();
for (relative_path, content) in contents {
let file = sandbox_file(&relative_path, content.len() as u64);
files.insert(file.path.clone(), content);
discovered.push(file);
}
MockSandbox {
files,
..MockSandbox::linux()
}
.with_walk_files(discovered)
}
fn workspace_globs(patterns: &[&str]) -> WorkspaceGlobSet {
WorkspaceGlobSet::try_new(patterns).unwrap()
}
#[test]
fn select_files_skips_oversized_files() {
let selected = select_files_to_collect(vec![sandbox_file("huge.xml", MAX_FILE_SIZE + 1)]);
assert!(selected.is_empty());
}
#[test]
fn select_files_sorts_smallest_first() {
let discovered = vec![
sandbox_file("a.xml", 3000),
sandbox_file("b.xml", 1000),
sandbox_file("c.xml", 2000),
];
let selected = select_files_to_collect(discovered);
assert_eq!(
selected
.iter()
.map(|file| file.relative_path.as_str())
.collect::<Vec<_>>(),
vec!["b.xml", "c.xml", "a.xml"]
);
}
#[test]
fn select_files_enforces_total_budget() {
let discovered = (0..6)
.map(|index| sandbox_file(&format!("file{index}.xml"), 9 * 1024 * 1024))
.collect::<Vec<_>>();
let selected = select_files_to_collect(discovered);
assert_eq!(selected.len(), 5);
}
#[test]
fn select_files_enforces_count_limit() {
let discovered = (0..150)
.map(|index| sandbox_file(&format!("file{index}.txt"), 100))
.collect::<Vec<_>>();
let selected = select_files_to_collect(discovered);
assert_eq!(selected.len(), MAX_FILE_COUNT);
}
#[tokio::test]
async fn collect_artifacts_matches_workspace_relative_paths() {
let stage_dir = tempfile::tempdir().unwrap();
let contents = HashMap::from([
(".ai/reports/summary.md".to_string(), "summary".to_string()),
(
".ai/reports/nested/ignored.md".to_string(),
"nested".to_string(),
),
(
".ai/plans/2026-07-25-globbing.md".to_string(),
"plan".to_string(),
),
(".ai/plans/DRAFTING.md".to_string(), "drafting".to_string()),
("README.md".to_string(), "readme".to_string()),
]);
let sandbox = asset_sandbox(contents);
let globs = workspace_globs(&[".ai/reports/*.md", ".ai/plans/????-??-??-*.md"]);
let summary = collect_artifacts(&sandbox.sandbox(), stage_dir.path(), &globs)
.await
.unwrap();
assert_eq!(summary.files_copied, 2);
assert_eq!(
summary
.captured_assets
.iter()
.map(|asset| asset.path.as_str())
.collect::<BTreeSet<_>>(),
BTreeSet::from([".ai/plans/2026-07-25-globbing.md", ".ai/reports/summary.md",])
);
assert!(!stage_dir.path().join("manifest.json").exists());
}
#[tokio::test]
async fn collect_artifacts_preserves_content_metadata() {
let stage_dir = tempfile::tempdir().unwrap();
let sandbox = asset_sandbox(HashMap::from([(
"test-results/r.xml".to_string(),
"<test/>".to_string(),
)]));
let globs = workspace_globs(&["test-results/**"]);
let summary = collect_artifacts(&sandbox.sandbox(), stage_dir.path(), &globs)
.await
.unwrap();
assert_eq!(summary.files_copied, 1);
assert_eq!(summary.total_bytes, 7);
assert_eq!(summary.download_errors, 0);
assert_eq!(summary.hash_errors, 0);
assert_eq!(summary.captured_assets.len(), 1);
let asset = &summary.captured_assets[0];
assert_eq!(asset.path, "test-results/r.xml");
assert_eq!(asset.mime, "text/xml");
assert_eq!(asset.bytes, 7);
assert_eq!(asset.content_md5, "f1430934c390c118ed2f148e1d44d36c");
assert_eq!(
asset.content_sha256,
"28e51ddac37391b99c2b9053f1122d0bf84b02365e6fd8c6e8667378bd00f436"
);
assert_eq!(
std::fs::read_to_string(stage_dir.path().join("test-results/r.xml")).unwrap(),
"<test/>"
);
}
#[tokio::test]
async fn collect_artifacts_rechecks_downloaded_file_size() {
let stage_dir = tempfile::tempdir().unwrap();
let content = "x".repeat(usize::try_from(MAX_FILE_SIZE + 1).unwrap());
let file = sandbox_file("test-results/grew.bin", 1);
let sandbox = MockSandbox {
files: HashMap::from([(file.path.clone(), content)]),
..MockSandbox::linux()
}
.with_walk_files(vec![file]);
let globs = workspace_globs(&["test-results/**"]);
let summary = collect_artifacts(&sandbox.sandbox(), stage_dir.path(), &globs)
.await
.unwrap();
assert_eq!(summary.files_copied, 0);
assert_eq!(summary.files_skipped, 1);
assert!(summary.captured_assets.is_empty());
assert!(!stage_dir.path().join("test-results/grew.bin").exists());
}
#[tokio::test]
async fn collect_artifacts_prunes_dependency_and_build_directories() {
let stage_dir = tempfile::tempdir().unwrap();
let sandbox = asset_sandbox(HashMap::from([
(".ai/reports/keep.md".to_string(), "keep".to_string()),
("target/report.md".to_string(), "target".to_string()),
(
"nested/node_modules/report.md".to_string(),
"dependency".to_string(),
),
]));
let globs = workspace_globs(&["**/*.md"]);
let summary = collect_artifacts(&sandbox.sandbox(), stage_dir.path(), &globs)
.await
.unwrap();
assert_eq!(summary.files_copied, 1);
assert_eq!(summary.captured_assets[0].path, ".ai/reports/keep.md");
}
#[tokio::test]
async fn collect_artifacts_deduplicates_overlapping_patterns() {
let stage_dir = tempfile::tempdir().unwrap();
let sandbox = asset_sandbox(HashMap::from([(
".ai/reports/summary.md".to_string(),
"summary".to_string(),
)]));
let globs = workspace_globs(&[".ai/**/*.md", ".ai/reports/*.md"]);
let summary = collect_artifacts(&sandbox.sandbox(), stage_dir.path(), &globs)
.await
.unwrap();
assert_eq!(summary.files_copied, 1);
assert_eq!(summary.captured_assets.len(), 1);
}
#[tokio::test]
async fn collect_artifacts_reports_traversal_errors() {
let stage_dir = tempfile::tempdir().unwrap();
let sandbox = asset_sandbox(HashMap::new()).with_walk_files_error("permission denied");
let globs = workspace_globs(&["test-results/**"]);
let error = collect_artifacts(&sandbox.sandbox(), stage_dir.path(), &globs)
.await
.expect_err("failed traversal should fail artifact collection");
assert!(error.contains("artifact file traversal failed"), "{error}");
assert!(error.contains("permission denied"), "{error}");
}
#[tokio::test]
async fn collect_artifacts_keeps_download_errors_non_fatal() {
let stage_dir = tempfile::tempdir().unwrap();
let sandbox = asset_sandbox(HashMap::new()).with_walk_files(vec![
sandbox_file("test-results/missing.xml", 100),
sandbox_file("test-results/also-missing.xml", 200),
]);
let globs = workspace_globs(&["test-results/**"]);
let summary = collect_artifacts(&sandbox.sandbox(), stage_dir.path(), &globs)
.await
.unwrap();
assert_eq!(summary.files_copied, 0);
assert_eq!(summary.download_errors, 2);
assert_eq!(summary.hash_errors, 0);
}
}

23
lib/components/fabro-workflow/src/artifact_upload.rs
View file

@ -1,23 +0,0 @@
use std::path::Path;
use std::sync::Arc;
use anyhow::Result;
use async_trait::async_trait;
use fabro_store::ArtifactStore;
use fabro_types::{ArtifactUpload, StageId};
#[async_trait]
pub trait StageArtifactUploader: Send + Sync {
async fn upload_stage_artifacts(
&self,
stage_id: &StageId,
retry: u32,
artifact_capture_dir: &Path,
artifacts: &[ArtifactUpload],
) -> Result<()>;
}
pub enum ArtifactSink {
Store(ArtifactStore),
Uploader(Arc<dyn StageArtifactUploader>),
}

163
lib/components/fabro-workflow/src/command_log.rs
View file

@ -1,163 +0,0 @@
use std::path::{Path, PathBuf};
use std::sync::Arc;
use fabro_config::RunScratch;
use fabro_store::stage_storage_segment;
use fabro_types::{StageId, format_blob_ref};
use serde_json::Value;
use tokio::fs::{self, File, OpenOptions};
use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt};
use tokio::sync::Mutex;
use crate::error::{Error, Result};
use crate::runtime_store::RunStoreHandle;
#[derive(Debug, Clone)]
pub struct FinalizedCommandLogs {
pub output_ref: String,
pub output_bytes: u64,
pub output_text: String,
}
pub struct CommandLogRecorder {
output: Mutex<File>,
output_path: PathBuf,
}
impl CommandLogRecorder {
pub async fn create(run_dir: &Path, stage_id: &StageId) -> Result<Arc<Self>> {
let output_path = command_log_path(run_dir, stage_id);
if let Some(parent) = output_path.parent() {
fs::create_dir_all(parent).await.map_err(|err| {
Error::Io(format!(
"creating command log directory {}: {err}",
parent.display()
))
})?;
}
let output = open_truncated(&output_path).await?;
Ok(Arc::new(Self {
output: Mutex::new(output),
output_path,
}))
}
pub async fn append(&self, bytes: &[u8]) -> Result<()> {
if bytes.is_empty() {
return Ok(());
}
let mut file = self.output.lock().await;
file.write_all(bytes)
.await
.map_err(|err| Error::Io(format!("writing command output log failed: {err}")))?;
Ok(())
}
pub async fn finalize(&self, run_store: &RunStoreHandle) -> Result<FinalizedCommandLogs> {
self.flush_all().await?;
let (output_text, output_bytes) = read_lossy_text(&self.output_path).await?;
let output_ref = write_json_string_blob(run_store, &output_text).await?;
Ok(FinalizedCommandLogs {
output_ref,
output_bytes,
output_text,
})
}
pub async fn discard(self: Arc<Self>) -> Result<()> {
self.flush_all().await?;
let output_path = self.output_path.clone();
drop(self);
remove_if_exists(&output_path).await
}
async fn flush_all(&self) -> Result<()> {
self.output
.lock()
.await
.flush()
.await
.map_err(|err| Error::Io(format!("flushing command output log failed: {err}")))?;
Ok(())
}
}
pub fn command_log_path(run_dir: &Path, stage_id: &StageId) -> PathBuf {
RunScratch::new(run_dir)
.runtime_dir()
.join("stages")
.join(stage_storage_segment(stage_id))
.join("output.log")
}
pub async fn read_log_slice(
path: &Path,
offset: u64,
limit: u64,
) -> std::io::Result<(Vec<u8>, u64)> {
let mut file = fs::File::open(path).await?;
let total = file.metadata().await?.len();
let start = offset.min(total);
file.seek(std::io::SeekFrom::Start(start)).await?;
let take = limit.min(total.saturating_sub(start));
let mut buf = vec![0; usize::try_from(take).unwrap_or(usize::MAX)];
file.read_exact(&mut buf).await?;
Ok((buf, total))
}
pub async fn read_json_string_blob(
run_store: &RunStoreHandle,
blob_ref: &str,
) -> Result<Option<String>> {
let Some(blob_hash) = fabro_types::parse_blob_ref(blob_ref) else {
return Ok(None);
};
let bytes = run_store
.read_blob(&blob_hash)
.await
.map_err(|err| Error::engine_with_anyhow("command log blob read failed", err))?
.ok_or_else(|| Error::engine(format!("command log blob missing: {blob_hash}")))?;
let text = serde_json::from_slice::<String>(&bytes)
.map_err(|err| Error::engine_with_source("command log blob was not a JSON string", err))?;
Ok(Some(text))
}
async fn open_truncated(path: &Path) -> Result<File> {
OpenOptions::new()
.create(true)
.write(true)
.truncate(true)
.open(path)
.await
.map_err(|err| Error::Io(format!("opening command log {}: {err}", path.display())))
}
async fn read_lossy_text(path: &Path) -> Result<(String, u64)> {
let bytes = fs::read(path)
.await
.map_err(|err| Error::Io(format!("reading command log {}: {err}", path.display())))?;
let len = u64::try_from(bytes.len()).unwrap_or(u64::MAX);
Ok((String::from_utf8_lossy(&bytes).into_owned(), len))
}
async fn remove_if_exists(path: &Path) -> Result<()> {
match fs::remove_file(path).await {
Ok(()) => Ok(()),
Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(()),
Err(err) => Err(Error::Io(format!(
"removing command log {}: {err}",
path.display()
))),
}
}
async fn write_json_string_blob(run_store: &RunStoreHandle, text: &str) -> Result<String> {
let value = Value::String(text.to_string());
let bytes = serde_json::to_vec(&value)
.map_err(|err| Error::engine_with_source("command log JSON serialization failed", err))?;
let blob_hash = run_store
.write_blob(&bytes)
.await
.map_err(|err| Error::engine_with_anyhow("command log blob write failed", err))?;
Ok(format_blob_ref(&blob_hash))
}

632
lib/components/fabro-workflow/src/condition.rs
View file

@ -1,632 +0,0 @@
/// Condition expression evaluator for edge guards (spec Section 10).
///
/// The parser lives in `fabro_graphviz::condition`; this module re-exports
/// `parse_condition` and provides runtime evaluation against
/// `Outcome`/`Context`.
use fabro_graphviz::condition::{Clause, ConditionExpr, Op};
use crate::context::{self, Context, keys};
use crate::outcome::Outcome;
// ---------------------------------------------------------------------------
// Evaluator
// ---------------------------------------------------------------------------
fn resolve_key(key: &str, outcome: &Outcome, context: &Context) -> String {
if key == keys::OUTCOME {
return outcome.status.to_string();
}
if key == keys::PREFERRED_LABEL {
return outcome.preferred_label.as_deref().unwrap_or("").to_string();
}
context::lookup_flat(context, key).map_or_else(String::new, |val| json_value_to_string(&val))
}
fn resolve_key_value(key: &str, outcome: &Outcome, context: &Context) -> serde_json::Value {
if key == keys::OUTCOME {
return serde_json::Value::String(outcome.status.to_string());
}
if key == keys::PREFERRED_LABEL {
return outcome
.preferred_label
.as_deref()
.map_or(serde_json::Value::Null, |s| {
serde_json::Value::String(s.to_string())
});
}
context::lookup_flat(context, key).unwrap_or(serde_json::Value::Null)
}
fn json_value_to_string(val: &serde_json::Value) -> String {
match val {
serde_json::Value::String(s) => s.clone(),
serde_json::Value::Bool(b) => b.to_string(),
serde_json::Value::Number(n) => n.to_string(),
serde_json::Value::Null => String::new(),
other => other.to_string(),
}
}
fn is_truthy(s: &str) -> bool {
!s.is_empty() && s != "false" && s != "0"
}
fn eval_expr(expr: &ConditionExpr, outcome: &Outcome, context: &Context) -> bool {
match expr {
ConditionExpr::And(children) => {
if children.is_empty() {
return true;
}
children.iter().all(|c| eval_expr(c, outcome, context))
}
ConditionExpr::Or(children) => children.iter().any(|c| eval_expr(c, outcome, context)),
ConditionExpr::Not(inner) => !eval_expr(inner, outcome, context),
ConditionExpr::Clause(clause) => eval_clause(clause, outcome, context),
}
}
fn eval_clause(clause: &Clause, outcome: &Outcome, context: &Context) -> bool {
match &clause.op {
Op::Truthy => {
let resolved = resolve_key(&clause.key, outcome, context);
is_truthy(&resolved)
}
Op::Eq => {
let resolved = resolve_key(&clause.key, outcome, context);
resolved == clause.value
}
Op::NotEq => {
let resolved = resolve_key(&clause.key, outcome, context);
resolved != clause.value
}
Op::Gt | Op::Lt | Op::Gte | Op::Lte => {
let resolved = resolve_key(&clause.key, outcome, context);
let lhs: f64 = match resolved.parse() {
Ok(v) => v,
Err(_) => return false,
};
let rhs: f64 = match clause.value.parse() {
Ok(v) => v,
Err(_) => return false,
};
match &clause.op {
Op::Gt => lhs > rhs,
Op::Lt => lhs < rhs,
Op::Gte => lhs >= rhs,
Op::Lte => lhs <= rhs,
_ => unreachable!("outer match arm already restricts to Gt, Lt, Gte, and Lte"),
}
}
Op::Contains => {
let raw = resolve_key_value(&clause.key, outcome, context);
if let serde_json::Value::Array(arr) = &raw {
arr.iter()
.any(|elem| json_value_to_string(elem) == clause.value)
} else {
let s = json_value_to_string(&raw);
s.contains(&clause.value)
}
}
Op::Matches => {
let resolved = resolve_key(&clause.key, outcome, context);
// Regex was validated at parse time, so unwrap is safe
regex::Regex::new(&clause.value).is_ok_and(|re| re.is_match(&resolved))
}
}
}
/// Evaluate a condition expression against an outcome and context.
/// Empty conditions always return true.
#[must_use]
pub(crate) fn evaluate_condition(expr: &str, outcome: &Outcome, context: &Context) -> bool {
use fabro_graphviz::condition::parse_condition_expr;
let Ok(parsed) = parse_condition_expr(expr) else {
return false;
};
eval_expr(&parsed, outcome, context)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::outcome::StageOutcome;
fn make_outcome(status: StageOutcome) -> Outcome {
Outcome {
status,
..Outcome::success()
}
}
// -----------------------------------------------------------------------
// Phase 0: Existing behavior preserved
// -----------------------------------------------------------------------
#[test]
fn empty_condition_is_true() {
let outcome = make_outcome(StageOutcome::Succeeded);
let context = Context::new();
assert!(evaluate_condition("", &outcome, &context));
assert!(evaluate_condition(" ", &outcome, &context));
}
#[test]
fn outcome_equals_success() {
let outcome = make_outcome(StageOutcome::Succeeded);
let context = Context::new();
assert!(evaluate_condition("outcome=succeeded", &outcome, &context));
assert!(!evaluate_condition("outcome=failed", &outcome, &context));
}
#[test]
fn outcome_not_equals() {
let outcome = make_outcome(StageOutcome::Succeeded);
let context = Context::new();
assert!(evaluate_condition("outcome!=failed", &outcome, &context));
assert!(!evaluate_condition(
"outcome!=succeeded",
&outcome,
&context
));
}
#[test]
fn preferred_label_match() {
let mut outcome = make_outcome(StageOutcome::Succeeded);
outcome.preferred_label = Some("Fix".to_string());
let context = Context::new();
assert!(evaluate_condition(
"preferred_label=Fix",
&outcome,
&context
));
assert!(!evaluate_condition(
"preferred_label=Approve",
&outcome,
&context
));
}
#[test]
fn context_key_with_prefix() {
let outcome = make_outcome(StageOutcome::Succeeded);
let context = Context::new();
context.set("tests_passed", serde_json::json!("true"));
assert!(evaluate_condition(
"context.tests_passed=true",
&outcome,
&context
));
}
#[test]
fn bare_key_context_lookup() {
let outcome = make_outcome(StageOutcome::Succeeded);
let context = Context::new();
context.set("custom_key", serde_json::json!("custom_value"));
assert!(evaluate_condition(
"custom_key=custom_value",
&outcome,
&context
));
}
#[test]
fn missing_key_compares_as_empty() {
let outcome = make_outcome(StageOutcome::Succeeded);
let context = Context::new();
assert!(!evaluate_condition(
"missing_key=something",
&outcome,
&context
));
assert!(evaluate_condition("missing_key=", &outcome, &context));
}
#[test]
fn multiple_clauses_and() {
let outcome = make_outcome(StageOutcome::Succeeded);
let context = Context::new();
context.set("tests_passed", serde_json::json!("true"));
assert!(evaluate_condition(
"outcome=succeeded && context.tests_passed=true",
&outcome,
&context
));
assert!(!evaluate_condition(
"outcome=failed && context.tests_passed=true",
&outcome,
&context
));
}
#[test]
fn context_dotted_fallback() {
let outcome = make_outcome(StageOutcome::Succeeded);
let context = Context::new();
context.set("loop_state", serde_json::json!("exhausted"));
assert!(evaluate_condition(
"context.loop_state=exhausted",
&outcome,
&context
));
}
#[test]
fn bare_key_truthy_when_non_empty() {
let outcome = make_outcome(StageOutcome::Succeeded);
let context = Context::new();
context.set("my_flag", serde_json::json!("yes"));
assert!(evaluate_condition("my_flag", &outcome, &context));
}
#[test]
fn bare_key_falsy_when_empty() {
let outcome = make_outcome(StageOutcome::Succeeded);
let context = Context::new();
assert!(!evaluate_condition("missing_key", &outcome, &context));
}
#[test]
fn bare_key_falsy_when_false_string() {
let outcome = make_outcome(StageOutcome::Succeeded);
let context = Context::new();
context.set("my_flag", serde_json::json!("false"));
assert!(!evaluate_condition("my_flag", &outcome, &context));
}
#[test]
fn bare_key_falsy_when_zero_string() {
let outcome = make_outcome(StageOutcome::Succeeded);
let context = Context::new();
context.set("my_flag", serde_json::json!("0"));
assert!(!evaluate_condition("my_flag", &outcome, &context));
}
#[test]
fn bare_key_with_and_clause() {
let outcome = make_outcome(StageOutcome::Succeeded);
let context = Context::new();
context.set("flag", serde_json::json!("yes"));
assert!(evaluate_condition(
"outcome=succeeded && flag",
&outcome,
&context
));
}
#[test]
fn context_failure_class_matches_when_set() {
let outcome = make_outcome(StageOutcome::Failed {
retry_requested: false,
});
let context = Context::new();
context.set(keys::FAILURE_CLASS, serde_json::json!("budget_exhausted"));
assert!(evaluate_condition(
"context.failure_class=budget_exhausted",
&outcome,
&context
));
}
#[test]
fn context_failure_class_not_equals_on_success() {
let outcome = make_outcome(StageOutcome::Succeeded);
let context = Context::new();
context.set(keys::FAILURE_CLASS, serde_json::json!(""));
assert!(evaluate_condition(
"context.failure_class!=transient_infra",
&outcome,
&context
));
}
#[test]
fn context_failure_class_combined_with_outcome() {
let outcome = make_outcome(StageOutcome::Failed {
retry_requested: false,
});
let context = Context::new();
context.set(keys::FAILURE_CLASS, serde_json::json!("transient_infra"));
assert!(evaluate_condition(
"outcome=failed && context.failure_class=transient_infra",
&outcome,
&context
));
assert!(!evaluate_condition(
"outcome=failed && context.failure_class=deterministic",
&outcome,
&context
));
}
// -----------------------------------------------------------------------
// Phase 1: Numeric comparisons
// -----------------------------------------------------------------------
#[test]
fn numeric_gt() {
let outcome = make_outcome(StageOutcome::Succeeded);
let context = Context::new();
context.set("score", serde_json::json!(90));
assert!(evaluate_condition("context.score > 80", &outcome, &context));
context.set("score", serde_json::json!(70));
assert!(!evaluate_condition(
"context.score > 80",
&outcome,
&context
));
}
#[test]
fn numeric_gte() {
let outcome = make_outcome(StageOutcome::Succeeded);
let context = Context::new();
context.set("score", serde_json::json!(80));
assert!(evaluate_condition(
"context.score >= 80",
&outcome,
&context
));
}
#[test]
fn numeric_lte() {
let outcome = make_outcome(StageOutcome::Succeeded);
let context = Context::new();
context.set("score", serde_json::json!(80));
assert!(evaluate_condition(
"context.score <= 80",
&outcome,
&context
));
}
#[test]
fn numeric_lt() {
let outcome = make_outcome(StageOutcome::Succeeded);
let context = Context::new();
context.set("count", serde_json::json!(3));
assert!(evaluate_condition("context.count < 5", &outcome, &context));
}
#[test]
fn numeric_float() {
let outcome = make_outcome(StageOutcome::Succeeded);
let context = Context::new();
context.set("ratio", serde_json::json!(0.75));
assert!(evaluate_condition(
"context.ratio > 0.5",
&outcome,
&context
));
}
#[test]
fn numeric_non_numeric_returns_false() {
let outcome = make_outcome(StageOutcome::Succeeded);
let context = Context::new();
context.set("score", serde_json::json!("not_a_number"));
assert!(!evaluate_condition(
"context.score > 80",
&outcome,
&context
));
}
// -----------------------------------------------------------------------
// Phase 2: contains operator
// -----------------------------------------------------------------------
#[test]
fn contains_substring() {
let outcome = make_outcome(StageOutcome::Succeeded);
let context = Context::new();
context.set("message", serde_json::json!("an error occurred"));
assert!(evaluate_condition(
"context.message contains error",
&outcome,
&context
));
context.set("message", serde_json::json!("all good"));
assert!(!evaluate_condition(
"context.message contains error",
&outcome,
&context
));
}
#[test]
fn contains_case_sensitive() {
let outcome = make_outcome(StageOutcome::Succeeded);
let context = Context::new();
context.set("message", serde_json::json!("an error occurred"));
assert!(!evaluate_condition(
"context.message contains Error",
&outcome,
&context
));
}
#[test]
fn contains_json_array() {
let outcome = make_outcome(StageOutcome::Succeeded);
let context = Context::new();
context.set("tags", serde_json::json!(["urgent", "low"]));
assert!(evaluate_condition(
"context.tags contains urgent",
&outcome,
&context
));
assert!(!evaluate_condition(
"context.tags contains critical",
&outcome,
&context
));
}
// -----------------------------------------------------------------------
// Phase 3: matches operator (regex)
// -----------------------------------------------------------------------
#[test]
fn matches_regex() {
let outcome = make_outcome(StageOutcome::Succeeded);
let context = Context::new();
context.set("version", serde_json::json!("v2.0"));
assert!(evaluate_condition(
r"context.version matches ^v\d+",
&outcome,
&context
));
context.set("version", serde_json::json!("beta"));
assert!(!evaluate_condition(
r"context.version matches ^v\d+",
&outcome,
&context
));
}
// -----------------------------------------------------------------------
// Phase 4: OR (||)
// -----------------------------------------------------------------------
#[test]
fn or_disjunction() {
let outcome = make_outcome(StageOutcome::Succeeded);
let context = Context::new();
assert!(evaluate_condition(
"outcome=succeeded || outcome=partially_succeeded",
&outcome,
&context
));
let outcome = make_outcome(StageOutcome::Failed {
retry_requested: false,
});
assert!(!evaluate_condition(
"outcome=succeeded || outcome=partially_succeeded",
&outcome,
&context
));
}
#[test]
fn or_precedence_and_binds_tighter() {
// a=1 && b=2 || c=3 is (a=1 AND b=2) OR c=3
let outcome = make_outcome(StageOutcome::Succeeded);
let context = Context::new();
context.set("a", serde_json::json!("0"));
context.set("b", serde_json::json!("2"));
context.set("c", serde_json::json!("3"));
// a=1 is false, b=2 is true => AND is false; c=3 is true => OR is true
assert!(evaluate_condition("a=1 && b=2 || c=3", &outcome, &context));
}
#[test]
fn or_precedence_right_and() {
// a=1 || b=2 && c=3 is a=1 OR (b=2 AND c=3)
let outcome = make_outcome(StageOutcome::Succeeded);
let context = Context::new();
context.set("a", serde_json::json!("0"));
context.set("b", serde_json::json!("2"));
context.set("c", serde_json::json!("0"));
// a=1 false; b=2 true, c=3 false => AND false; OR false
assert!(!evaluate_condition("a=1 || b=2 && c=3", &outcome, &context));
}
// -----------------------------------------------------------------------
// Phase 5: NOT (!)
// -----------------------------------------------------------------------
#[test]
fn not_negation() {
let outcome = make_outcome(StageOutcome::Succeeded);
let context = Context::new();
assert!(evaluate_condition("!outcome=failed", &outcome, &context));
assert!(!evaluate_condition(
"!outcome=succeeded",
&outcome,
&context
));
}
#[test]
fn not_missing_key_is_true() {
let outcome = make_outcome(StageOutcome::Succeeded);
let context = Context::new();
assert!(evaluate_condition("!missing_key", &outcome, &context));
}
#[test]
fn not_with_and() {
let outcome = make_outcome(StageOutcome::Succeeded);
let context = Context::new();
context.set("ready", serde_json::json!("true"));
assert!(evaluate_condition(
"!outcome=failed && context.ready=true",
&outcome,
&context
));
}
// -----------------------------------------------------------------------
// Phase 6: Quoted literal values (spec parse_literal)
// -----------------------------------------------------------------------
#[test]
fn quoted_value_matches_bare_value() {
let outcome = make_outcome(StageOutcome::Succeeded);
let context = Context::new();
assert!(evaluate_condition(
r#"outcome="succeeded""#,
&outcome,
&context
));
assert!(!evaluate_condition(
r#"outcome="failed""#,
&outcome,
&context
));
}
#[test]
fn quoted_not_eq_matches() {
let outcome = make_outcome(StageOutcome::Succeeded);
let context = Context::new();
assert!(evaluate_condition(
r#"outcome!="failed""#,
&outcome,
&context
));
assert!(!evaluate_condition(
r#"outcome!="succeeded""#,
&outcome,
&context
));
}
#[test]
fn quoted_context_value() {
let outcome = make_outcome(StageOutcome::Succeeded);
let context = Context::new();
context.set("env", serde_json::json!("production"));
assert!(evaluate_condition(
r#"context.env="production""#,
&outcome,
&context
));
}
#[test]
fn quoted_and_bare_equivalent_in_compound() {
let outcome = make_outcome(StageOutcome::Succeeded);
let context = Context::new();
context.set("ready", serde_json::json!("true"));
// Mix bare and quoted in a compound expression
assert!(evaluate_condition(
r#"outcome=succeeded && context.ready="true""#,
&outcome,
&context
));
}
}

617
lib/components/fabro-workflow/src/context.rs
View file

@ -1,617 +0,0 @@
pub mod keys {
//! Static context key constants and helper functions for dynamic keys.
//!
//! All context keys used across the engine, handlers, and preamble are
//! defined here to prevent typos and improve discoverability.
// --- Top-level keys ---
pub const CURRENT_NODE: &str = "current_node";
pub const OUTCOME: &str = "outcome";
pub const FAILURE_CLASS: &str = "failure_class";
pub const FAILURE_SIGNATURE: &str = "failure_signature";
pub const PREFERRED_LABEL: &str = "preferred_label";
pub const LAST_STAGE: &str = "last_stage";
pub const LAST_RESPONSE: &str = "last_response";
pub const REVIEW_TARGET: &str = "review_target";
// --- graph.* keys ---
pub const GRAPH_GOAL: &str = "graph.goal";
// --- internal.* keys ---
pub const INTERNAL_RUN_ID: &str = "internal.run_id";
pub const INTERNAL_WORK_DIR: &str = "internal.work_dir";
pub const INTERNAL_FIDELITY: &str = "internal.fidelity";
pub const INTERNAL_THREAD_ID: &str = "internal.thread_id";
pub const INTERNAL_NODE_VISIT_COUNT: &str = "internal.node_visit_count";
/// 1-based stage execution ordinal for the currently-executing node — the
/// numeric component of the external `StageId`. Runtime-only: reserved by
/// the lifecycle when a stage execution first becomes observable and
/// stripped from durable context snapshots, unlike
/// [`INTERNAL_NODE_VISIT_COUNT`], which remains the checkpointed graph
/// visit.
pub const INTERNAL_STAGE_EXECUTION_ORDINAL: &str = "internal.stage_execution_ordinal";
pub const INTERNAL_PARENT_PREAMBLE: &str = "internal.parent_preamble";
pub const INTERNAL_PARALLEL_GROUP_ID: &str = "internal.parallel_group_id";
pub const INTERNAL_PARALLEL_BRANCH_ID: &str = "internal.parallel_branch_id";
/// Stash of pre-rendered per-branch preambles for a parallel node; see
/// [`super::ParallelBranchPreamble`] for the entry shape and the
/// producer/consumer contract.
pub const INTERNAL_PARALLEL_BRANCH_PREAMBLES: &str = "internal.parallel_branch_preambles";
// --- current.* keys ---
pub const CURRENT_PREAMBLE: &str = "current.preamble";
// --- command.* keys ---
pub const COMMAND_OUTPUT: &str = "command.output";
// --- human.gate.* keys ---
pub const HUMAN_GATE_SELECTED: &str = "human.gate.selected";
pub const HUMAN_GATE_LABEL: &str = "human.gate.label";
pub const HUMAN_GATE_TEXT: &str = "human.gate.text";
// --- parallel.* keys ---
pub const PARALLEL_RESULTS: &str = "parallel.results";
pub const PARALLEL_BRANCH_COUNT: &str = "parallel.branch_count";
/// Runtime-only keys stripped from durable context projections.
pub(crate) const TRANSIENT_CONTEXT_KEYS: &[&str] = &[
CURRENT_PREAMBLE,
INTERNAL_PARALLEL_BRANCH_PREAMBLES,
INTERNAL_STAGE_EXECUTION_ORDINAL,
];
// --- Prefix constants (for filtering and dynamic keys) ---
pub const GRAPH_PREFIX: &str = "graph.";
pub const INTERNAL_PREFIX: &str = "internal.";
pub const CURRENT_PREFIX: &str = "current";
pub const THREAD_PREFIX: &str = "thread.";
pub const RESPONSE_PREFIX: &str = "response.";
pub const INTERNAL_RETRY_COUNT_PREFIX: &str = "internal.retry_count.";
/// Keys the prompt preamble never renders as context values: engine
/// bookkeeping, per-thread cursors, and values the per-stage sections
/// already present.
#[must_use]
pub(crate) fn is_preamble_hidden_key(key: &str) -> bool {
is_engine_internal_key(key)
|| key.starts_with(RESPONSE_PREFIX)
|| key == OUTCOME
|| key == LAST_STAGE
|| key == LAST_RESPONSE
|| key == PREFERRED_LABEL
}
// --- Helper functions for dynamic keys ---
#[must_use]
pub fn response_key(node_id: &str) -> String {
format!("{RESPONSE_PREFIX}{node_id}")
}
#[must_use]
pub fn thread_current_node_key(thread_id: &str) -> String {
format!("{THREAD_PREFIX}{thread_id}.current_node")
}
#[must_use]
pub fn graph_attr_key(attr: &str) -> String {
format!("{GRAPH_PREFIX}{attr}")
}
#[must_use]
pub fn retry_count_key(node_id: &str) -> String {
format!("{INTERNAL_RETRY_COUNT_PREFIX}{node_id}")
}
/// Returns `true` for engine-internal keys that should not propagate from
/// child to parent workflow contexts.
#[must_use]
pub fn is_engine_internal_key(key: &str) -> bool {
key.starts_with(INTERNAL_PREFIX)
|| key.starts_with(GRAPH_PREFIX)
|| key.starts_with(THREAD_PREFIX)
|| key.starts_with(CURRENT_PREFIX)
}
pub use fabro_graphviz::Fidelity;
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn response_key_formats_correctly() {
assert_eq!(response_key("plan"), "response.plan");
}
#[test]
fn thread_current_node_key_formats_correctly() {
assert_eq!(thread_current_node_key("main"), "thread.main.current_node");
}
#[test]
fn graph_attr_key_formats_correctly() {
assert_eq!(graph_attr_key("goal"), "graph.goal");
}
#[test]
fn retry_count_key_formats_correctly() {
assert_eq!(retry_count_key("plan"), "internal.retry_count.plan");
}
#[test]
fn is_engine_internal_key_classifies_correctly() {
// Keys that ARE engine-internal (should not propagate)
assert!(is_engine_internal_key("internal.run_id"));
assert!(is_engine_internal_key("internal.fidelity"));
assert!(is_engine_internal_key("internal.parent_preamble"));
assert!(is_engine_internal_key("graph.goal"));
assert!(is_engine_internal_key("thread.main.current_node"));
assert!(is_engine_internal_key("current.preamble"));
assert!(is_engine_internal_key("current_node"));
// Keys that are NOT engine-internal (should propagate)
assert!(!is_engine_internal_key("response.plan"));
assert!(!is_engine_internal_key("command.output"));
assert!(!is_engine_internal_key("outcome"));
assert!(!is_engine_internal_key("last_stage"));
assert!(!is_engine_internal_key("review.result"));
assert!(!is_engine_internal_key(REVIEW_TARGET));
assert!(!is_engine_internal_key("user.name"));
}
}
}
use std::collections::HashMap;
pub use fabro_core::Context;
use fabro_graphviz::Fidelity;
use fabro_types::{ParallelBranchId, RunId, StageId};
use serde::{Deserialize, Serialize};
use crate::error::{Error, FailureSignature, FailureSignatureExt};
use crate::event::StageScope;
use crate::outcome::{Outcome, OutcomeExt};
/// Applies the context values derived from a completed node result.
///
/// Edge-policy projection and the durable `after_record` lifecycle use this
/// same function so conditional routes observe identical values.
pub(crate) fn apply_recorded_outcome_context(
context: &Context,
node_id: &str,
outcome: &Outcome,
retry_count: u32,
) {
let failure_class = outcome.classified_failure_category();
let failure_signature = failure_class
.map(|category| {
let signature_hint = outcome
.failure
.as_ref()
.and_then(|failure| failure.signature.as_deref());
FailureSignature::new(node_id, category, signature_hint, outcome.failure_reason())
.to_string()
})
.unwrap_or_default();
context.set(
keys::retry_count_key(node_id),
serde_json::json!(retry_count),
);
context.set(keys::OUTCOME, serde_json::json!(outcome.status.to_string()));
context.set(
keys::FAILURE_CLASS,
serde_json::json!(failure_class.map_or(String::new(), |class| class.to_string())),
);
context.set(
keys::FAILURE_SIGNATURE,
serde_json::json!(failure_signature),
);
if let Some(preferred_label) = &outcome.preferred_label {
context.set(keys::PREFERRED_LABEL, serde_json::json!(preferred_label));
}
}
/// Keys whose values changed or were added in `after` relative to `before`.
/// Takes `after` by value so changed entries move instead of clone.
pub(crate) fn context_diff(
before: &HashMap<String, serde_json::Value>,
after: HashMap<String, serde_json::Value>,
) -> HashMap<String, serde_json::Value> {
after
.into_iter()
.filter(|(key, value)| before.get(key) != Some(value))
.collect()
}
/// [`context_diff`] restricted to user-visible keys: the diff that should
/// propagate outside the executing scope (to a parent workflow or across a
/// parallel fork), with engine-internal keys removed.
pub(crate) fn context_diff_public(
before: &HashMap<String, serde_json::Value>,
after: HashMap<String, serde_json::Value>,
) -> HashMap<String, serde_json::Value> {
context_diff(before, after)
.into_iter()
.filter(|(key, _)| !keys::is_engine_internal_key(key))
.collect()
}
/// Read a context key the way workflow authors write one: the declared key
/// first, then the same key with a leading `context.` stripped.
///
/// The lookup is flat. `context.plan.title` reads the literal keys
/// `context.plan.title` and `plan.title`; it never walks into a nested object.
pub(crate) fn lookup_flat(context: &Context, key: &str) -> Option<serde_json::Value> {
if let Some(bare) = key.strip_prefix("context.") {
return context.get(key).or_else(|| context.get(bare));
}
context.get(key)
}
/// One entry of the [`keys::INTERNAL_PARALLEL_BRANCH_PREAMBLES`] stash.
///
/// The stash is a JSON array indexed by the parallel node's outgoing-edge
/// order. `null` entries mean the branch inherits the fork's preamble.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub(crate) struct ParallelBranchPreamble {
pub(crate) fidelity: Fidelity,
pub(crate) preamble: String,
}
/// Domain-specific typed accessors for workflow context values.
pub trait WorkflowContext {
fn fidelity(&self) -> Fidelity;
fn thread_id(&self) -> Option<String>;
fn preamble(&self) -> String;
fn run_id(&self) -> String;
/// Parse `internal.run_id`, failing when the engine did not seed a
/// valid run ID.
fn parsed_run_id(&self) -> Result<RunId, Error>;
fn parallel_group_id(&self) -> Option<StageId>;
fn parallel_branch_id(&self) -> Option<ParallelBranchId>;
/// Build the stage-level emit scope from the currently-executing node and
/// its execution ordinal. Returns `None` for run-level emissions
/// where no stage is active (i.e., `CURRENT_NODE` is unset).
fn current_stage_scope(&self) -> Option<StageScope>;
}
impl WorkflowContext for Context {
fn fidelity(&self) -> Fidelity {
self.get_string(keys::INTERNAL_FIDELITY, "")
.parse()
.unwrap_or_default()
}
fn thread_id(&self) -> Option<String> {
self.get(keys::INTERNAL_THREAD_ID)
.and_then(|v| v.as_str().map(String::from))
}
fn preamble(&self) -> String {
self.get_string(keys::CURRENT_PREAMBLE, "")
}
fn run_id(&self) -> String {
self.get_string(keys::INTERNAL_RUN_ID, "unknown")
}
fn parsed_run_id(&self) -> Result<RunId, Error> {
self.run_id()
.parse()
.map_err(|err| Error::handler_with_source("invalid internal run_id", err))
}
fn parallel_group_id(&self) -> Option<StageId> {
self.get(keys::INTERNAL_PARALLEL_GROUP_ID)
.and_then(|value| serde_json::from_value(value).ok())
}
fn parallel_branch_id(&self) -> Option<ParallelBranchId> {
self.get(keys::INTERNAL_PARALLEL_BRANCH_ID)
.and_then(|value| serde_json::from_value(value).ok())
}
fn current_stage_scope(&self) -> Option<StageScope> {
let node_id = self
.get(keys::CURRENT_NODE)
.and_then(|value| value.as_str().map(String::from))?;
Some(StageScope::from_context(self, node_id))
}
}
#[cfg(test)]
mod tests {
use std::collections::HashMap;
use super::*;
#[test]
fn new_context_is_empty() {
let ctx = Context::new();
assert!(ctx.snapshot().is_empty());
}
#[test]
fn set_and_get() {
let ctx = Context::new();
ctx.set("key", serde_json::json!("value"));
assert_eq!(ctx.get("key"), Some(serde_json::json!("value")));
}
#[test]
fn lookup_flat_prefers_the_exact_key_then_strips_the_context_prefix() {
let ctx = Context::new();
ctx.set("context.items", serde_json::json!(["exact"]));
ctx.set("items", serde_json::json!(["fallback"]));
assert_eq!(
lookup_flat(&ctx, "context.items"),
Some(serde_json::json!(["exact"]))
);
// An explicit null is a value, not a miss, so it wins over the bare key.
ctx.set("context.items", serde_json::Value::Null);
assert_eq!(
lookup_flat(&ctx, "context.items"),
Some(serde_json::Value::Null)
);
let bare_only = Context::new();
bare_only.set("items", serde_json::json!(["fallback"]));
assert_eq!(
lookup_flat(&bare_only, "context.items"),
Some(serde_json::json!(["fallback"]))
);
assert_eq!(
lookup_flat(&bare_only, "items"),
Some(serde_json::json!(["fallback"]))
);
assert_eq!(lookup_flat(&bare_only, "context.missing"), None);
}
#[test]
fn get_missing_key() {
let ctx = Context::new();
assert_eq!(ctx.get("missing"), None);
}
#[test]
fn context_diff_detects_additions() {
let before = HashMap::new();
let mut after = HashMap::new();
after.insert("key".to_string(), serde_json::json!("value"));
let diff = context_diff(&before, after);
assert_eq!(diff.len(), 1);
assert_eq!(diff.get("key"), Some(&serde_json::json!("value")));
}
#[test]
fn context_diff_detects_changes() {
let mut before = HashMap::new();
before.insert("key".to_string(), serde_json::json!("old"));
let mut after = HashMap::new();
after.insert("key".to_string(), serde_json::json!("new"));
let diff = context_diff(&before, after);
assert_eq!(diff.len(), 1);
assert_eq!(diff.get("key"), Some(&serde_json::json!("new")));
}
#[test]
fn context_diff_ignores_unchanged() {
let mut before = HashMap::new();
before.insert("key".to_string(), serde_json::json!("same"));
let mut after = HashMap::new();
after.insert("key".to_string(), serde_json::json!("same"));
let diff = context_diff(&before, after);
assert!(diff.is_empty());
}
#[test]
fn context_diff_ignores_deletions() {
let mut before = HashMap::new();
before.insert("removed".to_string(), serde_json::json!("gone"));
let after = HashMap::new();
let diff = context_diff(&before, after);
assert!(diff.is_empty());
}
#[test]
fn context_diff_public_excludes_engine_internal_keys() {
let before = HashMap::new();
let mut after = HashMap::new();
after.insert("graph.goal".to_string(), serde_json::json!("child goal"));
after.insert(
"internal.run_id".to_string(),
serde_json::json!("child-run"),
);
after.insert(
"thread.main.current_node".to_string(),
serde_json::json!("exit"),
);
after.insert("current_node".to_string(), serde_json::json!("exit"));
after.insert("response.plan".to_string(), serde_json::json!("the plan"));
after.insert("review.result".to_string(), serde_json::json!("approved"));
let filtered = context_diff_public(&before, after);
assert_eq!(filtered.len(), 2);
assert!(filtered.contains_key("response.plan"));
assert!(filtered.contains_key("review.result"));
}
#[test]
fn get_string_with_value() {
let ctx = Context::new();
ctx.set("name", serde_json::json!("alice"));
assert_eq!(ctx.get_string("name", "default"), "alice");
}
#[test]
fn get_string_missing_key() {
let ctx = Context::new();
assert_eq!(ctx.get_string("missing", "fallback"), "fallback");
}
#[test]
fn get_string_non_string_value() {
let ctx = Context::new();
ctx.set("num", serde_json::json!(42));
assert_eq!(ctx.get_string("num", "default"), "default");
}
#[test]
fn snapshot_is_independent() {
let ctx = Context::new();
ctx.set("a", serde_json::json!(1));
let snap = ctx.snapshot();
ctx.set("b", serde_json::json!(2));
assert!(snap.contains_key("a"));
assert!(!snap.contains_key("b"));
}
#[test]
fn fork_is_independent() {
let ctx = Context::new();
ctx.set("shared", serde_json::json!("original"));
let forked = ctx.fork();
forked.set("shared", serde_json::json!("modified"));
assert_eq!(ctx.get("shared"), Some(serde_json::json!("original")));
assert_eq!(forked.get("shared"), Some(serde_json::json!("modified")));
}
#[test]
fn apply_updates() {
let ctx = Context::new();
ctx.set("existing", serde_json::json!("old"));
let mut updates = HashMap::new();
updates.insert("existing".to_string(), serde_json::json!("new"));
updates.insert("added".to_string(), serde_json::json!(true));
ctx.apply_updates(&updates);
assert_eq!(ctx.get("existing"), Some(serde_json::json!("new")));
assert_eq!(ctx.get("added"), Some(serde_json::json!(true)));
}
#[test]
fn default_creates_empty_context() {
let ctx = Context::default();
assert!(ctx.snapshot().is_empty());
}
#[test]
fn run_id_default() {
let ctx = Context::new();
assert_eq!(ctx.run_id(), "unknown");
}
#[test]
fn run_id_set() {
let ctx = Context::new();
ctx.set(keys::INTERNAL_RUN_ID, serde_json::json!("abc-123"));
assert_eq!(ctx.run_id(), "abc-123");
}
#[test]
fn fidelity_default() {
let ctx = Context::new();
assert_eq!(ctx.fidelity(), keys::Fidelity::Compact);
}
#[test]
fn fidelity_set() {
let ctx = Context::new();
ctx.set(keys::INTERNAL_FIDELITY, serde_json::json!("full"));
assert_eq!(ctx.fidelity(), keys::Fidelity::Full);
}
#[test]
fn preamble_default() {
let ctx = Context::new();
assert_eq!(ctx.preamble(), "");
}
#[test]
fn preamble_set() {
let ctx = Context::new();
ctx.set(keys::CURRENT_PREAMBLE, serde_json::json!("hello"));
assert_eq!(ctx.preamble(), "hello");
}
#[test]
fn thread_id_default() {
let ctx = Context::new();
assert_eq!(ctx.thread_id(), None);
}
#[test]
fn thread_id_null() {
let ctx = Context::new();
ctx.set(keys::INTERNAL_THREAD_ID, serde_json::Value::Null);
assert_eq!(ctx.thread_id(), None);
}
#[test]
fn thread_id_set() {
let ctx = Context::new();
ctx.set(keys::INTERNAL_THREAD_ID, serde_json::json!("main"));
assert_eq!(ctx.thread_id(), Some("main".to_string()));
}
#[test]
fn parallel_ids_default() {
let ctx = Context::new();
assert_eq!(ctx.parallel_group_id(), None);
assert_eq!(ctx.parallel_branch_id(), None);
}
#[test]
fn parallel_ids_set() {
let ctx = Context::new();
ctx.set(
keys::INTERNAL_PARALLEL_GROUP_ID,
serde_json::json!("fanout@2"),
);
ctx.set(
keys::INTERNAL_PARALLEL_BRANCH_ID,
serde_json::json!("fanout@2:1"),
);
assert_eq!(ctx.parallel_group_id(), Some(StageId::new("fanout", 2)));
assert_eq!(
ctx.parallel_branch_id(),
Some(ParallelBranchId::new(StageId::new("fanout", 2), 1))
);
}
#[test]
fn node_visit_count_default() {
let ctx = Context::new();
// fabro-core returns 0 for missing; workflow code expects 1 as default
// when used in workflow context. The raw core accessor returns 0.
assert_eq!(ctx.node_visit_count(), 0);
}
#[test]
fn node_visit_count_set() {
let ctx = Context::new();
ctx.set(keys::INTERNAL_NODE_VISIT_COUNT, serde_json::json!(3));
assert_eq!(ctx.node_visit_count(), 3);
}
#[test]
fn current_node_id_default() {
let ctx = Context::new();
assert_eq!(ctx.current_node_id(), "");
}
#[test]
fn current_node_id_set() {
let ctx = Context::new();
ctx.set(keys::CURRENT_NODE, serde_json::json!("plan"));
assert_eq!(ctx.current_node_id(), "plan");
}
}

99
lib/components/fabro-workflow/src/error.rs
View file

@ -748,7 +748,6 @@ mod tests {
.with_retry(RetryClassification::Safe),
)
}
use crate::outcome::OutcomeExt;
#[derive(Debug)]
struct TestCause(&'static str);
@ -1941,18 +1940,6 @@ mod tests {
assert!(failure.signature.is_none());
}
#[test]
fn to_fail_outcome_includes_error_message_as_reason() {
let err = Error::from(transient_error(ErrorKind::Network, "connection refused"));
let outcome = err.to_fail_outcome();
assert!(
outcome
.failure_reason()
.unwrap()
.contains("connection refused")
);
}
#[test]
fn to_fail_outcome_no_context_updates() {
let err = Error::from(transient_error(ErrorKind::Network, "refused"));
@ -2097,75 +2084,8 @@ mod tests {
);
}
#[test]
fn to_fail_outcome_preserves_class() {
let err = Error::handler("timeout");
let outcome = err.to_fail_outcome();
assert_eq!(
outcome.failure_category(),
Some(FailureCategory::TransientInfra)
);
}
// --- E2E error pipeline tests ---
#[test]
fn e2e_llm_error_to_outcome_to_event_preserves_classification() {
use crate::event::Event;
// 1. Create SdkError → Error
let sdk_err = transient_error(ErrorKind::RateLimit, "too fast");
let arc_err = Error::from(sdk_err);
assert_eq!(arc_err.failure_category(), FailureCategory::TransientInfra);
// 2. Error → Outcome
let outcome = arc_err.to_fail_outcome();
assert_eq!(
outcome.failure_category(),
Some(FailureCategory::TransientInfra)
);
// 3. Outcome → StageFailed event
let failure = outcome.failure.clone().unwrap();
let event = Event::StageFailed {
node_id: "code".into(),
name: "code".into(),
index: 0,
failure: failure.clone(),
will_retry: false,
timing: fabro_types::StageTiming::wall_only(0),
usage_by_model: Vec::new(),
usage: None,
actor: None,
};
// 4. Verify classification survived all the way through
match &event {
Event::StageFailed { failure, .. } => {
assert_eq!(failure.category, FailureCategory::TransientInfra);
}
_ => panic!("expected StageFailed"),
}
}
#[test]
fn e2e_handler_error_classified_at_edge() {
// handler smart constructor classifies eagerly
let err = Error::handler("connection refused");
assert_eq!(err.failure_category(), FailureCategory::TransientInfra);
// to_fail_outcome preserves
let outcome = err.to_fail_outcome();
assert_eq!(
outcome.failure_category(),
Some(FailureCategory::TransientInfra)
);
// event preserves
let failure = outcome.failure.unwrap();
assert_eq!(failure.category, FailureCategory::TransientInfra);
}
#[test]
fn e2e_handler_retryable_checks() {
assert!(Error::handler("timeout").is_retryable());
@ -2182,23 +2102,4 @@ mod tests {
assert_eq!(failure.reason, FailureReason::WorkflowError);
assert_eq!(failure.detail.category, FailureCategory::TransientInfra);
}
#[test]
fn e2e_failure_detail_in_outcome_serde_roundtrip() {
use crate::outcome::Outcome;
let outcome = Outcome::fail_classify("rate limit exceeded")
.with_signature(Some("api_transient|openai|rate_limited"));
let json = serde_json::to_string(&outcome).unwrap();
let deserialized: Outcome = serde_json::from_str(&json).unwrap();
let failure = deserialized.failure.unwrap();
assert_eq!(failure.message, "rate limit exceeded");
assert_eq!(failure.category, FailureCategory::TransientInfra);
assert_eq!(
failure.signature.as_deref(),
Some("api_transient|openai|rate_limited")
);
}
}

7
lib/components/fabro-workflow/src/event/emitter.rs
View file

@ -1,6 +1,5 @@
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Duration;
use ::fabro_types::{ExecOutputTail, RunEvent, RunId, RunNoticeCode, RunNoticeLevel};
use chrono::Utc;
@ -198,12 +197,6 @@ impl Emitter {
}
}
/// Returns the monotonic instant of the last `emit()` or `touch()` call,
/// or the emitter's creation instant if neither has been called.
pub(crate) fn last_activity(&self) -> Instant {
self.activity_origin + Duration::from_millis(self.last_activity_ms.load(Ordering::Relaxed))
}
/// Manually record activity (e.g. to seed the watchdog at workflow run
/// start, or for agent stream deltas that are not emitted as run events).
pub fn touch(&self) {

438
lib/components/fabro-workflow/src/git_bridge.rs
View file

@ -1,438 +0,0 @@
//! Secret-free Git bridging environment for additional-repository access.
//!
//! When a run declares additional GitHub repositories, every resolved
//! command/tool/ACP environment receives `GIT_CONFIG_COUNT` /
//! `GIT_CONFIG_KEY_n` / `GIT_CONFIG_VALUE_n` entries that make plain Git
//! commands work against the declared set through the managed
//! `GITHUB_TOKEN`:
//!
//! - a credential helper for `https://github.com` that reads `$GITHUB_TOKEN`
//! from the invoking Git process's environment at invocation time, so token
//! refresh flows through per-stage environment resolution with no bridging
//! update;
//! - per-repository `url.<https>.insteadOf` rewrites for the
//! `git@github.com:owner/repo[.git]` and
//! `ssh://git@github.com/owner/repo[.git]` SSH spellings of each effective
//! repository.
//!
//! None of the values contain a secret; the token lives only in
//! `GITHUB_TOKEN`.
//!
//! The credential helper is host-scoped to `https://github.com`, not
//! path-scoped. This is safe because the token is scoped server-side to the
//! declared repository set and is only ever offered to github.com. It does
//! change one failure mode for *undeclared* repositories: public HTTPS
//! clones are unaffected (Git tries unauthenticated first), while private
//! undeclared HTTPS repositories fail with a GitHub authorization error
//! instead of a missing-credential error. Both fail; only the diagnostic
//! differs.
//!
//! `insteadOf` matches by string prefix, not exactly: a rule for
//! `owner/repo` also matches `owner/repo-other`. An undeclared repository
//! that shares a declared prefix is therefore rewritten to HTTPS; the scoped
//! token is invalid for it at GitHub, so authority is unchanged, but its Git
//! transport changes from SSH to HTTPS.
use std::collections::HashMap;
use fabro_github::{GITHUB_CREDENTIAL_HELPER, GITHUB_CREDENTIAL_HELPER_KEY};
use fabro_types::GitHubRepositorySlug;
use crate::error::Error;
/// Section base for the effective repositories' HTTPS routes.
const GITHUB_HTTPS_BASE: &str = "https://github.com/";
/// Merge the bridging entries into `env` for the effective repository set
/// (primary first). Appends after any valid user-provided `GIT_CONFIG_COUNT`
/// overlay without overwriting it, and fails with a configuration error when
/// the user overlay is malformed rather than silently replacing it.
pub(crate) fn merge_git_bridge_env(
env: &mut HashMap<String, String>,
targets: &[&GitHubRepositorySlug],
) -> Result<(), Error> {
let start = user_git_config_count(env)?;
let entries = bridge_entries(targets, GITHUB_HTTPS_BASE);
let total = start + entries.len();
for (offset, (key, value)) in entries.into_iter().enumerate() {
let index = start + offset;
env.insert(format!("GIT_CONFIG_KEY_{index}"), key);
env.insert(format!("GIT_CONFIG_VALUE_{index}"), value);
}
env.insert("GIT_CONFIG_COUNT".to_string(), total.to_string());
// Fail instead of hanging when access is missing or invalid; a user who
// explicitly configured prompting keeps their value.
env.entry("GIT_TERMINAL_PROMPT".to_string())
.or_insert_with(|| "0".to_string());
Ok(())
}
/// The bridge's Git config entries in order: the credential helper, then two
/// SSH-to-HTTPS rewrites per repository. `https_base` is
/// [`GITHUB_HTTPS_BASE`] in production; contract tests substitute a local
/// `file://` root to prove real Git applies the generated entries without
/// touching the network.
fn bridge_entries(targets: &[&GitHubRepositorySlug], https_base: &str) -> Vec<(String, String)> {
let mut entries = Vec::with_capacity(1 + targets.len() * 2);
entries.push((
GITHUB_CREDENTIAL_HELPER_KEY.to_string(),
GITHUB_CREDENTIAL_HELPER.to_string(),
));
for slug in targets {
let owner = slug.owner();
let repo = slug.repo();
let https = format!("{https_base}{owner}/{repo}");
// One prefix rule per SSH spelling covers both the bare and `.git`
// suffixed forms.
entries.push((
format!("url.{https}.insteadOf"),
format!("git@github.com:{owner}/{repo}"),
));
entries.push((
format!("url.{https}.insteadOf"),
format!("ssh://git@github.com/{owner}/{repo}"),
));
}
entries
}
/// Validate and measure a user-provided `GIT_CONFIG_COUNT` overlay so the
/// bridge appends after it. Orphaned `GIT_CONFIG_KEY_n` entries without a
/// count are inert to Git and are treated as absent.
fn user_git_config_count(env: &HashMap<String, String>) -> Result<usize, Error> {
let Some(raw) = env.get("GIT_CONFIG_COUNT") else {
return Ok(0);
};
let count: usize = raw.trim().parse().map_err(|_| {
Error::Precondition(format!(
"environment variable GIT_CONFIG_COUNT must be a non-negative integer to combine \
with Fabro's Git bridging entries, got `{raw}`"
))
})?;
for index in 0..count {
let key = format!("GIT_CONFIG_KEY_{index}");
let value = format!("GIT_CONFIG_VALUE_{index}");
if !env.contains_key(&key) || !env.contains_key(&value) {
return Err(Error::Precondition(format!(
"GIT_CONFIG_COUNT is {count} but {key} or {value} is missing; fix the indexed \
Git config overlay so Fabro can append its bridging entries after it"
)));
}
}
Ok(count)
}
#[cfg(test)]
#[expect(
clippy::disallowed_methods,
clippy::disallowed_types,
reason = "contract tests drive the installed git binary synchronously in non-async tests"
)]
mod tests {
use std::path::Path;
use std::process::Command;
use super::*;
fn slug(value: &str) -> GitHubRepositorySlug {
value.parse().expect("test slug should parse")
}
fn bridged_env(
base_env: HashMap<String, String>,
targets: &[&GitHubRepositorySlug],
) -> HashMap<String, String> {
let mut env = base_env;
merge_git_bridge_env(&mut env, targets).expect("bridge entries should merge");
env
}
/// Run `git` with ONLY the bridge-relevant environment: the inherited
/// user/system/global Git config is disabled so assertions observe just
/// the generated entries.
fn git(args: &[&str], env: &HashMap<String, String>, cwd: &Path) -> std::process::Output {
let mut command = Command::new("git");
command
.args(args)
.current_dir(cwd)
.env("GIT_CONFIG_NOSYSTEM", "1")
.env("GIT_CONFIG_GLOBAL", "/dev/null")
.env("GIT_TERMINAL_PROMPT", "0")
.env("GIT_ASKPASS", "true");
for (key, value) in env {
command.env(key, value);
}
command.output().expect("git should run")
}
/// Create a bare fixture answering both the bare and `.git`-suffixed
/// routes, the way GitHub serves both HTTPS spellings.
fn init_bare_fixture(root: &Path, owner_repo: &str) -> String {
let fixture = root.join(format!("{owner_repo}.git"));
std::fs::create_dir_all(&fixture).unwrap();
let init = Command::new("git")
.args(["init", "--bare", "--initial-branch=main"])
.arg(&fixture)
.output()
.expect("git init should run");
assert!(init.status.success(), "{init:?}");
#[cfg(unix)]
std::os::unix::fs::symlink(&fixture, root.join(owner_repo)).unwrap();
format!("file://{}/", root.display())
}
#[test]
fn no_targets_means_no_bridge_call_and_empty_env_stays_empty() {
// The caller only bridges when the additional set is non-empty; the
// pure entry builder is still total for the primary-only case.
assert_eq!(bridge_entries(&[], GITHUB_HTTPS_BASE).len(), 1);
let env: HashMap<String, String> = HashMap::new();
assert!(!env.contains_key("GIT_CONFIG_COUNT"));
}
#[test]
fn merges_helper_rewrites_count_and_terminal_prompt() {
let keystone = slug("fabro-sh/keystone");
let fabro = slug("fabro-sh/fabro");
let env = bridged_env(HashMap::new(), &[&fabro, &keystone]);
assert_eq!(env.get("GIT_CONFIG_COUNT").map(String::as_str), Some("5"));
assert_eq!(
env.get("GIT_CONFIG_KEY_0").map(String::as_str),
Some("credential.https://github.com.helper")
);
assert_eq!(
env.get("GIT_CONFIG_KEY_1").map(String::as_str),
Some("url.https://github.com/fabro-sh/fabro.insteadOf")
);
assert_eq!(
env.get("GIT_CONFIG_VALUE_1").map(String::as_str),
Some("git@github.com:fabro-sh/fabro")
);
assert_eq!(
env.get("GIT_CONFIG_VALUE_2").map(String::as_str),
Some("ssh://git@github.com/fabro-sh/fabro")
);
assert_eq!(
env.get("GIT_TERMINAL_PROMPT").map(String::as_str),
Some("0")
);
// No secrets anywhere in the generated values.
for (key, value) in &env {
assert!(!value.contains("ghs_"), "{key}={value}");
}
}
#[test]
fn respects_an_explicit_user_terminal_prompt() {
let keystone = slug("fabro-sh/keystone");
let env = bridged_env(
HashMap::from([("GIT_TERMINAL_PROMPT".to_string(), "1".to_string())]),
&[&keystone],
);
assert_eq!(
env.get("GIT_TERMINAL_PROMPT").map(String::as_str),
Some("1")
);
}
#[test]
fn appends_after_a_valid_user_git_config_overlay() {
let keystone = slug("fabro-sh/keystone");
let env = bridged_env(
HashMap::from([
("GIT_CONFIG_COUNT".to_string(), "1".to_string()),
("GIT_CONFIG_KEY_0".to_string(), "user.name".to_string()),
("GIT_CONFIG_VALUE_0".to_string(), "Overlay User".to_string()),
]),
&[&keystone],
);
assert_eq!(env.get("GIT_CONFIG_COUNT").map(String::as_str), Some("4"));
assert_eq!(
env.get("GIT_CONFIG_KEY_0").map(String::as_str),
Some("user.name"),
"user entry must survive at its original index"
);
assert_eq!(
env.get("GIT_CONFIG_KEY_1").map(String::as_str),
Some("credential.https://github.com.helper")
);
// Real Git sees both the user's entry and the appended bridge entry.
let dir = tempfile::tempdir().unwrap();
let output = git(&["config", "--list"], &env, dir.path());
assert!(output.status.success(), "{output:?}");
let listed = String::from_utf8_lossy(&output.stdout);
assert!(listed.contains("user.name=Overlay User"), "{listed}");
assert!(
listed.contains("credential.https://github.com.helper"),
"{listed}"
);
}
#[test]
fn rejects_a_malformed_user_git_config_overlay() {
let keystone = slug("fabro-sh/keystone");
let mut non_numeric = HashMap::from([("GIT_CONFIG_COUNT".to_string(), "two".to_string())]);
let err = merge_git_bridge_env(&mut non_numeric, &[&keystone]).unwrap_err();
assert!(err.to_string().contains("GIT_CONFIG_COUNT"), "{err}");
let mut missing_index = HashMap::from([
("GIT_CONFIG_COUNT".to_string(), "2".to_string()),
("GIT_CONFIG_KEY_0".to_string(), "user.name".to_string()),
("GIT_CONFIG_VALUE_0".to_string(), "Overlay".to_string()),
]);
let err = merge_git_bridge_env(&mut missing_index, &[&keystone]).unwrap_err();
assert!(err.to_string().contains("GIT_CONFIG_KEY_1"), "{err}");
}
/// With the bridge active, `git credential fill` for github.com resolves
/// through the generated helper and reads `$GITHUB_TOKEN` from the
/// invoking process environment at invocation time.
#[test]
fn credential_helper_reads_github_token_at_invocation_time() {
use std::io::Write as _;
let keystone = slug("fabro-sh/keystone");
let mut env = bridged_env(HashMap::new(), &[&keystone]);
env.insert("GITHUB_TOKEN".to_string(), "test-token-value".to_string());
let dir = tempfile::tempdir().unwrap();
let mut command = Command::new("git");
command
.args(["credential", "fill"])
.current_dir(dir.path())
.env("GIT_CONFIG_NOSYSTEM", "1")
.env("GIT_CONFIG_GLOBAL", "/dev/null")
.stdin(std::process::Stdio::piped())
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::piped());
for (key, value) in &env {
command.env(key, value);
}
let mut child = command.spawn().expect("git credential fill should spawn");
child
.stdin
.as_mut()
.unwrap()
.write_all(b"protocol=https\nhost=github.com\npath=fabro-sh/keystone\n\n")
.unwrap();
let output = child.wait_with_output().unwrap();
assert!(output.status.success(), "{output:?}");
let filled = String::from_utf8_lossy(&output.stdout);
assert!(filled.contains("username=x-access-token"), "{filled}");
assert!(filled.contains("password=test-token-value"), "{filled}");
}
/// Real Git applies the generated `insteadOf` rewrites: the exact SSH
/// spellings of a declared repository resolve to their HTTPS-analog
/// route (a local `file://` fixture here, so no network is involved),
/// while `GIT_SSH_COMMAND=false` proves SSH is never attempted.
#[test]
fn declared_ssh_urls_rewrite_to_the_https_route() {
let root = tempfile::tempdir().unwrap();
let base = init_bare_fixture(root.path(), "fabro-sh/keystone");
let keystone = slug("fabro-sh/keystone");
let mut env: HashMap<String, String> = HashMap::new();
for (offset, (key, value)) in bridge_entries(&[&keystone], &base).into_iter().enumerate() {
env.insert(format!("GIT_CONFIG_KEY_{offset}"), key);
env.insert(format!("GIT_CONFIG_VALUE_{offset}"), value);
}
env.insert("GIT_CONFIG_COUNT".to_string(), "3".to_string());
env.insert("GIT_SSH_COMMAND".to_string(), "false".to_string());
for url in [
"ssh://git@github.com/fabro-sh/keystone.git",
"ssh://git@github.com/fabro-sh/keystone",
"git@github.com:fabro-sh/keystone.git",
"git@github.com:fabro-sh/keystone",
] {
let output = git(&["ls-remote", url], &env, root.path());
assert!(
output.status.success(),
"{url} should rewrite to the fixture route: {}",
String::from_utf8_lossy(&output.stderr)
);
}
}
/// An undeclared SSH URL that shares no declared prefix is not
/// rewritten: Git still routes it to SSH, where the scripted
/// `GIT_SSH_COMMAND=false` fails immediately without network access.
#[test]
fn undeclared_ssh_urls_are_not_rewritten() {
let root = tempfile::tempdir().unwrap();
let base = init_bare_fixture(root.path(), "fabro-sh/keystone");
let keystone = slug("fabro-sh/keystone");
let mut env: HashMap<String, String> = HashMap::new();
for (offset, (key, value)) in bridge_entries(&[&keystone], &base).into_iter().enumerate() {
env.insert(format!("GIT_CONFIG_KEY_{offset}"), key);
env.insert(format!("GIT_CONFIG_VALUE_{offset}"), value);
}
env.insert("GIT_CONFIG_COUNT".to_string(), "3".to_string());
env.insert("GIT_SSH_COMMAND".to_string(), "false".to_string());
env.insert("GITHUB_TOKEN".to_string(), "test-token-value".to_string());
let output = git(
&["ls-remote", "git@github.com:fabro-sh/undeclared"],
&env,
root.path(),
);
assert!(!output.status.success(), "{output:?}");
let stderr = String::from_utf8_lossy(&output.stderr);
// Not rewritten: the failure never mentions the local HTTPS-analog
// fixture route, so Git still chose the SSH transport.
assert!(
!stderr.contains(&root.path().display().to_string()),
"undeclared URL must not be rewritten to the fixture route: {stderr}"
);
assert!(!stderr.contains("test-token-value"), "{stderr}");
}
/// Prefix collision: with `fabro-sh/keystone` declared, both SSH
/// spellings of `fabro-sh/keystone-other` are rewritten to the HTTPS
/// route (prefix match), where access fails — at GitHub this is an
/// authorization error for the scoped token — and no token leaks into
/// the output.
#[test]
fn prefix_colliding_undeclared_repositories_rewrite_and_fail_without_token_leak() {
let root = tempfile::tempdir().unwrap();
let base = init_bare_fixture(root.path(), "fabro-sh/keystone");
let keystone = slug("fabro-sh/keystone");
let mut env: HashMap<String, String> = HashMap::new();
for (offset, (key, value)) in bridge_entries(&[&keystone], &base).into_iter().enumerate() {
env.insert(format!("GIT_CONFIG_KEY_{offset}"), key);
env.insert(format!("GIT_CONFIG_VALUE_{offset}"), value);
}
env.insert("GIT_CONFIG_COUNT".to_string(), "3".to_string());
env.insert("GIT_SSH_COMMAND".to_string(), "false".to_string());
env.insert("GITHUB_TOKEN".to_string(), "test-token-value".to_string());
for url in [
"git@github.com:fabro-sh/keystone-other",
"ssh://git@github.com/fabro-sh/keystone-other.git",
] {
let output = git(&["ls-remote", url], &env, root.path());
assert!(!output.status.success(), "{url}: {output:?}");
let stderr = String::from_utf8_lossy(&output.stderr);
// The failure names the (missing) HTTPS-analog fixture route,
// proving the prefix rule rewrote the URL away from SSH.
assert!(
stderr.contains("keystone-other"),
"{url} must be rewritten away from SSH, got: {stderr}"
);
assert!(
stderr.contains(&root.path().display().to_string()),
"{url} must land on the rewritten route, got: {stderr}"
);
assert!(!stderr.contains("test-token-value"), "{stderr}");
}
}
}

152
lib/components/fabro-workflow/src/graph.rs
View file

@ -1,152 +0,0 @@
mod routing;
use std::collections::HashMap;
use std::sync::Arc;
use fabro_core::error::{Error as CoreError, Result as CoreResult};
use fabro_core::graph::{EdgeSelection as CoreEdgeSelection, EdgeSpec, Graph, NodeSpec};
use fabro_core::outcome::NodeResult;
use fabro_graphviz::graph::types::{Edge as GvEdge, Graph as GvGraph, Node as GvNode};
use fabro_types::ResolvedOnFailure;
use crate::context::{self, Context};
use crate::outcome::{ModelUsage, Outcome};
// ---- WorkflowNode ----
#[derive(Debug, Clone)]
pub(crate) struct WorkflowNode(pub Arc<GvNode>);
impl WorkflowNode {
pub(crate) fn inner(&self) -> &GvNode {
&self.0
}
}
impl NodeSpec for WorkflowNode {
fn id(&self) -> &str {
&self.0.id
}
fn is_terminal(&self) -> bool {
routing::is_terminal(&self.0)
}
fn max_visits(&self) -> Option<usize> {
self.0
.max_visits()
.map(|v| usize::try_from(v.max(0)).unwrap_or(usize::MAX))
}
}
// ---- WorkflowEdge ----
#[derive(Debug, Clone)]
pub(crate) struct WorkflowEdge(pub Arc<GvEdge>);
impl WorkflowEdge {
pub(crate) fn inner(&self) -> &GvEdge {
&self.0
}
}
impl EdgeSpec for WorkflowEdge {
fn target(&self) -> &str {
&self.0.to
}
fn label(&self) -> Option<&str> {
self.0.label()
}
fn is_loop_restart(&self) -> bool {
self.0.loop_restart()
}
}
// ---- WorkflowGraph ----
#[derive(Debug, Clone)]
pub(crate) struct WorkflowGraph(pub Arc<GvGraph>);
impl WorkflowGraph {
pub(crate) fn inner(&self) -> &GvGraph {
&self.0
}
}
impl Graph for WorkflowGraph {
type Node = WorkflowNode;
type Edge = WorkflowEdge;
type Meta = Option<ModelUsage>;
fn get_node(&self, id: &str) -> Option<Self::Node> {
self.0
.nodes
.get(id)
.map(|n| WorkflowNode(Arc::new(n.clone())))
}
fn find_start_node(&self) -> CoreResult<Self::Node> {
self.0
.find_start_node()
.map(|n| WorkflowNode(Arc::new(n.clone())))
.ok_or(CoreError::NoStartNode)
}
fn outgoing_edges(&self, node_id: &str) -> Vec<Self::Edge> {
self.0
.outgoing_edges(node_id)
.into_iter()
.map(|e| WorkflowEdge(Arc::new(e.clone())))
.collect()
}
fn select_edge(
&self,
node: &Self::Node,
outcome: &Outcome,
context: &Context,
) -> Option<CoreEdgeSelection<Self>> {
let selection = routing::select_edge(
node.inner(),
outcome,
context,
self.inner(),
node.inner().selection(),
);
selection.map(|sel| CoreEdgeSelection {
edge: WorkflowEdge(Arc::new(sel.edge.clone())),
reason: sel.reason,
})
}
fn project_result_context(
&self,
node: &Self::Node,
result: &NodeResult<Self::Meta>,
context: &Context,
) {
context::apply_recorded_outcome_context(
context,
node.id(),
&result.outcome,
result.attempts.saturating_sub(1),
);
}
fn check_goal_gates(
&self,
outcomes: &HashMap<String, Outcome>,
) -> std::result::Result<(), String> {
routing::check_goal_gates(self.inner(), outcomes)
}
fn get_retry_target(&self, failed_node_id: &str) -> Option<String> {
routing::get_retry_target(failed_node_id, self.inner())
}
fn resolve_on_failure(&self, node: &Self::Node) -> ResolvedOnFailure {
self.inner().resolve_on_failure(node.inner())
}
}

851
lib/components/fabro-workflow/src/graph/routing.rs
View file

@ -1,851 +0,0 @@
use std::collections::HashMap;
use fabro_core::graph::EdgeSelectionReason;
use fabro_graphviz::graph::types::{Edge as GvEdge, Graph as GvGraph, Node as GvNode};
use rand::Rng;
use crate::condition::evaluate_condition;
use crate::context::Context;
use crate::outcome::Outcome;
/// Result of edge selection: the chosen edge and the reason it was selected.
pub(crate) struct SelectedGraphEdge<'a> {
pub(crate) edge: &'a GvEdge,
pub(crate) reason: EdgeSelectionReason,
}
/// Check whether a node is a terminal (exit) node.
pub(crate) fn is_terminal(node: &GvNode) -> bool {
node.shape() == "Msquare" || node.handler_type() == Some("exit")
}
/// Select the next edge from a node's outgoing edges (spec Section 3.3).
pub(crate) fn select_edge<'a>(
node: &GvNode,
outcome: &Outcome,
context: &Context,
graph: &'a GvGraph,
selection: &str,
) -> Option<SelectedGraphEdge<'a>> {
let node_id = &node.id;
let edges = graph.outgoing_edges(node_id);
if edges.is_empty() {
return None;
}
let condition_matched: Vec<&GvEdge> = edges
.iter()
.filter(|e| {
e.condition()
.is_some_and(|c| !c.is_empty() && evaluate_condition(c, outcome, context))
})
.copied()
.collect();
if !condition_matched.is_empty() {
return pick_edge(&condition_matched, selection).map(|edge| SelectedGraphEdge {
edge,
reason: EdgeSelectionReason::Condition,
});
}
if let Some(pref) = &outcome.preferred_label {
let normalized_pref = normalize_label(pref);
for edge in &edges {
if edge.condition().is_none_or(str::is_empty) {
if let Some(label) = edge.label() {
if normalize_label(label) == normalized_pref {
return Some(SelectedGraphEdge {
edge,
reason: EdgeSelectionReason::PreferredLabel,
});
}
}
}
}
}
for suggested_id in &outcome.suggested_next_ids {
for edge in &edges {
if edge.condition().is_none_or(str::is_empty) && edge.to == *suggested_id {
return Some(SelectedGraphEdge {
edge,
reason: EdgeSelectionReason::SuggestedNext,
});
}
}
}
if blocks_unconditional_failure_fallthrough(node, outcome) {
return None;
}
let unconditional: Vec<&GvEdge> = edges
.iter()
.filter(|e| e.condition().is_none_or(str::is_empty))
.copied()
.collect();
if !unconditional.is_empty() {
return pick_edge(&unconditional, selection).map(|edge| SelectedGraphEdge {
edge,
reason: EdgeSelectionReason::Unconditional,
});
}
None
}
/// Check if all goal gates have been satisfied.
/// Returns Ok(()) if all gates passed, or Err with the failed node ID.
pub(crate) fn check_goal_gates(
graph: &GvGraph,
node_outcomes: &HashMap<String, Outcome>,
) -> std::result::Result<(), String> {
let mut goal_gate_ids: Vec<&String> = graph
.nodes
.iter()
.filter_map(|(node_id, node)| node.goal_gate().then_some(node_id))
.collect();
goal_gate_ids.sort();
for node_id in goal_gate_ids {
if !node_outcomes
.get(node_id)
.is_some_and(|outcome| outcome.status.is_successful())
{
return Err(node_id.clone());
}
}
Ok(())
}
/// Resolve the retry target for a failed goal gate node.
pub(crate) fn get_retry_target(failed_node_id: &str, graph: &GvGraph) -> Option<String> {
if let Some(node) = graph.nodes.get(failed_node_id) {
if let Some(target) = node.retry_target() {
if graph.nodes.contains_key(target) {
return Some(target.to_string());
}
}
if let Some(target) = node.fallback_retry_target() {
if graph.nodes.contains_key(target) {
return Some(target.to_string());
}
}
}
if let Some(target) = graph.retry_target() {
if graph.nodes.contains_key(target) {
return Some(target.to_string());
}
}
if let Some(target) = graph.fallback_retry_target() {
if graph.nodes.contains_key(target) {
return Some(target.to_string());
}
}
None
}
/// Normalize a label for comparison: lowercase, trim, strip accelerator
/// prefixes. Patterns: "[Y] ", "Y) ", "Y - "
fn normalize_label(label: &str) -> String {
let s = label.trim().to_lowercase();
if s.starts_with('[') {
if let Some(rest) = s
.strip_prefix('[')
.and_then(|s| s.find(']').map(|i| s[i + 1..].trim_start().to_string()))
{
return rest;
}
}
if s.len() >= 2 {
let bytes = s.as_bytes();
if bytes.get(1) == Some(&b')') {
return s[2..].trim_start().to_string();
}
}
if s.len() >= 3 {
if let Some(rest) = s.get(1..).and_then(|r| r.strip_prefix(" - ")) {
return rest.to_string();
}
}
s
}
/// Pick the best edge by highest weight, then lexical target node ID tiebreak.
fn best_by_weight_then_lexical<'a>(edges: &[&'a GvEdge]) -> Option<&'a GvEdge> {
if edges.is_empty() {
return None;
}
let mut best = edges[0];
for &edge in &edges[1..] {
if edge.weight() > best.weight() || (edge.weight() == best.weight() && edge.to < best.to) {
best = edge;
}
}
Some(best)
}
/// Pick a random edge using weighted-random selection.
/// Edges with `weight <= 0` are treated as weight 1 for probability
/// calculation.
fn weighted_random<'a>(edges: &[&'a GvEdge]) -> Option<&'a GvEdge> {
if edges.is_empty() {
return None;
}
if edges.len() == 1 {
return Some(edges[0]);
}
let weights: Vec<f64> = edges
.iter()
.map(|e| {
let w = e.weight();
if w <= 0 { 1.0 } else { w as f64 }
})
.collect();
let total: f64 = weights.iter().sum();
let mut rng = rand::rng();
let mut roll: f64 = rng.random_range(0.0..total);
for (i, &w) in weights.iter().enumerate() {
roll -= w;
if roll < 0.0 {
return Some(edges[i]);
}
}
Some(edges[edges.len() - 1])
}
/// Dispatch to the appropriate edge-picking strategy.
fn pick_edge<'a>(edges: &[&'a GvEdge], selection: &str) -> Option<&'a GvEdge> {
match selection {
"random" => weighted_random(edges),
_ => best_by_weight_then_lexical(edges),
}
}
fn blocks_unconditional_failure_fallthrough(node: &GvNode, outcome: &Outcome) -> bool {
node.handler_type() == Some("human")
&& outcome.status.is_failure()
&& outcome.preferred_label.is_none()
&& outcome.suggested_next_ids.is_empty()
}
#[cfg(test)]
mod tests {
use std::collections::HashMap;
use fabro_graphviz::graph::{AttrValue, Edge, Graph, Node};
use fabro_types::{OnFailure, ResolvedOnFailure};
use super::*;
use crate::context::Context;
use crate::outcome::{Outcome, OutcomeExt, StageOutcome};
fn make_graph_with_edges(edges: Vec<Edge>) -> Graph {
let mut g = Graph::new("test");
for edge in &edges {
if !g.nodes.contains_key(&edge.from) {
g.nodes.insert(edge.from.clone(), Node::new(&edge.from));
}
if !g.nodes.contains_key(&edge.to) {
g.nodes.insert(edge.to.clone(), Node::new(&edge.to));
}
}
g.edges = edges;
g
}
#[test]
fn normalize_label_lowercase_and_trim() {
assert_eq!(normalize_label(" Yes "), "yes");
}
#[test]
fn normalize_label_strip_bracket_prefix() {
assert_eq!(normalize_label("[A] Approve"), "approve");
assert_eq!(normalize_label("[F] Fix"), "fix");
}
#[test]
fn normalize_label_strip_paren_prefix() {
assert_eq!(normalize_label("Y) Yes"), "yes");
}
#[test]
fn normalize_label_strip_dash_prefix() {
assert_eq!(normalize_label("Y - Yes"), "yes");
}
#[test]
fn normalize_label_plain() {
assert_eq!(normalize_label("next"), "next");
}
#[test]
fn best_by_weight_highest_wins() {
let e1 = Edge::new("a", "x");
let mut e2 = Edge::new("a", "y");
e2.attrs.insert("weight".to_string(), AttrValue::Integer(5));
let result = best_by_weight_then_lexical(&[&e1, &e2]).unwrap();
assert_eq!(result.to, "y");
}
#[test]
fn best_by_weight_lexical_tiebreak() {
let e1 = Edge::new("a", "beta");
let e2 = Edge::new("a", "alpha");
let result = best_by_weight_then_lexical(&[&e1, &e2]).unwrap();
assert_eq!(result.to, "alpha");
}
#[test]
fn best_by_weight_empty_returns_none() {
let result = best_by_weight_then_lexical(&[]);
assert!(result.is_none());
}
#[test]
fn weighted_random_empty_returns_none() {
assert!(weighted_random(&[]).is_none());
}
#[test]
fn weighted_random_single_edge() {
let e = Edge::new("a", "b");
let result = weighted_random(&[&e]).unwrap();
assert_eq!(result.to, "b");
}
#[test]
fn weighted_random_zero_weight_all_selected() {
let e1 = Edge::new("a", "b");
let e2 = Edge::new("a", "c");
let edges = vec![&e1, &e2];
let mut seen_b = false;
let mut seen_c = false;
for _ in 0..200 {
let pick = weighted_random(&edges).unwrap();
if pick.to == "b" {
seen_b = true;
}
if pick.to == "c" {
seen_c = true;
}
}
assert!(seen_b, "expected target 'b' to be selected at least once");
assert!(seen_c, "expected target 'c' to be selected at least once");
}
#[test]
fn weighted_random_high_weight_dominates() {
let mut heavy = Edge::new("a", "heavy");
heavy
.attrs
.insert("weight".to_string(), AttrValue::Integer(100));
let mut light = Edge::new("a", "light");
light
.attrs
.insert("weight".to_string(), AttrValue::Integer(1));
let edges = vec![&heavy, &light];
let mut heavy_count = 0;
for _ in 0..500 {
let pick = weighted_random(&edges).unwrap();
if pick.to == "heavy" {
heavy_count += 1;
}
}
let ratio = f64::from(heavy_count) / 500.0;
assert!(
ratio > 0.90,
"expected heavy edge to win >90% of the time, got {ratio:.2}"
);
}
#[test]
fn select_edge_no_edges() {
let g = Graph::new("test");
let node = Node::new("a");
let outcome = Outcome::success();
let context = Context::new();
assert!(select_edge(&node, &outcome, &context, &g, "deterministic").is_none());
}
#[test]
fn select_edge_single_unconditional() {
let g = make_graph_with_edges(vec![Edge::new("a", "b")]);
let node = g.nodes.get("a").unwrap();
let outcome = Outcome::success();
let context = Context::new();
let sel = select_edge(node, &outcome, &context, &g, "deterministic").unwrap();
assert_eq!(sel.edge.to, "b");
assert_eq!(sel.reason, EdgeSelectionReason::Unconditional);
}
#[test]
fn failed_outcome_selects_unconditional_edge() {
let graph = make_graph_with_edges(vec![Edge::new("a", "b")]);
let node = graph.nodes.get("a").unwrap();
let outcome = Outcome::fail_classify("boom");
let selected =
select_edge(node, &outcome, &Context::new(), &graph, "deterministic").unwrap();
assert_eq!(selected.edge.to, "b");
assert_eq!(selected.reason, EdgeSelectionReason::Unconditional);
}
#[test]
fn non_failed_outcomes_select_unconditional_edge() {
let graph = make_graph_with_edges(vec![Edge::new("a", "b")]);
let node = graph.nodes.get("a").unwrap();
let mut partial = Outcome::success();
partial.status = StageOutcome::PartiallySucceeded;
for outcome in [Outcome::success(), partial, Outcome::skipped("not needed")] {
let selected =
select_edge(node, &outcome, &Context::new(), &graph, "deterministic").unwrap();
assert_eq!(selected.edge.to, "b");
assert_eq!(selected.reason, EdgeSelectionReason::Unconditional);
}
}
#[test]
fn failure_condition_is_an_explicit_selection() {
let mut recovery = Edge::new("a", "recover");
recovery.attrs.insert(
"condition".to_string(),
AttrValue::String("outcome=failed".to_string()),
);
let graph = make_graph_with_edges(vec![recovery, Edge::new("a", "fallback")]);
let node = graph.nodes.get("a").unwrap();
let outcome = Outcome::fail_classify("boom");
let selected =
select_edge(node, &outcome, &Context::new(), &graph, "deterministic").unwrap();
assert_eq!(selected.edge.to, "recover");
assert_eq!(selected.reason, EdgeSelectionReason::Condition);
}
#[test]
fn preferred_and_suggested_routes_are_explicit_selections() {
let mut preferred = Edge::new("a", "preferred");
preferred.attrs.insert(
"label".to_string(),
AttrValue::String("Recover".to_string()),
);
let graph = make_graph_with_edges(vec![preferred, Edge::new("a", "suggested")]);
let node = graph.nodes.get("a").unwrap();
let mut preferred_outcome = Outcome::fail_classify("boom");
preferred_outcome.preferred_label = Some("Recover".to_string());
let selected = select_edge(
node,
&preferred_outcome,
&Context::new(),
&graph,
"deterministic",
)
.unwrap();
assert_eq!(selected.edge.to, "preferred");
assert_eq!(selected.reason, EdgeSelectionReason::PreferredLabel);
let mut suggested_outcome = Outcome::fail_classify("boom");
suggested_outcome.suggested_next_ids = vec!["suggested".to_string()];
let selected = select_edge(
node,
&suggested_outcome,
&Context::new(),
&graph,
"deterministic",
)
.unwrap();
assert_eq!(selected.edge.to, "suggested");
assert_eq!(selected.reason, EdgeSelectionReason::SuggestedNext);
}
#[test]
fn promoted_outcome_selects_succeeded_condition() {
let mut on_success = Edge::new("a", "next");
on_success.attrs.insert(
"condition".to_string(),
AttrValue::String("outcome=succeeded".to_string()),
);
let graph = make_graph_with_edges(vec![on_success, Edge::new("a", "fallback")]);
let node = graph.nodes.get("a").unwrap();
let mut outcome = Outcome::fail_classify("boom");
outcome.apply_on_failure(ResolvedOnFailure::node(OnFailure::Succeed));
let selected =
select_edge(node, &outcome, &Context::new(), &graph, "deterministic").unwrap();
assert_eq!(selected.edge.to, "next");
assert_eq!(selected.reason, EdgeSelectionReason::Condition);
}
#[test]
fn select_edge_condition_match() {
let mut e1 = Edge::new("a", "fail_path");
e1.attrs.insert(
"condition".to_string(),
AttrValue::String("outcome=failed".to_string()),
);
let mut e2 = Edge::new("a", "success_path");
e2.attrs.insert(
"condition".to_string(),
AttrValue::String("outcome=succeeded".to_string()),
);
let g = make_graph_with_edges(vec![e1, e2]);
let node = g.nodes.get("a").unwrap();
let outcome = Outcome::success();
let context = Context::new();
let sel = select_edge(node, &outcome, &context, &g, "deterministic").unwrap();
assert_eq!(sel.edge.to, "success_path");
assert_eq!(sel.reason, EdgeSelectionReason::Condition);
}
#[test]
fn select_edge_preferred_label() {
let mut e1 = Edge::new("a", "approve");
e1.attrs.insert(
"label".to_string(),
AttrValue::String("[A] Approve".to_string()),
);
let mut e2 = Edge::new("a", "fix");
e2.attrs.insert(
"label".to_string(),
AttrValue::String("[F] Fix".to_string()),
);
let g = make_graph_with_edges(vec![e1, e2]);
let node = g.nodes.get("a").unwrap();
let mut outcome = Outcome::success();
outcome.preferred_label = Some("Fix".to_string());
let context = Context::new();
let sel = select_edge(node, &outcome, &context, &g, "deterministic").unwrap();
assert_eq!(sel.edge.to, "fix");
assert_eq!(sel.reason, EdgeSelectionReason::PreferredLabel);
}
#[test]
fn select_edge_suggested_next_ids() {
let e1 = Edge::new("a", "path1");
let e2 = Edge::new("a", "path2");
let g = make_graph_with_edges(vec![e1, e2]);
let node = g.nodes.get("a").unwrap();
let mut outcome = Outcome::success();
outcome.suggested_next_ids = vec!["path2".to_string()];
let context = Context::new();
let sel = select_edge(node, &outcome, &context, &g, "deterministic").unwrap();
assert_eq!(sel.edge.to, "path2");
assert_eq!(sel.reason, EdgeSelectionReason::SuggestedNext);
}
#[test]
fn select_edge_weight_tiebreak() {
let mut e1 = Edge::new("a", "low");
e1.attrs.insert("weight".to_string(), AttrValue::Integer(1));
let mut e2 = Edge::new("a", "high");
e2.attrs
.insert("weight".to_string(), AttrValue::Integer(10));
let g = make_graph_with_edges(vec![e1, e2]);
let node = g.nodes.get("a").unwrap();
let outcome = Outcome::success();
let context = Context::new();
let sel = select_edge(node, &outcome, &context, &g, "deterministic").unwrap();
assert_eq!(sel.edge.to, "high");
assert_eq!(sel.reason, EdgeSelectionReason::Unconditional);
}
#[test]
fn select_edge_lexical_tiebreak() {
let e1 = Edge::new("a", "charlie");
let e2 = Edge::new("a", "alpha");
let g = make_graph_with_edges(vec![e1, e2]);
let node = g.nodes.get("a").unwrap();
let outcome = Outcome::success();
let context = Context::new();
let sel = select_edge(node, &outcome, &context, &g, "deterministic").unwrap();
assert_eq!(sel.edge.to, "alpha");
assert_eq!(sel.reason, EdgeSelectionReason::Unconditional);
}
#[test]
fn select_edge_condition_beats_unconditional() {
let mut e_cond = Edge::new("a", "cond_path");
e_cond.attrs.insert(
"condition".to_string(),
AttrValue::String("outcome=succeeded".to_string()),
);
let e_uncond = Edge::new("a", "uncond_path");
let g = make_graph_with_edges(vec![e_cond, e_uncond]);
let node = g.nodes.get("a").unwrap();
let outcome = Outcome::success();
let context = Context::new();
let sel = select_edge(node, &outcome, &context, &g, "deterministic").unwrap();
assert_eq!(sel.edge.to, "cond_path");
assert_eq!(sel.reason, EdgeSelectionReason::Condition);
}
#[test]
fn select_edge_random_returns_some_edge() {
let e1 = Edge::new("a", "b");
let e2 = Edge::new("a", "c");
let g = make_graph_with_edges(vec![e1, e2]);
let node = g.nodes.get("a").unwrap();
let outcome = Outcome::success();
let context = Context::new();
let sel = select_edge(node, &outcome, &context, &g, "random").unwrap();
assert!(sel.edge.to == "b" || sel.edge.to == "c");
assert_eq!(sel.reason, EdgeSelectionReason::Unconditional);
}
#[test]
fn select_edge_random_preferred_label_still_wins() {
let mut e1 = Edge::new("a", "approve");
e1.attrs.insert(
"label".to_string(),
AttrValue::String("Approve".to_string()),
);
let e2 = Edge::new("a", "other");
let g = make_graph_with_edges(vec![e1, e2]);
let node = g.nodes.get("a").unwrap();
let mut outcome = Outcome::success();
outcome.preferred_label = Some("Approve".to_string());
let context = Context::new();
let sel = select_edge(node, &outcome, &context, &g, "random").unwrap();
assert_eq!(sel.edge.to, "approve");
assert_eq!(sel.reason, EdgeSelectionReason::PreferredLabel);
}
#[test]
fn select_edge_failed_human_gate_does_not_fall_through_to_unconditional() {
let graph = make_graph_with_edges(vec![
Edge::new("gate", "approve"),
Edge::new("gate", "skip"),
]);
let mut node = graph.nodes.get("gate").unwrap().clone();
node.attrs.insert(
"shape".to_string(),
AttrValue::String("hexagon".to_string()),
);
let outcome = Outcome::fail_deterministic(
"human interaction interrupted before an answer was provided",
);
let context = Context::new();
assert!(select_edge(&node, &outcome, &context, &graph, "deterministic").is_none());
}
#[test]
fn select_edge_failed_human_gate_routes_via_fail_condition() {
let mut fail = Edge::new("gate", "retry");
fail.attrs.insert(
"condition".to_string(),
AttrValue::String("outcome=failed".to_string()),
);
let approve = Edge::new("gate", "approve");
let graph = make_graph_with_edges(vec![fail, approve]);
let mut node = graph.nodes.get("gate").unwrap().clone();
node.attrs.insert(
"shape".to_string(),
AttrValue::String("hexagon".to_string()),
);
let outcome = Outcome::fail_deterministic(
"human interaction interrupted before an answer was provided",
);
let context = Context::new();
let sel = select_edge(&node, &outcome, &context, &graph, "deterministic").unwrap();
assert_eq!(sel.edge.to, "retry");
assert_eq!(sel.reason, EdgeSelectionReason::Condition);
}
#[test]
fn select_edge_deterministic_no_fallback_when_no_condition_matches() {
let mut e1 = Edge::new("a", "path1");
e1.attrs.insert(
"condition".to_string(),
AttrValue::String("outcome=failed".to_string()),
);
let mut e2 = Edge::new("a", "path2");
e2.attrs.insert(
"condition".to_string(),
AttrValue::String("outcome=error".to_string()),
);
let g = make_graph_with_edges(vec![e1, e2]);
let node = g.nodes.get("a").unwrap();
let outcome = Outcome::success();
let context = Context::new();
assert!(select_edge(node, &outcome, &context, &g, "deterministic").is_none());
}
#[test]
fn select_edge_random_no_fallback_when_no_condition_matches() {
let mut e1 = Edge::new("a", "path1");
e1.attrs.insert(
"condition".to_string(),
AttrValue::String("outcome=failed".to_string()),
);
let mut e2 = Edge::new("a", "path2");
e2.attrs.insert(
"condition".to_string(),
AttrValue::String("outcome=error".to_string()),
);
let g = make_graph_with_edges(vec![e1, e2]);
let node = g.nodes.get("a").unwrap();
let outcome = Outcome::success();
let context = Context::new();
assert!(select_edge(node, &outcome, &context, &g, "random").is_none());
}
#[test]
fn goal_gates_all_satisfied() {
let mut g = Graph::new("test");
let mut n = Node::new("work");
n.attrs
.insert("goal_gate".to_string(), AttrValue::Boolean(true));
g.nodes.insert("work".to_string(), n);
let mut outcomes = HashMap::new();
outcomes.insert("work".to_string(), Outcome::success());
assert!(check_goal_gates(&g, &outcomes).is_ok());
}
#[test]
fn goal_gates_partial_success_counts() {
let mut g = Graph::new("test");
let mut n = Node::new("work");
n.attrs
.insert("goal_gate".to_string(), AttrValue::Boolean(true));
g.nodes.insert("work".to_string(), n);
let mut outcomes = HashMap::new();
let mut o = Outcome::success();
o.status = StageOutcome::PartiallySucceeded;
outcomes.insert("work".to_string(), o);
assert!(check_goal_gates(&g, &outcomes).is_ok());
}
#[test]
fn goal_gates_failed_returns_node_id() {
let mut g = Graph::new("test");
let mut n = Node::new("work");
n.attrs
.insert("goal_gate".to_string(), AttrValue::Boolean(true));
g.nodes.insert("work".to_string(), n);
let mut outcomes = HashMap::new();
outcomes.insert("work".to_string(), Outcome::fail_classify("test"));
assert_eq!(check_goal_gates(&g, &outcomes), Err("work".to_string()));
}
#[test]
fn goal_gates_unvisited_returns_node_id() {
let mut g = Graph::new("test");
let mut n = Node::new("verify");
n.attrs
.insert("goal_gate".to_string(), AttrValue::Boolean(true));
g.nodes.insert("verify".to_string(), n);
let outcomes = HashMap::new();
assert_eq!(check_goal_gates(&g, &outcomes), Err("verify".to_string()));
}
#[test]
fn goal_gates_non_gate_nodes_ignored() {
let mut g = Graph::new("test");
g.nodes.insert("work".to_string(), Node::new("work"));
let mut outcomes = HashMap::new();
outcomes.insert("work".to_string(), Outcome::fail_classify("test"));
assert!(check_goal_gates(&g, &outcomes).is_ok());
}
#[test]
fn retry_target_from_node() {
let mut g = Graph::new("test");
let mut n = Node::new("work");
n.attrs.insert(
"retry_target".to_string(),
AttrValue::String("plan".to_string()),
);
g.nodes.insert("work".to_string(), n);
g.nodes.insert("plan".to_string(), Node::new("plan"));
assert_eq!(get_retry_target("work", &g), Some("plan".to_string()));
}
#[test]
fn retry_target_from_fallback() {
let mut g = Graph::new("test");
let mut n = Node::new("work");
n.attrs.insert(
"fallback_retry_target".to_string(),
AttrValue::String("plan".to_string()),
);
g.nodes.insert("work".to_string(), n);
g.nodes.insert("plan".to_string(), Node::new("plan"));
assert_eq!(get_retry_target("work", &g), Some("plan".to_string()));
}
#[test]
fn retry_target_from_graph() {
let mut g = Graph::new("test");
g.nodes.insert("work".to_string(), Node::new("work"));
g.nodes.insert("plan".to_string(), Node::new("plan"));
g.attrs.insert(
"retry_target".to_string(),
AttrValue::String("plan".to_string()),
);
assert_eq!(get_retry_target("work", &g), Some("plan".to_string()));
}
#[test]
fn retry_target_none_when_missing() {
let mut g = Graph::new("test");
g.nodes.insert("work".to_string(), Node::new("work"));
assert!(get_retry_target("work", &g).is_none());
}
#[test]
fn retry_target_skips_nonexistent_node() {
let mut g = Graph::new("test");
let mut n = Node::new("work");
n.attrs.insert(
"retry_target".to_string(),
AttrValue::String("nonexistent".to_string()),
);
g.nodes.insert("work".to_string(), n);
assert!(get_retry_target("work", &g).is_none());
}
#[test]
fn terminal_by_shape() {
let mut n = Node::new("exit");
n.attrs.insert(
"shape".to_string(),
AttrValue::String("Msquare".to_string()),
);
assert!(is_terminal(&n));
}
#[test]
fn terminal_by_type() {
let mut n = Node::new("end");
n.attrs
.insert("type".to_string(), AttrValue::String("exit".to_string()));
assert!(is_terminal(&n));
}
#[test]
fn non_terminal_node() {
let n = Node::new("work");
assert!(!is_terminal(&n));
}
}

1606
lib/components/fabro-workflow/src/handler/agent.rs
View file

File diff suppressed because it is too large Load diff

1956
lib/components/fabro-workflow/src/handler/command.rs
View file

File diff suppressed because it is too large Load diff

55
lib/components/fabro-workflow/src/handler/conditional.rs
View file

@ -1,55 +0,0 @@
use std::path::Path;
use async_trait::async_trait;
use fabro_graphviz::graph::{Graph, Node};
use super::{EngineServices, Handler};
use crate::context::Context;
use crate::error::Error;
use crate::outcome::Outcome;
/// Conditional routing handler. Returns SUCCESS with a note; actual routing
/// is handled by the engine's edge selection algorithm.
pub struct ConditionalHandler;
#[async_trait]
impl Handler for ConditionalHandler {
async fn execute(
&self,
node: &Node,
_context: &Context,
_graph: &Graph,
_run_dir: &Path,
_services: &EngineServices,
) -> Result<Outcome, Error> {
let mut outcome = Outcome::success();
outcome.notes = Some(format!("Conditional node evaluated: {}", node.id));
Ok(outcome)
}
}
#[cfg(test)]
mod tests {
use super::*;
fn make_services() -> EngineServices {
EngineServices::test_default()
}
#[tokio::test]
async fn conditional_handler_returns_success_with_note() {
let handler = ConditionalHandler;
let node = Node::new("gate");
let context = Context::new();
let graph = Graph::new("test");
let run_dir = Path::new("/tmp/test");
let outcome = handler
.execute(&node, &context, &graph, run_dir, &make_services())
.await
.unwrap();
assert_eq!(outcome.status, crate::outcome::StageOutcome::Succeeded);
assert_eq!(
outcome.notes.as_deref(),
Some("Conditional node evaluated: gate")
);
}
}

48
lib/components/fabro-workflow/src/handler/exit.rs
View file

@ -1,48 +0,0 @@
use std::path::Path;
use async_trait::async_trait;
use fabro_graphviz::graph::{Graph, Node};
use super::{EngineServices, Handler};
use crate::context::Context;
use crate::error::Error;
use crate::outcome::Outcome;
/// No-op handler for pipeline exit point. Returns SUCCESS immediately.
pub struct ExitHandler;
#[async_trait]
impl Handler for ExitHandler {
async fn execute(
&self,
_node: &Node,
_context: &Context,
_graph: &Graph,
_run_dir: &Path,
_services: &EngineServices,
) -> Result<Outcome, Error> {
Ok(Outcome::success())
}
}
#[cfg(test)]
mod tests {
use super::*;
fn make_services() -> EngineServices {
EngineServices::test_default()
}
#[tokio::test]
async fn exit_handler_returns_success() {
let handler = ExitHandler;
let node = Node::new("exit");
let context = Context::new();
let graph = Graph::new("test");
let run_dir = Path::new("/tmp/test");
let outcome = handler
.execute(&node, &context, &graph, run_dir, &make_services())
.await
.unwrap();
assert_eq!(outcome.status, crate::outcome::StageOutcome::Succeeded);
}
}

255
lib/components/fabro-workflow/src/handler/fan_in.rs
View file

@ -1,255 +0,0 @@
use std::path::Path;
use std::sync::Arc;
use async_trait::async_trait;
use fabro_graphviz::graph::{Graph, Node};
use fabro_types::ParallelBranchResult;
use super::agent::CodergenBackend;
use super::prompt::PromptHandler;
use super::{EngineServices, Handler};
use crate::context::{Context, keys};
use crate::error::Error;
use crate::event::Emitter;
use crate::outcome::Outcome;
/// Joins results from a preceding parallel node.
///
/// Promptless fan-in nodes are barriers. Prompted fan-in nodes use the same
/// execution path as standard prompt stages and synthesize the full ordered
/// branch result set without selecting workspace state.
pub struct FanInHandler {
prompt_handler: PromptHandler,
}
impl FanInHandler {
#[must_use]
pub fn new(backend: Option<Box<dyn CodergenBackend>>) -> Self {
Self {
prompt_handler: PromptHandler::new(backend),
}
}
}
impl FanInHandler {
async fn run_join(
&self,
node: &Node,
context: &Context,
graph: &Graph,
run_dir: &Path,
services: &EngineServices,
simulated: bool,
) -> Result<Outcome, Error> {
let branch_count = validated_branch_count(context)?;
if node
.prompt()
.is_some_and(|prompt| !prompt.trim().is_empty())
{
return if simulated {
self.prompt_handler
.simulate(node, context, graph, run_dir, services)
.await
} else {
self.prompt_handler
.execute(node, context, graph, run_dir, services)
.await
};
}
Ok(joined_outcome(branch_count, simulated))
}
}
#[async_trait]
impl Handler for FanInHandler {
async fn shutdown(&self, emitter: &Arc<Emitter>) {
self.prompt_handler.shutdown(emitter).await;
}
async fn simulate(
&self,
node: &Node,
context: &Context,
graph: &Graph,
run_dir: &Path,
services: &EngineServices,
) -> Result<Outcome, Error> {
self.run_join(node, context, graph, run_dir, services, true)
.await
}
async fn execute(
&self,
node: &Node,
context: &Context,
graph: &Graph,
run_dir: &Path,
services: &EngineServices,
) -> Result<Outcome, Error> {
self.run_join(node, context, graph, run_dir, services, false)
.await
}
}
/// Validate that `parallel.results` exists and has the typed shape.
fn validated_branch_count(context: &Context) -> Result<usize, Error> {
let value = context
.get(keys::PARALLEL_RESULTS)
.ok_or_else(|| Error::handler("No parallel results to join"))?;
let results: Vec<ParallelBranchResult> = serde_json::from_value(value)
.map_err(|err| Error::handler_with_source("Invalid parallel results", err))?;
Ok(results.len())
}
fn joined_outcome(branch_count: usize, simulated: bool) -> Outcome {
let mut outcome = Outcome::success();
let prefix = if simulated { "[Simulated] " } else { "" };
outcome.notes = Some(format!(
"{prefix}Joined {branch_count} parallel {}",
if branch_count == 1 {
"branch"
} else {
"branches"
}
));
outcome
}
#[cfg(test)]
mod tests {
use fabro_graphviz::graph::AttrValue;
use fabro_types::StageTiming;
use tempfile::TempDir;
use super::*;
use crate::handler::agent::{CodergenResult, CodergenRunRequest, OneShotRequest};
use crate::outcome::StageOutcome;
fn make_services() -> EngineServices {
EngineServices::test_default()
}
fn context_with_results() -> Context {
let context = Context::new();
context.set(
keys::PARALLEL_RESULTS,
serde_json::json!([
{
"id": "branch_a",
"status": "failed",
"context_updates": {"command.output": "failure details"}
},
{
"id": "branch_b",
"status": "succeeded",
"context_updates": {"response.branch_b": "complete response"}
}
]),
);
context
}
#[tokio::test]
async fn promptless_fan_in_is_a_noop_barrier() {
let outcome = FanInHandler::new(None)
.execute(
&Node::new("fan_in"),
&context_with_results(),
&Graph::new("test"),
Path::new("/tmp/test"),
&make_services(),
)
.await
.unwrap();
assert_eq!(outcome.status, StageOutcome::Succeeded);
assert_eq!(outcome.notes.as_deref(), Some("Joined 2 parallel branches"));
assert!(outcome.context_updates.is_empty());
}
#[tokio::test]
async fn fan_in_requires_typed_parallel_results() {
let context = Context::new();
let missing = FanInHandler::new(None)
.execute(
&Node::new("fan_in"),
&context,
&Graph::new("test"),
Path::new("/tmp/test"),
&make_services(),
)
.await;
assert!(missing.is_err());
context.set(keys::PARALLEL_RESULTS, serde_json::json!([{"id": "a"}]));
let invalid = FanInHandler::new(None)
.execute(
&Node::new("fan_in"),
&context,
&Graph::new("test"),
Path::new("/tmp/test"),
&make_services(),
)
.await;
assert!(invalid.is_err());
}
#[tokio::test]
async fn prompted_fan_in_uses_standard_prompt_response_fields() {
struct ReducerBackend;
#[async_trait]
impl CodergenBackend for ReducerBackend {
async fn run(&self, _request: CodergenRunRequest<'_>) -> Result<CodergenResult, Error> {
panic!("prompted fan-in must use one_shot like a standard prompt")
}
async fn one_shot(&self, request: OneShotRequest<'_>) -> Result<CodergenResult, Error> {
assert!(request.prompt.contains("Synthesize every result"));
Ok(CodergenResult::Text {
text: "combined result".to_string(),
usage_by_model: Vec::new(),
usage: None,
files_touched: Vec::new(),
last_file_touched: None,
timing: StageTiming::new(0, 20, 30),
})
}
}
let handler = FanInHandler::new(Some(Box::new(ReducerBackend)));
let mut node = Node::new("fan_in");
node.attrs.insert(
"prompt".to_string(),
AttrValue::String("Synthesize every result".to_string()),
);
let run_dir = TempDir::new().unwrap();
let outcome = handler
.execute(
&node,
&context_with_results(),
&Graph::new("test"),
run_dir.path(),
&make_services(),
)
.await
.unwrap();
assert_eq!(outcome.status, StageOutcome::Succeeded);
assert_eq!(
outcome.context_updates.get(&keys::response_key("fan_in")),
Some(&serde_json::json!("combined result"))
);
assert_eq!(
outcome.context_updates.get(keys::LAST_RESPONSE),
Some(&serde_json::json!("combined result"))
);
assert_eq!(outcome.timing, Some(StageTiming::new(0, 20, 30)));
assert!(
outcome
.context_updates
.keys()
.all(|key| !key.starts_with("parallel.fan_in.best_"))
);
}
}

1318
lib/components/fabro-workflow/src/handler/human.rs
View file

File diff suppressed because it is too large Load diff

1450
lib/components/fabro-workflow/src/handler/llm/acp.rs
View file

File diff suppressed because it is too large Load diff

313
lib/components/fabro-workflow/src/handler/llm/activation_lease.rs
View file

@ -1,313 +0,0 @@
//! A stage's session on the steering bus, with fabro's lifecycle events.
//!
//! Activating attaches the session at its stage, records
//! `agent.session.activated` with the route and capabilities the run should
//! show, and then drains steers that waited for it. Releasing detaches and
//! records `agent.session.deactivated` once, however many times it is asked.
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use fabro_types::{PermissionLevel, SessionCapability, StageId};
use lithos_llm::types::{ReasoningEffort, Speed};
use pebble_coding_agent::steering::SteerableSession;
use crate::error::Error;
use crate::event::{Emitter, Event};
use crate::steering_hub::SteeringHub;
pub struct ActivationLease {
stage_id: StageId,
session_id: String,
hub: Arc<SteeringHub>,
emitter: Arc<Emitter>,
released: AtomicBool,
}
pub struct ActivationLeaseOptions {
pub stage_id: StageId,
pub session_id: String,
pub thread_id: Option<String>,
pub provider: Option<String>,
pub model: Option<String>,
pub reasoning_effort: Option<ReasoningEffort>,
pub speed: Option<Speed>,
pub permission_level: Option<PermissionLevel>,
pub capabilities: Vec<SessionCapability>,
pub hub: Arc<SteeringHub>,
pub emitter: Arc<Emitter>,
}
impl ActivationLease {
pub fn activate(
options: ActivationLeaseOptions,
session: Arc<dyn SteerableSession>,
) -> Result<Arc<Self>, Error> {
options
.hub
.attach(&options.stage_id, &options.session_id, session)
.map_err(|_| {
Error::Precondition(format!(
"stage {} already has a different active agent session",
options.stage_id
))
})?;
options.emitter.emit(&Event::AgentSessionActivated {
node_id: options.stage_id.node_id().to_string(),
visit: options.stage_id.visit(),
session_id: options.session_id.clone(),
thread_id: options.thread_id,
provider: options.provider,
model: options.model,
reasoning_effort: options.reasoning_effort,
speed: options.speed,
permission_level: options.permission_level,
capabilities: options.capabilities,
});
options.hub.drain_pending_into(&options.stage_id);
Ok(Arc::new(Self {
stage_id: options.stage_id,
session_id: options.session_id,
hub: options.hub,
emitter: options.emitter,
released: AtomicBool::new(false),
}))
}
pub fn release(&self) {
if !self.mark_released() {
return;
}
self.hub.detach(&self.stage_id, &self.session_id);
}
/// The close-the-door check: release only if the session has no steering
/// waiting. Returns whether the lease is released.
pub fn release_if_idle(&self) -> bool {
if self.released.load(Ordering::Acquire) {
return true;
}
if !self.hub.detach_if_idle(&self.stage_id, &self.session_id) {
return false;
}
self.mark_released();
true
}
pub fn is_pair_active(&self) -> bool {
!self.released.load(Ordering::Acquire)
&& self
.hub
.pair_is_active_for(&self.stage_id, &self.session_id)
}
fn mark_released(&self) -> bool {
if self
.released
.compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
.is_err()
{
return false;
}
self.emitter.emit(&Event::AgentSessionDeactivated {
node_id: self.stage_id.node_id().to_string(),
visit: self.stage_id.visit(),
session_id: self.session_id.clone(),
});
true
}
}
impl Drop for ActivationLease {
fn drop(&mut self) {
self.release();
}
}
#[cfg(test)]
mod tests {
use std::sync::{Arc, Mutex};
use fabro_types::RunId;
use pebble_coding_agent::{SteeringMessage, SteeringOutcome};
use super::*;
#[derive(Default)]
struct SessionControlHandle {
queue: Mutex<Vec<SteeringMessage>>,
}
impl SessionControlHandle {
fn queue_len(&self) -> usize {
self.queue.lock().unwrap().len()
}
}
impl SteerableSession for SessionControlHandle {
fn steer(&self, message: SteeringMessage) -> SteeringOutcome {
self.queue.lock().unwrap().push(message);
SteeringOutcome::Accepted
}
fn interrupt(&self) -> bool {
false
}
fn steer_now(&self, message: SteeringMessage) -> SteeringOutcome {
self.steer(message)
}
fn has_pending_steering(&self) -> bool {
!self.queue.lock().unwrap().is_empty()
}
}
fn collect_event_names(emitter: &Arc<Emitter>) -> Arc<Mutex<Vec<String>>> {
let names = Arc::new(Mutex::new(Vec::new()));
let names_for_listener = Arc::clone(&names);
emitter.on_event(move |event| {
names_for_listener
.lock()
.unwrap()
.push(event.event_name().to_string());
});
names
}
fn options(
stage_id: StageId,
session_id: &str,
hub: Arc<SteeringHub>,
emitter: Arc<Emitter>,
) -> ActivationLeaseOptions {
ActivationLeaseOptions {
stage_id,
session_id: session_id.to_string(),
thread_id: None,
provider: Some("openai".to_string()),
model: Some("gpt-5.4".to_string()),
reasoning_effort: None,
speed: None,
permission_level: None,
capabilities: vec![SessionCapability::Steer],
hub,
emitter,
}
}
fn session(handle: &Arc<SessionControlHandle>) -> Arc<dyn SteerableSession> {
Arc::clone(handle) as Arc<dyn SteerableSession>
}
#[test]
fn activate_emits_activated_before_draining_pending() {
let emitter = Arc::new(Emitter::new(RunId::new()));
let names = collect_event_names(&emitter);
let hub = Arc::new(SteeringHub::new(Arc::clone(&emitter)));
let stage_id = StageId::new("agent", 1);
let handle = Arc::new(SessionControlHandle::default());
hub.deliver_steer("queued".to_string(), None);
let _lease = ActivationLease::activate(
options(
stage_id.clone(),
"session-a",
Arc::clone(&hub),
Arc::clone(&emitter),
),
session(&handle),
)
.unwrap();
assert_eq!(handle.queue_len(), 1);
assert_eq!(names.lock().unwrap().as_slice(), [
"run.steer",
"agent.steer.buffered",
"agent.session.activated"
]);
}
#[test]
fn activate_rejects_mismatched_existing_session() {
let emitter = Arc::new(Emitter::new(RunId::new()));
let names = collect_event_names(&emitter);
let hub = Arc::new(SteeringHub::new(Arc::clone(&emitter)));
let stage_id = StageId::new("agent", 1);
let handle_a = Arc::new(SessionControlHandle::default());
let handle_b = Arc::new(SessionControlHandle::default());
let _lease = ActivationLease::activate(
options(
stage_id.clone(),
"session-a",
Arc::clone(&hub),
Arc::clone(&emitter),
),
session(&handle_a),
)
.unwrap();
let result = ActivationLease::activate(
options(
stage_id,
"session-b",
Arc::clone(&hub),
Arc::clone(&emitter),
),
session(&handle_b),
);
assert!(result.is_err());
assert_eq!(handle_b.queue_len(), 0);
assert_eq!(
names
.lock()
.unwrap()
.iter()
.filter(|name| name.as_str() == "agent.session.activated")
.count(),
1
);
}
#[test]
fn release_is_idempotent_and_release_if_idle_waits_for_steering() {
let emitter = Arc::new(Emitter::new(RunId::new()));
let names = collect_event_names(&emitter);
let hub = Arc::new(SteeringHub::new(Arc::clone(&emitter)));
let stage_id = StageId::new("agent", 1);
let handle = Arc::new(SessionControlHandle::default());
let lease = ActivationLease::activate(
options(
stage_id,
"session-a",
Arc::clone(&hub),
Arc::clone(&emitter),
),
session(&handle),
)
.unwrap();
hub.deliver_steer("late".to_string(), None);
assert!(
!lease.release_if_idle(),
"a waiting steer keeps the door open"
);
handle.queue.lock().unwrap().clear();
assert!(lease.release_if_idle());
assert!(lease.release_if_idle(), "released stays released");
lease.release();
assert_eq!(
names
.lock()
.unwrap()
.iter()
.filter(|name| name.as_str() == "agent.session.deactivated")
.count(),
1
);
}
}

63
lib/components/fabro-workflow/src/handler/llm/changed_files.rs
View file

@ -1,63 +0,0 @@
use std::collections::HashSet;
use std::sync::Arc;
use fabro_sandbox::RunSandbox;
use fabro_util::shell;
use sandbox_driver::{Git as _, GitDiffOptions, GitRevisionRange};
/// The paths the working tree changed against `HEAD`, plus the untracked
/// files git does not ignore, sorted and deduplicated. A sandbox without
/// git, or a working directory that is not a repository, has no changed
/// files.
pub async fn detect_changed_files(sandbox: &Arc<RunSandbox>) -> Vec<String> {
let Ok(git) = sandbox.git() else {
return Vec::new();
};
let repo = sandbox.working_directory();
let mut files: Vec<String> = Vec::new();
if let Ok(entries) = git
.diff_entries(repo, &GitDiffOptions::new(GitRevisionRange::new("HEAD")))
.await
{
files.extend(entries.into_iter().map(|entry| entry.path));
}
if let Ok(untracked) = git.untracked_files(repo).await {
files.extend(untracked);
}
files.sort();
files.dedup();
files
}
pub async fn files_touched_since(
sandbox: &Arc<RunSandbox>,
files_before: &[String],
) -> (Vec<String>, Option<String>) {
let files_after = detect_changed_files(sandbox).await;
let files_before: HashSet<&str> = files_before.iter().map(String::as_str).collect();
let files_touched: Vec<String> = files_after
.into_iter()
.filter(|file| !files_before.contains(file.as_str()))
.collect();
let last_file_touched = if files_touched.is_empty() {
None
} else {
let quoted_files: Vec<String> = files_touched
.iter()
.map(|file| shell::shell_quote(file))
.collect();
let cmd = format!("ls -t {} | head -1", quoted_files.join(" "));
sandbox
.exec_command(&cmd, 5_000, None, None, None)
.await
.ok()
.and_then(|result| {
let trimmed = result.stdout_lossy().trim().to_string();
(result.success() && !trimmed.is_empty()).then_some(trimmed)
})
};
(files_touched, last_file_touched)
}

137
lib/components/fabro-workflow/src/handler/llm/controls.rs
View file

@ -1,137 +0,0 @@
//! Per-request model controls: the reasoning effort and speed a stage asks
//! for, resolved from the node's attributes over the run-level defaults.
use fabro_graphviz::graph::{AttrValue, Node};
use fabro_types::settings::run::RunModelControls;
use lithos_llm::types::{ReasoningEffort, Speed};
use crate::error::Error;
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct EffectiveRequestControls {
pub(crate) reasoning_effort: Option<ReasoningEffort>,
pub(crate) speed: Option<Speed>,
}
pub(crate) fn effective_request_controls(
run_model_controls: &RunModelControls,
node: &Node,
) -> Result<EffectiveRequestControls, Error> {
let reasoning_effort = match control_attr(node, "reasoning_effort")
.or(run_model_controls.reasoning_effort.as_deref())
{
Some(value) => Some(parse_reasoning_effort(node, value)?),
None => None,
};
let speed = control_attr(node, "speed")
.or(run_model_controls.speed.as_deref())
.map(|value| parse_speed(node, value))
.transpose()?;
Ok(EffectiveRequestControls {
reasoning_effort,
speed,
})
}
fn control_attr<'a>(node: &'a Node, key: &str) -> Option<&'a str> {
node.attrs.get(key).and_then(AttrValue::as_str)
}
fn parse_reasoning_effort(node: &Node, value: &str) -> Result<ReasoningEffort, Error> {
value.parse().map_err(|_| {
Error::handler(format!(
"Invalid reasoning_effort \"{value}\" for node \"{}\"; expected one of: {}",
node.id,
expected_values(
ReasoningEffort::ALL
.into_iter()
.map(ReasoningEffort::as_str)
),
))
})
}
fn parse_speed(node: &Node, value: &str) -> Result<Speed, Error> {
value.parse().map_err(|_| {
Error::handler(format!(
"Invalid speed \"{value}\" for node \"{}\"; expected one of: {}",
node.id,
expected_values(Speed::ALL.into_iter().map(Speed::as_str)),
))
})
}
fn expected_values<'a>(values: impl Iterator<Item = &'a str>) -> String {
values.collect::<Vec<_>>().join(", ")
}
/// Node-level `max_tokens`, as the client's `u32` output budget.
pub(crate) fn node_max_output_tokens(node: &Node) -> Option<u32> {
node.max_tokens()
.and_then(|tokens| u32::try_from(tokens).ok())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn run_model_controls_apply_when_node_omits_controls() {
let run_controls = RunModelControls {
reasoning_effort: Some("low".to_string()),
speed: Some("fast".to_string()),
};
let node = Node::new("work");
let controls = effective_request_controls(&run_controls, &node).unwrap();
assert_eq!(controls.reasoning_effort, Some(ReasoningEffort::Low));
assert_eq!(controls.speed, Some(Speed::Fast));
}
#[test]
fn node_controls_override_run_model_controls() {
let run_controls = RunModelControls {
reasoning_effort: Some("low".to_string()),
speed: Some("fast".to_string()),
};
let mut node = Node::new("work");
node.attrs.insert(
"reasoning_effort".to_string(),
AttrValue::String("high".to_string()),
);
node.attrs.insert(
"speed".to_string(),
AttrValue::String("balanced".to_string()),
);
let controls = effective_request_controls(&run_controls, &node).unwrap();
assert_eq!(controls.reasoning_effort, Some(ReasoningEffort::High));
assert_eq!(controls.speed, Some(Speed::Balanced));
}
#[test]
fn omitted_reasoning_effort_stays_unset() {
let node = Node::new("work");
let controls = effective_request_controls(&RunModelControls::default(), &node).unwrap();
assert_eq!(controls.reasoning_effort, None);
assert_eq!(controls.speed, None);
}
#[test]
fn invalid_reasoning_effort_names_the_node() {
let mut node = Node::new("work");
node.attrs.insert(
"reasoning_effort".to_string(),
AttrValue::String("maximal".to_string()),
);
let error = effective_request_controls(&RunModelControls::default(), &node).unwrap_err();
assert!(error.to_string().contains("node \"work\""), "{error}");
}
}

391
lib/components/fabro-workflow/src/handler/llm/fallback.rs
View file

@ -1,391 +0,0 @@
//! The fixed fallback plan a stage follows when its model fails.
//!
//! The plan belongs to the originally requested model: advancing it never
//! activates a target model's own chain. `model_fallback.rs` decides the
//! policy; this module resolves it against the catalog and records each
//! failover as a run event. Agent stages hand the resolved routes to pebble
//! ([`FallbackPlan::pebble_routes`]), which executes them and reports each
//! move as `RouteFailover`; one-shot prompt stages walk the plan themselves.
use fabro_graphviz::graph::Node;
use fabro_llm::FallbackTarget;
use fabro_llm::lithos_catalog::Catalog;
use fabro_types::FailoverProps;
use lithos_llm::catalog::ProviderId;
use lithos_llm::types::ReasoningEffort;
use pebble_coding_agent::FallbackRoute;
use super::controls::EffectiveRequestControls;
use crate::event::{Emitter, Event, StageScope};
use crate::model_fallback::{ModelFallbackNotice, ModelFallbackPolicy, canonical_model_id};
#[derive(Clone, Debug)]
pub(crate) struct LlmRoute {
pub(crate) target: FallbackTarget,
pub(crate) controls: EffectiveRequestControls,
}
impl LlmRoute {
/// The `provider/model` selector the client resolves for this route.
pub(crate) fn selector(&self) -> String {
format!("{}/{}", self.target.provider, self.target.model)
}
}
#[derive(Clone, Debug)]
pub(crate) struct FallbackPlan {
pub(crate) original: LlmRoute,
pub(crate) remaining: Vec<LlmRoute>,
/// 0 addresses the original route; N addresses `remaining[N - 1]`.
pub(crate) position: usize,
}
impl FallbackPlan {
pub(crate) fn current(&self) -> &LlmRoute {
self.route_at(self.position)
}
/// The route that was active before the most recent [`Self::advance`].
pub(crate) fn previous(&self) -> &LlmRoute {
self.route_at(self.position.saturating_sub(1))
}
fn route_at(&self, position: usize) -> &LlmRoute {
position
.checked_sub(1)
.map_or(&self.original, |index| &self.remaining[index])
}
pub(crate) fn attempt(&self) -> u32 {
u32::try_from(self.position).unwrap_or(u32::MAX)
}
#[must_use]
pub(crate) fn has_next(&self) -> bool {
self.position < self.remaining.len()
}
/// Move to the next fallback route. Returns false when the plan is
/// exhausted.
pub(crate) fn advance(&mut self) -> bool {
if self.has_next() {
self.position += 1;
true
} else {
false
}
}
/// Moves to the route whose `provider/model` selector is `selector`, the
/// route pebble reports a prompt ended on. Returns whether the position
/// changed; a selector the plan does not know leaves it where it was.
pub(crate) fn advance_to(&mut self, selector: &str) -> bool {
if self.current().selector() == selector {
return false;
}
match self
.remaining
.iter()
.position(|route| route.selector() == selector)
{
Some(index) => {
self.position = index + 1;
true
}
None => false,
}
}
/// The routes after the current one, as pebble executes them: each with
/// its own controls and the stage's output limit.
pub(crate) fn pebble_routes(&self, max_tokens: Option<i64>) -> Vec<FallbackRoute> {
self.remaining
.iter()
.skip(self.position)
.map(|route| {
FallbackRoute::new(route.selector())
.with_reasoning_effort(route.controls.reasoning_effort)
.with_speed(route.controls.speed)
.with_max_tokens(max_tokens)
})
.collect()
}
}
/// The `prompt.failover` payload for a one-shot stage's move from `from` to
/// `to`, both `provider/model` selectors.
///
/// `from` may be a route that failed during activation without serving
/// traffic; `error` says why it was abandoned. Consecutive payloads chain:
/// one's `to` is the next one's `from`.
pub(crate) fn failover_props(from: &str, to: &str, attempt: u32, error: &str) -> FailoverProps {
let (from_provider, from_model) = split_selector(from);
let (to_provider, to_model) = split_selector(to);
FailoverProps {
from_provider,
from_model,
to_provider,
to_model,
attempt: Some(attempt),
error: error.to_string(),
}
}
/// A `provider/model` selector split at its first slash; a selector with no
/// slash is all model.
fn split_selector(selector: &str) -> (String, String) {
match selector.split_once('/') {
Some((provider, model)) => (provider.to_string(), model.to_string()),
None => (String::new(), selector.to_string()),
}
}
/// Request controls resolved for one fallback target.
enum FallbackControls {
/// The target can serve the request with these controls.
Usable(EffectiveRequestControls),
/// The target advertises reasoning levels, but none is near the requested
/// effort.
NoNearbyReasoningLevel(ReasoningEffort),
}
fn fallback_controls_for_target(
catalog: &Catalog,
target: &FallbackTarget,
requested: EffectiveRequestControls,
) -> FallbackControls {
let Some(requested_effort) = requested.reasoning_effort else {
return FallbackControls::Usable(requested);
};
let Some(offering) = catalog
.enabled_provider(target.provider.as_str())
.and_then(|provider| provider.offering(target.model.as_str()))
else {
// A catalog-unknown passthrough target has no advertised controls.
// Preserve the request and let the provider validate it.
return FallbackControls::Usable(requested);
};
let capabilities = offering.model.capabilities();
let effective_effort = capabilities.closest_supported_effort(requested_effort);
match effective_effort {
Some(effort) => FallbackControls::Usable(EffectiveRequestControls {
reasoning_effort: Some(effort),
speed: requested.speed,
}),
// No level is verified. Unless the requested one is verified
// unsupported, preserve it and let the provider validate, as for
// a passthrough target.
None if !capabilities
.reasoning_effort(requested_effort)
.is_unsupported() =>
{
FallbackControls::Usable(requested)
}
None => FallbackControls::NoNearbyReasoningLevel(requested_effort),
}
}
/// The plan for `model` on `provider`, and the configuration notices the
/// caller should surface once per run.
pub(crate) fn fallback_plan(
catalog: &Catalog,
fallbacks: &ModelFallbackPolicy,
model: &str,
provider: &ProviderId,
requested_controls: EffectiveRequestControls,
) -> (FallbackPlan, Vec<ModelFallbackNotice>) {
let primary_model = canonical_model_id(catalog, provider, model);
let original = LlmRoute {
target: FallbackTarget::new(provider, &primary_model),
controls: requested_controls,
};
let Some(configured) = fallbacks.chain_for_canonical(&primary_model) else {
return (
FallbackPlan {
original,
remaining: Vec::new(),
position: 0,
},
Vec::new(),
);
};
let mut remaining = Vec::new();
let mut notices = Vec::new();
for target in configured {
// The resolver already de-duplicated the chain; only the primary
// target, which the resolver cannot know, needs filtering here.
if *target == original.target {
continue;
}
let controls = match fallback_controls_for_target(catalog, target, requested_controls) {
FallbackControls::Usable(controls) => controls,
FallbackControls::NoNearbyReasoningLevel(requested_effort) => {
notices.push(ModelFallbackNotice::NoNearbyReasoningLevel {
requested_model: original.target.model.to_string(),
target: target.clone(),
requested_effort,
});
continue;
}
};
remaining.push(LlmRoute {
target: target.clone(),
controls,
});
}
if !configured.is_empty() && remaining.is_empty() {
notices.push(ModelFallbackNotice::ChainEmpty {
requested_model: original.target.model.to_string(),
});
}
(
FallbackPlan {
original,
remaining,
position: 0,
},
notices,
)
}
/// Emit `prompt.failover` for the plan's most recent
/// [`FallbackPlan::advance`], on a one-shot stage that walks the plan itself.
/// An agent stage never emits it: pebble walks the routes and reports each
/// move as `agent.route.failover`.
pub(crate) fn emit_failover(
node: &Node,
emitter: &Emitter,
stage_scope: &StageScope,
plan: &FallbackPlan,
error: &str,
) {
emitter.emit_scoped(
&Event::Failover {
stage: node.id.clone(),
props: failover_props(
&plan.previous().selector(),
&plan.current().selector(),
plan.attempt(),
error,
),
},
stage_scope,
);
}
#[cfg(test)]
mod tests {
use std::collections::BTreeMap;
use fabro_llm::test_support::test_catalog_with_overlay;
use lithos_llm::catalog::builtin;
use super::*;
/// Modal and OpenRouter ship disabled; enable them the way an operator
/// would so their models become fallback targets.
fn enabled_fallback_catalog() -> Catalog {
test_catalog_with_overlay(
"[providers.modal]\nenabled = true\n\n[providers.openrouter]\nenabled = true\n",
)
}
#[test]
fn fallback_plan_maps_reasoning_to_each_target_and_rounds_ties_up() {
let policy = ModelFallbackPolicy::new(BTreeMap::from([("kimi-k3".to_string(), vec![
FallbackTarget::new("moonshot", "kimi-k3"),
FallbackTarget::new("openrouter", "kimi-k3"),
FallbackTarget::new("anthropic", "claude-opus-5"),
])]));
let (plan, notices) = fallback_plan(
&enabled_fallback_catalog(),
&policy,
"kimi-k3",
&ProviderId::new("modal"),
EffectiveRequestControls {
reasoning_effort: Some(ReasoningEffort::Medium),
speed: None,
},
);
assert!(notices.is_empty());
assert_eq!(
plan.remaining
.iter()
.map(|route| route.controls.reasoning_effort)
.collect::<Vec<_>>(),
vec![
Some(ReasoningEffort::High),
Some(ReasoningEffort::High),
Some(ReasoningEffort::Medium),
]
);
}
#[test]
fn advancing_a_fallback_plan_never_activates_the_target_models_chain() {
let policy = ModelFallbackPolicy::new(BTreeMap::from([
("claude-fable-5".to_string(), vec![
FallbackTarget::new("openai", "gpt-5.6-sol"),
FallbackTarget::new("anthropic", "claude-opus-5"),
]),
("gpt-5.6-sol".to_string(), vec![FallbackTarget::new(
"anthropic",
"claude-sonnet-5",
)]),
]));
let (mut plan, notices) = fallback_plan(
&enabled_fallback_catalog(),
&policy,
"claude-fable-5",
&builtin::anthropic(),
EffectiveRequestControls::default(),
);
assert!(notices.is_empty());
assert!(plan.advance(), "Sol should be first");
assert_eq!(
plan.current().target,
FallbackTarget::new("openai", "gpt-5.6-sol")
);
assert_eq!(plan.current().selector(), "openai/gpt-5.6-sol");
assert_eq!(plan.attempt(), 1);
assert!(plan.advance(), "Opus should be second");
assert_eq!(
plan.current().target,
FallbackTarget::new("anthropic", "claude-opus-5")
);
assert_eq!(plan.attempt(), 2);
assert!(!plan.has_next());
assert!(!plan.advance());
}
#[test]
fn failover_props_name_both_routes_and_the_attempt() {
let props = failover_props(
"anthropic/claude-fable-5",
"openai/gpt-5.6-sol",
1,
"overloaded",
);
assert_eq!(props, FailoverProps {
from_provider: "anthropic".to_string(),
from_model: "claude-fable-5".to_string(),
to_provider: "openai".to_string(),
to_model: "gpt-5.6-sol".to_string(),
attempt: Some(1),
error: "overloaded".to_string(),
});
// A selector with no slash is all model.
let bare = failover_props("local-model", "openai/gpt-5.6-sol", 2, "down");
assert_eq!(bare.from_provider, "");
assert_eq!(bare.from_model, "local-model");
assert_eq!(bare.attempt, Some(2));
}
}

16
lib/components/fabro-workflow/src/handler/llm/mod.rs
View file

@ -1,16 +0,0 @@
pub mod acp;
pub mod activation_lease;
pub mod changed_files;
pub mod controls;
pub mod fabro_tools;
pub mod fallback;
pub mod pebble;
pub mod preamble;
pub mod router;
pub mod routing;
pub use acp::AgentAcpBackend;
pub use controls::EffectiveRequestControls;
pub use fabro_tools::{register_fabro_run_tools, register_named_fabro_run_tools};
pub use pebble::PebbleBackend;
pub use router::BackendRouter;

1480
lib/components/fabro-workflow/src/handler/llm/pebble.rs
View file

File diff suppressed because it is too large Load diff

2360
lib/components/fabro-workflow/src/handler/llm/preamble.rs
View file

File diff suppressed because it is too large Load diff

192
lib/components/fabro-workflow/src/handler/llm/router.rs
View file

@ -1,192 +0,0 @@
use std::sync::Arc;
use async_trait::async_trait;
use fabro_graphviz::graph::Node;
use fabro_types::AgentBackend;
use super::super::agent::{CodergenBackend, CodergenResult, CodergenRunRequest, OneShotRequest};
use super::acp::AgentAcpBackend;
use super::controls::EffectiveRequestControls;
use super::routing;
use crate::error::Error;
use crate::event::Emitter;
use crate::handler::NodeTimeoutPolicy;
/// Routes codergen invocations to API or ACP backends based on node attributes.
pub struct BackendRouter {
api: Box<dyn CodergenBackend>,
acp: AgentAcpBackend,
}
impl BackendRouter {
#[must_use]
pub fn new(api_backend: Box<dyn CodergenBackend>, acp_backend: AgentAcpBackend) -> Self {
Self {
api: api_backend,
acp: acp_backend,
}
}
fn select_backend(node: &Node) -> Result<AgentBackend, Error> {
routing::select_run_backend(node)
}
fn select_one_shot_backend(node: &Node) -> Result<AgentBackend, Error> {
routing::select_one_shot_backend(node)
}
}
#[async_trait]
impl CodergenBackend for BackendRouter {
async fn run(&self, request: CodergenRunRequest<'_>) -> Result<CodergenResult, Error> {
match Self::select_backend(request.node)? {
AgentBackend::Api => self.api.run(request).await,
AgentBackend::Acp => self.acp.run(request).await,
}
}
async fn one_shot(&self, request: OneShotRequest<'_>) -> Result<CodergenResult, Error> {
match Self::select_one_shot_backend(request.node)? {
AgentBackend::Api => self.api.one_shot(request).await,
AgentBackend::Acp => {
unreachable!("ACP one-shot is rejected by select_one_shot_backend")
}
}
}
async fn shutdown(&self, emitter: &Arc<Emitter>) {
self.api.shutdown(emitter).await;
}
fn effective_request_controls(&self, node: &Node) -> Result<EffectiveRequestControls, Error> {
match Self::select_backend(node)? {
AgentBackend::Api => self.api.effective_request_controls(node),
AgentBackend::Acp => self.acp.effective_request_controls(node),
}
}
fn node_timeout_policy(&self, node: &Node) -> NodeTimeoutPolicy {
match Self::select_backend(node) {
Ok(AgentBackend::Api) => self.api.node_timeout_policy(node),
Ok(AgentBackend::Acp) => self.acp.node_timeout_policy(node),
Err(_) => NodeTimeoutPolicy::ExecutorEnforced,
}
}
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use async_trait::async_trait;
use fabro_graphviz::graph::{AttrValue, Node};
use fabro_sandbox::{RunSandbox, local_sandbox};
use lithos_llm::types::{ReasoningEffort, Speed};
use tokio_util::sync::CancellationToken;
use super::*;
use crate::context::Context;
use crate::event::{Emitter, StageScope};
#[test]
fn router_uses_api_by_default() {
let node = Node::new("test");
assert_eq!(
BackendRouter::select_backend(&node).unwrap(),
AgentBackend::Api
);
}
#[test]
fn router_rejects_cli_backend() {
let mut node = Node::new("test");
node.attrs
.insert("backend".to_string(), AttrValue::String("cli".to_string()));
let err = BackendRouter::select_backend(&node).unwrap_err();
assert_eq!(
err.to_string(),
"Validation error: unsupported agent backend \"cli\"; expected one of: api, acp"
);
}
#[tokio::test]
async fn router_routes_one_shot_to_api_by_default() {
let node = Node::new("test");
let sandbox: Arc<RunSandbox> = Arc::new(
local_sandbox(tempfile::tempdir().unwrap().path().to_path_buf())
.await
.unwrap(),
);
let context = Context::new();
let router = BackendRouter::new(Box::new(StubBackend), AgentAcpBackend::new());
let emitter = Arc::new(Emitter::default());
let stage_scope = StageScope::for_handler(&context, "test");
let result = router
.one_shot(OneShotRequest {
node: &node,
prompt: "prompt",
system_prompt: None,
emitter: &emitter,
stage_scope: &stage_scope,
sandbox: &sandbox,
cancel_token: CancellationToken::new(),
})
.await
.unwrap();
let CodergenResult::Text { text, .. } = result else {
panic!("expected text result");
};
assert_eq!(text, "api one-shot");
}
#[test]
fn router_delegates_effective_request_controls_to_api_backend() {
let node = Node::new("test");
let router = BackendRouter::new(Box::new(StubBackend), AgentAcpBackend::new());
let controls = router.effective_request_controls(&node).unwrap();
assert_eq!(controls.reasoning_effort, Some(ReasoningEffort::High));
assert_eq!(controls.speed, Some(Speed::Fast));
}
struct StubBackend;
#[async_trait]
impl CodergenBackend for StubBackend {
async fn run(&self, _request: CodergenRunRequest<'_>) -> Result<CodergenResult, Error> {
Ok(CodergenResult::Text {
text: "api run".to_string(),
usage_by_model: Vec::new(),
usage: None,
files_touched: Vec::new(),
last_file_touched: None,
timing: fabro_types::StageTiming::default(),
})
}
async fn one_shot(&self, _request: OneShotRequest<'_>) -> Result<CodergenResult, Error> {
Ok(CodergenResult::Text {
text: "api one-shot".to_string(),
usage_by_model: Vec::new(),
usage: None,
files_touched: Vec::new(),
last_file_touched: None,
timing: fabro_types::StageTiming::default(),
})
}
fn effective_request_controls(
&self,
_node: &Node,
) -> Result<EffectiveRequestControls, Error> {
Ok(EffectiveRequestControls {
reasoning_effort: Some(ReasoningEffort::High),
speed: Some(Speed::Fast),
})
}
}
}

110
lib/components/fabro-workflow/src/handler/llm/routing.rs
View file

@ -1,110 +0,0 @@
use fabro_graphviz::graph::{self, Node};
use fabro_llm::lithos_catalog::Catalog;
use fabro_llm::{ModelSelectionError, catalog, selection};
use fabro_types::{AgentBackend, AgentProfileKind};
use lithos_llm::catalog::ProviderId;
use crate::error::Error;
pub(crate) fn select_run_backend(node: &Node) -> Result<AgentBackend, Error> {
match node.agent_backend() {
None => Ok(AgentBackend::Api),
Some(Ok(backend)) => Ok(backend),
Some(Err(_)) => Err(unsupported_backend_error(
node.backend().unwrap_or_default(),
)),
}
}
pub(crate) fn select_one_shot_backend(node: &Node) -> Result<AgentBackend, Error> {
match node.agent_backend() {
Some(Ok(AgentBackend::Acp)) => Err(Error::Validation(
"backend=\"acp\" is only valid on agent nodes; prompt nodes are API-only".to_string(),
)),
Some(Ok(AgentBackend::Api)) | None => Ok(AgentBackend::Api),
Some(Err(_)) => Err(unsupported_backend_error(
node.backend().unwrap_or_default(),
)),
}
}
pub(crate) fn node_needs_api_backend(node: &Node) -> bool {
if !graph::is_llm_handler_type(node.handler_type()) {
return false;
}
match node.handler_type() {
Some("prompt") => true,
_ => matches!(select_run_backend(node), Ok(AgentBackend::Api)),
}
}
#[derive(Clone)]
pub(crate) struct ProviderContext {
pub(crate) provider_id: ProviderId,
pub(crate) profile_kind: AgentProfileKind,
}
pub(crate) fn resolve_provider_context(
catalog: &Catalog,
default_provider_id: &ProviderId,
model: &str,
provider_attr: Option<&str>,
) -> Result<ProviderContext, Error> {
let provider_id = if let Some(provider) = provider_attr {
catalog
.enabled_provider(provider)
.map(|found| found.id().clone())
.ok_or_else(|| {
Error::Precondition(format!("Provider \"{provider}\" is not configured"))
})?
} else if catalog
.enabled_provider(default_provider_id.as_str())
.and_then(|provider| provider.offering(model))
.is_some()
{
// The run's selected provider is a pin whenever it offers the model.
default_provider_id.clone()
} else {
match selection::select(
catalog,
model,
None,
&catalog.enabled_provider_ids().into_iter().collect(),
) {
Ok(entry) => entry.provider.id().clone(),
Err(ModelSelectionError::UnknownSelector { .. }) => default_provider_id.clone(),
Err(error) => return Err(error.into()),
}
};
let provider_id = catalog
.enabled_provider(provider_id.as_str())
.map(|provider| provider.id().clone())
.ok_or_else(|| {
Error::Precondition(format!("Provider \"{provider_id}\" is not configured"))
})?;
let profile_kind = catalog::agent_profile(catalog, provider_id.as_str(), Some(model))
.expect("validated provider should resolve an agent profile");
Ok(ProviderContext {
provider_id,
profile_kind,
})
}
pub(crate) fn resolve_node_provider_context(
catalog: &Catalog,
default_provider_id: &ProviderId,
default_model: &str,
node: &Node,
) -> Result<ProviderContext, Error> {
let model = node.model().unwrap_or(default_model);
resolve_provider_context(catalog, default_provider_id, model, node.provider())
}
fn unsupported_backend_error(raw: &str) -> Error {
Error::Validation(format!(
"unsupported agent backend \"{raw}\"; expected one of: {}",
AgentBackend::expected_values()
))
}

1032
lib/components/fabro-workflow/src/handler/manager_loop.rs
View file

File diff suppressed because it is too large Load diff

417
lib/components/fabro-workflow/src/handler/mod.rs
View file

@ -1,417 +0,0 @@
pub mod agent;
pub mod command;
pub mod conditional;
pub mod exit;
pub mod fan_in;
pub mod human;
pub mod llm;
pub mod manager_loop;
pub mod parallel;
pub mod prompt;
pub mod start;
pub mod structured_output;
pub mod wait;
use std::any::Any;
use std::collections::HashMap;
use std::path::Path;
use std::sync::Arc;
use async_trait::async_trait;
use fabro_graphviz::graph::{Graph, Node, shape_to_handler_type};
use fabro_interview::Interviewer;
use crate::context::Context;
use crate::error::Error;
use crate::event::Emitter;
use crate::outcome::{Outcome, OutcomeExt};
pub use crate::services::{EngineServices, RunServices};
/// The handler interface for node execution.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum NodeTimeoutPolicy {
/// The workflow executor wraps the whole handler future in the node
/// timeout.
ExecutorEnforced,
/// The handler consumes the node timeout and is responsible for surfacing
/// timeout-specific outcome and events.
HandlerManaged,
}
#[async_trait]
pub trait Handler: Send + Sync {
async fn execute(
&self,
node: &Node,
context: &Context,
graph: &Graph,
run_dir: &Path,
services: &EngineServices,
) -> Result<Outcome, Error>;
/// Produce a simulated result for dry-run mode.
/// Override for handlers that need custom context updates.
async fn simulate(
&self,
node: &Node,
_context: &Context,
_graph: &Graph,
_run_dir: &Path,
_services: &EngineServices,
) -> Result<Outcome, Error> {
Ok(Outcome::simulated(&node.id))
}
/// Determines whether an error should be retried.
/// Default implementation retries transient errors only.
fn should_retry(&self, err: &Error) -> bool {
err.is_retryable()
}
fn node_timeout_policy(&self, _node: &Node) -> NodeTimeoutPolicy {
NodeTimeoutPolicy::ExecutorEnforced
}
async fn shutdown(&self, _emitter: &Arc<Emitter>) {}
}
/// Extract a human-readable message from a panic payload.
pub(crate) fn format_panic_message(payload: &Box<dyn Any + Send>) -> String {
if let Some(s) = payload.downcast_ref::<&str>() {
format!("handler panicked: {s}")
} else if let Some(s) = payload.downcast_ref::<String>() {
format!("handler panicked: {s}")
} else {
"handler panicked".to_string()
}
}
/// Route to [`Handler::simulate`] when `services.dry_run` is true, otherwise
/// [`Handler::execute`].
pub async fn dispatch_handler(
handler: &dyn Handler,
node: &Node,
context: &Context,
graph: &Graph,
run_dir: &Path,
services: &EngineServices,
) -> Result<Outcome, Error> {
if services.dry_run {
handler
.simulate(node, context, graph, run_dir, services)
.await
} else {
handler
.execute(node, context, graph, run_dir, services)
.await
}
}
/// Maps handler type strings to handler implementations.
pub struct HandlerRegistry {
handlers: HashMap<String, Box<dyn Handler>>,
default_handler: Box<dyn Handler>,
}
impl HandlerRegistry {
#[must_use]
pub fn new(default_handler: Box<dyn Handler>) -> Self {
Self {
handlers: HashMap::new(),
default_handler,
}
}
/// Register a handler for a given type string.
pub fn register(&mut self, type_string: impl Into<String>, handler: Box<dyn Handler>) {
self.handlers.insert(type_string.into(), handler);
}
/// Resolve which handler should execute for a given node.
/// Priority: explicit type -> shape-based -> default.
#[must_use]
pub fn resolve(&self, node: &Node) -> &dyn Handler {
// 1. Explicit type attribute
if let Some(node_type) = node.node_type() {
if let Some(handler) = self.handlers.get(node_type) {
return handler.as_ref();
}
}
// 2. Shape-based resolution
if let Some(handler_type) = shape_to_handler_type(node.shape()) {
if let Some(handler) = self.handlers.get(handler_type) {
return handler.as_ref();
}
}
// 3. Default
self.default_handler.as_ref()
}
pub async fn shutdown_all(&self, emitter: &Arc<Emitter>) {
self.default_handler.shutdown(emitter).await;
for handler in self.handlers.values() {
handler.shutdown(emitter).await;
}
}
}
/// Build a [`HandlerRegistry`] with all built-in handler types registered.
///
/// The `make_backend` closure is called for each handler that needs a backend
/// (default, `"agent"`, `"prompt"`, and `"parallel.fan_in"`).
#[must_use]
pub fn default_registry(
interviewer: Arc<dyn Interviewer>,
make_backend: impl Fn() -> Option<Box<dyn agent::CodergenBackend>>,
) -> HandlerRegistry {
let mut registry = HandlerRegistry::new(Box::new(agent::AgentHandler::new(make_backend())));
registry.register("start", Box::new(start::StartHandler));
registry.register("exit", Box::new(exit::ExitHandler));
registry.register("agent", Box::new(agent::AgentHandler::new(make_backend())));
registry.register(
"prompt",
Box::new(prompt::PromptHandler::new(make_backend())),
);
registry.register("conditional", Box::new(conditional::ConditionalHandler));
registry.register("human", Box::new(human::HumanHandler::new(interviewer)));
registry.register("command", Box::new(command::CommandHandler));
registry.register("tool", Box::new(command::CommandHandler));
registry.register("parallel", Box::new(parallel::ParallelHandler));
registry.register(
"parallel.fan_in",
Box::new(fan_in::FanInHandler::new(make_backend())),
);
registry.register(
"stack.manager_loop",
Box::new(manager_loop::SubWorkflowHandler),
);
registry.register("wait", Box::new(wait::WaitHandler));
registry
}
#[cfg(test)]
mod tests {
use fabro_graphviz::graph::AttrValue;
use fabro_interview::AutoApproveInterviewer;
use super::*;
use crate::handler::agent::CodergenBackend;
struct TestHandler {
_name: String,
}
#[async_trait]
impl Handler for TestHandler {
async fn execute(
&self,
_node: &Node,
_context: &Context,
_graph: &Graph,
_run_dir: &Path,
_services: &EngineServices,
) -> Result<Outcome, Error> {
Ok(Outcome::success())
}
}
#[test]
fn resolve_by_explicit_type() {
let mut registry = HandlerRegistry::new(Box::new(TestHandler {
_name: "default".to_string(),
}));
registry.register(
"human",
Box::new(TestHandler {
_name: "human".to_string(),
}),
);
let mut node = Node::new("gate");
node.attrs
.insert("type".to_string(), AttrValue::String("human".to_string()));
let handler = registry.resolve(&node);
// We can verify it returns the right handler by checking it doesn't panic
// and returns a valid reference
let _ = handler;
}
#[test]
fn resolve_by_shape() {
let mut registry = HandlerRegistry::new(Box::new(TestHandler {
_name: "default".to_string(),
}));
registry.register(
"start",
Box::new(TestHandler {
_name: "start".to_string(),
}),
);
let mut node = Node::new("entry");
node.attrs.insert(
"shape".to_string(),
AttrValue::String("Mdiamond".to_string()),
);
let handler = registry.resolve(&node);
let _ = handler;
}
#[test]
fn resolve_falls_back_to_default() {
let registry = HandlerRegistry::new(Box::new(TestHandler {
_name: "default".to_string(),
}));
let node = Node::new("work");
let handler = registry.resolve(&node);
let _ = handler;
}
#[test]
fn default_should_retry_uses_is_retryable() {
let handler = TestHandler {
_name: "test".to_string(),
};
assert!(handler.should_retry(&Error::handler("timeout".to_string())));
assert!(!handler.should_retry(&Error::Parse("bad".to_string())));
}
#[test]
fn timeout_policy_defaults_to_executor_enforced() {
let handler = TestHandler {
_name: "test".to_string(),
};
let node = Node::new("work");
assert_eq!(
handler.node_timeout_policy(&node),
NodeTimeoutPolicy::ExecutorEnforced
);
}
#[test]
fn built_in_handlers_that_consume_node_timeout_manage_it_themselves() {
let node = Node::new("work");
let human = human::HumanHandler::new(Arc::new(AutoApproveInterviewer::engine()));
let acp = llm::AgentAcpBackend::new();
assert_eq!(
human.node_timeout_policy(&node),
NodeTimeoutPolicy::HandlerManaged
);
assert_eq!(
command::CommandHandler.node_timeout_policy(&node),
NodeTimeoutPolicy::HandlerManaged
);
assert_eq!(
acp.node_timeout_policy(&node),
NodeTimeoutPolicy::HandlerManaged
);
}
#[test]
fn agent_handler_delegates_timeout_policy_to_backend() {
let node = Node::new("work");
let handler = agent::AgentHandler::new(Some(Box::new(llm::AgentAcpBackend::new())));
assert_eq!(
handler.node_timeout_policy(&node),
NodeTimeoutPolicy::HandlerManaged
);
}
struct NeverRetryHandler;
#[async_trait]
impl Handler for NeverRetryHandler {
async fn execute(
&self,
_node: &Node,
_context: &Context,
_graph: &Graph,
_run_dir: &Path,
_services: &EngineServices,
) -> Result<Outcome, Error> {
Ok(Outcome::success())
}
fn should_retry(&self, _err: &Error) -> bool {
false
}
}
#[test]
fn custom_should_retry_override() {
let handler = NeverRetryHandler;
assert!(!handler.should_retry(&Error::handler("timeout".to_string())));
assert!(!handler.should_retry(&Error::Io("connection reset".to_string())));
}
#[test]
fn register_replaces_existing() {
let mut registry = HandlerRegistry::new(Box::new(TestHandler {
_name: "default".to_string(),
}));
registry.register(
"start",
Box::new(TestHandler {
_name: "first".to_string(),
}),
);
registry.register(
"start",
Box::new(TestHandler {
_name: "second".to_string(),
}),
);
// Should not panic
let mut node = Node::new("s");
node.attrs.insert(
"shape".to_string(),
AttrValue::String("Mdiamond".to_string()),
);
let handler = registry.resolve(&node);
let _ = handler;
}
#[tokio::test]
async fn dispatch_handler_routes_to_simulate_when_dry_run() {
let handler = TestHandler {
_name: "test".to_string(),
};
let node = Node::new("my_node");
let context = Context::new();
let graph = Graph::new("test");
let run_dir = std::path::Path::new("/tmp/test");
let mut services = EngineServices::test_default();
services.dry_run = true;
let outcome = dispatch_handler(&handler, &node, &context, &graph, run_dir, &services)
.await
.unwrap();
assert_eq!(outcome.status, crate::outcome::StageOutcome::Succeeded);
assert_eq!(outcome.notes.as_deref(), Some("[Simulated] my_node"));
}
#[tokio::test]
async fn dispatch_handler_routes_to_execute_when_not_dry_run() {
let handler = TestHandler {
_name: "test".to_string(),
};
let node = Node::new("my_node");
let context = Context::new();
let graph = Graph::new("test");
let run_dir = std::path::Path::new("/tmp/test");
let mut services = EngineServices::test_default();
services.dry_run = false;
let outcome = dispatch_handler(&handler, &node, &context, &graph, run_dir, &services)
.await
.unwrap();
assert_eq!(outcome.status, crate::outcome::StageOutcome::Succeeded);
// execute() returns success with no notes
assert!(outcome.notes.is_none());
}
}

2496
lib/components/fabro-workflow/src/handler/parallel.rs
View file

File diff suppressed because it is too large Load diff

854
lib/components/fabro-workflow/src/handler/prompt.rs
View file

@ -1,854 +0,0 @@
use std::path::Path;
use std::sync::Arc;
use async_trait::async_trait;
use fabro_graphviz::graph::{Graph, Node};
use fabro_types::{StageModelUsage, StageTiming};
use super::agent::{
CodergenBackend, CodergenResult, OneShotRequest, emit_stage_prompt, extract_status_fields,
truncate,
};
use super::llm::routing;
use super::{EngineServices, Handler, structured_output};
use crate::agent_memory;
use crate::context::{Context, WorkflowContext, keys};
use crate::error::Error;
use crate::event::{Emitter, Event};
use crate::outcome::Outcome;
/// Handler for single-shot LLM calls (no tools, no agent loop).
pub struct PromptHandler {
backend: Option<Box<dyn CodergenBackend>>,
}
impl PromptHandler {
#[must_use]
pub fn new(backend: Option<Box<dyn CodergenBackend>>) -> Self {
Self { backend }
}
}
#[async_trait]
impl Handler for PromptHandler {
async fn shutdown(&self, emitter: &Arc<Emitter>) {
if let Some(backend) = self.backend.as_ref() {
backend.shutdown(emitter).await;
}
}
async fn simulate(
&self,
node: &Node,
_context: &Context,
_graph: &Graph,
_run_dir: &Path,
_services: &EngineServices,
) -> Result<Outcome, Error> {
Ok(super::agent::simulate_llm_handler(node))
}
async fn execute(
&self,
node: &Node,
context: &Context,
_graph: &Graph,
_run_dir: &Path,
services: &EngineServices,
) -> Result<Outcome, Error> {
// 1. Build prompt (prepend fidelity preamble if present)
let raw_prompt = node.prompt_or_label();
let preamble = context.preamble();
let prompt = if preamble.is_empty() {
raw_prompt.to_string()
} else {
format!("{preamble}\n\n{raw_prompt}")
};
// 1b. Discover project docs for system prompt when project_memory is enabled
let system_prompt = if node.project_memory() {
let profile_kind = routing::resolve_node_provider_context(
services.run.catalog.as_ref(),
&services.run.provider_id,
&services.run.model,
node,
)?
.profile_kind;
agent_memory::load_memory_text(
&services.run.sandbox,
profile_kind,
&services.run.cancel_token(),
)
.await?
} else {
None
};
let stage_scope = emit_stage_prompt(
services,
context,
node,
&prompt,
StageModelUsage::MODE_PROMPT,
self.backend.as_deref(),
)?;
// 3. Call LLM backend (one_shot)
let (response_text, stage_usage, backend_files_touched, timing) =
if let Some(backend) = &self.backend {
let result = backend
.one_shot(OneShotRequest {
node,
prompt: &prompt,
system_prompt: system_prompt.as_deref(),
emitter: &services.run.emitter,
stage_scope: &stage_scope,
sandbox: &services.run.sandbox,
cancel_token: services.run.cancel_token(),
})
.await;
match result {
Ok(CodergenResult::Full(outcome)) => return Ok(*outcome),
Ok(CodergenResult::Text {
text,
usage,
files_touched,
timing,
..
}) => (text, usage, files_touched, timing),
Err(Error::Cancelled) => return Err(Error::Cancelled),
Err(e) if e.is_retryable() => {
return Err(e);
}
Err(e) => {
return Ok(e.to_fail_outcome());
}
}
} else {
(
format!("[Simulated] Response for stage: {}", node.id),
None,
Vec::new(),
StageTiming::default(),
)
};
let response_model = stage_usage
.as_ref()
.map(|usage| usage.model_id().to_string())
.or_else(|| node.model().map(String::from))
.unwrap_or_default();
let response_provider = node
.provider()
.map(String::from)
.or_else(|| Some(services.run.provider_id.to_string()))
.unwrap_or_default();
services.run.emitter.emit_scoped(
&Event::PromptCompleted {
node_id: node.id.clone(),
response: response_text.clone(),
model: response_model,
provider: response_provider,
usage: stage_usage.clone(),
},
&stage_scope,
);
// 4. Build and write status
let mut outcome = Outcome::success();
outcome.notes = Some(format!("Stage completed: {}", node.id));
outcome
.context_updates
.insert(keys::LAST_STAGE.to_string(), serde_json::json!(node.id));
outcome.context_updates.insert(
keys::LAST_RESPONSE.to_string(),
serde_json::json!(truncate(&response_text, 200)),
);
outcome.context_updates.insert(
keys::response_key(&node.id),
serde_json::json!(&response_text),
);
if let Some(schema) = structured_output::parse_node_output_schema(node)? {
if let Ok(validated) =
structured_output::validate_response_text(&schema, &response_text)
{
structured_output::apply_validated_output(node, &schema, &validated, &mut outcome);
} else {
let mut failed =
structured_output::exhausted_failure_outcome(node.output_retries());
failed.timing = Some(timing);
failed.usage = stage_usage;
failed.files_touched = backend_files_touched;
return Ok(failed);
}
} else {
extract_status_fields(&response_text, &mut outcome);
}
outcome.usage = stage_usage;
outcome.files_touched = backend_files_touched;
outcome.timing = Some(timing);
Ok(outcome)
}
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use std::time::Duration;
use fabro_graphviz::graph::AttrValue;
use fabro_store::{Database, RunDatabase, StageId};
use fabro_types::{PetriAdmission, fixtures, test_support};
use lithos_llm::catalog::ProviderId;
use lithos_llm::types::{ReasoningEffort, Speed};
use object_store::memory::InMemory;
use tempfile::TempDir;
use super::*;
use crate::event::Emitter;
use crate::handler::agent::CodergenRunRequest;
use crate::outcome::OutcomeExt;
fn make_services() -> EngineServices {
EngineServices::test_default()
}
fn test_store() -> Arc<Database> {
Arc::new(fabro_store::test_support::test_database(
Arc::new(InMemory::new()),
"",
Duration::from_millis(1),
None,
))
}
async fn make_services_with_run_store() -> (
EngineServices,
RunDatabase,
crate::event::StoreProgressLogger,
) {
let store = test_store();
let run_store = store.create_run(&fixtures::RUN_1).await.unwrap();
seed_created(&run_store).await;
let mut services = EngineServices::test_default();
services.run = services
.run
.with_emitter(Arc::new(Emitter::new(fixtures::RUN_1)))
.with_run_store(run_store.clone().into());
let logger = crate::event::StoreProgressLogger::new(run_store.clone());
logger.register(services.run.emitter.as_ref());
(services, run_store, logger)
}
async fn seed_created(run_store: &RunDatabase) {
crate::event::append_event(
run_store,
&fixtures::RUN_1,
&crate::event::Event::RunCreated {
run_id: fixtures::RUN_1,
title: None,
settings: serde_json::to_value(fabro_types::WorkflowSettings::default())
.unwrap(),
graph: serde_json::to_value(fabro_types::Graph::new("test")).unwrap(),
workflow_source: None,
labels: std::collections::BTreeMap::default(),
source_directory: None,
workflow_slug: None,
workflow_version_id: None,
target: None,
automation: None,
provenance: test_support::test_run_provenance(),
spec_blob: None,
git: None,
fork_source_ref: None,
retried_from: None,
parent_id: None,
web_url: None,
admission: PetriAdmission::default(),
},
)
.await
.unwrap();
}
#[tokio::test]
async fn prompt_handler_simulate() {
let handler = PromptHandler::new(None);
let node = Node::new("classify");
let context = Context::new();
let graph = Graph::new("test");
let tmp = TempDir::new().unwrap();
let outcome = handler
.simulate(&node, &context, &graph, tmp.path(), &make_services())
.await
.unwrap();
assert_eq!(outcome.status, crate::outcome::StageOutcome::Succeeded);
assert_eq!(outcome.notes.as_deref(), Some("[Simulated] classify"));
assert_eq!(
outcome
.context_updates
.get(crate::context::keys::LAST_STAGE),
Some(&serde_json::json!("classify"))
);
assert!(
outcome
.context_updates
.contains_key(crate::context::keys::LAST_RESPONSE)
);
assert_eq!(
outcome
.context_updates
.get(&crate::context::keys::response_key("classify")),
Some(&serde_json::json!(
"[Simulated] Response for stage: classify"
))
);
}
#[tokio::test]
async fn prompt_handler_dispatches_to_backend_one_shot() {
struct OneShotBackend;
#[async_trait]
impl CodergenBackend for OneShotBackend {
async fn run(&self, _request: CodergenRunRequest<'_>) -> Result<CodergenResult, Error> {
panic!("run() should not be called for prompt handler");
}
async fn one_shot(
&self,
_request: OneShotRequest<'_>,
) -> Result<CodergenResult, Error> {
Ok(CodergenResult::Text {
text: "one-shot response".to_string(),
usage_by_model: Vec::new(),
usage: None,
files_touched: Vec::new(),
last_file_touched: None,
timing: StageTiming::default(),
})
}
fn effective_request_controls(
&self,
_node: &Node,
) -> Result<crate::handler::llm::EffectiveRequestControls, Error> {
Ok(crate::handler::llm::EffectiveRequestControls {
reasoning_effort: Some(ReasoningEffort::High),
speed: Some(Speed::Fast),
})
}
}
let handler = PromptHandler::new(Some(Box::new(OneShotBackend)));
let mut node = Node::new("classify");
node.attrs.insert(
"prompt".to_string(),
AttrValue::String("Classify this".to_string()),
);
let context = Context::new();
let graph = Graph::new("test");
let tmp = TempDir::new().unwrap();
let outcome = handler
.execute(&node, &context, &graph, tmp.path(), &make_services())
.await
.unwrap();
assert_eq!(outcome.status, crate::outcome::StageOutcome::Succeeded);
assert_eq!(
outcome
.context_updates
.get(&crate::context::keys::response_key("classify")),
Some(&serde_json::json!("one-shot response"))
);
}
#[tokio::test]
async fn prompt_handler_copies_backend_timing_to_outcome() {
struct TimingBackend;
#[async_trait]
impl CodergenBackend for TimingBackend {
async fn run(&self, _request: CodergenRunRequest<'_>) -> Result<CodergenResult, Error> {
panic!("run() should not be called for prompt handler");
}
async fn one_shot(
&self,
_request: OneShotRequest<'_>,
) -> Result<CodergenResult, Error> {
Ok(CodergenResult::Text {
text: "one-shot response".to_string(),
usage_by_model: Vec::new(),
usage: None,
files_touched: Vec::new(),
last_file_touched: None,
timing: StageTiming::new(0, 200, 300),
})
}
}
let handler = PromptHandler::new(Some(Box::new(TimingBackend)));
let node = Node::new("classify");
let context = Context::new();
let graph = Graph::new("test");
let tmp = TempDir::new().unwrap();
let outcome = handler
.execute(&node, &context, &graph, tmp.path(), &make_services())
.await
.unwrap();
assert_eq!(outcome.timing, Some(StageTiming::new(0, 200, 300)));
}
#[tokio::test]
async fn prompt_handler_custom_output_schema_updates_output_context_key() {
struct CustomOutputBackend;
#[async_trait]
impl CodergenBackend for CustomOutputBackend {
async fn run(&self, _request: CodergenRunRequest<'_>) -> Result<CodergenResult, Error> {
panic!("run() should not be called for prompt handler");
}
async fn one_shot(
&self,
_request: OneShotRequest<'_>,
) -> Result<CodergenResult, Error> {
Ok(CodergenResult::Text {
text: r#"{"passed": true}"#.to_string(),
usage_by_model: Vec::new(),
usage: None,
files_touched: Vec::new(),
last_file_touched: None,
timing: StageTiming::default(),
})
}
}
let handler = PromptHandler::new(Some(Box::new(CustomOutputBackend)));
let mut node = Node::new("audit");
node.attrs.insert(
"output_schema".to_string(),
AttrValue::String(
r#"{"type":"object","required":["passed"],"properties":{"passed":{"type":"boolean"}}}"#
.to_string(),
),
);
let context = Context::new();
let graph = Graph::new("test");
let tmp = TempDir::new().unwrap();
let outcome = handler
.execute(&node, &context, &graph, tmp.path(), &make_services())
.await
.unwrap();
assert_eq!(
outcome.context_updates.get("output.audit"),
Some(&serde_json::json!({"passed": true})),
);
}
#[tokio::test]
async fn prompt_handler_routing_output_schema_requires_valid_routing_json() {
struct BadRoutingBackend;
#[async_trait]
impl CodergenBackend for BadRoutingBackend {
async fn run(&self, _request: CodergenRunRequest<'_>) -> Result<CodergenResult, Error> {
panic!("run() should not be called for prompt handler");
}
async fn one_shot(
&self,
_request: OneShotRequest<'_>,
) -> Result<CodergenResult, Error> {
Ok(CodergenResult::Text {
text: r#"{"outcome": 123}"#.to_string(),
usage_by_model: Vec::new(),
usage: None,
files_touched: Vec::new(),
last_file_touched: None,
timing: StageTiming::default(),
})
}
}
let handler = PromptHandler::new(Some(Box::new(BadRoutingBackend)));
let mut node = Node::new("route");
node.attrs.insert(
"output_schema".to_string(),
AttrValue::String("routing".to_string()),
);
node.attrs
.insert("output_retries".to_string(), AttrValue::Integer(0));
let context = Context::new();
let graph = Graph::new("test");
let tmp = TempDir::new().unwrap();
let outcome = handler
.execute(&node, &context, &graph, tmp.path(), &make_services())
.await
.unwrap();
assert_eq!(outcome.status, crate::outcome::StageOutcome::Failed {
retry_requested: false,
});
assert_eq!(
outcome.failure_reason(),
Some("output schema validation failed after 0 repair attempt(s)")
);
}
#[tokio::test]
async fn prompt_handler_projects_provider_used_from_prompt_events() {
struct ProviderOneShotBackend;
#[async_trait]
impl CodergenBackend for ProviderOneShotBackend {
async fn run(&self, _request: CodergenRunRequest<'_>) -> Result<CodergenResult, Error> {
panic!("run() should not be called for prompt handler");
}
async fn one_shot(
&self,
_request: OneShotRequest<'_>,
) -> Result<CodergenResult, Error> {
Ok(CodergenResult::Text {
text: "one-shot response".to_string(),
usage_by_model: Vec::new(),
usage: None,
files_touched: Vec::new(),
last_file_touched: None,
timing: StageTiming::default(),
})
}
fn effective_request_controls(
&self,
_node: &Node,
) -> Result<crate::handler::llm::EffectiveRequestControls, Error> {
Ok(crate::handler::llm::EffectiveRequestControls {
reasoning_effort: Some(ReasoningEffort::High),
speed: Some(Speed::Fast),
})
}
}
let handler = PromptHandler::new(Some(Box::new(ProviderOneShotBackend)));
let mut node = Node::new("classify");
node.attrs.insert(
"prompt".to_string(),
AttrValue::String("Classify this".to_string()),
);
let context = Context::new();
let graph = Graph::new("test");
let tmp = TempDir::new().unwrap();
let (services, run_store, logger) = make_services_with_run_store().await;
handler
.execute(&node, &context, &graph, tmp.path(), &services)
.await
.unwrap();
logger.flush().await.unwrap();
let state = run_store.state().await.unwrap();
let node_state = state.stage(&StageId::new("classify", 1)).unwrap();
let provider_used = node_state.provider_used.as_ref().unwrap();
assert_eq!(provider_used.mode, StageModelUsage::MODE_PROMPT);
assert_eq!(provider_used.reasoning_effort, Some(ReasoningEffort::High));
assert_eq!(provider_used.speed, Some(Speed::Fast));
}
struct OneShotCapturingBackend {
captured_prompt: Arc<std::sync::Mutex<Option<String>>>,
captured_system_prompt: Arc<std::sync::Mutex<Option<Option<String>>>>,
}
#[async_trait]
impl CodergenBackend for OneShotCapturingBackend {
async fn run(&self, _request: CodergenRunRequest<'_>) -> Result<CodergenResult, Error> {
panic!("run() should not be called for prompt handler");
}
async fn one_shot(&self, request: OneShotRequest<'_>) -> Result<CodergenResult, Error> {
*self.captured_prompt.lock().unwrap() = Some(request.prompt.to_string());
*self.captured_system_prompt.lock().unwrap() =
Some(request.system_prompt.map(String::from));
Ok(CodergenResult::Text {
text: "classified".to_string(),
usage_by_model: Vec::new(),
usage: None,
files_touched: Vec::new(),
last_file_touched: None,
timing: StageTiming::default(),
})
}
}
#[tokio::test]
async fn prompt_handler_prepends_preamble() {
use std::sync::Mutex;
let captured = Arc::new(Mutex::new(None));
let backend = OneShotCapturingBackend {
captured_prompt: captured.clone(),
captured_system_prompt: Arc::new(Mutex::new(None)),
};
let handler = PromptHandler::new(Some(Box::new(backend)));
let mut node = Node::new("classify");
node.attrs.insert(
"prompt".to_string(),
AttrValue::String("Classify this".to_string()),
);
let context = Context::new();
context.set(
keys::CURRENT_PREAMBLE,
serde_json::json!("Prior output here"),
);
let graph = Graph::new("test");
let tmp = TempDir::new().unwrap();
handler
.execute(&node, &context, &graph, tmp.path(), &make_services())
.await
.unwrap();
let prompt = captured.lock().unwrap().clone().unwrap();
assert!(
prompt.starts_with("Prior output here"),
"one_shot prompt should start with preamble, got: {prompt}"
);
assert!(prompt.ends_with("Classify this"));
}
#[tokio::test]
async fn prompt_handler_passes_system_prompt_when_project_memory_enabled() {
use std::sync::Mutex;
let captured_sys = Arc::new(Mutex::new(None));
let backend = OneShotCapturingBackend {
captured_prompt: Arc::new(Mutex::new(None)),
captured_system_prompt: captured_sys.clone(),
};
let handler = PromptHandler::new(Some(Box::new(backend)));
// project_memory defaults to true; sandbox working_directory points to cwd
// which likely has no AGENTS.md/CLAUDE.md, so system_prompt should be None
let mut node = Node::new("classify");
node.attrs.insert(
"prompt".to_string(),
AttrValue::String("Classify this".to_string()),
);
let context = Context::new();
let graph = Graph::new("test");
let tmp = TempDir::new().unwrap();
handler
.execute(&node, &context, &graph, tmp.path(), &make_services())
.await
.unwrap();
// With project_memory=true (default), one_shot is called (system_prompt
// captured)
let sys = captured_sys.lock().unwrap().clone();
assert!(sys.is_some(), "one_shot should have been called");
}
#[tokio::test]
async fn prompt_handler_project_memory_uses_model_agent_profile_override() {
use std::sync::Mutex;
let captured_sys = Arc::new(Mutex::new(None));
let backend = OneShotCapturingBackend {
captured_prompt: Arc::new(Mutex::new(None)),
captured_system_prompt: captured_sys.clone(),
};
let handler = PromptHandler::new(Some(Box::new(backend)));
let workspace = TempDir::new().unwrap();
tokio::fs::write(workspace.path().join("CLAUDE.md"), "anthropic memory")
.await
.unwrap();
let catalog = Arc::new(fabro_llm::test_support::test_catalog_with_overlay(
r#"
[providers.acme]
display_name = "Acme"
adapter = "openai-compatible"
codec = "openai-chat"
base_url = "https://api.acme.test/v1"
auth = { type = "bearer" }
default_model = "acme-claude"
[providers.acme.metadata.agent]
profile = "openai"
[providers.acme.models.acme-claude]
display_name = "Acme Claude"
aliases = ["ac"]
api_model = "acme-claude"
limits = { context_tokens = 1000, max_output_tokens = 500 }
capabilities = { text = true, tools = true }
family = "claude"
[providers.acme.models.acme-claude.metadata.agent]
profile = "anthropic"
"#,
));
let mut services = make_services();
services.run = services
.run
.with_sandbox(Arc::new(
fabro_sandbox::local_sandbox(workspace.path().to_path_buf())
.await
.unwrap(),
))
.with_catalog_context(
Arc::clone(&catalog),
ProviderId::new("acme"),
"acme-claude".to_string(),
);
let mut node = Node::new("classify");
node.attrs.insert(
"prompt".to_string(),
AttrValue::String("Classify this".to_string()),
);
node.attrs
.insert("model".to_string(), AttrValue::String("ac".to_string()));
let context = Context::new();
let graph = Graph::new("test");
handler
.execute(&node, &context, &graph, workspace.path(), &services)
.await
.unwrap();
let sys = captured_sys.lock().unwrap().clone();
assert!(
sys.flatten()
.is_some_and(|system_prompt| system_prompt.contains("anthropic memory")),
"project memory should use model-level Anthropic profile and read CLAUDE.md"
);
}
#[tokio::test]
async fn prompt_handler_project_memory_uses_default_model_profile_for_provider_attr() {
use std::sync::Mutex;
let captured_sys = Arc::new(Mutex::new(None));
let backend = OneShotCapturingBackend {
captured_prompt: Arc::new(Mutex::new(None)),
captured_system_prompt: captured_sys.clone(),
};
let handler = PromptHandler::new(Some(Box::new(backend)));
let workspace = TempDir::new().unwrap();
tokio::fs::write(workspace.path().join("CLAUDE.md"), "anthropic memory")
.await
.unwrap();
let catalog = Arc::new(fabro_llm::test_support::test_catalog_with_overlay(
r#"
[providers.acme]
display_name = "Acme"
adapter = "openai-compatible"
codec = "openai-chat"
base_url = "https://api.acme.test/v1"
auth = { type = "bearer" }
default_model = "acme-claude"
[providers.acme.metadata.agent]
profile = "openai"
[providers.acme.models.acme-claude]
display_name = "Acme Claude"
aliases = ["ac"]
api_model = "acme-claude"
limits = { context_tokens = 1000, max_output_tokens = 500 }
capabilities = { text = true, tools = true }
family = "claude"
[providers.acme.models.acme-claude.metadata.agent]
profile = "anthropic"
"#,
));
let mut services = make_services();
services.run = services
.run
.with_sandbox(Arc::new(
fabro_sandbox::local_sandbox(workspace.path().to_path_buf())
.await
.unwrap(),
))
.with_catalog_context(
Arc::clone(&catalog),
ProviderId::new("acme"),
"acme-claude".to_string(),
);
let mut node = Node::new("classify");
node.attrs.insert(
"prompt".to_string(),
AttrValue::String("Classify this".to_string()),
);
node.attrs.insert(
"provider".to_string(),
AttrValue::String("acme".to_string()),
);
let context = Context::new();
let graph = Graph::new("test");
handler
.execute(&node, &context, &graph, workspace.path(), &services)
.await
.unwrap();
let sys = captured_sys.lock().unwrap().clone();
assert!(
sys.flatten()
.is_some_and(|system_prompt| system_prompt.contains("anthropic memory")),
"project memory should use the default model's Anthropic profile when only the matching provider is set"
);
}
#[tokio::test]
async fn prompt_handler_passes_none_system_prompt_when_project_memory_false() {
use std::sync::Mutex;
let captured_sys = Arc::new(Mutex::new(None));
let backend = OneShotCapturingBackend {
captured_prompt: Arc::new(Mutex::new(None)),
captured_system_prompt: captured_sys.clone(),
};
let handler = PromptHandler::new(Some(Box::new(backend)));
let mut node = Node::new("classify");
node.attrs.insert(
"prompt".to_string(),
AttrValue::String("Classify this".to_string()),
);
node.attrs
.insert("project_memory".to_string(), AttrValue::Boolean(false));
let context = Context::new();
let graph = Graph::new("test");
let tmp = TempDir::new().unwrap();
handler
.execute(&node, &context, &graph, tmp.path(), &make_services())
.await
.unwrap();
let sys = captured_sys.lock().unwrap().clone();
assert_eq!(
sys,
Some(None),
"system_prompt should be None when project_memory=false"
);
}
}

48
lib/components/fabro-workflow/src/handler/start.rs
View file

@ -1,48 +0,0 @@
use std::path::Path;
use async_trait::async_trait;
use fabro_graphviz::graph::{Graph, Node};
use super::{EngineServices, Handler};
use crate::context::Context;
use crate::error::Error;
use crate::outcome::Outcome;
/// No-op handler for pipeline entry point. Returns SUCCESS immediately.
pub struct StartHandler;
#[async_trait]
impl Handler for StartHandler {
async fn execute(
&self,
_node: &Node,
_context: &Context,
_graph: &Graph,
_run_dir: &Path,
_services: &EngineServices,
) -> Result<Outcome, Error> {
Ok(Outcome::success())
}
}
#[cfg(test)]
mod tests {
use super::*;
fn make_services() -> EngineServices {
EngineServices::test_default()
}
#[tokio::test]
async fn start_handler_returns_success() {
let handler = StartHandler;
let node = Node::new("start");
let context = Context::new();
let graph = Graph::new("test");
let run_dir = Path::new("/tmp/test");
let outcome = handler
.execute(&node, &context, &graph, run_dir, &make_services())
.await
.unwrap();
assert_eq!(outcome.status, crate::outcome::StageOutcome::Succeeded);
}
}

1143
lib/components/fabro-workflow/src/handler/structured_output.rs
View file

File diff suppressed because it is too large Load diff

79
lib/components/fabro-workflow/src/handler/wait.rs
View file

@ -1,79 +0,0 @@
use std::path::Path;
use async_trait::async_trait;
use fabro_graphviz::graph::{AttrValue, Graph, Node};
use tokio::time::sleep;
use super::{EngineServices, Handler};
use crate::context::Context;
use crate::error::Error;
use crate::outcome::Outcome;
/// Sleeps for a configured duration before proceeding.
pub struct WaitHandler;
#[async_trait]
impl Handler for WaitHandler {
async fn execute(
&self,
node: &Node,
_context: &Context,
_graph: &Graph,
_run_dir: &Path,
_services: &EngineServices,
) -> Result<Outcome, Error> {
let duration = node
.attrs
.get("duration")
.and_then(AttrValue::as_duration)
.ok_or_else(|| {
Error::Validation(format!(
"wait node {:?} is missing a valid `duration` attribute",
node.id
))
})?;
sleep(duration).await;
Ok(Outcome::success())
}
}
#[cfg(test)]
mod tests {
use std::time::Duration;
use super::*;
fn make_services() -> EngineServices {
EngineServices::test_default()
}
#[tokio::test]
async fn wait_timer_success_with_short_duration() {
let handler = WaitHandler;
let mut node = Node::new("wait60");
node.attrs.insert(
"duration".to_string(),
AttrValue::Duration(Duration::from_millis(1)),
);
let context = Context::new();
let graph = Graph::new("test");
let run_dir = Path::new("/tmp/test");
let outcome = handler
.execute(&node, &context, &graph, run_dir, &make_services())
.await
.unwrap();
assert_eq!(outcome.status, crate::outcome::StageOutcome::Succeeded);
}
#[tokio::test]
async fn wait_timer_errors_without_duration() {
let handler = WaitHandler;
let node = Node::new("wait_no_dur");
let context = Context::new();
let graph = Graph::new("test");
let run_dir = Path::new("/tmp/test");
let result = handler
.execute(&node, &context, &graph, run_dir, &make_services())
.await;
assert!(result.is_err());
}
}

36
lib/components/fabro-workflow/src/hook_context.rs
View file

@ -1,36 +0,0 @@
use fabro_graphviz::graph::types::Node as GvNode;
use fabro_hooks::HookContext;
/// Populate node-related fields on a `HookContext` from a graph node.
pub(crate) fn set_hook_node(ctx: &mut HookContext, node: &GvNode) {
ctx.node_id = Some(node.id.clone());
ctx.node_label = Some(node.label().to_string());
ctx.handler_type = node.handler_type().map(String::from);
}
#[cfg(test)]
mod tests {
use fabro_graphviz::graph::{AttrValue, Node};
use fabro_hooks::HookEvent;
use fabro_types::fixtures;
use super::*;
#[test]
fn set_hook_node_populates_hook_context_fields() {
let mut node = Node::new("approve");
node.attrs.insert(
"label".to_string(),
AttrValue::String("Approve PR".to_string()),
);
node.attrs
.insert("type".to_string(), AttrValue::String("human".to_string()));
let mut ctx = HookContext::new(HookEvent::StageStart, fixtures::RUN_1, "graph".into());
set_hook_node(&mut ctx, &node);
assert_eq!(ctx.node_id.as_deref(), Some("approve"));
assert_eq!(ctx.node_label.as_deref(), Some("Approve PR"));
assert_eq!(ctx.handler_type.as_deref(), Some("human"));
}
}

772
lib/components/fabro-workflow/src/interview_runtime.rs
View file

@ -1,772 +0,0 @@
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
use std::time::Instant;
use async_trait::async_trait;
use fabro_interview::{Answer, AnswerSubmission, AnswerValue, Interviewer, Question};
use fabro_types::{
BlockedReason, InterviewOption, Principal, QuestionType, StageId, SystemActorKind,
};
use futures::future;
use pebble_coding_agent::extensions::{
Answer as AgentAnswer, AnswerStatus, HumanInputError, HumanInputProvider,
Question as AgentQuestion, QuestionKind,
};
use tokio::sync::watch;
use tokio_util::sync::CancellationToken;
use ulid::Ulid;
use crate::event::{Emitter, Event, StageScope};
use crate::millis_u64;
/// Unresolved interviews per stage. A stage is present only while it has at
/// least one, so the run is blocked exactly when the map is non-empty.
#[derive(Debug, Default)]
pub(crate) struct InterviewBlockState {
blocked_stages: HashMap<StageId, usize>,
}
impl InterviewBlockState {
pub(crate) fn is_run_blocked(&self) -> bool {
!self.blocked_stages.is_empty()
}
pub(crate) fn is_stage_blocked(&self, stage_id: &StageId) -> bool {
self.blocked_stages.contains_key(stage_id)
}
fn block(&mut self, stage_id: StageId) {
*self.blocked_stages.entry(stage_id).or_default() += 1;
}
/// `RunInterviewGuard` resolves at most once, so an unknown stage here
/// means the state is already clear. Runs from `Drop`, so it must not
/// panic.
fn resolve(&mut self, stage_id: &StageId) {
let Some(count) = self.blocked_stages.get_mut(stage_id) else {
return;
};
*count = count.saturating_sub(1);
if *count == 0 {
self.blocked_stages.remove(stage_id);
}
}
}
/// Run-scoped state for unresolved human input. Emits `run.blocked` on the
/// first unresolved human/agent interview and `run.unblocked` after the last
/// one resolves. Subscribers use the same state to suspend run and stage
/// timeout budgets without deriving runtime control from persisted events.
///
/// Both transitions publish the new state before emitting the event, so a
/// listener that reads `subscribe()` from an event callback always sees state
/// that agrees with the event it just received.
pub(crate) struct RunInterviewBlocker {
state: watch::Sender<InterviewBlockState>,
/// Serializes state change plus event emission so concurrent guards cannot
/// interleave into an out-of-order `run.blocked` / `run.unblocked` pair.
transitions: Mutex<()>,
}
impl RunInterviewBlocker {
#[must_use]
pub(crate) fn new() -> Self {
let (state, _) = watch::channel(InterviewBlockState::default());
Self {
state,
transitions: Mutex::new(()),
}
}
pub(crate) fn subscribe(&self) -> watch::Receiver<InterviewBlockState> {
self.state.subscribe()
}
pub(crate) fn block(
self: &Arc<Self>,
emitter: Arc<Emitter>,
stage_id: StageId,
) -> RunInterviewGuard {
let _transition = self
.transitions
.lock()
.expect("interview transition mutex should not be poisoned");
let mut newly_blocked = false;
self.state.send_modify(|state| {
newly_blocked = !state.is_run_blocked();
state.block(stage_id.clone());
});
if newly_blocked {
emitter.emit(&Event::RunBlocked {
blocked_reason: BlockedReason::HumanInputRequired,
});
}
RunInterviewGuard {
blocker: Arc::clone(self),
emitter,
stage_id,
resolved: false,
}
}
fn resolved(&self, emitter: &Emitter, stage_id: &StageId) {
let _transition = self
.transitions
.lock()
.expect("interview transition mutex should not be poisoned");
let mut fully_unblocked = false;
self.state.send_modify(|state| {
state.resolve(stage_id);
fully_unblocked = !state.is_run_blocked();
});
if fully_unblocked {
emitter.emit(&Event::RunUnblocked);
}
}
}
pub(crate) struct RunInterviewGuard {
blocker: Arc<RunInterviewBlocker>,
emitter: Arc<Emitter>,
stage_id: StageId,
resolved: bool,
}
impl RunInterviewGuard {
pub(crate) fn resolve(mut self) {
self.resolve_in_place();
}
fn resolve_in_place(&mut self) {
if !self.resolved {
self.blocker.resolved(self.emitter.as_ref(), &self.stage_id);
self.resolved = true;
}
}
}
impl Drop for RunInterviewGuard {
fn drop(&mut self) {
self.resolve_in_place();
}
}
/// Pebble's human-input provider for a workflow stage: the `ask_user`
/// tool's questions go to the run's interviewer and are recorded as
/// interview events, blocking the run's timeout budgets while they wait.
pub(crate) struct WorkflowHumanInput {
interviewer: Arc<dyn Interviewer>,
emitter: Arc<Emitter>,
stage_scope: StageScope,
/// Graph node id, reported as the `stage` on interview events. Distinct
/// from `stage_scope.stage_id()`, which is the visit-qualified `StageId`
/// used to key block state.
node_id: String,
blocker: Arc<RunInterviewBlocker>,
}
impl WorkflowHumanInput {
#[must_use]
pub(crate) fn new(
interviewer: Arc<dyn Interviewer>,
emitter: Arc<Emitter>,
stage_scope: StageScope,
node_id: impl Into<String>,
blocker: Arc<RunInterviewBlocker>,
) -> Self {
Self {
interviewer,
emitter,
stage_scope,
node_id: node_id.into(),
blocker,
}
}
}
struct PreparedQuestion {
agent_question: AgentQuestion,
question: Question,
}
struct PendingAgentQuestionBatch {
emitter: Arc<Emitter>,
stage_scope: StageScope,
node_id: String,
questions: Vec<(String, String)>,
started_at: Instant,
guard: Option<RunInterviewGuard>,
}
impl PendingAgentQuestionBatch {
fn new(
emitter: Arc<Emitter>,
stage_scope: StageScope,
node_id: String,
prepared: &[PreparedQuestion],
guard: RunInterviewGuard,
started_at: Instant,
) -> Self {
Self {
emitter,
stage_scope,
node_id,
questions: prepared
.iter()
.map(|prepared_question| {
(
prepared_question.question.id.clone(),
prepared_question.question.text.clone(),
)
})
.collect(),
started_at,
guard: Some(guard),
}
}
fn resolve(mut self) {
if let Some(guard) = self.guard.take() {
guard.resolve();
}
}
}
impl Drop for PendingAgentQuestionBatch {
fn drop(&mut self) {
if self.guard.is_none() {
return;
}
let duration_ms = millis_u64(self.started_at.elapsed());
for (question_id, question) in &self.questions {
self.emitter.emit_scoped(
&Event::InterviewInterrupted {
actor: Some(Principal::System {
system_kind: SystemActorKind::Engine,
}),
question_id: question_id.clone(),
question: question.clone(),
stage: self.node_id.clone(),
reason: "interrupted".to_string(),
duration_ms,
},
&self.stage_scope,
);
}
if let Some(guard) = self.guard.take() {
guard.resolve();
}
}
}
#[async_trait]
impl HumanInputProvider for WorkflowHumanInput {
async fn ask_questions(
&self,
tool_call_id: &str,
questions: Vec<AgentQuestion>,
cancel_token: CancellationToken,
) -> Result<Vec<AgentAnswer>, HumanInputError> {
if questions.is_empty() {
return Ok(Vec::new());
}
let prepared = questions
.into_iter()
.enumerate()
.map(|(index, question)| self.prepare_question(tool_call_id, index, question))
.collect::<Vec<_>>();
for prepared_question in &prepared {
let question = &prepared_question.question;
self.emitter.emit_scoped(
&Event::InterviewStarted {
question_id: question.id.clone(),
question: question.text.clone(),
stage: self.node_id.clone(),
question_type: question.question_type.to_string(),
options: question.options.clone(),
allow_freeform: question.allow_freeform,
timeout_seconds: None,
context_display: question.context_display.clone(),
review_target: question.review_target.clone(),
},
&self.stage_scope,
);
}
let interview_start = Instant::now();
let cleanup = PendingAgentQuestionBatch::new(
Arc::clone(&self.emitter),
self.stage_scope.clone(),
self.node_id.clone(),
&prepared,
self.blocker
.block(Arc::clone(&self.emitter), self.stage_scope.stage_id()),
interview_start,
);
let ask_all = future::join_all(
prepared
.iter()
.map(|prepared_question| self.interviewer.ask(prepared_question.question.clone())),
);
tokio::pin!(ask_all);
let answers = tokio::select! {
submissions = &mut ask_all => Some(submissions),
() = cancel_token.cancelled() => None,
};
let results = match answers {
Some(submissions) => prepared
.iter()
.zip(submissions)
.map(|(prepared_question, submission)| {
self.emit_submission_event(
prepared_question,
&submission,
millis_u64(interview_start.elapsed()),
);
answer_from_submission(&prepared_question.agent_question, &submission)
})
.collect::<Vec<_>>(),
None => prepared
.iter()
.map(|prepared_question| {
self.emit_interrupted(
prepared_question,
Some(Principal::System {
system_kind: SystemActorKind::Engine,
}),
"interrupted",
millis_u64(interview_start.elapsed()),
);
AgentAnswer::unanswered(
&prepared_question.agent_question,
AnswerStatus::Interrupted,
)
})
.collect::<Vec<_>>(),
};
cleanup.resolve();
Ok(results)
}
}
impl WorkflowHumanInput {
fn prepare_question(
&self,
tool_call_id: &str,
index: usize,
agent_question: AgentQuestion,
) -> PreparedQuestion {
let question_type = match agent_question.kind {
QuestionKind::MultiSelect => QuestionType::MultiSelect,
// Pebble may add kinds; anything else is one choice from a list.
QuestionKind::MultipleChoice | _ => QuestionType::MultipleChoice,
};
let mut question = Question::new(agent_question.text.clone(), question_type);
question.id = internal_question_id(&self.stage_scope, tool_call_id, index);
question.options = agent_question
.options
.iter()
.map(|option| InterviewOption {
key: option.key.clone(),
label: option.label.clone(),
description: option.description.clone(),
preview: option.preview.clone(),
})
.collect();
question.allow_freeform = agent_question.allow_freeform;
question.stage.clone_from(&self.node_id);
question.metadata.insert(
"agent.tool_call_id".to_string(),
serde_json::json!(tool_call_id),
);
question.metadata.insert(
"agent.original_question".to_string(),
serde_json::json!(agent_question.original_question),
);
if let Some(original_id) = &agent_question.original_id {
question.metadata.insert(
"agent.original_id".to_string(),
serde_json::json!(original_id),
);
}
if let Some(header) = &agent_question.header {
question
.metadata
.insert("agent.header".to_string(), serde_json::json!(header));
}
PreparedQuestion {
agent_question,
question,
}
}
fn emit_submission_event(
&self,
prepared: &PreparedQuestion,
submission: &AnswerSubmission,
duration_ms: u64,
) {
match submission.answer.value {
AnswerValue::Timeout => self.emitter.emit_scoped(
&Event::InterviewTimeout {
actor: Some(Principal::System {
system_kind: SystemActorKind::Timeout,
}),
question_id: prepared.question.id.clone(),
question: prepared.question.text.clone(),
stage: self.node_id.clone(),
duration_ms,
},
&self.stage_scope,
),
AnswerValue::Interrupted => self.emit_interrupted(
prepared,
Some(submission.actor.clone()),
"interrupted",
duration_ms,
),
AnswerValue::Cancelled => self.emit_interrupted(
prepared,
Some(submission.actor.clone()),
"cancelled",
duration_ms,
),
_ => self.emitter.emit_scoped(
&Event::InterviewCompleted {
actor: Some(submission.actor.clone()),
question_id: prepared.question.id.clone(),
question: prepared.question.text.clone(),
answer: answer_labels(&prepared.question.options, &submission.answer)
.join(", "),
duration_ms,
},
&self.stage_scope,
),
}
}
fn emit_interrupted(
&self,
prepared: &PreparedQuestion,
actor: Option<Principal>,
reason: &str,
duration_ms: u64,
) {
self.emitter.emit_scoped(
&Event::InterviewInterrupted {
actor,
question_id: prepared.question.id.clone(),
question: prepared.question.text.clone(),
stage: self.node_id.clone(),
reason: reason.to_string(),
duration_ms,
},
&self.stage_scope,
);
}
}
fn answer_from_submission(
agent_question: &AgentQuestion,
submission: &AnswerSubmission,
) -> AgentAnswer {
let status = match &submission.answer.value {
AnswerValue::Cancelled => Some(AnswerStatus::Cancelled),
AnswerValue::Interrupted => Some(AnswerStatus::Interrupted),
AnswerValue::Skipped => Some(AnswerStatus::Skipped),
AnswerValue::Timeout => Some(AnswerStatus::Timeout),
_ => None,
};
match status {
Some(status) => AgentAnswer::unanswered(agent_question, status),
None => AgentAnswer::answered(
agent_question,
answer_labels(&interview_options(agent_question), &submission.answer),
),
}
}
fn interview_options(agent_question: &AgentQuestion) -> Vec<InterviewOption> {
agent_question
.options
.iter()
.map(|option| InterviewOption {
key: option.key.clone(),
label: option.label.clone(),
description: option.description.clone(),
preview: option.preview.clone(),
})
.collect()
}
fn answer_labels(options: &[InterviewOption], answer: &Answer) -> Vec<String> {
match &answer.value {
AnswerValue::Selected(key) => vec![label_for_key(options, key)],
AnswerValue::MultiSelected(keys) => {
keys.iter().map(|key| label_for_key(options, key)).collect()
}
AnswerValue::Text(text) => vec![text.clone()],
AnswerValue::Yes => vec!["yes".to_string()],
AnswerValue::No => vec!["no".to_string()],
AnswerValue::Cancelled => vec!["cancelled".to_string()],
AnswerValue::Interrupted => vec!["interrupted".to_string()],
AnswerValue::Skipped => vec!["skipped".to_string()],
AnswerValue::Timeout => vec!["timeout".to_string()],
}
}
fn label_for_key(options: &[InterviewOption], key: &str) -> String {
options
.iter()
.find(|option| option.key == key)
.map_or_else(|| key.to_string(), |option| option.label.clone())
}
fn internal_question_id(scope: &StageScope, tool_call_id: &str, index: usize) -> String {
format!(
"agentq-{}-v{}-{}-{}-{}",
slug(&scope.node_id),
scope.visit,
slug(tool_call_id),
index + 1,
Ulid::new(),
)
}
fn slug(value: &str) -> String {
let mut out = value
.chars()
.filter_map(|ch| {
if ch.is_ascii_alphanumeric() {
Some(ch.to_ascii_lowercase())
} else if matches!(ch, '-' | '_') {
Some(ch)
} else {
None
}
})
.take(48)
.collect::<String>();
if out.is_empty() {
out.push('x');
}
out
}
#[cfg(test)]
mod tests {
use fabro_interview::ControlInterviewer;
use fabro_types::{EventBody, RunId};
use pebble_coding_agent::extensions::QuestionOption;
use super::*;
#[test]
fn answer_labels_return_user_facing_labels_in_submission_order() {
let options = vec![
InterviewOption {
key: "a".to_string(),
label: "Alpha".to_string(),
..InterviewOption::default()
},
InterviewOption {
key: "b".to_string(),
label: "Beta".to_string(),
..InterviewOption::default()
},
];
let answer = Answer::multi_selected(vec!["b".to_string(), "a".to_string()]);
assert_eq!(answer_labels(&options, &answer), vec!["Beta", "Alpha"]);
}
#[test]
fn internal_question_id_includes_stage_visit_and_tool_call_context() {
let scope = StageScope {
node_id: "Review Changes".to_string(),
visit: 3,
parallel_group_id: None,
parallel_branch_id: None,
};
let id = internal_question_id(&scope, "call_123", 1);
assert!(id.starts_with("agentq-reviewchanges-v3-call_123-2-"));
let ulid = id
.rsplit('-')
.next()
.expect("question id should include a ULID suffix");
assert_eq!(ulid.len(), 26);
}
#[tokio::test]
async fn batch_questions_are_all_started_before_run_is_blocked_and_return_labels() {
let interviewer = Arc::new(ControlInterviewer::new());
let emitter = Arc::new(Emitter::new(RunId::new()));
let events = Arc::new(std::sync::Mutex::new(Vec::new()));
emitter.on_event({
let events = Arc::clone(&events);
move |event| events.lock().unwrap().push(event.clone())
});
let stage_scope = StageScope {
node_id: "ask".to_string(),
visit: 1,
parallel_group_id: None,
parallel_branch_id: None,
};
let stage_id = stage_scope.stage_id();
let blocker = Arc::new(RunInterviewBlocker::new());
let block_state = blocker.subscribe();
let runtime = WorkflowHumanInput::new(
interviewer.clone(),
Arc::clone(&emitter),
stage_scope,
"ask",
blocker,
);
let option = QuestionOption {
key: "ship".to_string(),
label: "Ship it".to_string(),
description: Some("Deploy".to_string()),
preview: Some("preview".to_string()),
};
let ask = tokio::spawn(async move {
runtime
.ask_questions(
"call_1",
vec![
AgentQuestion {
original_id: Some("q1".to_string()),
original_question: "First?".to_string(),
header: None,
text: "First?".to_string(),
kind: QuestionKind::MultipleChoice,
options: vec![option.clone()],
allow_freeform: true,
},
AgentQuestion {
original_id: Some("q2".to_string()),
original_question: "Second?".to_string(),
header: None,
text: "Second?".to_string(),
kind: QuestionKind::MultipleChoice,
options: vec![option.clone()],
allow_freeform: true,
},
],
CancellationToken::new(),
)
.await
.unwrap()
});
tokio::task::yield_now().await;
assert!(block_state.borrow().is_run_blocked());
assert!(block_state.borrow().is_stage_blocked(&stage_id));
let question_ids = {
let events = events.lock().unwrap();
assert!(matches!(events[0].body, EventBody::InterviewStarted(_)));
assert!(matches!(events[1].body, EventBody::InterviewStarted(_)));
assert!(matches!(events[2].body, EventBody::RunBlocked(_)));
events
.iter()
.filter_map(|event| match &event.body {
EventBody::InterviewStarted(props) => Some(props.question_id.clone()),
_ => None,
})
.collect::<Vec<_>>()
};
for question_id in question_ids {
let option = InterviewOption {
key: "ship".to_string(),
label: "Ship it".to_string(),
..InterviewOption::default()
};
interviewer
.submit(
&question_id,
AnswerSubmission::system(
Answer::selected("ship", option),
SystemActorKind::Engine,
),
)
.await
.unwrap();
}
let answers = ask.await.unwrap();
assert_eq!(answers.len(), 2);
assert_eq!(answers[0].answers, vec!["Ship it"]);
assert_eq!(answers[1].answers, vec!["Ship it"]);
assert!(
events
.lock()
.unwrap()
.iter()
.any(|event| matches!(event.body, EventBody::RunUnblocked(_)))
);
assert!(!block_state.borrow().is_run_blocked());
assert!(!block_state.borrow().is_stage_blocked(&stage_id));
}
#[tokio::test]
async fn cancelling_agent_question_unblocks_its_stage() {
let interviewer = Arc::new(ControlInterviewer::new());
let emitter = Arc::new(Emitter::new(RunId::new()));
let stage_scope = StageScope {
node_id: "ask".to_string(),
visit: 1,
parallel_group_id: None,
parallel_branch_id: None,
};
let stage_id = stage_scope.stage_id();
let blocker = Arc::new(RunInterviewBlocker::new());
let block_state = blocker.subscribe();
let runtime = WorkflowHumanInput::new(
interviewer,
emitter,
stage_scope,
"ask",
Arc::clone(&blocker),
);
let cancel_token = CancellationToken::new();
let ask_cancel_token = cancel_token.clone();
let ask = tokio::spawn(async move {
runtime
.ask_questions(
"call_1",
vec![AgentQuestion {
original_id: Some("q1".to_string()),
original_question: "Continue?".to_string(),
header: None,
text: "Continue?".to_string(),
kind: QuestionKind::MultipleChoice,
options: Vec::new(),
allow_freeform: true,
}],
ask_cancel_token,
)
.await
.unwrap()
});
tokio::task::yield_now().await;
assert!(block_state.borrow().is_run_blocked());
assert!(block_state.borrow().is_stage_blocked(&stage_id));
cancel_token.cancel();
let answers = ask.await.unwrap();
assert_eq!(answers[0].status, AnswerStatus::Interrupted);
assert!(!block_state.borrow().is_run_blocked());
assert!(!block_state.borrow().is_stage_blocked(&stage_id));
}
}

316
lib/components/fabro-workflow/src/lib.rs
View file

@ -1,3 +1,16 @@
//! Fabro's platform half of a workflow run: what Fabro does around the
//! engine.
//!
//! Petri executes every run (`fabro-petri` is the seam). This crate keeps
//! what Fabro itself owns: the create-time compile of the Fabro graph the
//! read side displays (`pipeline`, `transforms`, `operations`), the run
//! records and status vocabulary (`records`, `run_status`), the Git
//! helpers a run's platform effects use (`git`, `git_identity`,
//! `sandbox_git`), pull request creation (`pull_request`), the run tools an
//! agent session calls (`run_tools`, `services`), the built-in web search
//! backend (`web_search`), and, until the legacy event store is deleted,
//! the legacy run event vocabulary (`event`, `runtime_store`).
#![cfg_attr(
test,
allow(
@ -15,331 +28,40 @@
)
)]
use std::collections::HashMap;
use std::sync::Arc;
use fabro_store::EventEnvelope;
use fabro_types::{EventBody, StageId};
/// Callback invoked when a workflow node starts executing.
pub type OnNodeCallback = Option<Arc<dyn Fn(&str) + Send + Sync>>;
/// Convert a Duration's milliseconds to u64, saturating on overflow.
pub(crate) fn millis_u64(d: std::time::Duration) -> u64 {
u64::try_from(d.as_millis()).unwrap_or(u64::MAX)
}
/// Extract the timing breakdown from a `stage.completed` / `stage.failed`
/// event body, or `None` for any other variant.
fn stage_completion_timing(body: &EventBody) -> Option<fabro_types::StageTiming> {
match body {
EventBody::StageCompleted(props) => Some(props.timing),
EventBody::StageFailed(props) => Some(props.timing),
_ => None,
}
}
/// Extract per-stage (node_id, visit) timing from `stage.completed` /
/// `stage.failed` events. Keys on the full [`StageId`] so multi-visit stages
/// (e.g. a looped `verify` node) keep distinct timings.
///
/// This is the canonical primitive; [`total_stage_timing_by_node`] and
/// [`latest_stage_timing_by_node`] are explicit rollups built on top of it.
pub fn extract_stage_timings_by_stage_id(
events: &[EventEnvelope],
) -> HashMap<StageId, fabro_types::StageTiming> {
let mut timings = HashMap::new();
for envelope in events {
let Some(timing) = stage_completion_timing(&envelope.event.body) else {
continue;
};
let Some(stage_id) = envelope.event.stage_id.as_ref() else {
continue;
};
timings.insert(stage_id.clone(), timing);
}
timings
}
/// Sum of timing in each node across every visit. Use for usage
/// where a retried node should count its full time. `wall_time_ms`,
/// `inference_time_ms`, `tool_time_ms`, and `active_time_ms` are all summed
/// per node.
pub fn total_stage_timing_by_node(
events: &[EventEnvelope],
) -> HashMap<String, fabro_types::StageTiming> {
let mut totals: HashMap<String, fabro_types::StageTiming> = HashMap::new();
for (stage_id, timing) in extract_stage_timings_by_stage_id(events) {
let entry = totals.entry(stage_id.node_id().to_string()).or_default();
*entry = entry.saturating_add(&timing);
}
totals
}
/// Timing of each node's most recent visit (the highest visit number). Use
/// for run summaries where the table shows one row per node and "the last
/// attempt" is the right representative.
pub fn latest_stage_timing_by_node(
events: &[EventEnvelope],
) -> HashMap<String, fabro_types::StageTiming> {
let mut entries: Vec<(StageId, fabro_types::StageTiming)> =
extract_stage_timings_by_stage_id(events)
.into_iter()
.collect();
entries.sort_by_key(|(stage_id, _)| stage_id.visit());
let mut latest = HashMap::new();
for (stage_id, timing) in entries {
latest.insert(stage_id.node_id().to_string(), timing);
}
latest
}
#[cfg(test)]
mod duration_tests {
use chrono::{TimeZone, Utc};
use fabro_store::EventEnvelope;
use fabro_types::run_event::{StageCompletedProps, StageFailedProps};
use fabro_types::{EventBody, RunEvent, StageId, StageOutcome, StageTiming, fixtures};
use super::{
extract_stage_timings_by_stage_id, latest_stage_timing_by_node, total_stage_timing_by_node,
};
fn completed_event(seq: u32, node: &str, visit: u32, wall_time_ms: u64) -> EventEnvelope {
let event = RunEvent {
id: format!("evt_{seq}"),
ts: Utc.with_ymd_and_hms(2026, 1, 1, 0, 0, 0).unwrap(),
run_id: fixtures::RUN_1,
node_id: Some(node.to_string()),
node_label: None,
stage_id: Some(StageId::new(node, visit)),
parallel_group_id: None,
parallel_branch_id: None,
session_id: None,
parent_session_id: None,
tool_call_id: None,
actor: None,
body: EventBody::StageCompleted(StageCompletedProps {
index: 0,
timing: StageTiming::wall_only(wall_time_ms),
status: StageOutcome::Succeeded,
preferred_label: None,
suggested_next_ids: vec![],
usage_by_model: Vec::new(),
usage: None,
failure: None,
notes: None,
files_touched: vec![],
context_updates: None,
jump_to_node: None,
context_values: None,
node_visits: None,
loop_failure_signatures: None,
restart_failure_signatures: None,
response: None,
attempt: 1,
max_attempts: 1,
}),
};
EventEnvelope { seq, event }
}
fn failed_event(seq: u32, node: &str, visit: u32, wall_time_ms: u64) -> EventEnvelope {
let event = RunEvent {
id: format!("evt_{seq}"),
ts: Utc.with_ymd_and_hms(2026, 1, 1, 0, 0, 0).unwrap(),
run_id: fixtures::RUN_1,
node_id: Some(node.to_string()),
node_label: None,
stage_id: Some(StageId::new(node, visit)),
parallel_group_id: None,
parallel_branch_id: None,
session_id: None,
parent_session_id: None,
tool_call_id: None,
actor: None,
body: EventBody::StageFailed(StageFailedProps {
index: 0,
failure: None,
will_retry: true,
timing: StageTiming::wall_only(wall_time_ms),
usage_by_model: Vec::new(),
usage: None,
}),
};
EventEnvelope { seq, event }
}
#[test]
fn extract_keys_timings_by_full_stage_id() {
let events = vec![
completed_event(1, "verify", 1, 100),
completed_event(2, "verify", 2, 200),
];
let timings = extract_stage_timings_by_stage_id(&events);
assert_eq!(
timings
.get(&StageId::new("verify", 1))
.map(|t| t.wall_time_ms),
Some(100)
);
assert_eq!(
timings
.get(&StageId::new("verify", 2))
.map(|t| t.wall_time_ms),
Some(200)
);
}
#[test]
fn total_sums_wall_time_across_visits_per_node() {
let events = vec![
completed_event(1, "verify", 1, 100),
completed_event(2, "verify", 2, 200),
completed_event(3, "build", 1, 50),
];
let totals = total_stage_timing_by_node(&events);
assert_eq!(totals.get("verify").map(|t| t.wall_time_ms), Some(300));
assert_eq!(totals.get("build").map(|t| t.wall_time_ms), Some(50));
}
#[test]
fn latest_picks_highest_visit_regardless_of_input_order() {
// Visit 2 appears in the events vector before visit 1; the result
// must still reflect visit 2's timing (the latest visit).
let events = vec![
completed_event(1, "verify", 2, 999),
completed_event(2, "verify", 1, 100),
];
let latest = latest_stage_timing_by_node(&events);
assert_eq!(latest.get("verify").map(|t| t.wall_time_ms), Some(999));
}
#[test]
fn stage_failed_timings_are_included() {
let events = vec![failed_event(1, "verify", 1, 75)];
let timings = extract_stage_timings_by_stage_id(&events);
assert_eq!(
timings
.get(&StageId::new("verify", 1))
.map(|t| t.wall_time_ms),
Some(75)
);
}
#[test]
fn total_sums_active_breakdown_across_visits() {
// Same node visited twice with different inference/tool breakdowns:
// the rollup must add inference, tool, and active fields, not just
// wall time. This guards against accidentally summing wall only.
fn timed_completed(seq: u32, visit: u32, timing: StageTiming) -> EventEnvelope {
let event = RunEvent {
id: format!("evt_{seq}"),
ts: Utc.with_ymd_and_hms(2026, 1, 1, 0, 0, 0).unwrap(),
run_id: fixtures::RUN_1,
node_id: Some("agent".to_string()),
node_label: None,
stage_id: Some(StageId::new("agent", visit)),
parallel_group_id: None,
parallel_branch_id: None,
session_id: None,
parent_session_id: None,
tool_call_id: None,
actor: None,
body: EventBody::StageCompleted(StageCompletedProps {
index: 0,
timing,
status: StageOutcome::Succeeded,
preferred_label: None,
suggested_next_ids: vec![],
usage_by_model: Vec::new(),
usage: None,
failure: None,
notes: None,
files_touched: vec![],
context_updates: None,
jump_to_node: None,
context_values: None,
node_visits: None,
loop_failure_signatures: None,
restart_failure_signatures: None,
response: None,
attempt: 1,
max_attempts: 1,
}),
};
EventEnvelope { seq, event }
}
let events = vec![
timed_completed(1, 1, StageTiming::new(1000, 600, 300)),
timed_completed(2, 2, StageTiming::new(700, 400, 200)),
];
let totals = total_stage_timing_by_node(&events);
let agent = totals.get("agent").copied().unwrap();
assert_eq!(agent.wall_time_ms, 1700);
assert_eq!(agent.inference_time_ms, 1000);
assert_eq!(agent.tool_time_ms, 500);
assert_eq!(agent.active_time_ms, 1500);
}
}
#[doc(hidden)]
pub mod agent_memory;
pub mod artifact;
pub mod artifact_snapshot;
pub mod artifact_upload;
pub mod command_log;
pub(crate) mod condition;
pub mod context;
pub mod error;
pub mod event;
pub mod file_resolver;
pub mod git;
pub(crate) mod git_bridge;
pub mod git_identity;
pub(crate) mod graph;
pub mod handler;
mod hook_context;
mod interview_runtime;
#[allow(
dead_code,
reason = "The lifecycle module remains crate-visible for tests and pending integrations."
)]
pub(crate) mod lifecycle;
pub mod model_fallback;
pub(crate) mod node_handler;
pub mod operations;
pub mod outcome;
pub mod pipeline;
pub mod pull_request;
pub mod records;
mod retry;
pub mod run_control;
pub(crate) mod run_dir;
pub mod run_lookup;
pub mod usage_rollup;
pub use error::{Error, FailureCategory, FailureSignature, FailureSignatureExt, Result};
pub use fabro_types::ManifestPath;
pub use steering_hub::{PairControlError, SteeringHub};
pub use usage_rollup::{
ProjectionUsageByModel, ProjectionUsageRollup, ProjectionUsageStage,
usage_rollup_from_projection,
};
pub mod run_materialization;
pub mod run_options;
pub mod run_status;
pub mod run_tools;
pub mod runtime_store;
pub mod sandbox_git;
pub(crate) mod sandbox_git_runtime;
pub mod services;
pub(crate) mod stage_execution;
mod stage_scope;
pub mod steering_hub;
#[cfg(any(test, feature = "test-support"))]
pub mod test_support;
#[doc(hidden)]
pub mod transforms;
pub mod web_search;
pub mod workflow_bundle;
/// Convert a Duration's milliseconds to u64, saturating on overflow.
pub(crate) fn millis_u64(d: std::time::Duration) -> u64 {
u64::try_from(d.as_millis()).unwrap_or(u64::MAX)
}

359
lib/components/fabro-workflow/src/lifecycle/artifact.rs
View file

@ -1,359 +0,0 @@
use std::collections::HashSet;
use std::sync::Arc;
use std::time::Duration;
use anyhow::{Context as _, Result, anyhow};
use async_trait::async_trait;
use fabro_core::error::{Error as CoreError, Result as CoreResult};
use fabro_core::graph::NodeSpec;
use fabro_core::lifecycle::{AttemptResultContext, RunLifecycle};
use fabro_core::outcome::NodeResult;
use fabro_core::state::ExecutionState;
use fabro_store::{ArtifactKey, ArtifactStore};
use fabro_types::{ArtifactUpload, EventBody, RunId, StageId};
use fabro_util::error::collect_chain;
use fabro_util::workspace_glob::{WorkspaceGlobError, WorkspaceGlobSet};
use tokio::fs;
use tokio::sync::OnceCell;
use tokio::time::sleep;
use crate::artifact::{normalize_durable_updates, offload_large_values, sync_artifacts_to_env};
use crate::artifact_snapshot::{ArtifactCollectionSummary, collect_artifacts};
use crate::artifact_upload::ArtifactSink;
use crate::event::{Emitter, Event, RunNoticeCode, RunNoticeLevel};
use crate::graph::{WorkflowGraph, WorkflowNode};
use crate::lifecycle::event::stage_scope_for;
use crate::outcome::ModelUsage;
use crate::runtime_store::RunStoreHandle;
use crate::stage_execution::StageExecutionTracker;
type WfRunState = ExecutionState<Option<ModelUsage>>;
type WfNodeResult = NodeResult<Option<ModelUsage>>;
type ArtifactIdentity = (String, String);
const ARTIFACT_UPLOAD_RETRY_DELAYS: [Duration; 3] = [
Duration::from_millis(100),
Duration::from_millis(250),
Duration::from_millis(500),
];
/// Sub-lifecycle responsible for artifact collection, offloading, and syncing.
pub(crate) struct ArtifactLifecycle {
pub sandbox: Arc<fabro_sandbox::RunSandbox>,
pub run_store: RunStoreHandle,
pub emitter: Arc<Emitter>,
pub run_id: RunId,
artifact_globs: std::result::Result<WorkspaceGlobSet, WorkspaceGlobError>,
pub artifact_sink: Option<ArtifactSink>,
captured_artifacts: std::sync::Mutex<HashSet<ArtifactIdentity>>,
ledger_initialized: OnceCell<()>,
/// Run-scoped stage execution allocator shared with `RunServices`.
stage_executions: StageExecutionTracker,
}
impl ArtifactLifecycle {
pub(crate) fn new(
sandbox: Arc<fabro_sandbox::RunSandbox>,
run_store: RunStoreHandle,
emitter: Arc<Emitter>,
run_id: RunId,
artifact_globs: &[String],
artifact_sink: Option<ArtifactSink>,
stage_executions: StageExecutionTracker,
) -> Self {
Self {
sandbox,
run_store,
emitter,
run_id,
artifact_globs: WorkspaceGlobSet::try_new(artifact_globs),
artifact_sink,
captured_artifacts: std::sync::Mutex::new(HashSet::new()),
ledger_initialized: OnceCell::new(),
stage_executions,
}
}
fn artifact_globs(&self) -> CoreResult<&WorkspaceGlobSet> {
self.artifact_globs.as_ref().map_err(|error| {
CoreError::Other(format!("invalid run.artifacts.include pattern: {error}"))
})
}
}
#[async_trait]
impl RunLifecycle<WorkflowGraph> for ArtifactLifecycle {
async fn on_run_start(&self, _graph: &WorkflowGraph, _state: &WfRunState) -> CoreResult<()> {
let artifact_globs = self.artifact_globs()?;
if artifact_globs.is_empty() {
return Ok(());
}
self.ledger_initialized
.get_or_try_init(|| async {
let ledger = self
.rebuild_captured_artifact_ledger()
.await
.map_err(|err| {
let rendered = collect_chain(err.as_ref()).join(": ");
CoreError::Other(format!(
"failed to rebuild captured artifact ledger: {rendered}"
))
})?;
*self.captured_artifacts.lock().expect(
"artifact mutex should not be poisoned: no code panics while holding this lock",
) = ledger;
Ok::<(), CoreError>(())
})
.await?;
Ok(())
}
async fn after_attempt(
&self,
ctx: &AttemptResultContext<'_, WorkflowGraph>,
state: &WfRunState,
) -> CoreResult<()> {
let artifact_globs = self.artifact_globs()?;
if artifact_globs.is_empty() {
return Ok(());
}
let node_id = ctx.node.id();
// Artifact identity follows the stage execution ordinal so a resumed
// reexecution stores its captures under the new `StageId`.
let scope = stage_scope_for(&self.stage_executions, state, node_id);
let visit = scope.visit;
let node_slug = if visit <= 1 {
node_id.to_string()
} else {
format!("{node_id}-visit_{visit}")
};
let artifact_capture_dir =
tempfile::tempdir().map_err(|err| CoreError::Other(err.to_string()))?;
match collect_artifacts(&self.sandbox, artifact_capture_dir.path(), artifact_globs).await {
Ok(summary) => {
self.emit_collection_problem_notice(node_id, &summary);
let new_assets = self.new_captured_assets(&summary.captured_assets);
if new_assets.is_empty() {
return Ok(());
}
let stage_id = scope.stage_id();
if let Err(err) = self
.persist_artifacts(
&stage_id,
ctx.attempt,
artifact_capture_dir.path(),
&new_assets,
)
.await
{
self.emitter.notice(
RunNoticeLevel::Warn,
RunNoticeCode::ArtifactUploadFailed,
format!("[node: {node_id}] artifact upload failed: {err}"),
);
return Ok(());
}
self.record_captured_assets(&new_assets);
for asset in &new_assets {
self.emitter.emit_scoped(
&Event::ArtifactCaptured {
node_id: node_id.to_string(),
attempt: ctx.attempt,
node_slug: node_slug.clone(),
path: asset.path.clone(),
mime: asset.mime.clone(),
content_md5: asset.content_md5.clone(),
content_sha256: asset.content_sha256.clone(),
bytes: asset.bytes,
},
&scope,
);
}
}
Err(e) => {
self.emitter.notice(
RunNoticeLevel::Warn,
RunNoticeCode::ArtifactCollectionFailed,
format!("[node: {node_id}] artifact collection failed: {e}"),
);
}
}
Ok(())
}
async fn after_node(
&self,
node: &WorkflowNode,
result: &mut WfNodeResult,
_state: &WfRunState,
) -> CoreResult<()> {
let node_id = node.id();
// Offload large context_updates values to artifact store
if let Err(e) =
offload_large_values(&mut result.outcome.context_updates, &self.run_store).await
{
self.emitter.notice(
RunNoticeLevel::Warn,
RunNoticeCode::ArtifactOffloadFailed,
format!("[node: {node_id}] artifact offload failed: {e}"),
);
}
normalize_durable_updates(&mut result.outcome.context_updates);
// Sync file-backed artifacts to sandbox environment
if let Err(e) =
sync_artifacts_to_env(&mut result.outcome.context_updates, &self.sandbox).await
{
self.emitter.notice(
RunNoticeLevel::Warn,
RunNoticeCode::ArtifactSyncFailed,
format!("[node: {node_id}] artifact sync failed: {e}"),
);
}
Ok(())
}
}
impl ArtifactLifecycle {
async fn rebuild_captured_artifact_ledger(&self) -> Result<HashSet<ArtifactIdentity>> {
let events = self
.run_store
.list_events()
.await
.context("failed to list run events")?;
Ok(events
.into_iter()
.filter_map(|envelope| match envelope.event.body {
EventBody::ArtifactCaptured(props) => Some((props.path, props.content_sha256)),
_ => None,
})
.collect())
}
fn emit_collection_problem_notice(&self, node_id: &str, summary: &ArtifactCollectionSummary) {
if summary.download_errors == 0 && summary.hash_errors == 0 {
return;
}
let mut parts = Vec::new();
if summary.download_errors > 0 {
parts.push(format!("{} download error(s)", summary.download_errors));
}
if summary.hash_errors > 0 {
parts.push(format!("{} hash/read error(s)", summary.hash_errors));
}
self.emitter.notice(
RunNoticeLevel::Warn,
RunNoticeCode::ArtifactCollectionFailed,
format!(
"[node: {node_id}] artifact collection completed with {}",
parts.join(", ")
),
);
}
fn new_captured_assets(&self, artifacts: &[ArtifactUpload]) -> Vec<ArtifactUpload> {
let ledger = self.captured_artifacts.lock().expect(
"artifact mutex should not be poisoned: no code panics while holding this lock",
);
artifacts
.iter()
.filter(|artifact| !ledger.contains(&artifact_identity(artifact)))
.cloned()
.collect()
}
fn record_captured_assets(&self, artifacts: &[ArtifactUpload]) {
let mut ledger = self.captured_artifacts.lock().expect(
"artifact mutex should not be poisoned: no code panics while holding this lock",
);
for artifact in artifacts {
ledger.insert(artifact_identity(artifact));
}
}
async fn persist_artifacts(
&self,
stage_id: &StageId,
retry: u32,
artifact_capture_dir: &std::path::Path,
artifacts: &[ArtifactUpload],
) -> Result<()> {
let Some(sink) = self.artifact_sink.as_ref() else {
return Err(anyhow!("artifact sink is not configured"));
};
let mut last_error = None;
for attempt in 0..=ARTIFACT_UPLOAD_RETRY_DELAYS.len() {
match self
.persist_artifacts_once(sink, stage_id, retry, artifact_capture_dir, artifacts)
.await
{
Ok(()) => return Ok(()),
Err(err) => last_error = Some(err),
}
if let Some(delay) = ARTIFACT_UPLOAD_RETRY_DELAYS.get(attempt) {
sleep(*delay).await;
}
}
Err(last_error.unwrap_or_else(|| anyhow!("artifact upload failed")))
}
async fn persist_artifacts_once(
&self,
sink: &ArtifactSink,
stage_id: &StageId,
retry: u32,
artifact_capture_dir: &std::path::Path,
artifacts: &[ArtifactUpload],
) -> Result<()> {
match sink {
ArtifactSink::Store(store) => {
self.store_artifacts(store, stage_id, retry, artifact_capture_dir, artifacts)
.await
}
ArtifactSink::Uploader(uploader) => {
uploader
.upload_stage_artifacts(stage_id, retry, artifact_capture_dir, artifacts)
.await
}
}
}
async fn store_artifacts(
&self,
store: &ArtifactStore,
stage_id: &StageId,
retry: u32,
artifact_capture_dir: &std::path::Path,
artifacts: &[ArtifactUpload],
) -> Result<()> {
for artifact in artifacts {
let local_path = artifact_capture_dir.join(&artifact.path);
let bytes = fs::read(&local_path)
.await
.with_context(|| format!("failed to read artifact {}", local_path.display()))?;
store
.put(
&self.run_id,
&ArtifactKey::new(stage_id.clone(), retry, artifact.path.clone()),
&bytes,
)
.await
.map_err(anyhow::Error::new)?;
}
Ok(())
}
}
fn artifact_identity(artifact: &ArtifactUpload) -> ArtifactIdentity {
(artifact.path.clone(), artifact.content_sha256.clone())
}

160
lib/components/fabro-workflow/src/lifecycle/circuit_breaker.rs
View file

@ -1,160 +0,0 @@
use std::collections::HashMap;
use std::sync::Mutex;
use async_trait::async_trait;
use fabro_core::error::{Error as CoreError, Result as CoreResult};
use fabro_core::lifecycle::{EdgeContext, EdgeDecision, RunLifecycle};
use fabro_core::outcome::NodeResult;
use fabro_core::state::ExecutionState;
use crate::error::{FailureCategory, FailureSignature, FailureSignatureExt};
use crate::graph::{WorkflowGraph, WorkflowNode};
use crate::outcome::{ModelUsage, OutcomeExt};
type WfRunState = ExecutionState<Option<ModelUsage>>;
type WfNodeResult = NodeResult<Option<ModelUsage>>;
/// Sub-lifecycle responsible for tracking failure signatures and tripping the
/// circuit breaker when deterministic failure cycles are detected.
pub(crate) struct CircuitBreakerLifecycle {
loop_failure_signatures: Mutex<HashMap<FailureSignature, usize>>,
restart_failure_signatures: Mutex<HashMap<FailureSignature, usize>>,
loop_restart_signature_limit: usize,
}
impl CircuitBreakerLifecycle {
pub(crate) fn new(loop_restart_signature_limit: usize) -> Self {
Self {
loop_failure_signatures: Mutex::new(HashMap::new()),
restart_failure_signatures: Mutex::new(HashMap::new()),
loop_restart_signature_limit,
}
}
/// Restore circuit breaker state from a checkpoint (for resume).
pub(crate) fn restore(
&self,
loop_sigs: HashMap<FailureSignature, usize>,
restart_sigs: HashMap<FailureSignature, usize>,
) {
*self.loop_failure_signatures.lock().expect(
"circuit breaker mutex should not be poisoned: no code panics while holding this lock",
) = loop_sigs;
*self.restart_failure_signatures.lock().expect(
"circuit breaker mutex should not be poisoned: no code panics while holding this lock",
) = restart_sigs;
}
/// Snapshot current state for checkpoint building.
pub(crate) fn snapshot(
&self,
) -> (
HashMap<FailureSignature, usize>,
HashMap<FailureSignature, usize>,
) {
let loop_sigs = self.loop_failure_signatures.lock()
.expect("circuit breaker mutex should not be poisoned: no code panics while holding this lock")
.clone();
let restart_sigs = self.restart_failure_signatures.lock()
.expect("circuit breaker mutex should not be poisoned: no code panics while holding this lock")
.clone();
(loop_sigs, restart_sigs)
}
}
#[async_trait]
impl RunLifecycle<WorkflowGraph> for CircuitBreakerLifecycle {
async fn after_node(
&self,
node: &WorkflowNode,
result: &mut WfNodeResult,
_state: &WfRunState,
) -> CoreResult<()> {
let gv = node.inner();
let outcome = &result.outcome;
let outcome_failure_category = if outcome.status.is_failure() {
outcome.classified_failure_category()
} else {
None
};
if let Some(fc) = outcome_failure_category {
let sig_hint = outcome
.failure
.as_ref()
.and_then(|f| f.signature.as_deref());
let sig = FailureSignature::new(
&gv.id,
fc,
sig_hint,
outcome.failure.as_ref().map(|f| f.message.as_str()),
);
if fc.is_signature_tracked() {
let mut sigs = self.loop_failure_signatures.lock()
.expect("circuit breaker mutex should not be poisoned: no code panics while holding this lock");
let count = sigs.entry(sig.clone()).or_insert(0);
*count += 1;
let limit = self.loop_restart_signature_limit;
if *count >= limit {
return Err(CoreError::Other(format!(
"deterministic failure cycle detected: signature {sig} repeated {count} times (limit {limit})"
)));
}
}
}
Ok(())
}
async fn on_edge_selected(
&self,
ctx: &EdgeContext<'_, WorkflowGraph>,
_state: &WfRunState,
) -> CoreResult<EdgeDecision> {
// Only guard loop_restart edges
let Some(ref edge) = ctx.edge else {
return Ok(EdgeDecision::Continue);
};
if !edge.inner().loop_restart() {
return Ok(EdgeDecision::Continue);
}
let outcome = ctx.outcome;
// Guard: only TransientInfra failures may trigger loop_restart
let failure_class = outcome.classified_failure_category();
if let Some(fc) = failure_class {
if fc != FailureCategory::TransientInfra {
return Ok(EdgeDecision::Block(format!(
"loop_restart blocked: failure_class={fc} (requires transient_infra), failure_reason={}",
outcome.failure_reason().unwrap_or("none"),
)));
}
}
// Circuit breaker: check restart failure signatures
if let Some(ref failure) = outcome.failure {
let sig = FailureSignature::new(
ctx.from,
failure.category,
failure.signature.as_deref(),
Some(failure.message.as_str()),
);
if failure.category.is_signature_tracked() {
let mut sigs = self.restart_failure_signatures.lock()
.expect("circuit breaker mutex should not be poisoned: no code panics while holding this lock");
let count = sigs.entry(sig.clone()).or_insert(0);
*count += 1;
let limit = self.loop_restart_signature_limit;
if *count >= limit {
return Ok(EdgeDecision::Block(format!(
"loop_restart circuit breaker: signature {sig} repeated {count} times (limit {limit})"
)));
}
}
}
Ok(EdgeDecision::Continue)
}
}

540
lib/components/fabro-workflow/src/lifecycle/event.rs
View file

@ -1,540 +0,0 @@
use std::collections::BTreeMap;
use std::sync::{Arc, Mutex};
use std::time::Instant;
use async_trait::async_trait;
use fabro_core::error::Result as CoreResult;
use fabro_core::graph::NodeSpec;
use fabro_core::lifecycle::{
AttemptContext, AttemptResultContext, EdgeContext, EdgeDecision, NodeDecision, RunLifecycle,
};
use fabro_core::outcome::NodeResult;
use fabro_core::state::ExecutionState;
use fabro_types::{Principal, RunId, StageTiming};
use super::circuit_breaker::CircuitBreakerLifecycle;
use super::git::GitCheckpointResult;
use crate::context::{Context, WorkflowContext};
use crate::event::{Emitter, Event, StageScope};
use crate::graph::{WorkflowGraph, WorkflowNode};
use crate::outcome::{FailureCategory, FailureDetail, ModelUsage, Outcome, StageOutcome};
use crate::stage_execution::{StageExecution, StageExecutionTracker};
use crate::{artifact, context};
type WfRunState = ExecutionState<Option<ModelUsage>>;
type WfNodeResult = NodeResult<Option<ModelUsage>>;
type FailureSignatureSnapshot = (
Option<BTreeMap<String, usize>>,
Option<BTreeMap<String, usize>>,
);
/// Sub-lifecycle responsible for emitting workflow run events.
pub(crate) struct EventLifecycle {
pub emitter: Arc<Emitter>,
pub graph_name: String,
pub run_id: RunId,
pub run_start: Mutex<Instant>,
/// Set in on_edge_selected when loop_restart approved; emitted+cleared in
/// on_run_start.
pub restarted_from: Arc<Mutex<Option<(String, String)>>>,
// Config for WorkflowRunStarted payload
pub base_branch: Option<String>,
pub base_sha: Option<String>,
pub run_branch: Option<String>,
pub worktree_dir: Option<String>,
pub goal: Option<String>,
/// Shared git checkpoint result (written by GitLifecycle, read by
/// EventLifecycle when emitting CheckpointCompleted).
pub checkpoint_git_result: Arc<Mutex<Option<GitCheckpointResult>>>,
pub circuit_breaker: Arc<CircuitBreakerLifecycle>,
/// Run-scoped stage execution allocator shared with `RunServices`.
pub stage_executions: StageExecutionTracker,
}
fn snapshot_failure_signatures(
circuit_breaker: &CircuitBreakerLifecycle,
) -> FailureSignatureSnapshot {
let (loop_sigs, restart_sigs) = circuit_breaker.snapshot();
let loop_sigs = (!loop_sigs.is_empty()).then(|| {
loop_sigs
.into_iter()
.map(|(sig, count)| (sig.to_string(), count))
.collect::<BTreeMap<_, _>>()
});
let restart_sigs = (!restart_sigs.is_empty()).then(|| {
restart_sigs
.into_iter()
.map(|(sig, count)| (sig.to_string(), count))
.collect::<BTreeMap<_, _>>()
});
(loop_sigs, restart_sigs)
}
fn actor_for_stage_failure(failure: &FailureDetail) -> Option<Principal> {
failure
.system_actor
.map(|system_kind| Principal::System { system_kind })
}
/// Build a [`StageTiming`] from a [`WfNodeResult`]. Inference and tool time
/// flow from the executor's `NodeResult` fields, which are populated from
/// `outcome.timing` by [`fabro_core`]. Handlers without an active-time
/// breakdown produce a wall-only timing.
fn node_result_timing(result: &WfNodeResult) -> StageTiming {
StageTiming::new(
crate::millis_u64(result.wall_time),
crate::millis_u64(result.inference_time),
crate::millis_u64(result.tool_time),
)
}
fn response_from_outcome(node_id: &str, outcome: &Outcome) -> Option<String> {
outcome
.context_updates
.get(&context::keys::response_key(node_id))
.and_then(|value| value.as_str().map(ToOwned::to_owned))
}
/// Context values for `StageCompleted` events. Runtime-only keys are stripped.
fn stage_context_values(workflow_context: &Context) -> Option<BTreeMap<String, serde_json::Value>> {
let mut snapshot = workflow_context.snapshot();
artifact::strip_transient_keys(&mut snapshot);
(!snapshot.is_empty()).then(|| snapshot.into_iter().collect())
}
pub(super) fn stage_visit(state: &WfRunState, node_id: &str) -> u32 {
let visits = state.node_visits.get(node_id).copied().unwrap_or(1);
u32::try_from(visits).unwrap_or(u32::MAX)
}
fn stage_scope_from_execution(
execution: Option<&StageExecution>,
state: &WfRunState,
node_id: &str,
) -> StageScope {
let (node_id, visit) = execution.map_or_else(
|| (node_id.to_owned(), stage_visit(state, node_id)),
|execution| {
(
execution.stage_id.node_id().to_owned(),
execution.stage_id.visit(),
)
},
);
StageScope {
node_id,
visit,
parallel_group_id: state.context.parallel_group_id(),
parallel_branch_id: state.context.parallel_branch_id(),
}
}
/// Build the emission scope for a node from its active stage execution.
/// Falls back to the graph visit for direct unit-test call sites that emit
/// without a reservation; the two are equal for a first execution.
pub(crate) fn stage_scope_for(
stage_executions: &StageExecutionTracker,
state: &WfRunState,
node_id: &str,
) -> StageScope {
let execution = stage_executions.active(node_id);
stage_scope_from_execution(execution.as_deref(), state, node_id)
}
#[async_trait]
impl RunLifecycle<WorkflowGraph> for EventLifecycle {
async fn on_run_start(&self, _graph: &WorkflowGraph, _state: &WfRunState) -> CoreResult<()> {
// If restarted_from is Some, emit LoopRestart and clear it
{
let mut restarted = self.restarted_from.lock()
.expect("event lifecycle mutex should not be poisoned: no code panics while holding this lock");
if let Some((from_node, to_node)) = restarted.take() {
self.emitter
.emit(&Event::LoopRestart { from_node, to_node });
}
}
// Reset run_start for duration measurement
*self.run_start.lock().expect(
"event lifecycle mutex should not be poisoned: no code panics while holding this lock",
) = Instant::now();
// Emit RunStarted
self.emitter.emit(&Event::WorkflowRunStarted {
name: self.graph_name.clone(),
run_id: self.run_id,
base_branch: self.base_branch.clone(),
base_sha: self.base_sha.clone(),
run_branch: self.run_branch.clone(),
worktree_dir: self.worktree_dir.clone(),
goal: self.goal.clone(),
});
self.emitter.emit(&Event::RunRunning);
Ok(())
}
async fn on_terminal_reached(
&self,
node: &WorkflowNode,
goal_gates_passed: bool,
state: &WfRunState,
) {
if !goal_gates_passed {
return;
}
let gv = node.inner();
let stage_index = state.stage_index;
// Terminal nodes bypass `before_node`/`before_attempt`, so their
// synthetic paired events reserve an execution here.
let execution = self
.stage_executions
.reserve(&gv.id, stage_visit(state, &gv.id));
let scope = stage_scope_from_execution(Some(&execution), state, &gv.id);
let (loop_failure_signatures, restart_failure_signatures) =
snapshot_failure_signatures(&self.circuit_breaker);
self.emitter.emit_scoped(
&Event::StageStarted {
node_id: gv.id.clone(),
name: gv.label().to_string(),
index: stage_index,
handler_type: gv.handler_type().unwrap_or_default().to_string(),
attempt: 1,
max_attempts: 1,
graph_visit: Some(execution.graph_visit),
resumed_from_stage_id: execution.resumed_from.clone(),
},
&scope,
);
self.emitter.emit_scoped(
&Event::StageCompleted {
node_id: gv.id.clone(),
name: gv.label().to_string(),
index: stage_index,
timing: StageTiming::wall_only(0),
status: StageOutcome::Succeeded.to_string(),
preferred_label: None,
suggested_next_ids: Vec::new(),
usage_by_model: Vec::new(),
usage: None,
failure: None,
notes: None,
files_touched: Vec::new(),
context_updates: None,
jump_to_node: None,
context_values: None,
node_visits: None,
loop_failure_signatures,
restart_failure_signatures,
response: state
.context
.get(&context::keys::response_key(&gv.id))
.and_then(|value| value.as_str().map(ToOwned::to_owned)),
attempt: 1,
max_attempts: 1,
},
&scope,
);
}
async fn before_attempt(
&self,
ctx: &AttemptContext<'_, WorkflowGraph>,
state: &WfRunState,
) -> CoreResult<NodeDecision<Option<ModelUsage>>> {
let gv = ctx.node.inner();
let execution = self.stage_executions.active(&gv.id);
let scope = stage_scope_from_execution(execution.as_deref(), state, &gv.id);
let graph_visit = execution
.as_ref()
.map_or_else(|| stage_visit(state, &gv.id), |e| e.graph_visit);
self.emitter.emit_scoped(
&Event::StageStarted {
node_id: gv.id.clone(),
name: gv.label().to_string(),
index: state.stage_index,
handler_type: gv.handler_type().unwrap_or_default().to_string(),
attempt: ctx.attempt as usize,
max_attempts: ctx.max_attempts as usize,
graph_visit: Some(graph_visit),
resumed_from_stage_id: execution
.as_ref()
.and_then(|execution| execution.resumed_from.clone()),
},
&scope,
);
Ok(NodeDecision::Continue)
}
async fn after_attempt(
&self,
ctx: &AttemptResultContext<'_, WorkflowGraph>,
state: &WfRunState,
) -> CoreResult<()> {
if ctx.will_retry {
let gv = ctx.node.inner();
let outcome = &ctx.result.outcome;
let stage_index = state.stage_index;
let scope = stage_scope_for(&self.stage_executions, state, &gv.id);
let timing = node_result_timing(ctx.result);
let failure = outcome.failure.clone().unwrap_or_else(|| {
FailureDetail::new("handler failed", FailureCategory::TransientInfra)
});
let actor = actor_for_stage_failure(&failure);
self.emitter.emit_scoped(
&Event::StageFailed {
node_id: gv.id.clone(),
name: gv.label().to_string(),
index: stage_index,
failure,
will_retry: true,
timing,
usage: outcome.usage.clone(),
usage_by_model: outcome.usage_by_model.clone(),
actor,
},
&scope,
);
self.emitter.emit_scoped(
&Event::StageRetrying {
node_id: gv.id.clone(),
name: gv.label().to_string(),
index: stage_index,
attempt: ctx.attempt as usize,
max_attempts: ctx.result.max_attempts as usize,
delay_ms: ctx.backoff_delay.map_or(0, crate::millis_u64),
},
&scope,
);
}
Ok(())
}
async fn after_node(
&self,
node: &WorkflowNode,
result: &mut WfNodeResult,
state: &WfRunState,
) -> CoreResult<()> {
let outcome = &result.outcome;
// Skipped nodes had no StageStarted, so skip completion events (engine.rs:2080)
if outcome.status == StageOutcome::Skipped {
return Ok(());
}
let gv = node.inner();
let stage_index = state.stage_index;
let scope = stage_scope_for(&self.stage_executions, state, &gv.id);
let timing = node_result_timing(result);
let (loop_failure_signatures, restart_failure_signatures) =
snapshot_failure_signatures(&self.circuit_breaker);
if outcome.status.is_failure() {
let failure = outcome.failure.clone().unwrap_or_else(|| {
FailureDetail::new("handler failed", FailureCategory::Deterministic)
});
let actor = actor_for_stage_failure(&failure);
self.emitter.emit_scoped(
&Event::StageFailed {
node_id: gv.id.clone(),
name: gv.label().to_string(),
index: stage_index,
failure,
will_retry: false,
timing,
usage: outcome.usage.clone(),
usage_by_model: outcome.usage_by_model.clone(),
actor,
},
&scope,
);
} else {
self.emitter.emit_scoped(
&Event::StageCompleted {
node_id: gv.id.clone(),
name: gv.label().to_string(),
index: stage_index,
timing,
status: outcome.status.to_string(),
preferred_label: outcome.preferred_label.clone(),
suggested_next_ids: outcome.suggested_next_ids.clone(),
usage: outcome.usage.clone(),
usage_by_model: outcome.usage_by_model.clone(),
failure: outcome.failure.clone(),
notes: outcome.notes.clone(),
files_touched: outcome.files_touched.clone(),
context_updates: (!outcome.context_updates.is_empty()).then(|| {
outcome
.context_updates
.clone()
.into_iter()
.collect::<BTreeMap<_, _>>()
}),
jump_to_node: outcome.jump_to_node.clone(),
context_values: stage_context_values(&state.context),
node_visits: (!state.node_visits.is_empty()).then(|| {
state
.node_visits
.clone()
.into_iter()
.collect::<BTreeMap<_, _>>()
}),
loop_failure_signatures,
restart_failure_signatures,
response: response_from_outcome(&gv.id, outcome),
attempt: result.attempts as usize,
max_attempts: result.max_attempts as usize,
},
&scope,
);
}
Ok(())
}
async fn on_edge_selected(
&self,
ctx: &EdgeContext<'_, WorkflowGraph>,
_state: &WfRunState,
) -> CoreResult<EdgeDecision> {
let outcome = ctx.outcome;
let label = ctx
.edge
.as_ref()
.and_then(|e| e.inner().label().map(String::from));
let condition = ctx
.edge
.as_ref()
.and_then(|e| e.inner().condition().map(String::from));
self.emitter.emit(&Event::EdgeSelected {
from_node: ctx.from.to_string(),
to_node: ctx.to.to_string(),
label,
condition,
reason: ctx.reason.to_string(),
preferred_label: outcome.preferred_label.clone(),
suggested_next_ids: outcome.suggested_next_ids.clone(),
stage_status: outcome.status.to_string(),
is_jump: ctx.is_jump,
});
Ok(EdgeDecision::Continue)
}
async fn on_checkpoint(
&self,
node: &WorkflowNode,
result: &WfNodeResult,
next_node_id: Option<&str>,
state: &WfRunState,
) -> CoreResult<()> {
let status = result.outcome.status.to_string();
// Read git checkpoint result (set by GitLifecycle)
let git_result = self.checkpoint_git_result.lock()
.expect("event lifecycle mutex should not be poisoned: no code panics while holding this lock")
.clone();
let git_sha = git_result.as_ref().and_then(|r| r.commit_sha.clone());
let diff = git_result.as_ref().and_then(|r| r.diff.clone());
let diff_summary = git_result.as_ref().and_then(|r| r.diff_summary);
let (loop_failure_signatures, restart_failure_signatures) =
snapshot_failure_signatures(&self.circuit_breaker);
let context_values = artifact::durable_context_snapshot(&state.context);
let mut node_outcomes = state.node_outcomes.clone();
node_outcomes.insert(node.id().to_string(), result.outcome.clone());
artifact::normalize_durable_outcomes(&mut node_outcomes);
let execution = self.stage_executions.active(node.id());
let scope = stage_scope_from_execution(execution.as_deref(), state, node.id());
let graph_visit = execution
.as_ref()
.map_or_else(|| stage_visit(state, node.id()), |e| e.graph_visit);
self.emitter.emit_scoped(
&Event::CheckpointCompleted {
node_id: node.id().to_string(),
status,
current_node: node.id().to_string(),
completed_nodes: state.completed_nodes.clone(),
node_retries: state
.node_retries
.clone()
.into_iter()
.collect::<BTreeMap<_, _>>(),
context_values: context_values.into_iter().collect::<BTreeMap<_, _>>(),
node_outcomes: node_outcomes.into_iter().collect::<BTreeMap<_, _>>(),
next_node_id: next_node_id.map(ToOwned::to_owned),
git_commit_sha: git_sha.clone(),
loop_failure_signatures: loop_failure_signatures.unwrap_or_default(),
restart_failure_signatures: restart_failure_signatures.unwrap_or_default(),
node_visits: state
.node_visits
.clone()
.into_iter()
.collect::<BTreeMap<_, _>>(),
diff,
diff_summary,
graph_visit: Some(graph_visit),
resumed_from_stage_id: execution
.as_ref()
.and_then(|execution| execution.resumed_from.clone()),
},
&scope,
);
// Emit GitCommit + GitPush events if git produced results
if let Some(ref result) = git_result {
if let Some(ref sha) = result.commit_sha {
self.emitter.emit_scoped(
&Event::GitCommit {
node_id: Some(node.id().to_string()),
sha: sha.clone(),
},
&scope,
);
}
for push in &result.push_results {
self.emitter.emit(&Event::GitPush {
branch: push.branch.clone(),
success: push.success,
exec_output_tail: push.exec_output_tail.clone(),
attempts: push.attempts.clone(),
});
}
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn stage_context_values_drops_runtime_keys_including_current_preamble() {
let workflow_context = Context::new();
workflow_context.set(
context::keys::INTERNAL_PARALLEL_BRANCH_PREAMBLES,
serde_json::json!([{"fidelity": "summary:high", "preamble": "runtime only"}]),
);
workflow_context.set(
context::keys::INTERNAL_STAGE_EXECUTION_ORDINAL,
serde_json::json!(2),
);
workflow_context.set(
context::keys::CURRENT_PREAMBLE,
serde_json::json!("active preamble"),
);
workflow_context.set("response.work", serde_json::json!("durable"));
let values = stage_context_values(&workflow_context).expect("snapshot should not be empty");
assert!(!values.contains_key(context::keys::INTERNAL_PARALLEL_BRANCH_PREAMBLES));
assert!(!values.contains_key(context::keys::INTERNAL_STAGE_EXECUTION_ORDINAL));
assert!(!values.contains_key(context::keys::CURRENT_PREAMBLE));
assert_eq!(
values.get("response.work"),
Some(&serde_json::json!("durable"))
);
}
}

810
lib/components/fabro-workflow/src/lifecycle/fidelity.rs
View file

@ -1,810 +0,0 @@
use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::{Arc, Mutex};
use async_trait::async_trait;
use fabro_core::error::{Error as CoreError, Result as CoreResult};
use fabro_core::graph::NodeSpec;
use fabro_core::lifecycle::{EdgeContext, EdgeDecision, NodeDecision, RunLifecycle};
use fabro_core::state::ExecutionState;
use fabro_graphviz::graph::types::{Edge as GvEdge, Graph as GvGraph, Node as GvNode};
use fabro_sandbox::RunSandbox;
use crate::artifact;
use crate::context::{Context, ParallelBranchPreamble, keys};
use crate::graph::{WorkflowGraph, WorkflowNode};
use crate::handler::llm::preamble::build_preamble;
use crate::outcome::{ModelUsage, Outcome};
use crate::runtime_store::RunStoreHandle;
type WfRunState = ExecutionState<Option<ModelUsage>>;
type WfNodeDecision = NodeDecision<Option<ModelUsage>>;
/// Graphviz edge captured from edge selection, passed to the next node's
/// before_node for fidelity/thread resolution.
#[derive(Debug, Clone)]
struct IncomingEdgeData {
edge: Arc<GvEdge>,
}
/// Sub-lifecycle responsible for fidelity/thread resolution and context key
/// setup.
pub(crate) struct FidelityLifecycle {
pub graph: Arc<GvGraph>,
pub sandbox: Arc<RunSandbox>,
pub run_store: RunStoreHandle,
pub run_dir: PathBuf,
incoming_edge_data: Mutex<Option<IncomingEdgeData>>,
/// True on the first node after checkpoint resume when prior fidelity was
/// Full.
degrade_fidelity_on_resume: Mutex<bool>,
}
impl FidelityLifecycle {
pub(crate) fn new(
graph: Arc<GvGraph>,
sandbox: Arc<RunSandbox>,
run_store: RunStoreHandle,
run_dir: PathBuf,
) -> Self {
Self {
graph,
sandbox,
run_store,
run_dir,
incoming_edge_data: Mutex::new(None),
degrade_fidelity_on_resume: Mutex::new(false),
}
}
pub(crate) fn set_degrade_fidelity_on_resume(&self, flag: bool) {
*self.degrade_fidelity_on_resume.lock().expect(
"fidelity mutex should not be poisoned: no code panics while holding this lock",
) = flag;
}
/// Render the per-branch preamble stash for a parallel node, indexed by
/// outgoing-edge order (the same order `ParallelHandler` fans out in).
/// `Null` entries inherit the fork's preamble.
fn build_parallel_branch_preambles(
&self,
node_id: &str,
fork_fidelity: keys::Fidelity,
resolved_context: &Context,
resolved_outcomes: &HashMap<String, Outcome>,
completed_nodes: &[String],
) -> Vec<serde_json::Value> {
let edges = self.graph.outgoing_edges(node_id);
let mut preambles: Vec<serde_json::Value> = Vec::with_capacity(edges.len());
let mut rendered: HashMap<keys::Fidelity, usize> = HashMap::new();
for (branch_index, edge) in edges.into_iter().enumerate() {
let Some(target_node) = self.graph.nodes.get(&edge.to) else {
preambles.push(serde_json::Value::Null);
continue;
};
let resolution = resolve_parallel_branch_fidelity(edge, target_node, fork_fidelity);
if resolution.requested == Some(keys::Fidelity::Full) {
tracing::warn!(
parallel_node = %node_id,
branch = %edge.to,
branch_index,
effective_fidelity = %keys::Fidelity::Full.degraded(),
"Parallel branch fidelity degraded from full"
);
}
let Some(branch_fidelity) = resolution.effective else {
preambles.push(serde_json::Value::Null);
continue;
};
if let Some(&rendered_index) = rendered.get(&branch_fidelity) {
preambles.push(preambles[rendered_index].clone());
continue;
}
let entry = ParallelBranchPreamble {
fidelity: branch_fidelity,
preamble: build_preamble(
branch_fidelity,
resolved_context,
&self.graph,
completed_nodes,
resolved_outcomes,
),
};
rendered.insert(branch_fidelity, preambles.len());
preambles.push(
serde_json::to_value(entry)
.expect("ParallelBranchPreamble serialization cannot fail"),
);
}
preambles
}
}
#[async_trait]
impl RunLifecycle<WorkflowGraph> for FidelityLifecycle {
async fn on_run_start(&self, _graph: &WorkflowGraph, _state: &WfRunState) -> CoreResult<()> {
// Clear incoming edge data (restart target must not inherit pre-restart edge)
*self.incoming_edge_data.lock().expect(
"fidelity mutex should not be poisoned: no code panics while holding this lock",
) = None;
Ok(())
}
async fn before_node(
&self,
node: &WorkflowNode,
state: &WfRunState,
) -> CoreResult<WfNodeDecision> {
state.context.set(
keys::INTERNAL_PARALLEL_BRANCH_PREAMBLES,
serde_json::Value::Null,
);
let incoming = self
.incoming_edge_data
.lock()
.expect("fidelity mutex should not be poisoned: no code panics while holding this lock")
.take();
let gv_node = node.inner();
// 1. Fidelity resolution via resolve_fidelity: edge → node → graph default →
// Compact
let incoming_edge_ref = incoming.as_ref().map(|d| d.edge.as_ref());
let fidelity = resolve_fidelity(incoming_edge_ref, gv_node, &self.graph);
// 2. Fidelity degradation on resume (full → summary:high)
let fidelity = {
let mut degrade = self.degrade_fidelity_on_resume.lock().expect(
"fidelity mutex should not be poisoned: no code panics while holding this lock",
);
if *degrade {
*degrade = false;
fidelity.degraded()
} else {
fidelity
}
};
// 3. Set INTERNAL_FIDELITY
state.context.set(
keys::INTERNAL_FIDELITY,
serde_json::json!(fidelity.to_string()),
);
// 4. Preamble building: if Full, empty preamble; otherwise build from context
let mut resolved_values = artifact::resolved_context_snapshot(
&state.context,
&self.run_store,
&self.sandbox,
&self.run_dir,
)
.await
.map_err(|err| CoreError::Other(err.to_string()))?;
let mut resolved_outcomes = artifact::resolve_outcomes_for_execution(
&state.node_outcomes,
&self.run_store,
&self.sandbox,
&self.run_dir,
)
.await
.map_err(|err| CoreError::Other(err.to_string()))?;
// The resolved copies exist only to render prompt preambles, so bound
// what any one value may contribute before the builders see them.
// Full renders no preamble and Truncate renders no context values, so
// there is nothing to bound — except for a parallel node, whose branch
// stash may render at a richer fidelity.
let preamble_renders_values =
!matches!(fidelity, keys::Fidelity::Full | keys::Fidelity::Truncate)
|| gv_node.handler_type() == Some("parallel");
if preamble_renders_values {
artifact::demote_large_values_for_prompt(
&mut resolved_values,
&mut resolved_outcomes,
&self.run_store,
&self.sandbox,
&self.run_dir,
)
.await;
}
let resolved_context = Context::from_values(resolved_values);
let preamble = build_preamble(
fidelity,
&resolved_context,
&self.graph,
&state.completed_nodes,
&resolved_outcomes,
);
state
.context
.set(keys::CURRENT_PREAMBLE, serde_json::json!(preamble));
// 5. Parallel nodes: pre-render per-branch preambles into the stash that
// ParallelHandler consumes at fan-out.
if gv_node.handler_type() == Some("parallel") {
let branch_preambles = self.build_parallel_branch_preambles(
node.id(),
fidelity,
&resolved_context,
&resolved_outcomes,
&state.completed_nodes,
);
state.context.set(
keys::INTERNAL_PARALLEL_BRANCH_PREAMBLES,
serde_json::Value::Array(branch_preambles),
);
}
// 6. Thread ID resolution via resolve_thread_id: edge → node → graph default →
// class → previous
let thread_id = resolve_thread_id(
incoming_edge_ref,
gv_node,
&self.graph,
state.previous_node_id.as_deref(),
);
// 7. Set thread.{tid}.current_node
if let Some(ref tid) = thread_id {
let key = keys::thread_current_node_key(tid);
state.context.set(key, serde_json::json!(node.id()));
}
// 8. Set INTERNAL_THREAD_ID (or null)
match thread_id {
Some(tid) => {
state
.context
.set(keys::INTERNAL_THREAD_ID, serde_json::json!(tid));
}
None => {
state
.context
.set(keys::INTERNAL_THREAD_ID, serde_json::Value::Null);
}
}
// 9. Set INTERNAL_NODE_VISIT_COUNT and CURRENT_NODE
let visits = state.node_visits.get(node.id()).copied().unwrap_or(1);
state
.context
.set(keys::CURRENT_NODE, serde_json::json!(node.id()));
state
.context
.set(keys::INTERNAL_NODE_VISIT_COUNT, serde_json::json!(visits));
Ok(NodeDecision::Continue)
}
async fn on_edge_selected(
&self,
ctx: &EdgeContext<'_, WorkflowGraph>,
_state: &WfRunState,
) -> CoreResult<EdgeDecision> {
// Capture fidelity/thread from edge for next node
if let Some(ref edge) = ctx.edge {
let gv_edge = edge.inner();
let edge_data = IncomingEdgeData {
edge: Arc::new(gv_edge.clone()),
};
*self.incoming_edge_data.lock().expect(
"fidelity mutex should not be poisoned: no code panics while holding this lock",
) = Some(edge_data);
}
Ok(EdgeDecision::Continue)
}
}
#[derive(Debug, Clone, Copy)]
struct ParallelBranchFidelityResolution {
/// The explicit fidelity requested on the edge or node, pre-degradation.
requested: Option<keys::Fidelity>,
/// The fidelity to render an entry for; `None` inherits the fork preamble.
effective: Option<keys::Fidelity>,
}
/// Resolve explicit branch fidelity with edge-over-node precedence.
///
/// Branches with no explicit fidelity inherit the parallel node's preamble.
/// Explicit full fidelity is degraded because concurrent branches cannot share
/// an LLM session. An effective fidelity equal to the parallel node also
/// inherits, avoiding a redundant preamble render.
fn resolve_parallel_branch_fidelity(
edge: &GvEdge,
target_node: &GvNode,
parallel_fidelity: keys::Fidelity,
) -> ParallelBranchFidelityResolution {
let requested = explicit_fidelity(Some(edge), target_node).map(|(fidelity, _)| fidelity);
let effective = requested
.map(keys::Fidelity::degraded)
.filter(|fidelity| *fidelity != parallel_fidelity);
ParallelBranchFidelityResolution {
requested,
effective,
}
}
/// Explicit fidelity from the incoming edge attribute, else the node
/// attribute, with the winning source labeled for logging.
fn explicit_fidelity(
incoming_edge: Option<&GvEdge>,
node: &GvNode,
) -> Option<(keys::Fidelity, &'static str)> {
incoming_edge
.and_then(|e| e.fidelity())
.and_then(|s| s.parse().ok())
.map(|f| (f, "edge"))
.or_else(|| {
node.fidelity()
.and_then(|s| s.parse().ok())
.map(|f| (f, "node"))
})
}
/// Resolve the context fidelity for a node, following the precedence:
/// 1. Incoming edge `fidelity` attribute
/// 2. Target node `fidelity` attribute
/// 3. Graph `default_fidelity` attribute
/// 4. Default: Compact
fn resolve_fidelity(
incoming_edge: Option<&GvEdge>,
node: &GvNode,
graph: &GvGraph,
) -> keys::Fidelity {
let (resolved, source) = if let Some((f, source)) = explicit_fidelity(incoming_edge, node) {
(f, source)
} else if let Some(f) = graph.default_fidelity().and_then(|s| s.parse().ok()) {
(f, "graph")
} else {
(keys::Fidelity::default(), "default")
};
tracing::info!(
node = %node.id,
fidelity = %resolved,
source = source,
"Fidelity resolved"
);
resolved
}
/// Resolve the thread ID for a node, following the precedence:
/// 1. Incoming edge `thread_id` attribute
/// 2. Target node `thread_id` attribute
/// 3. Graph-level default thread
/// 4. Derived class from enclosing subgraph (first class from the node's
/// classes list)
/// 5. Fallback to previous node ID
fn resolve_thread_id(
incoming_edge: Option<&GvEdge>,
node: &GvNode,
graph: &GvGraph,
previous_node_id: Option<&str>,
) -> Option<String> {
if let Some(edge) = incoming_edge {
if let Some(tid) = edge.thread_id() {
return Some(tid.to_string());
}
}
if let Some(tid) = node.thread_id() {
return Some(tid.to_string());
}
if let Some(tid) = graph.default_thread() {
return Some(tid.to_string());
}
if let Some(first_class) = node.classes.first() {
return Some(first_class.clone());
}
previous_node_id.map(String::from)
}
#[cfg(test)]
mod tests {
use std::path::Path;
use std::time::Duration;
use fabro_core::graph::Graph as CoreGraph;
use fabro_graphviz::graph::{AttrValue, Edge, Graph, Node};
use fabro_types::fixtures;
use object_store::memory::InMemory;
use super::*;
use crate::context::WorkflowContext;
use crate::context::keys::Fidelity;
fn str_attr(value: &str) -> AttrValue {
AttrValue::String(value.to_string())
}
fn parallel_workflow_graph(
fork_fidelity: Option<&str>,
branch_a_fidelity: Option<&str>,
) -> WorkflowGraph {
let mut graph = Graph::new("parallel-fidelity");
let mut start = Node::new("start");
start
.attrs
.insert("shape".to_string(), str_attr("Mdiamond"));
let mut fork = Node::new("fork");
fork.attrs
.insert("shape".to_string(), str_attr("component"));
if let Some(fidelity) = fork_fidelity {
fork.attrs
.insert("fidelity".to_string(), str_attr(fidelity));
}
let mut branch_a = Node::new("branch_a");
if let Some(fidelity) = branch_a_fidelity {
branch_a
.attrs
.insert("fidelity".to_string(), str_attr(fidelity));
}
let branch_b = Node::new("branch_b");
let mut work = Node::new("work");
work.attrs.insert("shape".to_string(), str_attr("box"));
graph.nodes.insert(start.id.clone(), start);
graph.nodes.insert(fork.id.clone(), fork);
graph.nodes.insert(branch_a.id.clone(), branch_a);
graph.nodes.insert(branch_b.id.clone(), branch_b);
graph.nodes.insert(work.id.clone(), work);
graph.edges.push(Edge::new("start", "fork"));
graph.edges.push(Edge::new("fork", "branch_a"));
graph.edges.push(Edge::new("fork", "branch_b"));
WorkflowGraph(Arc::new(graph))
}
async fn test_lifecycle(graph: &WorkflowGraph, run_dir: &Path) -> FidelityLifecycle {
let store = Arc::new(fabro_store::test_support::test_database(
Arc::new(InMemory::new()),
"",
Duration::from_millis(1),
None,
));
let run_store = store.create_run(&fixtures::RUN_1).await.unwrap();
let sandbox: Arc<RunSandbox> = Arc::new(
fabro_sandbox::local_sandbox(run_dir.to_path_buf())
.await
.unwrap(),
);
FidelityLifecycle::new(
graph.0.clone(),
sandbox,
RunStoreHandle::local(run_store),
run_dir.to_path_buf(),
)
}
#[test]
fn parallel_branch_fidelity_edge_overrides_node() {
let mut node = Node::new("branch");
node.attrs
.insert("fidelity".to_string(), str_attr("compact"));
let mut edge = Edge::new("fork", "branch");
edge.attrs
.insert("fidelity".to_string(), str_attr("truncate"));
let resolved = resolve_parallel_branch_fidelity(&edge, &node, Fidelity::SummaryHigh);
assert_eq!(resolved.requested, Some(Fidelity::Truncate));
assert_eq!(resolved.effective, Some(Fidelity::Truncate));
}
#[test]
fn parallel_branch_fidelity_without_attribute_inherits() {
let node = Node::new("branch");
let edge = Edge::new("fork", "branch");
let resolution = resolve_parallel_branch_fidelity(&edge, &node, Fidelity::Compact);
assert_eq!(resolution.requested, None);
assert_eq!(resolution.effective, None);
}
#[test]
fn parallel_branch_full_fidelity_degrades_to_summary_high() {
let mut node = Node::new("branch");
node.attrs.insert("fidelity".to_string(), str_attr("full"));
let edge = Edge::new("fork", "branch");
let resolved = resolve_parallel_branch_fidelity(&edge, &node, Fidelity::Compact);
assert_eq!(resolved.requested, Some(Fidelity::Full));
assert_eq!(resolved.effective, Some(Fidelity::SummaryHigh));
}
#[test]
fn parallel_branch_fidelity_equal_to_fork_inherits() {
let mut node = Node::new("branch");
node.attrs
.insert("fidelity".to_string(), str_attr("summary:high"));
let edge = Edge::new("fork", "branch");
let resolution = resolve_parallel_branch_fidelity(&edge, &node, Fidelity::SummaryHigh);
assert_eq!(resolution.requested, Some(Fidelity::SummaryHigh));
assert_eq!(resolution.effective, None);
}
#[test]
fn explicit_full_branch_equal_to_degraded_fork_inherits() {
let mut node = Node::new("branch");
node.attrs.insert("fidelity".to_string(), str_attr("full"));
let edge = Edge::new("fork", "branch");
let resolution = resolve_parallel_branch_fidelity(&edge, &node, Fidelity::SummaryHigh);
assert_eq!(resolution.requested, Some(Fidelity::Full));
assert_eq!(resolution.effective, None);
}
#[test]
fn full_fork_without_branch_fidelity_does_not_create_entry() {
let node = Node::new("branch");
let edge = Edge::new("fork", "branch");
let resolution = resolve_parallel_branch_fidelity(&edge, &node, Fidelity::Full);
assert_eq!(resolution.requested, None);
assert_eq!(resolution.effective, None);
}
#[tokio::test]
async fn parallel_before_node_rebuilds_branch_preamble_stash() {
let graph = parallel_workflow_graph(None, Some("truncate"));
let run_dir = tempfile::tempdir().unwrap();
let lifecycle = test_lifecycle(&graph, run_dir.path()).await;
let state: WfRunState = ExecutionState::new(&graph).unwrap();
let fork = graph.get_node("fork").unwrap();
lifecycle.before_node(&fork, &state).await.unwrap();
state.context.set(
keys::INTERNAL_PARALLEL_BRANCH_PREAMBLES,
serde_json::json!(["stale", "entries", "must disappear"]),
);
lifecycle.before_node(&fork, &state).await.unwrap();
let stash = state
.context
.get(keys::INTERNAL_PARALLEL_BRANCH_PREAMBLES)
.expect("parallel stash should be set");
let entries = stash.as_array().expect("parallel stash should be an array");
assert_eq!(entries.len(), 2);
assert!(entries[0].is_object());
assert!(entries[1].is_null());
}
#[tokio::test]
async fn non_parallel_before_node_overwrites_branch_preamble_stash_with_null() {
let graph = parallel_workflow_graph(None, Some("truncate"));
let run_dir = tempfile::tempdir().unwrap();
let lifecycle = test_lifecycle(&graph, run_dir.path()).await;
let state: WfRunState = ExecutionState::new(&graph).unwrap();
let fork = graph.get_node("fork").unwrap();
let work = graph.get_node("work").unwrap();
lifecycle.before_node(&fork, &state).await.unwrap();
lifecycle.before_node(&work, &state).await.unwrap();
assert_eq!(
state.context.get(keys::INTERNAL_PARALLEL_BRANCH_PREAMBLES),
Some(serde_json::Value::Null)
);
}
#[tokio::test]
async fn resumed_full_fork_degrades_without_rendering_fallback_branches() {
let graph = parallel_workflow_graph(Some("full"), None);
let run_dir = tempfile::tempdir().unwrap();
let lifecycle = test_lifecycle(&graph, run_dir.path()).await;
lifecycle.set_degrade_fidelity_on_resume(true);
let state: WfRunState = ExecutionState::new(&graph).unwrap();
let fork = graph.get_node("fork").unwrap();
lifecycle.before_node(&fork, &state).await.unwrap();
assert_eq!(state.context.fidelity(), Fidelity::SummaryHigh);
assert_eq!(
state.context.get(keys::INTERNAL_PARALLEL_BRANCH_PREAMBLES),
Some(serde_json::json!([null, null]))
);
}
#[test]
fn fidelity_defaults_to_compact() {
let node = Node::new("work");
let graph = Graph::new("test");
assert_eq!(resolve_fidelity(None, &node, &graph), Fidelity::Compact);
}
#[test]
fn fidelity_from_graph_default() {
let node = Node::new("work");
let mut graph = Graph::new("test");
graph.attrs.insert(
"default_fidelity".to_string(),
AttrValue::String("truncate".to_string()),
);
assert_eq!(resolve_fidelity(None, &node, &graph), Fidelity::Truncate);
}
#[test]
fn fidelity_from_node_overrides_graph() {
let mut node = Node::new("work");
node.attrs.insert(
"fidelity".to_string(),
AttrValue::String("full".to_string()),
);
let mut graph = Graph::new("test");
graph.attrs.insert(
"default_fidelity".to_string(),
AttrValue::String("truncate".to_string()),
);
assert_eq!(resolve_fidelity(None, &node, &graph), Fidelity::Full);
}
#[test]
fn fidelity_from_edge_overrides_node() {
let mut node = Node::new("work");
node.attrs.insert(
"fidelity".to_string(),
AttrValue::String("full".to_string()),
);
let mut edge = Edge::new("a", "work");
edge.attrs.insert(
"fidelity".to_string(),
AttrValue::String("summary:high".to_string()),
);
let graph = Graph::new("test");
assert_eq!(
resolve_fidelity(Some(&edge), &node, &graph),
Fidelity::SummaryHigh
);
}
#[test]
fn thread_id_from_node_attribute() {
let mut node = Node::new("work");
node.attrs.insert(
"thread_id".to_string(),
AttrValue::String("main-thread".to_string()),
);
let graph = Graph::new("test");
assert_eq!(
resolve_thread_id(None, &node, &graph, Some("prev")),
Some("main-thread".to_string())
);
}
#[test]
fn thread_id_from_edge_attribute() {
let node = Node::new("work");
let mut edge = Edge::new("prev", "work");
edge.attrs.insert(
"thread_id".to_string(),
AttrValue::String("edge-thread".to_string()),
);
let graph = Graph::new("test");
assert_eq!(
resolve_thread_id(Some(&edge), &node, &graph, Some("prev")),
Some("edge-thread".to_string())
);
}
#[test]
fn thread_id_node_used_when_no_edge_thread() {
let mut node = Node::new("work");
node.attrs.insert(
"thread_id".to_string(),
AttrValue::String("node-thread".to_string()),
);
let edge = Edge::new("prev", "work");
let graph = Graph::new("test");
assert_eq!(
resolve_thread_id(Some(&edge), &node, &graph, Some("prev")),
Some("node-thread".to_string())
);
}
#[test]
fn thread_id_edge_overrides_node() {
let mut node = Node::new("work");
node.attrs.insert(
"thread_id".to_string(),
AttrValue::String("node-thread".to_string()),
);
let mut edge = Edge::new("prev", "work");
edge.attrs.insert(
"thread_id".to_string(),
AttrValue::String("edge-thread".to_string()),
);
let graph = Graph::new("test");
assert_eq!(
resolve_thread_id(Some(&edge), &node, &graph, Some("prev")),
Some("edge-thread".to_string()),
"edge thread_id should override node thread_id"
);
}
#[test]
fn thread_id_from_graph_default_thread() {
let node = Node::new("work");
let mut graph = Graph::new("test");
graph.attrs.insert(
"default_thread".to_string(),
AttrValue::String("shared-thread".to_string()),
);
assert_eq!(
resolve_thread_id(None, &node, &graph, Some("prev")),
Some("shared-thread".to_string())
);
}
#[test]
fn thread_id_edge_overrides_graph_default() {
let node = Node::new("work");
let mut edge = Edge::new("prev", "work");
edge.attrs.insert(
"thread_id".to_string(),
AttrValue::String("edge-thread".to_string()),
);
let mut graph = Graph::new("test");
graph.attrs.insert(
"default_thread".to_string(),
AttrValue::String("shared-thread".to_string()),
);
assert_eq!(
resolve_thread_id(Some(&edge), &node, &graph, Some("prev")),
Some("edge-thread".to_string())
);
}
#[test]
fn thread_id_graph_default_overrides_class() {
let mut node = Node::new("work");
node.classes = vec!["planning".to_string()];
let mut graph = Graph::new("test");
graph.attrs.insert(
"default_thread".to_string(),
AttrValue::String("shared-thread".to_string()),
);
assert_eq!(
resolve_thread_id(None, &node, &graph, Some("prev")),
Some("shared-thread".to_string())
);
}
#[test]
fn thread_id_from_node_class() {
let mut node = Node::new("work");
node.classes = vec!["planning".to_string(), "review".to_string()];
let graph = Graph::new("test");
assert_eq!(
resolve_thread_id(None, &node, &graph, Some("prev")),
Some("planning".to_string())
);
}
#[test]
fn thread_id_fallback_to_previous_node() {
let node = Node::new("work");
let graph = Graph::new("test");
assert_eq!(
resolve_thread_id(None, &node, &graph, Some("prev_node")),
Some("prev_node".to_string())
);
}
#[test]
fn thread_id_none_when_no_sources() {
let node = Node::new("start");
let graph = Graph::new("test");
assert_eq!(resolve_thread_id(None, &node, &graph, None), None);
}
}

530
lib/components/fabro-workflow/src/lifecycle/git.rs
View file

@ -1,530 +0,0 @@
use std::sync::{Arc, Mutex};
use async_trait::async_trait;
use fabro_core::error::{Error as CoreError, Result as CoreResult};
use fabro_core::graph::NodeSpec;
use fabro_core::lifecycle::RunLifecycle;
use fabro_core::outcome::NodeResult;
use fabro_core::state::ExecutionState;
use fabro_types::{DiffSummary, RunId};
use crate::event::{Emitter, Event, RunNoticeCode, RunNoticeLevel};
use crate::graph::{WorkflowGraph, WorkflowNode};
use crate::lifecycle::event::stage_scope_for;
use crate::outcome::ModelUsage;
use crate::run_options::RunOptions;
use crate::sandbox_git::{
checked_git_checkpoint, git_diff, list_diff_numstat, summarize_diff_numstat,
};
use crate::sandbox_git_runtime::SandboxGitRuntime;
use crate::stage_execution::StageExecutionTracker;
type WfRunState = ExecutionState<Option<ModelUsage>>;
type WfNodeResult = NodeResult<Option<ModelUsage>>;
/// Result of a git checkpoint operation, shared with EventLifecycle.
#[derive(Debug, Clone)]
pub(crate) struct GitCheckpointResult {
pub commit_sha: Option<String>,
pub push_results: Vec<PushResult>,
pub diff: Option<String>,
pub diff_summary: Option<DiffSummary>,
}
#[derive(Debug, Clone)]
pub(crate) struct PushResult {
pub branch: String,
pub success: bool,
pub exec_output_tail: Option<fabro_types::ExecOutputTail>,
pub attempts: Vec<fabro_sandbox::PushAttempt>,
}
/// Push a run branch to its remote counterpart.
///
/// Owns the refspec convention so the checkpoint push and the terminal publish
/// push cannot drift apart. The caller picks the retry budget: cheap for
/// checkpoint pushes (the next checkpoint re-pushes the same branch anyway),
/// generous for the terminal publish push.
pub(crate) async fn push_run_branch(
sandbox: &fabro_sandbox::RunSandbox,
branch: &str,
policy: &fabro_sandbox::GitRetryPolicy,
) -> Result<fabro_sandbox::PushReport, fabro_sandbox::PushError> {
sandbox
.git_push_ref(&format!("refs/heads/{branch}:refs/heads/{branch}"), policy)
.await
}
/// Sub-lifecycle responsible for git operations (checkpoint commits, pushes,
/// diffs).
pub(crate) struct GitLifecycle {
pub sandbox: Arc<fabro_sandbox::RunSandbox>,
pub emitter: Arc<Emitter>,
pub run_id: RunId,
pub run_options: Arc<RunOptions>,
pub sandbox_git: Arc<SandboxGitRuntime>,
pub start_node_id: Option<String>,
// Cross-lifecycle data (shared with EventLifecycle)
pub checkpoint_git_result: Arc<Mutex<Option<GitCheckpointResult>>>,
pub last_git_sha: Arc<Mutex<Option<String>>>,
/// Run-scoped stage execution allocator shared with `RunServices`.
pub stage_executions: StageExecutionTracker,
}
#[async_trait]
impl RunLifecycle<WorkflowGraph> for GitLifecycle {
async fn on_run_start(&self, _graph: &WorkflowGraph, _state: &WfRunState) -> CoreResult<()> {
// Reset last_git_sha (diff base parity)
*self.last_git_sha.lock().expect(
"git lifecycle mutex should not be poisoned: no code panics while holding this lock",
) = None;
*self.checkpoint_git_result.lock().expect(
"git lifecycle mutex should not be poisoned: no code panics while holding this lock",
) = None;
Ok(())
}
async fn on_checkpoint(
&self,
node: &WorkflowNode,
result: &WfNodeResult,
_next_node_id: Option<&str>,
state: &WfRunState,
) -> CoreResult<()> {
let node_id = node.id();
// Skip git checkpoint for the start node (always empty) or if git disabled
if self.start_node_id.as_deref() == Some(node_id) || self.run_options.git.is_none() {
*self.checkpoint_git_result.lock()
.expect("git lifecycle mutex should not be poisoned: no code panics while holding this lock") = None;
return Ok(());
}
// Run branch commit via sandbox
let completed_count = state.completed_nodes.len();
let git_author = self.run_options.git_author();
let commit_result = checked_git_checkpoint(
&self.sandbox_git,
&self.sandbox,
&self.run_id.to_string(),
node_id,
&result.outcome.status.to_string(),
completed_count,
self.run_options.checkpoint(),
&git_author,
)
.await;
match commit_result {
Ok(sha) => {
let mut git_result = GitCheckpointResult {
commit_sha: Some(sha.clone()),
push_results: Vec::new(),
diff: None,
diff_summary: None,
};
// Push run branch (skip in dry-run mode)
if !self.run_options.dry_run_enabled()
&& self.run_options.settings.run.run_branch.push
{
if let Some(branch) = self
.run_options
.git
.as_ref()
.and_then(|g| g.run_branch.as_ref())
{
let policy = fabro_sandbox::checkpoint_push_policy();
let (push_ok, exec_output_tail, attempts) =
match push_run_branch(self.sandbox.as_ref(), branch, &policy).await {
Ok(report) => {
self.sandbox_git.record_successful_push();
(true, None, report.attempts)
}
Err(push_error) => {
let exec_output_tail =
fabro_sandbox::default_redacted_output_tail(
&push_error.error,
);
tracing::warn!(
branch = %branch,
attempts = push_error.report.attempts.len(),
error = %fabro_sandbox::display_for_log(&push_error.error),
"git push from run lifecycle failed"
);
self.emitter.notice_with_tail(
RunNoticeLevel::Warn,
RunNoticeCode::GitPushFailed,
format!(
"Failed to push run branch {branch}: {}",
push_error.error
),
exec_output_tail.clone(),
);
(false, exec_output_tail, push_error.report.attempts)
}
};
git_result.push_results.push(PushResult {
branch: branch.clone(),
success: push_ok,
exec_output_tail,
attempts,
});
}
}
// Save diff.patch
let prev = self.last_git_sha.lock()
.expect("git lifecycle mutex should not be poisoned: no code panics while holding this lock")
.clone().or_else(|| {
self.run_options
.git
.as_ref()
.and_then(|g| g.base_sha.clone())
});
if let Some(prev) = prev.filter(|p| p != &sha) {
let summary_base = self
.run_options
.git
.as_ref()
.and_then(|git| git.base_sha.clone());
let (patch_result, numstat_result) =
tokio::join!(git_diff(&self.sandbox, &prev), async {
match summary_base.as_deref() {
Some(base) if base != sha => {
Some(list_diff_numstat(&self.sandbox, base, &sha).await)
}
_ => None,
}
},);
match patch_result {
Ok(patch) if !patch.is_empty() => {
git_result.diff = Some(patch);
}
Ok(_) => {}
Err(err) => {
let exec_output_tail =
fabro_sandbox::default_redacted_output_tail(&err);
self.emitter.notice_with_tail(
RunNoticeLevel::Warn,
RunNoticeCode::GitDiffFailed,
format!("[node: {node_id}] git diff failed: {err}"),
exec_output_tail,
);
}
}
match numstat_result {
Some(Ok(numstat)) => {
git_result.diff_summary = Some(summarize_diff_numstat(&numstat));
}
Some(Err(err)) => {
let exec_output_tail =
fabro_sandbox::default_redacted_output_tail(&err);
self.emitter.notice_with_tail(
RunNoticeLevel::Warn,
RunNoticeCode::GitDiffFailed,
format!("[node: {node_id}] git diff stats failed: {err}"),
exec_output_tail,
);
}
None => {}
}
}
// Update shared state
*self.last_git_sha.lock()
.expect("git lifecycle mutex should not be poisoned: no code panics while holding this lock") = Some(sha);
*self.checkpoint_git_result.lock()
.expect("git lifecycle mutex should not be poisoned: no code panics while holding this lock") = Some(git_result);
}
Err(e) => {
let exec_output_tail = fabro_sandbox::default_redacted_output_tail(&e);
let error = e.to_string();
// Emit CheckpointFailed and return error
let scope = stage_scope_for(&self.stage_executions, state, node_id);
self.emitter.emit_scoped(
&Event::CheckpointFailed {
node_id: node_id.to_string(),
error: error.clone(),
exec_output_tail,
},
&scope,
);
return Err(CoreError::Other(format!(
"git checkpoint commit failed for node '{node_id}': {error}"
)));
}
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use std::collections::HashMap;
use std::path::Path;
use std::sync::Arc;
use std::time::Duration;
use fabro_core::graph::Graph as CoreGraph;
use fabro_core::lifecycle::RunLifecycle;
use fabro_core::state::ExecutionState;
use fabro_graphviz::graph::types::{AttrValue, Edge, Graph, Node};
use fabro_types::{WorkflowSettings, fixtures};
use super::*;
use crate::outcome::Outcome;
use crate::run_options::GitCheckpointOptions;
#[expect(
clippy::disallowed_methods,
reason = "checkpoint tests use synchronous git commands to set up temporary repositories"
)]
fn init_git_repo(repo: &Path) {
let init = std::process::Command::new("git")
.args(["init", "-b", "main"])
.current_dir(repo)
.output()
.unwrap();
assert!(init.status.success());
for (key, value) in [("user.name", "Test"), ("user.email", "test@test.com")] {
let config = std::process::Command::new("git")
.args(["config", key, value])
.current_dir(repo)
.output()
.unwrap();
assert!(config.status.success());
}
let commit = std::process::Command::new("git")
.args(["commit", "--allow-empty", "-m", "initial"])
.current_dir(repo)
.output()
.unwrap();
assert!(commit.status.success());
}
#[expect(
clippy::disallowed_methods,
reason = "checkpoint tests use synchronous git commands to set up temporary repositories"
)]
fn git_commit_all(repo: &Path, msg: &str) -> String {
let add = std::process::Command::new("git")
.args(["add", "."])
.current_dir(repo)
.output()
.unwrap();
assert!(add.status.success());
let commit = std::process::Command::new("git")
.args(["commit", "-m", msg])
.current_dir(repo)
.output()
.unwrap();
assert!(
commit.status.success(),
"git commit failed: {}",
String::from_utf8_lossy(&commit.stderr)
);
let rev_parse = std::process::Command::new("git")
.args(["rev-parse", "HEAD"])
.current_dir(repo)
.output()
.unwrap();
assert!(rev_parse.status.success());
String::from_utf8(rev_parse.stdout)
.unwrap()
.trim()
.to_string()
}
fn workflow_graph() -> WorkflowGraph {
let mut graph = Graph::new("checkpoint");
let mut start = Node::new("start");
start.attrs.insert(
"shape".to_string(),
AttrValue::String("Mdiamond".to_string()),
);
graph.nodes.insert("start".to_string(), start);
let mut build = Node::new("build");
build
.attrs
.insert("shape".to_string(), AttrValue::String("box".to_string()));
graph.nodes.insert("build".to_string(), build);
let mut exit = Node::new("exit");
exit.attrs.insert(
"shape".to_string(),
AttrValue::String("Msquare".to_string()),
);
graph.nodes.insert("exit".to_string(), exit);
graph.edges.push(Edge::new("start", "build"));
graph.edges.push(Edge::new("build", "exit"));
WorkflowGraph(Arc::new(graph))
}
fn run_options(run_dir: &Path) -> Arc<RunOptions> {
Arc::new(RunOptions {
settings: WorkflowSettings::default(),
run_dir: run_dir.to_path_buf(),
cancel_token: tokio_util::sync::CancellationToken::new(),
run_id: fixtures::RUN_1,
labels: HashMap::new(),
workflow_slug: Some("checkpoint".to_string()),
github_app: None,
pre_run_git: None,
fork_source_ref: None,
base_branch: None,
display_base_sha: None,
git_identity: None,
git: Some(GitCheckpointOptions {
base_sha: None,
run_branch: None,
}),
})
}
async fn git_lifecycle(
repo: &Path,
emitter: Arc<Emitter>,
run_options: Arc<RunOptions>,
) -> GitLifecycle {
GitLifecycle {
stage_executions: StageExecutionTracker::default(),
sandbox: Arc::new(
fabro_sandbox::local_sandbox(repo.to_path_buf())
.await
.unwrap(),
),
emitter,
run_id: fixtures::RUN_1,
run_options,
sandbox_git: Arc::new(SandboxGitRuntime::new()),
start_node_id: Some("start".to_string()),
checkpoint_git_result: Arc::new(Mutex::new(None)),
last_git_sha: Arc::new(Mutex::new(None)),
}
}
#[tokio::test]
async fn checkpoint_git_result_includes_diff_summary() {
let repo_dir = tempfile::tempdir().unwrap();
let repo = repo_dir.path();
init_git_repo(repo);
tokio::fs::write(repo.join("notes.txt"), "one\n")
.await
.unwrap();
let base = git_commit_all(repo, "base");
tokio::fs::write(repo.join("notes.txt"), "one\ntwo\n")
.await
.unwrap();
let mut options = run_options(repo).as_ref().clone();
options.git = Some(GitCheckpointOptions {
base_sha: Some(base),
run_branch: None,
});
let lifecycle = git_lifecycle(
repo,
Arc::new(Emitter::new(fixtures::RUN_1)),
Arc::new(options),
)
.await;
let graph = workflow_graph();
let node = graph.get_node("build").unwrap();
let mut state = ExecutionState::new(&graph).unwrap();
state.increment_visits("build");
let result = WfNodeResult::new(
Outcome::success(),
Duration::from_millis(10),
Duration::ZERO,
Duration::ZERO,
1,
1,
);
lifecycle
.on_checkpoint(&node, &result, Some("exit"), &state)
.await
.unwrap();
let git_result = lifecycle
.checkpoint_git_result
.lock()
.unwrap()
.clone()
.unwrap();
let diff_summary = git_result.diff_summary.expect("diff summary");
assert_eq!(diff_summary.files_changed, 1);
assert_eq!(diff_summary.additions, 1);
assert_eq!(diff_summary.deletions, 0);
tokio::fs::write(repo.join("notes.txt"), "one\ntwo\nthree\n")
.await
.unwrap();
state.increment_visits("build");
lifecycle
.on_checkpoint(&node, &result, Some("exit"), &state)
.await
.unwrap();
let git_result = lifecycle
.checkpoint_git_result
.lock()
.unwrap()
.clone()
.unwrap();
let diff_summary = git_result.diff_summary.expect("diff summary");
assert_eq!(diff_summary.files_changed, 1);
assert_eq!(diff_summary.additions, 2);
assert_eq!(diff_summary.deletions, 0);
}
#[tokio::test]
async fn checkpoint_git_result_omits_push_when_run_branch_push_disabled() {
let repo_dir = tempfile::tempdir().unwrap();
let repo = repo_dir.path();
init_git_repo(repo);
tokio::fs::write(repo.join("notes.txt"), "checkpoint\n")
.await
.unwrap();
let mut options = run_options(repo).as_ref().clone();
options.settings.run.run_branch.push = false;
options.git = Some(GitCheckpointOptions {
base_sha: None,
run_branch: Some("fabro/run/test".to_string()),
});
let lifecycle = git_lifecycle(
repo,
Arc::new(Emitter::new(fixtures::RUN_1)),
Arc::new(options),
)
.await;
let graph = workflow_graph();
let node = graph.get_node("build").unwrap();
let mut state = ExecutionState::new(&graph).unwrap();
state.increment_visits("build");
let result = WfNodeResult::new(
Outcome::success(),
Duration::from_millis(10),
Duration::ZERO,
Duration::ZERO,
1,
1,
);
lifecycle
.on_checkpoint(&node, &result, Some("exit"), &state)
.await
.unwrap();
let git_result = lifecycle
.checkpoint_git_result
.lock()
.unwrap()
.clone()
.unwrap();
assert!(git_result.commit_sha.is_some());
assert!(git_result.push_results.is_empty());
}
}

177
lib/components/fabro-workflow/src/lifecycle/hook.rs
View file

@ -1,177 +0,0 @@
use std::sync::Arc;
use async_trait::async_trait;
use fabro_core::error::{Error as CoreError, Result as CoreResult};
use fabro_core::lifecycle::{
AttemptContext, EdgeContext, EdgeDecision, NodeDecision, RunLifecycle,
};
use fabro_core::outcome::NodeResult;
use fabro_core::state::ExecutionState;
use fabro_hooks::{HookContext, HookDecision, HookEvent, HookExecutionContext, HookRunner};
use fabro_sandbox::RunSandbox;
use fabro_types::RunId;
use crate::graph::{WorkflowGraph, WorkflowNode};
use crate::hook_context::set_hook_node;
use crate::outcome::{ModelUsage, Outcome, OutcomeExt, StageOutcome};
type WfRunState = ExecutionState<Option<ModelUsage>>;
type WfNodeResult = NodeResult<Option<ModelUsage>>;
type WfNodeDecision = NodeDecision<Option<ModelUsage>>;
/// Sub-lifecycle responsible for running workflow hooks.
pub(crate) struct HookLifecycle {
pub hook_runner: Option<Arc<HookRunner>>,
pub sandbox: Arc<RunSandbox>,
pub hook_execution_context: HookExecutionContext,
pub run_id: RunId,
pub graph_name: String,
}
impl HookLifecycle {
async fn run_hook(&self, hook_ctx: &HookContext) -> HookDecision {
let Some(ref runner) = self.hook_runner else {
return HookDecision::Proceed;
};
runner
.run(
hook_ctx,
self.sandbox.clone(),
self.hook_execution_context.clone(),
)
.await
}
}
#[async_trait]
impl RunLifecycle<WorkflowGraph> for HookLifecycle {
async fn on_run_start(&self, _graph: &WorkflowGraph, _state: &WfRunState) -> CoreResult<()> {
let hook_ctx = HookContext::new(HookEvent::RunStart, self.run_id, self.graph_name.clone());
let decision = self.run_hook(&hook_ctx).await;
if let HookDecision::Block { reason } = decision {
let msg = reason.unwrap_or_else(|| "blocked by RunStart hook".into());
return Err(CoreError::blocked(msg));
}
Ok(())
}
async fn before_attempt(
&self,
ctx: &AttemptContext<'_, WorkflowGraph>,
_state: &WfRunState,
) -> CoreResult<WfNodeDecision> {
let gv = ctx.node.inner();
let mut hook_ctx =
HookContext::new(HookEvent::StageStart, self.run_id, self.graph_name.clone());
hook_ctx.cwd = self
.hook_execution_context
.sandbox_work_dir
.as_ref()
.map(|path| path.display().to_string());
set_hook_node(&mut hook_ctx, gv);
hook_ctx.attempt = Some(ctx.attempt as usize);
hook_ctx.max_attempts = Some(ctx.max_attempts as usize);
let decision = self.run_hook(&hook_ctx).await;
match decision {
HookDecision::Skip { reason } => {
let msg = reason.unwrap_or_else(|| "skipped by StageStart hook".into());
Ok(NodeDecision::Skip(Box::new(Outcome::skipped(&msg))))
}
HookDecision::Block { reason } => {
let msg = reason.unwrap_or_else(|| "blocked by StageStart hook".into());
Err(CoreError::blocked(msg))
}
_ => Ok(NodeDecision::Continue),
}
}
async fn after_node(
&self,
node: &WorkflowNode,
result: &mut WfNodeResult,
_state: &WfRunState,
) -> CoreResult<()> {
let outcome = &result.outcome;
// Skipped nodes had no StageStarted, so skip hooks (engine.rs:2080)
if outcome.status == StageOutcome::Skipped {
return Ok(());
}
let hook_event = if outcome.status.is_failure() {
HookEvent::StageFailed
} else {
HookEvent::StageComplete
};
let mut hook_ctx = HookContext::new(hook_event, self.run_id, self.graph_name.clone());
set_hook_node(&mut hook_ctx, node.inner());
hook_ctx.status = Some(outcome.status.to_string());
hook_ctx.failure_reason = outcome.failure_reason().map(String::from);
let _ = self.run_hook(&hook_ctx).await;
Ok(())
}
async fn on_edge_selected(
&self,
ctx: &EdgeContext<'_, WorkflowGraph>,
_state: &WfRunState,
) -> CoreResult<EdgeDecision> {
let mut hook_ctx = HookContext::new(
HookEvent::EdgeSelected,
self.run_id,
self.graph_name.clone(),
);
hook_ctx.edge_from = Some(ctx.from.to_string());
hook_ctx.edge_to = Some(ctx.to.to_string());
hook_ctx.edge_label = ctx
.edge
.as_ref()
.and_then(|edge| edge.inner().label().map(String::from));
let decision = self.run_hook(&hook_ctx).await;
match decision {
HookDecision::Override { edge_to } => Ok(EdgeDecision::Override(edge_to)),
HookDecision::Block { reason } => {
let msg = reason.unwrap_or_else(|| "blocked by EdgeSelected hook".into());
Err(CoreError::blocked(msg))
}
_ => Ok(EdgeDecision::Continue),
}
}
async fn on_checkpoint(
&self,
node: &WorkflowNode,
_result: &WfNodeResult,
_next_node_id: Option<&str>,
_state: &WfRunState,
) -> CoreResult<()> {
let mut hook_ctx = HookContext::new(
HookEvent::CheckpointSaved,
self.run_id,
self.graph_name.clone(),
);
hook_ctx.node_id = Some(node.inner().id.clone());
let _ = self.run_hook(&hook_ctx).await;
Ok(())
}
async fn on_run_end(&self, outcome: &Outcome, state: &WfRunState) {
if state.cancelled {
return;
}
if outcome.status == StageOutcome::Succeeded
|| outcome.status == StageOutcome::PartiallySucceeded
{
let hook_ctx =
HookContext::new(HookEvent::RunComplete, self.run_id, self.graph_name.clone());
let _ = self.run_hook(&hook_ctx).await;
} else {
let error_msg = outcome
.failure
.as_ref()
.map_or_else(|| "run failed".to_string(), |f| f.message.clone());
let mut hook_ctx =
HookContext::new(HookEvent::RunFailed, self.run_id, self.graph_name.clone());
hook_ctx.failure_reason = Some(error_msg);
let _ = self.run_hook(&hook_ctx).await;
}
}
}

445
lib/components/fabro-workflow/src/lifecycle/mod.rs
View file

@ -1,445 +0,0 @@
pub(crate) mod artifact;
pub(crate) mod circuit_breaker;
pub(crate) mod event;
pub(crate) mod fidelity;
pub(crate) mod git;
pub(crate) mod hook;
use std::collections::HashMap;
use std::path::Path;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use std::time::Instant;
use async_trait::async_trait;
use fabro_core::error::{Error as CoreError, Result as CoreResult};
use fabro_core::graph::NodeSpec;
use fabro_core::lifecycle::{
AttemptContext, AttemptResultContext, EdgeContext, EdgeDecision, NodeDecision, RunLifecycle,
};
use fabro_core::outcome::NodeResult;
use fabro_core::state::ExecutionState;
use fabro_graphviz::graph::types::Graph as GvGraph;
use fabro_hooks::HookRunner;
use fabro_sandbox::RunSandbox;
use fabro_types::RunId;
use self::artifact::ArtifactLifecycle;
use self::circuit_breaker::CircuitBreakerLifecycle;
use self::event::EventLifecycle;
use self::fidelity::FidelityLifecycle;
use self::git::{GitCheckpointResult, GitLifecycle};
use self::hook::HookLifecycle;
use crate::artifact_upload::ArtifactSink;
use crate::context;
use crate::error::FailureSignature;
use crate::event::Emitter;
use crate::graph::{WorkflowGraph, WorkflowNode};
use crate::outcome::{ModelUsage, Outcome};
use crate::run_control::RunControlState;
use crate::run_options::RunOptions;
use crate::runtime_store::RunStoreHandle;
use crate::sandbox_git_runtime::SandboxGitRuntime;
use crate::services::RunLocations;
use crate::stage_execution::StageExecutionTracker;
type WfRunState = ExecutionState<Option<ModelUsage>>;
type WfNodeResult = NodeResult<Option<ModelUsage>>;
type WfNodeDecision = NodeDecision<Option<ModelUsage>>;
/// Orchestrates all sub-lifecycles with explicit per-callback ordering.
/// Implements `RunLifecycle<WorkflowGraph>` by delegating to focused structs.
pub(crate) struct WorkflowLifecycle {
event: EventLifecycle,
hook: HookLifecycle,
fidelity: FidelityLifecycle,
circuit_breaker: Arc<CircuitBreakerLifecycle>,
git: GitLifecycle,
artifact: ArtifactLifecycle,
sandbox: Arc<RunSandbox>,
on_node: crate::OnNodeCallback,
emitter: Arc<Emitter>,
run_control: Option<Arc<RunControlState>>,
/// Set in on_edge_selected when loop_restart approved; read+cleared by
/// EventLifecycle::on_run_start
restarted_from: Arc<Mutex<Option<(String, String)>>>,
/// Shared git checkpoint result (written by git, read by event)
checkpoint_git_result: Arc<Mutex<Option<GitCheckpointResult>>>,
/// True when constructed with a checkpoint; cleared after first
/// on_run_start. Gates context seeding on initial resume.
is_initial_resume: AtomicBool,
/// Run-scoped stage execution allocator shared with `RunServices`.
stage_executions: StageExecutionTracker,
// Config needed for context seeding
graph: Arc<GvGraph>,
run_id: RunId,
sandbox_work_dir: Option<String>,
}
impl WorkflowLifecycle {
#[allow(
clippy::too_many_arguments,
reason = "Workflow startup wires many run-scoped collaborators at once."
)]
pub(crate) fn new(
emitter: &Arc<Emitter>,
hook_runner: Option<Arc<HookRunner>>,
sandbox: &Arc<RunSandbox>,
graph: Arc<GvGraph>,
run_dir: &Path,
run_store: &RunStoreHandle,
artifact_sink: Option<ArtifactSink>,
locations: &RunLocations,
run_options: &Arc<RunOptions>,
sandbox_git: Arc<SandboxGitRuntime>,
is_resume: bool,
on_node: crate::OnNodeCallback,
run_control: Option<Arc<RunControlState>>,
stage_executions: StageExecutionTracker,
) -> Self {
let restarted_from: Arc<Mutex<Option<(String, String)>>> = Arc::new(Mutex::new(None));
let loop_restart_signature_limit = graph.loop_restart_signature_limit();
let checkpoint_git_result: Arc<Mutex<Option<GitCheckpointResult>>> =
Arc::new(Mutex::new(None));
let last_git_sha: Arc<Mutex<Option<String>>> = Arc::new(Mutex::new(None));
let circuit_breaker = Arc::new(CircuitBreakerLifecycle::new(loop_restart_signature_limit));
let has_run_branch = run_options
.git
.as_ref()
.and_then(|g| g.run_branch.as_ref())
.is_some();
let run_branch_sandbox_work_dir = if has_run_branch {
locations
.sandbox_work_dir
.as_ref()
.map(|path| path.display().to_string())
} else {
None
};
let event = EventLifecycle {
emitter: Arc::clone(emitter),
graph_name: graph.name.clone(),
run_id: run_options.run_id,
run_start: Mutex::new(Instant::now()),
restarted_from: Arc::clone(&restarted_from),
base_branch: run_options.base_branch.clone(),
base_sha: run_options.git.as_ref().and_then(|g| g.base_sha.clone()),
run_branch: run_options.git.as_ref().and_then(|g| g.run_branch.clone()),
worktree_dir: run_branch_sandbox_work_dir.clone(),
goal: (!graph.goal().is_empty()).then(|| graph.goal().to_string()),
checkpoint_git_result: Arc::clone(&checkpoint_git_result),
circuit_breaker: Arc::clone(&circuit_breaker),
stage_executions: stage_executions.clone(),
};
let hook = HookLifecycle {
hook_runner,
sandbox: Arc::clone(sandbox),
hook_execution_context: locations.hook_execution_context(),
run_id: run_options.run_id,
graph_name: graph.name.clone(),
};
let fidelity = FidelityLifecycle::new(
Arc::clone(&graph),
Arc::clone(sandbox),
run_store.clone(),
run_dir.to_path_buf(),
);
let start_node_id = graph.find_start_node().map(|n| n.id.clone());
let git = GitLifecycle {
sandbox: Arc::clone(sandbox),
emitter: Arc::clone(emitter),
run_id: run_options.run_id,
run_options: Arc::clone(run_options),
sandbox_git,
start_node_id,
checkpoint_git_result: Arc::clone(&checkpoint_git_result),
last_git_sha,
stage_executions: stage_executions.clone(),
};
let artifact = ArtifactLifecycle::new(
Arc::clone(sandbox),
run_store.clone(),
Arc::clone(emitter),
run_options.run_id,
run_options.artifact_glob_patterns(),
artifact_sink,
stage_executions.clone(),
);
Self {
event,
hook,
fidelity,
circuit_breaker,
git,
artifact,
sandbox: Arc::clone(sandbox),
on_node,
emitter: Arc::clone(emitter),
run_control,
restarted_from,
checkpoint_git_result,
is_initial_resume: AtomicBool::new(is_resume),
stage_executions,
graph,
run_id: run_options.run_id,
sandbox_work_dir: run_branch_sandbox_work_dir,
}
}
/// Restore circuit breaker state from a checkpoint (for resume).
pub(crate) fn restore_circuit_breaker(
&self,
loop_sigs: HashMap<FailureSignature, usize>,
restart_sigs: HashMap<FailureSignature, usize>,
) {
self.circuit_breaker.restore(loop_sigs, restart_sigs);
}
/// Set the fidelity degradation flag for checkpoint resume.
pub(crate) fn set_degrade_fidelity_on_resume(&self, flag: bool) {
self.fidelity.set_degrade_fidelity_on_resume(flag);
}
}
#[async_trait]
impl RunLifecycle<WorkflowGraph> for WorkflowLifecycle {
async fn on_run_start(&self, graph: &WorkflowGraph, state: &WfRunState) -> CoreResult<()> {
// Re-seed context keys (fires on initial start AND after every loop restart).
// Skip on initial checkpoint resume (context already has them).
if self.is_initial_resume.swap(false, Ordering::Relaxed) {
// First on_run_start after checkpoint resume — skip context seeding
} else {
// Mirror graph-level attributes into the core context
if !self.graph.goal().is_empty() {
state.context.set(
context::keys::GRAPH_GOAL,
serde_json::json!(self.graph.goal()),
);
}
for (key, val) in &self.graph.attrs {
state.context.set(
context::keys::graph_attr_key(key),
serde_json::json!(val.to_string_value()),
);
}
}
// Always set run_id and work_dir (idempotent)
state.context.set(
context::keys::INTERNAL_RUN_ID,
serde_json::json!(self.run_id),
);
if let Some(ref wd) = self.sandbox_work_dir {
state
.context
.set(context::keys::INTERNAL_WORK_DIR, serde_json::json!(wd));
}
// Reset restart-scoped state
self.fidelity.on_run_start(graph, state).await?;
self.artifact.on_run_start(graph, state).await?;
// Observable callbacks
self.event.on_run_start(graph, state).await?;
self.hook.on_run_start(graph, state).await?;
self.git.on_run_start(graph, state).await?;
Ok(())
}
async fn on_terminal_reached(
&self,
node: &WorkflowNode,
goal_gates_passed: bool,
state: &WfRunState,
) {
self.event
.on_terminal_reached(node, goal_gates_passed, state)
.await;
}
async fn before_node(
&self,
node: &WorkflowNode,
state: &WfRunState,
) -> CoreResult<WfNodeDecision> {
if let Some(run_control) = &self.run_control {
run_control.wait_if_paused(self.emitter.as_ref()).await;
}
// A provider may auto-stop while the run is paused between nodes.
self.sandbox.activate().await.map_err(|err| {
CoreError::context(
format!("failed to activate sandbox before node {}", node.id()),
err,
)
})?;
if let Some(on_node) = &self.on_node {
on_node(node.id());
}
// Node boundary: clear the prior execution scope so the next
// observable attempt reserves a fresh ordinal. No reservation happens
// here — a hook block or process exit before any stage-scoped event
// must not consume an ordinal.
self.stage_executions.begin_node(node.id());
state.context.set(
context::keys::INTERNAL_STAGE_EXECUTION_ORDINAL,
serde_json::Value::Null,
);
self.fidelity.before_node(node, state).await
}
async fn before_attempt(
&self,
ctx: &AttemptContext<'_, WorkflowGraph>,
state: &WfRunState,
) -> CoreResult<WfNodeDecision> {
// Hook first (can skip/block)
match self.hook.before_attempt(ctx, state).await? {
NodeDecision::Continue => {}
decision => return Ok(decision),
}
// Reserve the stage execution once per handler invocation: the first
// attempt allocates the ordinal and automatic retries reuse it.
let node_id = ctx.node.id();
let execution = self
.stage_executions
.ensure(node_id, event::stage_visit(state, node_id));
state.context.set(
context::keys::INTERNAL_STAGE_EXECUTION_ORDINAL,
serde_json::json!(execution.stage_id.visit()),
);
// Event emission
self.event.before_attempt(ctx, state).await?;
// Record epoch AFTER hook+event (engine.rs:968→1006)
self.artifact.before_attempt(ctx, state).await?;
Ok(NodeDecision::Continue)
}
async fn after_attempt(
&self,
ctx: &AttemptResultContext<'_, WorkflowGraph>,
state: &WfRunState,
) -> CoreResult<()> {
if let Some(run_control) = &self.run_control {
run_control.wait_if_paused(self.emitter.as_ref()).await;
}
// Human, wait, and paused stages can return after a long period with
// no sandbox traffic. Reactivate before artifact and checkpoint work.
self.sandbox.activate().await.map_err(|err| {
CoreError::context(
format!(
"failed to activate sandbox after node attempt {}",
ctx.node.id()
),
err,
)
})?;
self.artifact.after_attempt(ctx, state).await?;
self.event.after_attempt(ctx, state).await?;
Ok(())
}
async fn after_node(
&self,
node: &WorkflowNode,
result: &mut WfNodeResult,
state: &WfRunState,
) -> CoreResult<()> {
self.circuit_breaker.after_node(node, result, state).await?;
self.artifact.after_node(node, result, state).await?;
self.event.after_node(node, result, state).await?;
self.hook.after_node(node, result, state).await?;
Ok(())
}
async fn after_record(
&self,
node: &WorkflowNode,
result: &WfNodeResult,
state: &WfRunState,
) -> CoreResult<()> {
let retry_count = state.node_retries.get(node.id()).copied().unwrap_or(0);
context::apply_recorded_outcome_context(
&state.context,
node.id(),
&result.outcome,
retry_count,
);
Ok(())
}
async fn on_edge_selected(
&self,
ctx: &EdgeContext<'_, WorkflowGraph>,
state: &WfRunState,
) -> CoreResult<EdgeDecision> {
// Fidelity captures edge data
self.fidelity.on_edge_selected(ctx, state).await?;
// Event always fires first
self.event.on_edge_selected(ctx, state).await?;
// Hook can override/block
match self.hook.on_edge_selected(ctx, state).await? {
EdgeDecision::Continue => {
// Edge unchanged — check circuit breaker for loop_restart
let decision = self.circuit_breaker.on_edge_selected(ctx, state).await?;
// If loop_restart edge approved by both hook and circuit breaker, mark for
// LoopRestart emission
if matches!(decision, EdgeDecision::Continue) {
if let Some(ref edge) = ctx.edge {
if edge.inner().loop_restart() {
*self.restarted_from.lock()
.expect("lifecycle mutex should not be poisoned: no code panics while holding this lock") =
Some((ctx.from.to_string(), ctx.to.to_string()));
}
}
}
Ok(decision)
}
decision => Ok(decision), // Override/Block — skip circuit breaker
}
}
async fn on_checkpoint(
&self,
node: &WorkflowNode,
result: &WfNodeResult,
next_node_id: Option<&str>,
state: &WfRunState,
) -> CoreResult<()> {
// A StageStart hook can skip before any attempt reserved an execution
// scope. Ensure one exists so the `checkpoint.completed` envelope
// attaches to a concrete execution;
// an existing reservation from the attempt path is reused as-is.
let execution = self
.stage_executions
.ensure(node.id(), event::stage_visit(state, node.id()));
state.context.set(
context::keys::INTERNAL_STAGE_EXECUTION_ORDINAL,
serde_json::json!(execution.stage_id.visit()),
);
self.git
.on_checkpoint(node, result, next_node_id, state)
.await?;
self.event
.on_checkpoint(node, result, next_node_id, state)
.await?;
self.hook
.on_checkpoint(node, result, next_node_id, state)
.await?;
// Clear checkpoint result for next checkpoint
*self.checkpoint_git_result.lock().expect(
"lifecycle mutex should not be poisoned: no code panics while holding this lock",
) = None;
Ok(())
}
async fn on_run_end(&self, outcome: &Outcome, state: &WfRunState) {
self.hook.on_run_end(outcome, state).await;
}
}

589
lib/components/fabro-workflow/src/model_fallback.rs
View file

@ -1,589 +0,0 @@
use std::collections::{BTreeMap, HashMap, HashSet};
use fabro_llm::lithos_catalog::{Catalog, Offering};
use fabro_llm::{FallbackTarget, ModelSelectionError, selection};
use fabro_types::settings::{ModelRef, ResolvedModelRef};
use fabro_types::{RunNoticeCode, RunNoticeLevel};
use lithos_llm::catalog::ProviderId;
use lithos_llm::types::ReasoningEffort;
use crate::Error;
/// Catalog-resolved fallback chains keyed by canonical requested model ID.
///
/// A chain is selected from the original request only. Targets never cause
/// another chain lookup.
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct ModelFallbackPolicy {
chains: BTreeMap<String, Vec<FallbackTarget>>,
}
impl ModelFallbackPolicy {
#[cfg(test)]
#[must_use]
pub fn new(chains: BTreeMap<String, Vec<FallbackTarget>>) -> Self {
Self { chains }
}
#[must_use]
pub fn chain_for<'a>(
&'a self,
catalog: &Catalog,
provider: &ProviderId,
model: &str,
) -> Option<&'a [FallbackTarget]> {
self.chain_for_canonical(&canonical_model_id(catalog, provider, model))
}
/// Look up a chain by an already-canonicalized requested model ID.
#[must_use]
pub fn chain_for_canonical(&self, canonical_model: &str) -> Option<&[FallbackTarget]> {
self.chains.get(canonical_model).map(Vec::as_slice)
}
pub fn iter(&self) -> impl Iterator<Item = (&str, &[FallbackTarget])> {
self.chains
.iter()
.map(|(model, chain)| (model.as_str(), chain.as_slice()))
}
#[must_use]
pub fn len(&self) -> usize {
self.chains.len()
}
#[must_use]
pub fn is_empty(&self) -> bool {
self.chains.is_empty()
}
}
/// Server-side result of canonicalizing and filtering configured fallback
/// chains.
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct ResolvedModelFallbacks {
pub policy: ModelFallbackPolicy,
pub notices: Vec<ModelFallbackNotice>,
}
/// Why a configured fallback candidate was removed from one model's chain.
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum ModelFallbackNotice {
ProviderUnconfigured {
requested_model: String,
reference: ModelRef,
provider: ProviderId,
},
NoConfiguredOffering {
requested_model: String,
reference: ModelRef,
providers: Vec<ProviderId>,
},
PrimaryNotInCatalog {
requested_model: String,
reference: ModelRef,
primary: FallbackTarget,
},
NoCompatibleModel {
requested_model: String,
reference: ModelRef,
provider: ProviderId,
},
Duplicate {
requested_model: String,
reference: ModelRef,
target: FallbackTarget,
},
NoNearbyReasoningLevel {
requested_model: String,
target: FallbackTarget,
requested_effort: ReasoningEffort,
},
ChainEmpty {
requested_model: String,
},
}
impl ModelFallbackNotice {
#[must_use]
pub fn code(&self) -> RunNoticeCode {
match self {
Self::ChainEmpty { .. } => RunNoticeCode::ModelFallbackChainEmpty,
Self::ProviderUnconfigured { .. }
| Self::NoConfiguredOffering { .. }
| Self::PrimaryNotInCatalog { .. }
| Self::NoCompatibleModel { .. }
| Self::Duplicate { .. }
| Self::NoNearbyReasoningLevel { .. } => RunNoticeCode::ModelFallbackSkipped,
}
}
#[must_use]
pub fn level(&self) -> RunNoticeLevel {
match self {
Self::Duplicate { .. } => RunNoticeLevel::Info,
Self::ProviderUnconfigured { .. }
| Self::NoConfiguredOffering { .. }
| Self::PrimaryNotInCatalog { .. }
| Self::NoCompatibleModel { .. }
| Self::NoNearbyReasoningLevel { .. }
| Self::ChainEmpty { .. } => RunNoticeLevel::Warn,
}
}
#[must_use]
pub fn message(&self) -> String {
match self {
Self::ProviderUnconfigured {
requested_model,
reference,
provider,
} => format!(
"Model fallback `{reference}` for requested model `{requested_model}` was skipped because provider `{provider}` is not configured."
),
Self::NoConfiguredOffering {
requested_model,
reference,
providers,
} => {
let providers = providers
.iter()
.map(ProviderId::to_string)
.collect::<Vec<_>>()
.join(", ");
format!(
"Model fallback `{reference}` for requested model `{requested_model}` was skipped because none of its providers are configured. It is offered by: {providers}."
)
}
Self::PrimaryNotInCatalog {
requested_model,
reference,
primary,
} => format!(
"Model fallback `{reference}` for requested model `{requested_model}` was skipped because `{primary}` is not in the catalog, so there is no capability profile to match against."
),
Self::NoCompatibleModel {
requested_model,
reference,
provider,
} => format!(
"Model fallback `{reference}` for requested model `{requested_model}` was skipped because provider `{provider}` has no compatible model."
),
Self::Duplicate {
requested_model,
reference,
target,
} => format!(
"Model fallback `{reference}` for requested model `{requested_model}` was skipped because target `{target}` already appears in that chain."
),
Self::NoNearbyReasoningLevel {
requested_model,
target,
requested_effort,
} => {
format!(
"Model fallback `{target}` for requested model `{requested_model}` was skipped because it has no reasoning level near `{requested_effort}`."
)
}
Self::ChainEmpty { requested_model } => format!(
"No usable model fallbacks remain for requested model `{requested_model}` after filtering its configured candidates."
),
}
}
}
/// Resolve every model-keyed fallback chain against the server's catalog and
/// configured-provider snapshot.
///
/// This function must stay at server-side call sites. Offline validation only
/// parses the raw table and cannot canonicalize model aliases.
pub fn resolve_model_fallbacks(
catalog: &Catalog,
configured_providers: &[ProviderId],
configured: &BTreeMap<String, Vec<ModelRef>>,
) -> Result<ResolvedModelFallbacks, Error> {
let eligible = configured_providers.iter().cloned().collect::<HashSet<_>>();
let mut resolved = ResolvedModelFallbacks::default();
let mut raw_key_by_canonical = HashMap::<String, String>::new();
for (raw_key, references) in configured {
require_bare_model_key(catalog, raw_key)?;
let selected = selection::resolve_selection_with_catalog_fallback(
catalog,
Some(raw_key),
None,
&eligible,
)?;
let requested_model = selected.model;
if let Some(previous) =
raw_key_by_canonical.insert(requested_model.clone(), raw_key.clone())
{
return Err(Error::Precondition(format!(
"`run.model.fallbacks` keys `{previous}` and `{raw_key}` both resolve to requested model `{requested_model}`"
)));
}
let primary = FallbackTarget::new(&selected.provider, &requested_model);
let primary_model = catalog
.enabled_provider(selected.provider.as_str())
.and_then(|provider| provider.offering(&requested_model));
let mut targets = Vec::new();
for model_ref in references {
let target = match resolve_fallback_candidate(
catalog,
&requested_model,
&primary,
primary_model.as_ref(),
&eligible,
model_ref,
)? {
FallbackCandidate::Skipped(notice) => {
resolved.notices.push(notice);
continue;
}
FallbackCandidate::Target(target) => target,
};
if targets.contains(&target) {
resolved.notices.push(ModelFallbackNotice::Duplicate {
requested_model: requested_model.clone(),
reference: model_ref.clone(),
target,
});
} else {
targets.push(target);
}
}
if targets.is_empty() {
resolved.notices.push(ModelFallbackNotice::ChainEmpty {
requested_model: requested_model.clone(),
});
}
resolved.policy.chains.insert(requested_model, targets);
}
Ok(resolved)
}
/// Reject chain keys that name a provider. Keys are requested-model selectors;
/// a provider-qualified key can never match a dispatch-time canonical model
/// ID, so it would be silently dead configuration.
fn require_bare_model_key(catalog: &Catalog, raw_key: &str) -> Result<(), Error> {
let reference: ModelRef = raw_key
.parse()
.map_err(|error| Error::Precondition(format!("`run.model.fallbacks` key: {error}")))?;
match reference.resolve(catalog) {
Ok(ResolvedModelRef::Model { provider: None, .. }) => Ok(()),
Ok(ResolvedModelRef::Model {
provider: Some(_),
selector,
}) => Err(Error::Precondition(format!(
"`run.model.fallbacks` keys name a requested model; use `{selector}` instead of `{raw_key}`"
))),
Ok(ResolvedModelRef::Provider(provider)) => Err(Error::Precondition(format!(
"`run.model.fallbacks` key `{raw_key}` names provider `{provider}`; keys must name a requested model"
))),
Err(ambiguous) => Err(Error::Precondition(format!(
"`run.model.fallbacks` key: {ambiguous}"
))),
}
}
enum FallbackCandidate {
Target(FallbackTarget),
Skipped(ModelFallbackNotice),
}
/// The catalog id for `selector` on `provider`, else anywhere; the selector
/// itself for a passthrough model the catalog does not know.
pub(crate) fn canonical_model_id(
catalog: &Catalog,
provider: &ProviderId,
selector: &str,
) -> String {
catalog
.canonical_model_id(Some(provider), selector)
.map_or_else(|| selector.to_string(), ToString::to_string)
}
fn resolve_fallback_candidate(
catalog: &Catalog,
requested_model: &str,
primary: &FallbackTarget,
primary_model: Option<&Offering<'_>>,
eligible: &HashSet<ProviderId>,
model_ref: &ModelRef,
) -> Result<FallbackCandidate, Error> {
let reference = model_ref.clone();
Ok(match model_ref.resolve(catalog)? {
ResolvedModelRef::Provider(provider_name) => {
let provider = selection::require_provider(catalog, &provider_name)?;
if !eligible.contains(&provider) {
return Ok(FallbackCandidate::Skipped(
ModelFallbackNotice::ProviderUnconfigured {
requested_model: requested_model.to_string(),
reference,
provider,
},
));
}
let Some(primary_model) = primary_model else {
return Ok(FallbackCandidate::Skipped(
ModelFallbackNotice::PrimaryNotInCatalog {
requested_model: requested_model.to_string(),
reference,
primary: primary.clone(),
},
));
};
match catalog
.enabled_provider(provider.as_str())
.and_then(|target| target.closest_offering(primary_model.model))
{
Some(entry) => {
FallbackCandidate::Target(FallbackTarget::new(provider, entry.model.id()))
}
None => FallbackCandidate::Skipped(ModelFallbackNotice::NoCompatibleModel {
requested_model: requested_model.to_string(),
reference,
provider,
}),
}
}
ResolvedModelRef::Model {
provider: Some(provider_name),
selector,
} => {
let provider = selection::require_provider(catalog, &provider_name)?;
if !eligible.contains(&provider) {
return Ok(FallbackCandidate::Skipped(
ModelFallbackNotice::ProviderUnconfigured {
requested_model: requested_model.to_string(),
reference,
provider,
},
));
}
match selection::resolve_on_provider(catalog, &provider, &selector) {
Ok(entry) => FallbackCandidate::Target(FallbackTarget::new(
entry.provider.id(),
entry.model.id(),
)),
Err(ModelSelectionError::UnknownSelectorOnProvider { .. }) => {
FallbackCandidate::Target(FallbackTarget::new(provider, selector))
}
Err(error) => return Err(error.into()),
}
}
ResolvedModelRef::Model {
provider: None,
selector,
} => match selection::select(catalog, &selector, None, eligible) {
Ok(entry) => FallbackCandidate::Target(FallbackTarget::new(
entry.provider.id(),
entry.model.id(),
)),
Err(ModelSelectionError::NoEligibleOffering { providers, .. }) => {
FallbackCandidate::Skipped(ModelFallbackNotice::NoConfiguredOffering {
requested_model: requested_model.to_string(),
reference,
providers,
})
}
Err(ModelSelectionError::UnknownSelector { .. }) => {
FallbackCandidate::Target(FallbackTarget::new(&primary.provider, selector))
}
Err(error) => return Err(error.into()),
},
})
}
#[cfg(test)]
mod tests {
use std::collections::BTreeMap;
use fabro_llm::FallbackTarget;
use fabro_llm::lithos_catalog::Catalog;
use fabro_llm::test_support::test_catalog_with_overlay;
use lithos_llm::catalog::ProviderId;
use super::{ModelFallbackNotice, resolve_model_fallbacks};
fn references(values: &[&str]) -> Vec<fabro_types::settings::ModelRef> {
values
.iter()
.map(|value| value.parse().expect("fixture reference should parse"))
.collect()
}
fn openrouter_catalog() -> Catalog {
test_catalog_with_overlay("[providers.openrouter]\nenabled = true\n")
}
#[test]
fn canonicalizes_keys_and_keeps_each_chain_independent() {
let catalog = openrouter_catalog();
let eligible = [ProviderId::new("openrouter")];
let configured = BTreeMap::from([
("gpt-sol".to_string(), references(&["claude-opus"])),
(
"claude-fable".to_string(),
references(&["gpt-sol", "claude-opus"]),
),
]);
let resolved = resolve_model_fallbacks(&catalog, &eligible, &configured).unwrap();
assert_eq!(
resolved
.policy
.chain_for(&catalog, &ProviderId::new("openrouter"), "gpt-sol"),
Some([FallbackTarget::new("openrouter", "claude-opus-5")].as_slice())
);
assert_eq!(
resolved
.policy
.chain_for(&catalog, &ProviderId::new("openrouter"), "claude-fable"),
Some(
[
FallbackTarget::new("openrouter", "gpt-5.6-sol"),
FallbackTarget::new("openrouter", "claude-opus-5"),
]
.as_slice()
)
);
}
#[test]
fn rejects_aliases_that_define_the_same_requested_model_twice() {
let catalog = openrouter_catalog();
let eligible = [ProviderId::new("openrouter")];
let configured = BTreeMap::from([
("gpt-sol".to_string(), references(&["claude-opus"])),
("gpt-5.6-sol".to_string(), references(&["claude-fable"])),
]);
let error = resolve_model_fallbacks(&catalog, &eligible, &configured).unwrap_err();
assert!(
error
.to_string()
.contains("both resolve to requested model"),
"unexpected error: {error}"
);
}
#[test]
fn rejects_provider_qualified_keys() {
let catalog = openrouter_catalog();
let eligible = [ProviderId::new("openrouter")];
let configured = BTreeMap::from([(
"openrouter:gpt-sol".to_string(),
references(&["claude-opus"]),
)]);
let error = resolve_model_fallbacks(&catalog, &eligible, &configured).unwrap_err();
assert!(
error.to_string().contains("keys name a requested model"),
"unexpected error: {error}"
);
}
#[test]
fn skips_unconfigured_candidates_per_requested_model() {
let catalog = openrouter_catalog();
let eligible = [ProviderId::new("openrouter")];
let configured = BTreeMap::from([(
"kimi-k3".to_string(),
references(&["moonshot:kimi-k3", "openrouter:kimi-k3"]),
)]);
let resolved = resolve_model_fallbacks(&catalog, &eligible, &configured).unwrap();
assert_eq!(
resolved
.policy
.chain_for(&catalog, &ProviderId::new("openrouter"), "kimi-k3"),
Some([FallbackTarget::new("openrouter", "kimi-k3")].as_slice())
);
assert!(matches!(
resolved.notices.as_slice(),
[ModelFallbackNotice::ProviderUnconfigured {
requested_model,
provider,
..
}] if requested_model == "kimi-k3" && provider == &ProviderId::new("moonshot")
));
}
#[test]
fn resolves_the_requested_production_policy_as_independent_chains() {
let catalog = test_catalog_with_overlay(
"[providers.modal]\nenabled = true\n\n[providers.openrouter]\nenabled = true\n",
);
let eligible = [
ProviderId::new("modal"),
ProviderId::new("moonshot"),
ProviderId::new("openrouter"),
];
let configured = BTreeMap::from([
(
"kimi-k3".to_string(),
references(&["moonshot:kimi-k3", "openrouter:kimi-k3", "claude-opus"]),
),
("glm-5.2".to_string(), references(&["gpt-sol"])),
("gpt-sol".to_string(), references(&["claude-opus"])),
("claude-opus".to_string(), references(&["gpt-sol"])),
("gpt-terra".to_string(), references(&["claude-opus"])),
("gpt-luna".to_string(), references(&["claude-sonnet"])),
(
"claude-fable".to_string(),
references(&["gpt-sol", "claude-opus"]),
),
]);
let resolved = resolve_model_fallbacks(&catalog, &eligible, &configured).unwrap();
assert!(resolved.notices.is_empty());
let chain = |model: &str| {
resolved
.policy
.chain_for(&catalog, &ProviderId::new("openrouter"), model)
.expect("requested model should have a chain")
};
assert_eq!(chain("kimi-k3"), [
FallbackTarget::new("moonshot", "kimi-k3"),
FallbackTarget::new("openrouter", "kimi-k3"),
FallbackTarget::new("openrouter", "claude-opus-5"),
]);
assert_eq!(chain("glm-5.2"), [FallbackTarget::new(
"openrouter",
"gpt-5.6-sol"
)]);
assert_eq!(chain("gpt-sol"), [FallbackTarget::new(
"openrouter",
"claude-opus-5"
)]);
assert_eq!(chain("claude-opus"), [FallbackTarget::new(
"openrouter",
"gpt-5.6-sol"
)]);
assert_eq!(chain("gpt-terra"), [FallbackTarget::new(
"openrouter",
"claude-opus-5"
)]);
assert_eq!(chain("gpt-luna"), [FallbackTarget::new(
"openrouter",
"claude-sonnet-5"
)]);
assert_eq!(chain("claude-fable"), [
FallbackTarget::new("openrouter", "gpt-5.6-sol"),
FallbackTarget::new("openrouter", "claude-opus-5"),
]);
}
}

359
lib/components/fabro-workflow/src/node_handler.rs
View file

@ -1,359 +0,0 @@
use std::future::Future;
use std::panic::AssertUnwindSafe;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::Duration;
use async_trait::async_trait;
use fabro_core::error::{Error as CoreError, HandlerErrorDetail, Result as CoreResult};
use fabro_core::handler::NodeHandler;
use fabro_core::outcome::FailureCategory;
use fabro_core::retry::RetryPolicy as CoreRetryPolicy;
use fabro_graphviz::graph::types::{Graph as GvGraph, Node as GvNode};
use fabro_types::{StageId, SystemActorKind};
use futures::FutureExt;
use tokio::sync::watch;
use tokio::time::{Instant, sleep, timeout};
use crate::artifact;
use crate::context::Context;
use crate::error::Error;
use crate::event::StageScope;
use crate::graph::{WorkflowGraph, WorkflowNode};
use crate::handler::{EngineServices, NodeTimeoutPolicy, dispatch_handler, format_panic_message};
use crate::interview_runtime::InterviewBlockState;
use crate::outcome::{FailureDetail, Outcome, StageOutcome};
use crate::retry::build_retry_policy;
/// Runs `future` under a `duration` budget that only counts time when this
/// stage is not waiting on human input. A sibling stage's interview does not
/// pause this budget — the wait is keyed by `stage_id`.
///
/// Returns `None` if the budget runs out first.
async fn timeout_excluding_interview_wait<F>(
duration: Duration,
stage_id: &StageId,
mut interview_blocks: watch::Receiver<InterviewBlockState>,
future: F,
) -> Option<F::Output>
where
F: Future,
{
tokio::pin!(future);
let mut remaining = duration;
loop {
let blocked = interview_blocks
.borrow_and_update()
.is_stage_blocked(stage_id);
let active_started = Instant::now();
tokio::select! {
biased;
output = &mut future => return Some(output),
changed = interview_blocks.changed() => {
if changed.is_err() {
// The blocker outlives every handler. If it ever goes away,
// fall back to a plain deadline rather than spinning.
return timeout(remaining, future).await.ok();
}
if !blocked {
remaining = remaining.saturating_sub(active_started.elapsed());
}
}
() = sleep(remaining), if !blocked => return None,
}
}
}
/// Production node handler that bridges fabro-core's NodeHandler to the
/// existing fabro-workflow Handler trait via EngineServices.
///
/// On each `execute()` call, forks the context, runs the handler,
/// then diffs and applies changes back.
pub(crate) struct WorkflowNodeHandler {
pub services: Arc<EngineServices>,
pub run_dir: PathBuf,
pub graph: Arc<GvGraph>,
}
/// Execute one handler attempt through the workflow-owned artifact, panic, and
/// timeout envelope.
///
/// The core executor and direct parallel branch runner deliberately own their
/// retry loops separately, but both attempts must receive identical handler
/// semantics.
pub(crate) async fn execute_single_attempt(
node: &GvNode,
context: &Context,
graph: &GvGraph,
run_dir: &Path,
services: &EngineServices,
) -> CoreResult<Outcome> {
let handler = services.registry.resolve(node);
let wf_context = artifact::resolve_context_for_execution(
context,
&services.run.run_store,
&services.run.sandbox,
run_dir,
)
.await
.map_err(|err| {
CoreError::handler(HandlerErrorDetail {
retryable: true,
failure: err.to_failure_detail(),
})
})?;
let execution_snapshot = wf_context.snapshot();
let node_timeout = match handler.node_timeout_policy(node) {
NodeTimeoutPolicy::ExecutorEnforced => node.timeout(),
NodeTimeoutPolicy::HandlerManaged => None,
};
let future = dispatch_handler(handler, node, &wf_context, graph, run_dir, services);
let panic_safe = AssertUnwindSafe(future).catch_unwind();
let timed_result = if let Some(duration) = node_timeout {
let stage_id = StageScope::for_handler(&wf_context, &node.id).stage_id();
let Some(inner) = timeout_excluding_interview_wait(
duration,
&stage_id,
services.run.interview_blocker.subscribe(),
panic_safe,
)
.await
else {
let mut failure = FailureDetail::new(
format!("handler timed out after {}ms", duration.as_millis()),
FailureCategory::TransientInfra,
);
failure.system_actor = Some(SystemActorKind::Timeout);
return Err(CoreError::handler(HandlerErrorDetail {
retryable: true,
failure,
}));
};
inner
} else {
panic_safe.await
};
let mut new_values = wf_context.snapshot();
artifact::normalize_durable_updates(&mut new_values);
for (key, value) in &new_values {
if execution_snapshot.get(key) != Some(value) {
context.set(key.clone(), value.clone());
}
}
match timed_result {
Ok(Ok(wf_outcome)) => Ok(wf_outcome),
Ok(Err(Error::Cancelled)) => Err(CoreError::Cancelled),
Ok(Err(fabro_err)) => {
let retryable = handler.should_retry(&fabro_err);
Err(CoreError::handler(HandlerErrorDetail {
retryable,
failure: fabro_err.to_failure_detail(),
}))
}
Err(panic_payload) => {
let msg = format_panic_message(&panic_payload);
Err(CoreError::handler(HandlerErrorDetail {
retryable: false,
failure: FailureDetail::new(msg, FailureCategory::Deterministic),
}))
}
}
}
pub(crate) fn finalize_retries_exhausted(node: &GvNode, last_outcome: Outcome) -> Outcome {
if node.allow_partial() {
Outcome {
status: StageOutcome::PartiallySucceeded,
..last_outcome
}
} else {
Outcome {
status: StageOutcome::Failed {
retry_requested: false,
},
..last_outcome
}
}
}
#[async_trait]
impl NodeHandler<WorkflowGraph> for WorkflowNodeHandler {
async fn execute(
&self,
node: &WorkflowNode,
context: &Context,
_graph: &WorkflowGraph,
) -> CoreResult<Outcome> {
execute_single_attempt(
node.inner(),
context,
&self.graph,
&self.run_dir,
&self.services,
)
.await
}
async fn context_for_edge_selection(
&self,
context: &Context,
_graph: &WorkflowGraph,
) -> CoreResult<Context> {
artifact::resolve_context_for_edge_selection(context, &self.services.run.run_store)
.await
.map_err(|err| {
CoreError::handler(HandlerErrorDetail {
retryable: true,
failure: err.to_failure_detail(),
})
})
}
fn retry_policy(&self, node: &WorkflowNode, _graph: &WorkflowGraph) -> CoreRetryPolicy {
let gv_node = node.inner();
build_retry_policy(gv_node, &self.graph)
}
fn on_retries_exhausted(&self, node: &WorkflowNode, last_outcome: Outcome) -> Outcome {
finalize_retries_exhausted(node.inner(), last_outcome)
}
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use fabro_core::executor::ExecutorBuilder;
use fabro_core::lifecycle::NoopLifecycle;
use fabro_core::outcome::StageOutcome;
use fabro_core::state::ExecutionState;
use fabro_graphviz::graph::AttrValue;
use fabro_graphviz::graph::types::{Edge, Graph, Node};
use super::*;
use crate::event::Emitter;
use crate::graph::WorkflowGraph;
use crate::interview_runtime::RunInterviewBlocker;
/// Minimal spike handler that always succeeds — proves the trait plumbing.
pub(crate) struct SpikeHandler;
#[async_trait]
impl NodeHandler<WorkflowGraph> for SpikeHandler {
async fn execute(
&self,
_node: &WorkflowNode,
_context: &Context,
_graph: &WorkflowGraph,
) -> CoreResult<Outcome> {
Ok(Outcome::success())
}
fn retry_policy(&self, _node: &WorkflowNode, _graph: &WorkflowGraph) -> CoreRetryPolicy {
CoreRetryPolicy::none()
}
}
#[tokio::test]
async fn spike_core_executor_runs_start_to_exit() {
// Build a minimal graph: start [Mdiamond] → exit [Msquare]
let mut graph = Graph::new("test");
let mut start = Node::new("start");
start.attrs.insert(
"shape".to_string(),
AttrValue::String("Mdiamond".to_string()),
);
let mut exit = Node::new("exit");
exit.attrs.insert(
"shape".to_string(),
AttrValue::String("Msquare".to_string()),
);
graph.nodes.insert("start".to_string(), start);
graph.nodes.insert("exit".to_string(), exit);
graph.edges.push(Edge::new("start", "exit"));
let wf_graph = WorkflowGraph(Arc::new(graph));
let handler: Arc<dyn NodeHandler<WorkflowGraph>> = Arc::new(SpikeHandler);
let state = ExecutionState::new(&wf_graph).unwrap();
let executor = ExecutorBuilder::new(handler)
.lifecycle(Box::new(NoopLifecycle))
.build();
let (result, _) = executor.run(&wf_graph, state).await.unwrap();
assert_eq!(result.status, StageOutcome::Succeeded);
}
#[tokio::test(start_paused = true)]
async fn node_timeout_does_not_count_own_interview_wait() {
let blocker = Arc::new(RunInterviewBlocker::new());
let emitter = Arc::new(Emitter::default());
let stage_id = StageId::new("agent", 1);
let block_state = blocker.subscribe();
let guard = blocker.block(emitter, stage_id.clone());
let result = timeout_excluding_interview_wait(
Duration::from_millis(50),
&stage_id,
block_state,
async move {
sleep(Duration::from_millis(100)).await;
guard.resolve();
sleep(Duration::from_millis(40)).await;
"completed"
},
)
.await;
assert_eq!(result, Some("completed"));
}
#[tokio::test(start_paused = true)]
async fn node_timeout_still_limits_active_work_after_interview() {
let blocker = Arc::new(RunInterviewBlocker::new());
let emitter = Arc::new(Emitter::default());
let stage_id = StageId::new("agent", 1);
let block_state = blocker.subscribe();
let guard = blocker.block(emitter, stage_id.clone());
let result = timeout_excluding_interview_wait(
Duration::from_millis(50),
&stage_id,
block_state,
async move {
sleep(Duration::from_millis(100)).await;
guard.resolve();
sleep(Duration::from_millis(60)).await;
},
)
.await;
assert_eq!(result, None);
}
#[tokio::test(start_paused = true)]
async fn node_timeout_does_not_pause_for_another_stage_interview() {
let blocker = Arc::new(RunInterviewBlocker::new());
let emitter = Arc::new(Emitter::default());
let blocked_stage = StageId::new("agent_a", 1);
let active_stage = StageId::new("agent_b", 1);
let block_state = blocker.subscribe();
let guard = blocker.block(emitter, blocked_stage);
let result = timeout_excluding_interview_wait(
Duration::from_millis(50),
&active_stage,
block_state,
sleep(Duration::from_millis(100)),
)
.await;
guard.resolve();
assert_eq!(result, None);
}
}

150
lib/components/fabro-workflow/src/operations/create.rs
View file

@ -339,13 +339,16 @@ pub fn compile_admitted_run(input: CreateRunCompileInput) -> Result<CompiledRun,
})
}
/// Stage three for a run another engine admitted: no model pinning, since
/// the engine pinned every route at its own admission.
/// Stage three for a run Petri admitted: no model pinning, since Petri
/// pinned every route at its admission. The run's goal and pull request
/// settings are materialized as they were for every run: the graph's goal
/// becomes the inline `run.goal`, and a disabled pull request block is
/// dropped.
#[must_use]
pub fn materialize_admitted_run(compiled: CompiledRun) -> MaterializedRun {
let CompiledRun {
validated,
settings,
mut settings,
raw_source,
workflow_slug,
dot_path,
@ -354,6 +357,7 @@ pub fn materialize_admitted_run(compiled: CompiledRun) -> MaterializedRun {
labels,
configured_providers: _,
} = compiled;
run_materialization::materialize_goal_and_pull_request(&mut settings, validated.graph());
MaterializedRun {
validated,
settings,
@ -776,7 +780,6 @@ mod tests {
use fabro_types::{EventBody, PetriAdmission, WorkflowSettings, fixtures, test_support};
use fabro_util::error::collect_chain;
use fabro_validate::Severity;
use lithos_llm::catalog::builtin;
use object_store::local::LocalFileSystem;
use object_store::memory::InMemory;
@ -820,24 +823,6 @@ mod tests {
Arc::new(fabro_llm::test_support::test_catalog())
}
/// OpenAI and OpenRouter both offering GPT-5.6 Sol as their default, so a
/// portable selector resolves to whichever provider is ready.
fn portable_model_catalog() -> Arc<Catalog> {
Arc::new(fabro_llm::test_support::test_catalog_with_overlay(
r#"
[providers.openai]
priority = 90
default_model = "gpt-5.6-sol"
[providers.openrouter]
priority = 25
default_model = "gpt-5.6-sol"
enabled = true
"#,
))
}
fn test_provider_ids() -> Vec<ProviderId> {
fabro_llm::test_support::test_catalog()
.enabled_provider_ids()
@ -2087,127 +2072,6 @@ mod tests {
assert!(created.run_dir.is_dir());
}
#[tokio::test]
async fn create_materializes_portable_selectors_for_ready_provider_snapshot_and_pin() {
const MODEL_DOT: &str = r#"digraph Test {
graph [goal="Test"]
start [shape=Mdiamond]
work [prompt="Do work", model="MODEL_SELECTOR"]
exit [shape=Msquare]
start -> work -> exit
}"#;
let catalog = portable_model_catalog();
let cases = [
(vec![builtin::openai()], None, builtin::openai()),
(
vec![ProviderId::new("openrouter")],
None,
ProviderId::new("openrouter"),
),
(
vec![builtin::openai(), ProviderId::new("openrouter")],
None,
builtin::openai(),
),
(
vec![builtin::openai(), ProviderId::new("openrouter")],
Some("openrouter"),
ProviderId::new("openrouter"),
),
];
for selector in ["gpt-56-sol", "gpt-5.6"] {
for (ready, explicit_provider, expected_provider) in &cases {
let dir = tempfile::tempdir().unwrap();
let mut settings = test_default_settings();
settings.run.model.name = Some(selector.to_string());
settings.run.model.provider = explicit_provider.map(str::to_string);
let store = memory_store();
let created = create(
store.as_ref(),
CreateRunInput {
admission: PetriAdmission::default(),
workflow: WorkflowInput::DotSource {
source: MODEL_DOT.replace("MODEL_SELECTOR", selector),
base_dir: None,
},
settings,
vars: HashMap::new(),
cwd: dir.path().to_path_buf(),
workflow_slug: None,
workflow_path: None,
workflow_bundle: None,
target: None,
run_id: None,
title: None,
automation: None,
git: None,
fork_source_ref: None,
parent_id: None,
provenance: test_support::test_run_provenance(),
configured_providers: ready.clone(),
web_url: None,
},
dir.path().join("storage"),
Arc::clone(&catalog),
)
.await
.unwrap();
let run_spec = created.persisted.run_spec();
assert_eq!(
run_spec.settings.run.model.name.as_deref(),
Some("gpt-5.6-sol"),
"{selector}"
);
assert_eq!(
run_spec.settings.run.model.provider.as_deref(),
Some(expected_provider.as_str()),
"{selector}"
);
assert_eq!(
run_spec.graph.nodes["work"]
.attrs
.get("model")
.and_then(AttrValue::as_str),
Some("gpt-5.6-sol"),
"{selector}"
);
assert_eq!(
run_spec.graph.nodes["work"]
.attrs
.get("provider")
.and_then(AttrValue::as_str),
Some(expected_provider.as_str()),
"{selector}"
);
let run_store = store.open_run(&created.run_id).await.unwrap();
let run_store = run_store.into();
let reloaded = Persisted::load_from_store(&run_store, &created.run_dir)
.await
.unwrap();
assert_eq!(
reloaded.run_spec().settings.run.model.provider.as_deref(),
Some(expected_provider.as_str()),
"{selector}"
);
assert_eq!(
reloaded.run_spec().graph.nodes["work"]
.attrs
.get("provider")
.and_then(AttrValue::as_str),
Some(expected_provider.as_str()),
"{selector}"
);
assert!(
reloaded.source().contains(selector),
"persisted source should preserve the user's selector '{selector}'"
);
}
}
}
#[tokio::test]
async fn create_persists_secret_tokens_in_run_created_settings_source_form() {
let dir = tempfile::tempdir().unwrap();

505
lib/components/fabro-workflow/src/operations/fork.rs
View file

@ -1,505 +0,0 @@
use anyhow::Result as AnyResult;
use chrono::Utc;
use fabro_store::{Database, RunProjection, RunProjectionReducer};
use fabro_types::{EventBody, EventEnvelope, ForkSourceRef, RunId, RunTarget};
use super::timeline::{ForkTarget, RunTimeline, TimelineEntry, build_timeline};
use crate::error::Error;
use crate::event::{self, Event};
use crate::records::{Checkpoint, RunSpec};
#[derive(Debug, Clone)]
pub struct ForkRunInput {
pub source_run_id: RunId,
pub target: Option<ForkTarget>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ResolvedForkTarget {
pub checkpoint_ordinal: usize,
pub node_id: String,
pub visit: usize,
}
impl ResolvedForkTarget {
#[must_use]
pub fn response_target(&self) -> String {
format!("@{}", self.checkpoint_ordinal)
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ForkOutcome {
pub source_run_id: RunId,
pub new_run_id: RunId,
pub target: ResolvedForkTarget,
}
pub async fn fork_run(
store: &Database,
input: &ForkRunInput,
) -> std::result::Result<ForkOutcome, Error> {
let source_run_id = input.source_run_id;
let run_store = store
.open_run(&source_run_id)
.await
.map_err(|err| Error::engine(err.to_string()))?;
let state = run_store
.state()
.await
.map_err(|err| Error::engine(err.to_string()))?;
validate_target_support(state.spec.target.as_ref())?;
let timeline = build_timeline(&state).map_err(|err| Error::engine(err.to_string()))?;
let entry = resolve_fork_entry(&timeline, &source_run_id, input.target.as_ref())
.map_err(|err| Error::Validation(err.to_string()))?;
let checkpoint_sha = entry.run_commit_sha.clone().ok_or_else(|| {
Error::Validation(format!(
"checkpoint @{} has no git_commit_sha; cannot fork",
entry.ordinal
))
})?;
validate_source_spec(&state.spec, &checkpoint_sha)?;
let events = run_store
.list_events()
.await
.map_err(|err| Error::engine(err.to_string()))?;
let historical_events = events
.into_iter()
.filter(|event| event.seq <= entry.checkpoint_seq)
.collect::<Vec<_>>();
let mut projection = RunProjection::apply_events(&historical_events)
.map_err(|err| Error::engine(err.to_string()))?;
let mut run_spec = projection.spec.clone();
let new_run_id = RunId::new();
run_spec.run_id = new_run_id;
run_spec.fork_source_ref = Some(ForkSourceRef {
source_run_id,
checkpoint_sha: checkpoint_sha.clone(),
});
projection.spec = run_spec;
projection.start = None;
projection.sandbox = None;
projection.conclusion = None;
projection.pull_request = None;
projection.superseded_by = None;
if let Some(record) = projection.checkpoints.last_mut() {
record.checkpoint.git_commit_sha = Some(checkpoint_sha);
}
persist_forked_run(store, &projection, &historical_events).await?;
Ok(ForkOutcome {
source_run_id,
new_run_id,
target: ResolvedForkTarget {
checkpoint_ordinal: entry.ordinal,
node_id: entry.node_name.clone(),
visit: entry.visit,
},
})
}
fn validate_target_support(target: Option<&RunTarget>) -> std::result::Result<(), Error> {
if matches!(target, Some(RunTarget::Folder { .. })) {
return Err(Error::Validation(
"Local folder runs execute in place without Git checkpoints; cannot fork or rewind"
.to_string(),
));
}
Ok(())
}
fn validate_source_spec(spec: &RunSpec, checkpoint_sha: &str) -> std::result::Result<(), Error> {
if checkpoint_sha.trim().is_empty() {
return Err(Error::Validation(
"target checkpoint has an empty git_commit_sha; cannot fork".to_string(),
));
}
let Some(origin) = spec.repo_origin_url() else {
return Err(Error::Validation(
"source run has no repo_origin_url; cannot validate fork origin".to_string(),
));
};
if fabro_github::normalize_repo_origin_url(origin).is_empty() {
return Err(Error::Validation(
"source run has an empty repo_origin_url; cannot validate fork origin".to_string(),
));
}
Ok(())
}
fn resolve_fork_entry<'a>(
timeline: &'a RunTimeline,
source_run_id: &RunId,
target: Option<&ForkTarget>,
) -> AnyResult<&'a TimelineEntry> {
match target {
Some(target) => timeline.resolve(target),
None => timeline
.entries
.last()
.ok_or_else(|| anyhow::anyhow!("no checkpoints found for run {source_run_id}")),
}
}
async fn persist_forked_run(
store: &Database,
projection: &RunProjection,
historical_events: &[EventEnvelope],
) -> std::result::Result<(), Error> {
let spec = &projection.spec;
let checkpoint = projection
.current_checkpoint()
.ok_or_else(|| Error::engine("forked run projection has no checkpoint"))?;
let first_event = Event::RunCreated {
run_id: spec.run_id,
title: None,
settings: serde_json::to_value(&spec.settings)
.map_err(|err| Error::engine(err.to_string()))?,
graph: serde_json::to_value(&spec.graph)
.map_err(|err| Error::engine(err.to_string()))?,
workflow_source: projection.spec.graph_source.clone(),
labels: spec.labels.clone().into_iter().collect(),
source_directory: spec.source_directory.clone(),
workflow_slug: spec.workflow_slug.clone(),
workflow_version_id: spec.workflow_version_id,
target: spec.target.clone(),
automation: spec.automation.clone(),
provenance: spec.provenance.clone(),
// Content-addressed, so the forked run reads the source run's
// unredacted spec bytes through the same id.
spec_blob: spec.spec_blob,
git: spec.git.clone(),
fork_source_ref: spec.fork_source_ref.clone(),
retried_from: None,
parent_id: None,
web_url: None,
admission: spec.admission.clone(),
};
let run_store = event::create_run(store, &spec.run_id, &first_event, Utc::now())
.await
.map_err(|err| Error::engine(err.to_string()))?;
let replayed_checkpoint =
replay_historical_projection_events(&run_store, spec.run_id, historical_events).await?;
if !replayed_checkpoint {
event::append_event(
&run_store,
&spec.run_id,
&checkpoint_completed_event(checkpoint),
)
.await
.map_err(|err| Error::engine(err.to_string()))?;
}
event::append_event(&run_store, &spec.run_id, &Event::RunSubmitted {
definition_blob: spec.definition_blob,
})
.await
.map_err(|err| Error::engine(err.to_string()))
}
async fn replay_historical_projection_events(
run_store: &fabro_store::RunDatabase,
new_run_id: RunId,
historical_events: &[EventEnvelope],
) -> std::result::Result<bool, Error> {
let mut replayed_checkpoint = false;
for envelope in historical_events {
if !replay_event_for_fork_projection(&envelope.event.body) {
continue;
}
if matches!(envelope.event.body, EventBody::CheckpointCompleted(_)) {
replayed_checkpoint = true;
}
let mut event = envelope.event.clone();
event.id = format!("{new_run_id}-fork-{}", envelope.seq);
event.run_id = new_run_id;
let payload = event::build_redacted_event_payload(&event, &new_run_id)
.map_err(|err| Error::engine(err.to_string()))?;
run_store
.append_event(&payload)
.await
.map_err(|err| Error::engine(err.to_string()))?;
}
Ok(replayed_checkpoint)
}
fn replay_event_for_fork_projection(body: &EventBody) -> bool {
matches!(
body,
EventBody::StageCompleted(_)
| EventBody::StageFailed(_)
| EventBody::StagePrompt(_)
| EventBody::PromptCompleted(_)
| EventBody::CheckpointCompleted(_)
| EventBody::InterviewStarted(_)
| EventBody::InterviewCompleted(_)
| EventBody::InterviewTimeout(_)
| EventBody::InterviewInterrupted(_)
| EventBody::AgentSessionActivated(_)
| EventBody::AgentToolsAvailable(_)
| EventBody::AgentAcpStarted(_)
| EventBody::AgentAcpCancelled(_)
| EventBody::AgentAcpTimedOut(_)
| EventBody::CommandStarted(_)
| EventBody::CommandCompleted(_)
| EventBody::ParallelCompleted(_)
)
}
fn checkpoint_completed_event(checkpoint: &Checkpoint) -> Event {
let status = checkpoint
.node_outcomes
.get(&checkpoint.current_node)
.map_or_else(
|| "success".to_string(),
|outcome| outcome.status.to_string(),
);
Event::CheckpointCompleted {
node_id: checkpoint.current_node.clone(),
status,
current_node: checkpoint.current_node.clone(),
completed_nodes: checkpoint.completed_nodes.clone(),
node_retries: checkpoint.node_retries.clone().into_iter().collect(),
context_values: checkpoint.context_values.clone().into_iter().collect(),
node_outcomes: checkpoint.node_outcomes.clone().into_iter().collect(),
next_node_id: checkpoint.next_node_id.clone(),
git_commit_sha: checkpoint.git_commit_sha.clone(),
loop_failure_signatures: checkpoint
.loop_failure_signatures
.iter()
.map(|(signature, count)| (signature.to_string(), *count))
.collect(),
restart_failure_signatures: checkpoint
.restart_failure_signatures
.iter()
.map(|(signature, count)| (signature.to_string(), *count))
.collect(),
node_visits: checkpoint.node_visits.clone().into_iter().collect(),
diff: None,
diff_summary: None,
graph_visit: None,
resumed_from_stage_id: None,
}
}
#[cfg(test)]
mod tests {
use std::collections::BTreeMap;
use std::sync::Arc;
use std::time::Duration;
use fabro_graphviz::graph::Graph;
use fabro_store::{Database, RunProjectionReducer};
use fabro_types::{PetriAdmission, StageId, WorkflowSettings, fixtures, test_support};
use object_store::memory::InMemory;
use super::*;
fn test_store() -> Database {
fabro_store::test_support::test_database(
Arc::new(InMemory::new()),
"",
Duration::from_millis(1),
None,
)
}
#[test]
fn folder_targets_report_that_fork_and_rewind_are_unsupported() {
let target = RunTarget::Folder {
path: "/canonical/project".to_string(),
};
let error = validate_target_support(Some(&target)).unwrap_err();
assert!(error.to_string().contains("cannot fork or rewind"));
}
#[test]
fn fork_replay_keeps_stage_scoped_session_activation_only() {
assert!(replay_event_for_fork_projection(
&EventBody::AgentSessionActivated(fabro_types::run_event::AgentSessionActivatedProps {
thread_id: None,
provider: Some("openai".to_string()),
model: Some("gpt-5.4".to_string()),
reasoning_effort: None,
speed: None,
permission_level: None,
capabilities: vec![fabro_types::SessionCapability::Steer],
visit: 1,
})
));
assert!(replay_event_for_fork_projection(
&EventBody::AgentToolsAvailable(fabro_types::run_event::AgentToolsAvailableProps {
tools: Vec::new(),
visit: 1,
})
));
}
#[test]
fn fork_replay_preserves_agent_acp_projection_events() {
assert!(replay_event_for_fork_projection(
&EventBody::AgentAcpStarted(fabro_types::run_event::AgentAcpStartedProps {
visit: 1,
command: "python fake_agent.py".to_string(),
config_name: Some("fake".to_string()),
})
));
assert!(replay_event_for_fork_projection(
&EventBody::AgentAcpCancelled(fabro_types::run_event::AgentAcpCancelledProps {
stdout: "partial".to_string(),
stderr: "cancelled".to_string(),
duration_ms: 7,
})
));
assert!(replay_event_for_fork_projection(
&EventBody::AgentAcpTimedOut(fabro_types::run_event::AgentAcpTimedOutProps {
stdout: "partial".to_string(),
stderr: "timeout".to_string(),
duration_ms: 99,
})
));
assert!(!replay_event_for_fork_projection(
&EventBody::AgentAcpCompleted(fabro_types::run_event::AgentAcpCompletedProps {
stdout: "done".to_string(),
stderr: String::new(),
stop_reason: "end_turn".to_string(),
duration_ms: 42,
})
));
}
#[tokio::test]
async fn fork_persists_historical_node_projection_through_target_checkpoint() {
let store = test_store();
let source_run_id = fixtures::RUN_1;
let source = store.create_run(&source_run_id).await.unwrap();
let graph = Graph::new("fork-source");
let settings = WorkflowSettings::default();
let workflow_version_id = test_support::test_workflow_version_id();
event::append_event(&source, &source_run_id, &Event::RunCreated {
run_id: source_run_id,
title: None,
settings: serde_json::to_value(&settings).unwrap(),
graph: serde_json::to_value(&graph).unwrap(),
workflow_source: Some("digraph fork_source {}".to_string()),
labels: BTreeMap::new(),
source_directory: Some("/client/source".to_string()),
workflow_slug: Some("fork-source".to_string()),
workflow_version_id: Some(workflow_version_id),
target: Some(fabro_types::RunTarget::Git(fabro_types::GitRunTarget {
repo: "example/repo".to_string(),
branch: "main".to_string(),
tag: None,
sha: None,
})),
automation: None,
provenance: test_support::test_run_provenance(),
spec_blob: None,
git: Some(fabro_types::GitContext {
origin_url: "https://github.com/example/repo".to_string(),
branch: "main".to_string(),
sha: None,
dirty: fabro_types::DirtyStatus::Clean,
}),
fork_source_ref: None,
retried_from: None,
parent_id: None,
web_url: None,
admission: PetriAdmission::default(),
})
.await
.unwrap();
let mut node_visits = BTreeMap::new();
node_visits.insert("work".to_string(), 1);
event::append_event(&source, &source_run_id, &Event::StageCompleted {
node_id: "work".to_string(),
name: "Work".to_string(),
index: 1,
timing: fabro_types::StageTiming::wall_only(10),
status: "succeeded".to_string(),
preferred_label: None,
suggested_next_ids: Vec::new(),
usage_by_model: Vec::new(),
usage: None,
failure: None,
notes: None,
files_touched: Vec::new(),
context_updates: None,
jump_to_node: None,
context_values: None,
node_visits: Some(node_visits.clone()),
loop_failure_signatures: None,
restart_failure_signatures: None,
response: Some("historical response".to_string()),
attempt: 1,
max_attempts: 1,
})
.await
.unwrap();
event::append_event(&source, &source_run_id, &Event::CheckpointCompleted {
graph_visit: None,
resumed_from_stage_id: None,
node_id: "work".to_string(),
status: "succeeded".to_string(),
current_node: "work".to_string(),
completed_nodes: vec!["work".to_string()],
node_retries: BTreeMap::new(),
context_values: BTreeMap::new(),
node_outcomes: BTreeMap::new(),
next_node_id: None,
git_commit_sha: Some("abc123".to_string()),
loop_failure_signatures: BTreeMap::new(),
restart_failure_signatures: BTreeMap::new(),
node_visits,
diff: None,
diff_summary: None,
})
.await
.unwrap();
let outcome = fork_run(&store, &ForkRunInput {
source_run_id,
target: None,
})
.await
.unwrap();
let forked = store.open_run(&outcome.new_run_id).await.unwrap();
let forked_events = forked.list_events().await.unwrap();
let forked_state = fabro_store::RunProjection::apply_events(&forked_events).unwrap();
let node = forked_state
.stage(&StageId::new("work", 1))
.expect("forked state should retain historical node projection");
assert_eq!(node.response.as_deref(), Some("historical response"));
assert_eq!(forked_state.checkpoints.len(), 1);
assert_eq!(
forked_state.spec.workflow_version_id,
Some(workflow_version_id)
);
assert_eq!(
forked_state.spec.target,
Some(fabro_types::RunTarget::Git(fabro_types::GitRunTarget {
repo: "example/repo".to_string(),
branch: "main".to_string(),
tag: None,
sha: None,
}))
);
assert_eq!(
forked_state.spec.fork_source_ref.unwrap().source_run_id,
source_run_id
);
}
}

13
lib/components/fabro-workflow/src/operations/mod.rs
View file

@ -1,13 +1,7 @@
mod archive;
mod create;
mod fork;
mod resume;
mod retry;
mod rewind;
mod run_store;
mod source;
mod start;
mod timeline;
mod validate;
pub use archive::{
@ -20,14 +14,7 @@ pub use create::{
assemble_create_run_persistence_input, compile_admitted_run, compile_create_run, create,
make_run_dir, materialize_admitted_run, materialize_create_run, persist_create_run,
};
pub use fork::{ForkOutcome, ForkRunInput, ResolvedForkTarget, fork_run};
pub use resume::resume;
pub use retry::{RetryOutcome, RetryRunInput, retry_run};
pub use rewind::{RewindInput, RewindOutcome, rewind};
pub use source::WorkflowInput;
pub use start::{StartServices, Started, start};
pub use timeline::{ForkTarget, RunTimeline, TimelineEntry, build_timeline, timeline};
pub use validate::{ValidateInput, validate, validate_with_catalog, validate_with_ready_providers};
pub use crate::pipeline::{LlmSpec, SandboxEnvSpec};
pub use crate::transforms::RenderMode;

53
lib/components/fabro-workflow/src/operations/resume.rs
View file

@ -1,53 +0,0 @@
use std::path::Path;
use super::start::{StartServices, Started, execute_persisted_run};
use crate::error::Error;
use crate::event::{Event, append_event_to_sink};
use crate::outcome::StageOutcome;
use crate::pipeline::ResumeState;
use crate::run_status::RunStatus;
/// Resume a workflow run from its checkpoint. Errors if no checkpoint is found.
pub async fn resume(run_dir: &Path, services: StartServices) -> Result<Started, Error> {
let state = services
.run_store
.state()
.await
.map_err(|err| Error::engine(err.to_string()))?;
let status = state.status;
super::archive::ensure_not_archived(state.archived_at.is_some(), &services.run_id)?;
if matches!(status, RunStatus::Succeeded { .. }) {
return Err(Error::Precondition(
"run already finished successfully — nothing to resume".to_string(),
));
}
if let Some(conclusion) = state.conclusion.as_ref() {
if matches!(
conclusion.status,
StageOutcome::Succeeded | StageOutcome::PartiallySucceeded | StageOutcome::Skipped
) {
return Err(Error::Precondition(
"run already finished successfully — nothing to resume".to_string(),
));
}
}
let resume_state = ResumeState::from_projection(&state)
.ok_or_else(|| Error::Precondition("no checkpoint to resume from".to_string()))?;
let definition_blob = state.spec.definition_blob;
cleanup_resume_artifacts(run_dir);
append_event_to_sink(
&services.event_sink,
&services.run_id,
&Event::RunSubmitted { definition_blob },
)
.await?;
Box::pin(execute_persisted_run(run_dir, Some(resume_state), services)).await
}
fn cleanup_resume_artifacts(run_dir: &Path) {
let _ = run_dir;
}

692
lib/components/fabro-workflow/src/operations/retry.rs
View file

@ -1,692 +0,0 @@
use std::collections::BTreeMap;
use chrono::Utc;
use fabro_store::Database;
use fabro_types::{RunId, RunProvenance, RunSpec, RunStatus};
use super::archive::ensure_not_archived;
use super::run_store::map_open_run_error;
use crate::error::Error;
use crate::event::{self, Event};
#[derive(Debug, Clone)]
pub struct RetryRunInput {
pub source_run_id: RunId,
pub new_run_id: RunId,
pub provenance: RunProvenance,
pub web_url: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RetryOutcome {
pub source_run_id: RunId,
pub new_run_id: RunId,
}
pub async fn retry_run(
store: &Database,
input: &RetryRunInput,
) -> std::result::Result<RetryOutcome, Error> {
let source_run_id = input.source_run_id;
let new_run_id = input.new_run_id;
let source_store = store
.open_run(&source_run_id)
.await
.map_err(|err| map_open_run_error(&source_run_id, err))?;
let source = source_store
.state()
.await
.map_err(|err| Error::engine(err.to_string()))?;
ensure_not_archived(source.archived_at.is_some(), &source_run_id)?;
ensure_retryable(source.status, &source_run_id)?;
let title = source.title().into_owned();
let parent_id = source.parent_id;
let RunSpec {
run_id: _,
settings,
graph,
graph_source,
workflow_slug,
workflow_version_id,
target,
automation,
source_directory,
labels,
provenance: _,
definition_blob,
spec_blob,
git,
fork_source_ref,
admission,
} = source.spec;
let settings = serde_json::to_value(&settings).map_err(|err| Error::engine(err.to_string()))?;
let graph = serde_json::to_value(&graph).map_err(|err| Error::engine(err.to_string()))?;
let first_event = Event::RunCreated {
run_id: new_run_id,
title: Some(title),
settings,
graph,
workflow_source: graph_source,
labels: labels.into_iter().collect::<BTreeMap<_, _>>(),
source_directory,
workflow_slug,
workflow_version_id,
target,
automation,
provenance: input.provenance.clone(),
// Blobs are content-addressed, so the retried run reads the source
// run's unredacted spec bytes through the same id.
spec_blob,
git,
fork_source_ref,
retried_from: Some(source_run_id),
parent_id,
web_url: input.web_url.clone(),
// The admitted graph is content-addressed, so a retry runs from the
// same admission.
admission,
};
let retry_store = event::create_run(store, &new_run_id, &first_event, Utc::now())
.await
.map_err(|err| Error::engine(err.to_string()))?;
event::append_event(&retry_store, &new_run_id, &Event::RunSubmitted {
definition_blob,
})
.await
.map_err(|err| Error::engine(err.to_string()))?;
Ok(RetryOutcome {
source_run_id,
new_run_id,
})
}
fn ensure_retryable(status: RunStatus, run_id: &RunId) -> std::result::Result<(), Error> {
if status.is_terminal() {
Ok(())
} else {
Err(Error::Precondition(format!(
"run {run_id} cannot be retried from status {status}; expected terminal"
)))
}
}
#[cfg(test)]
mod tests {
use std::collections::{BTreeMap, HashMap};
use std::sync::Arc;
use std::time::Duration;
use fabro_store::{Database, RunProjectionReducer};
use fabro_types::{
AuthMethod, BlobHash, DirtyStatus, FailureReason, ForkSourceRef, GitContext, Graph,
IdpIdentity, PetriAdmission, Principal, PullRequestLink, RunRunnableSource,
RunServerProvenance, RunTarget, RunTiming, WorkflowSettings, fixtures, test_support,
};
use object_store::memory::InMemory;
use super::*;
fn memory_store() -> Database {
fabro_store::test_support::test_database(
Arc::new(InMemory::new()),
"",
Duration::from_millis(1),
None,
)
}
fn actor(login: &str) -> Principal {
Principal::user(
IdpIdentity::new("github", format!("user:{login}")).unwrap(),
login.to_string(),
AuthMethod::DevToken,
)
}
fn provenance(login: &str) -> RunProvenance {
RunProvenance {
server: Some(RunServerProvenance {
version: "test".to_string(),
}),
client: None,
subject: actor(login),
}
}
fn git_context() -> GitContext {
GitContext {
origin_url: "https://github.com/fabro-sh/fabro".to_string(),
branch: "main".to_string(),
sha: Some("abcdef0123456789abcdef0123456789abcdef01".to_string()),
dirty: DirtyStatus::Clean,
}
}
fn run_target() -> RunTarget {
RunTarget::Git(fabro_types::GitRunTarget {
repo: "fabro-sh/fabro".to_string(),
branch: "main".to_string(),
tag: None,
sha: Some("abcdef0123456789abcdef0123456789abcdef01".to_string()),
})
}
async fn append_created(
store: &fabro_store::RunDatabase,
run_id: RunId,
fork_source_ref: Option<ForkSourceRef>,
) {
let mut settings = WorkflowSettings::default();
settings
.run
.metadata
.insert("env".to_string(), "test".to_string());
let labels = HashMap::from([("team".to_string(), "core".to_string())]);
event::append_event(store, &run_id, &Event::RunCreated {
run_id,
title: Some("Original title".to_string()),
settings: serde_json::to_value(&settings).unwrap(),
graph: serde_json::to_value(Graph::new("retry_source")).unwrap(),
workflow_source: Some("digraph retry_source { start -> exit }".to_string()),
labels: labels.into_iter().collect(),
source_directory: Some("/workspace/source".to_string()),
workflow_slug: Some("retry-source".to_string()),
workflow_version_id: Some(test_support::test_workflow_version_id()),
target: Some(run_target()),
automation: None,
provenance: provenance("source-user"),
spec_blob: None,
git: Some(git_context()),
fork_source_ref,
retried_from: None,
parent_id: None,
web_url: None,
admission: PetriAdmission::default(),
})
.await
.unwrap();
}
async fn append_runnable(store: &fabro_store::RunDatabase, run_id: RunId) {
event::append_event(store, &run_id, &Event::RunRunnable {
source: RunRunnableSource::StartRequested,
actor: None,
})
.await
.unwrap();
}
async fn append_started(store: &fabro_store::RunDatabase, run_id: RunId) {
append_runnable(store, run_id).await;
event::append_event(store, &run_id, &Event::RunStarting)
.await
.unwrap();
event::append_event(store, &run_id, &Event::RunRunning)
.await
.unwrap();
}
async fn append_failed(store: &fabro_store::RunDatabase, run_id: RunId, reason: FailureReason) {
append_started(store, run_id).await;
let event = Event::workflow_run_failed_from_error(
&Error::engine("boom"),
RunTiming::wall_only(10),
reason,
None,
None,
None,
None,
);
event::append_event(store, &run_id, &event).await.unwrap();
}
async fn append_succeeded(store: &fabro_store::RunDatabase, run_id: RunId) {
append_started(store, run_id).await;
event::append_event(store, &run_id, &Event::WorkflowRunCompleted {
timing: RunTiming::wall_only(10),
artifact_count: 0,
status: "succeeded".to_string(),
reason: fabro_types::SuccessReason::Completed,
final_git_commit_sha: None,
final_patch: None,
diff_summary: None,
usage: None,
})
.await
.unwrap();
}
async fn seed_retryable_failed_source(
store: &Database,
source_run_id: RunId,
) -> (Option<BlobHash>, ForkSourceRef) {
let source_store = store.create_run(&source_run_id).await.unwrap();
let definition_blob = Some(
source_store
.write_blob(br#"{\"definition\":true}"#)
.await
.unwrap(),
);
let fork_source_ref = ForkSourceRef {
source_run_id: fixtures::RUN_3,
checkpoint_sha: "fork-sha".to_string(),
};
append_created(&source_store, source_run_id, Some(fork_source_ref.clone())).await;
event::append_event(&source_store, &source_run_id, &Event::RunSubmitted {
definition_blob,
})
.await
.unwrap();
event::append_event(&source_store, &source_run_id, &Event::RunParentLinked {
previous_parent_id: None,
parent_id: fixtures::RUN_2,
actor: None,
})
.await
.unwrap();
event::append_event(&source_store, &source_run_id, &Event::RunTitleUpdated {
title: "Current title".to_string(),
actor: None,
})
.await
.unwrap();
event::append_event(&source_store, &source_run_id, &Event::CheckpointCompleted {
graph_visit: None,
resumed_from_stage_id: None,
node_id: "work".to_string(),
status: "succeeded".to_string(),
current_node: "work".to_string(),
completed_nodes: vec!["work".to_string()],
node_retries: BTreeMap::new(),
context_values: BTreeMap::new(),
node_outcomes: BTreeMap::new(),
next_node_id: None,
git_commit_sha: Some("checkpoint-sha".to_string()),
loop_failure_signatures: BTreeMap::new(),
restart_failure_signatures: BTreeMap::new(),
node_visits: BTreeMap::new(),
diff: Some("diff --git a/file b/file".to_string()),
diff_summary: Some(fabro_types::DiffSummary {
files_changed: 1,
additions: 1,
deletions: 0,
}),
})
.await
.unwrap();
event::append_event(&source_store, &source_run_id, &Event::SandboxInitialized {
provider: fabro_types::SandboxProviderKind::LOCAL,
id: "sandbox-source".to_string(),
working_directory: "/tmp/source".to_string(),
image: None,
snapshot: None,
repo_cloned: None,
clone_origin_url: None,
clone_branch: None,
workspace_root: None,
repos_root: None,
primary_repo_path: None,
primary_repo_link: None,
})
.await
.unwrap();
event::append_event(&source_store, &source_run_id, &Event::PullRequestLinked {
pull_request: PullRequestLink {
owner: "fabro-sh".to_string(),
repo: "fabro".to_string(),
number: 42,
},
})
.await
.unwrap();
append_failed(&source_store, source_run_id, FailureReason::WorkflowError).await;
(definition_blob, fork_source_ref)
}
#[tokio::test]
async fn retry_creates_fresh_run_from_durable_definition_only() {
let store = memory_store();
let source_run_id = fixtures::RUN_1;
let (definition_blob, fork_source_ref) =
seed_retryable_failed_source(&store, source_run_id).await;
let source_event_count = store
.open_run(&source_run_id)
.await
.unwrap()
.list_events()
.await
.unwrap()
.len();
let outcome = retry_run(&store, &RetryRunInput {
source_run_id,
new_run_id: RunId::new(),
provenance: provenance("retry-user"),
web_url: Some("http://localhost:3000/runs/retry".to_string()),
})
.await
.unwrap();
assert_ne!(outcome.new_run_id, source_run_id);
assert_eq!(outcome.source_run_id, source_run_id);
let retry_store = store.open_run(&outcome.new_run_id).await.unwrap();
let retry_events = retry_store.list_events().await.unwrap();
let retry_state = fabro_store::RunProjection::apply_events(&retry_events).unwrap();
assert_eq!(retry_events.len(), 2);
assert_eq!(retry_state.status, RunStatus::Submitted);
assert_eq!(retry_state.retried_from, Some(source_run_id));
assert_eq!(retry_state.parent_id, Some(fixtures::RUN_2));
assert_eq!(retry_state.title(), "Current title");
assert_eq!(
retry_state.spec.labels.get("team"),
Some(&"core".to_string())
);
assert_eq!(
retry_state.spec.settings.run.metadata.get("env"),
Some(&"test".to_string())
);
assert_eq!(retry_state.spec.graph.name, "retry_source");
assert_eq!(
retry_state.spec.workflow_version_id,
Some(test_support::test_workflow_version_id())
);
assert_eq!(
retry_state.spec.graph_source.as_deref(),
Some("digraph retry_source { start -> exit }")
);
assert_eq!(retry_state.spec.git, Some(git_context()));
assert_eq!(retry_state.spec.target, Some(run_target()));
assert_eq!(retry_state.spec.definition_blob, definition_blob);
assert_eq!(retry_state.spec.fork_source_ref, Some(fork_source_ref));
assert_eq!(retry_state.spec.provenance.subject, actor("retry-user"));
assert_eq!(
retry_state.web_url.as_deref(),
Some("http://localhost:3000/runs/retry")
);
assert!(retry_state.checkpoints.is_empty());
assert!(retry_state.conclusion.is_none());
assert!(retry_state.pull_request.is_none());
assert!(retry_state.pending_interviews.is_empty());
assert!(retry_state.pending_control.is_none());
assert!(
retry_state
.sandbox
.as_ref()
.and_then(fabro_types::RunSandbox::instance)
.is_none()
);
let source_store = store.open_run(&source_run_id).await.unwrap();
assert_eq!(
source_store.list_events().await.unwrap().len(),
source_event_count
);
assert_eq!(
source_store.state().await.unwrap().status,
RunStatus::Failed {
reason: FailureReason::WorkflowError,
}
);
}
#[tokio::test]
async fn retry_preserves_none_target_without_git_or_source_directory() {
let store = memory_store();
let source_run_id = fixtures::RUN_1;
let source_store = store.create_run(&source_run_id).await.unwrap();
event::append_event(&source_store, &source_run_id, &Event::RunCreated {
run_id: source_run_id,
title: Some("None target".to_string()),
settings: serde_json::to_value(WorkflowSettings::default()).unwrap(),
graph: serde_json::to_value(Graph::new("none_target_retry")).unwrap(),
workflow_source: Some("digraph none_target_retry { start -> exit }".to_string()),
labels: BTreeMap::new(),
source_directory: None,
workflow_slug: Some("none-target-retry".to_string()),
workflow_version_id: Some(test_support::test_workflow_version_id()),
target: Some(RunTarget::None {}),
automation: None,
provenance: provenance("source-user"),
spec_blob: None,
git: None,
fork_source_ref: None,
retried_from: None,
parent_id: None,
web_url: None,
admission: PetriAdmission::default(),
})
.await
.unwrap();
event::append_event(&source_store, &source_run_id, &Event::RunSubmitted {
definition_blob: None,
})
.await
.unwrap();
append_failed(&source_store, source_run_id, FailureReason::WorkflowError).await;
let source_state = source_store.state().await.unwrap();
assert_eq!(source_state.status, RunStatus::Failed {
reason: FailureReason::WorkflowError,
});
assert_eq!(source_state.spec.target, Some(RunTarget::None {}));
assert_eq!(source_state.spec.git, None);
assert_eq!(source_state.spec.source_directory, None);
let outcome = retry_run(&store, &RetryRunInput {
source_run_id,
new_run_id: RunId::new(),
provenance: provenance("retry-user"),
web_url: None,
})
.await
.unwrap();
let retry_store = store.open_run(&outcome.new_run_id).await.unwrap();
let retry_events = retry_store.list_events().await.unwrap();
let retry_state = fabro_store::RunProjection::apply_events(&retry_events).unwrap();
assert_eq!(retry_events.len(), 2);
assert_eq!(retry_state.status, RunStatus::Submitted);
assert_eq!(retry_state.retried_from, Some(source_run_id));
assert_eq!(retry_state.spec.target, Some(RunTarget::None {}));
assert_eq!(retry_state.spec.git, None);
assert_eq!(retry_state.spec.source_directory, None);
}
#[tokio::test]
async fn retry_preserves_folder_target_and_source_directory_without_git() {
let store = memory_store();
let source_run_id = fixtures::RUN_1;
let source_store = store.create_run(&source_run_id).await.unwrap();
let path = "/canonical/local/folder".to_string();
let target = RunTarget::Folder { path: path.clone() };
event::append_event(&source_store, &source_run_id, &Event::RunCreated {
run_id: source_run_id,
title: Some("Folder target".to_string()),
settings: serde_json::to_value(WorkflowSettings::default()).unwrap(),
graph: serde_json::to_value(Graph::new("folder_target_retry")).unwrap(),
workflow_source: Some("digraph folder_target_retry { start -> exit }".to_string()),
labels: BTreeMap::new(),
source_directory: Some(path.clone()),
workflow_slug: Some("folder-target-retry".to_string()),
workflow_version_id: Some(test_support::test_workflow_version_id()),
target: Some(target.clone()),
automation: None,
provenance: provenance("source-user"),
spec_blob: None,
git: None,
fork_source_ref: None,
retried_from: None,
parent_id: None,
web_url: None,
admission: PetriAdmission::default(),
})
.await
.unwrap();
event::append_event(&source_store, &source_run_id, &Event::RunSubmitted {
definition_blob: None,
})
.await
.unwrap();
append_failed(&source_store, source_run_id, FailureReason::WorkflowError).await;
let outcome = retry_run(&store, &RetryRunInput {
source_run_id,
new_run_id: RunId::new(),
provenance: provenance("retry-user"),
web_url: None,
})
.await
.unwrap();
let retry_store = store.open_run(&outcome.new_run_id).await.unwrap();
let retry_state = retry_store.state().await.unwrap();
assert_eq!(retry_state.status, RunStatus::Submitted);
assert_eq!(retry_state.spec.target, Some(target));
assert_eq!(
retry_state.spec.source_directory.as_deref(),
Some(path.as_str())
);
assert_eq!(retry_state.spec.git, None);
}
#[tokio::test]
async fn retry_creates_fresh_run_from_succeeded_source() {
let store = memory_store();
let source_run_id = fixtures::RUN_1;
let source_store = store.create_run(&source_run_id).await.unwrap();
append_created(&source_store, source_run_id, None).await;
let definition_blob = Some(
source_store
.write_blob(br#"{\"definition\":true}"#)
.await
.unwrap(),
);
event::append_event(&source_store, &source_run_id, &Event::RunSubmitted {
definition_blob,
})
.await
.unwrap();
append_succeeded(&source_store, source_run_id).await;
let outcome = retry_run(&store, &RetryRunInput {
source_run_id,
new_run_id: RunId::new(),
provenance: provenance("retry-user"),
web_url: None,
})
.await
.unwrap();
let retry_store = store.open_run(&outcome.new_run_id).await.unwrap();
let retry_events = retry_store.list_events().await.unwrap();
let retry_state = fabro_store::RunProjection::apply_events(&retry_events).unwrap();
assert_eq!(retry_events.len(), 2);
assert_eq!(retry_state.status, RunStatus::Submitted);
assert_eq!(retry_state.retried_from, Some(source_run_id));
assert_eq!(retry_state.spec.definition_blob, definition_blob);
assert_eq!(
source_store.state().await.unwrap().status,
RunStatus::Succeeded {
reason: fabro_types::SuccessReason::Completed,
}
);
}
#[tokio::test]
async fn retry_rejects_active_and_archived_sources() {
let store = memory_store();
let active = fixtures::RUN_2;
let active_store = store.create_run(&active).await.unwrap();
append_created(&active_store, active, None).await;
event::append_event(&active_store, &active, &Event::RunSubmitted {
definition_blob: None,
})
.await
.unwrap();
event::append_event(&active_store, &active, &Event::RunRunnable {
source: RunRunnableSource::StartRequested,
actor: None,
})
.await
.unwrap();
let archived = fixtures::RUN_3;
let archived_store = store.create_run(&archived).await.unwrap();
append_created(&archived_store, archived, None).await;
append_failed(&archived_store, archived, FailureReason::WorkflowError).await;
event::append_event(&archived_store, &archived, &Event::RunArchived {
actor: None,
})
.await
.unwrap();
for run_id in [active, archived] {
let err = retry_run(&store, &RetryRunInput {
source_run_id: run_id,
new_run_id: RunId::new(),
provenance: provenance("retry-user"),
web_url: None,
})
.await
.unwrap_err();
assert!(
matches!(err, Error::Precondition(_)),
"unexpected error: {err:?}"
);
}
}
#[tokio::test]
async fn retry_reports_missing_source() {
let store = memory_store();
let err = retry_run(&store, &RetryRunInput {
source_run_id: fixtures::RUN_1,
new_run_id: RunId::new(),
provenance: provenance("retry-user"),
web_url: None,
})
.await
.unwrap_err();
assert!(
matches!(err, Error::RunNotFound(_)),
"unexpected error: {err:?}"
);
}
#[test]
fn dead_status_is_retryable() {
ensure_retryable(RunStatus::Dead, &fixtures::RUN_1).unwrap();
}
#[test]
fn succeeded_status_is_retryable() {
ensure_retryable(
RunStatus::Succeeded {
reason: fabro_types::SuccessReason::Completed,
},
&fixtures::RUN_1,
)
.unwrap();
}
#[test]
fn cancelled_status_is_retryable() {
ensure_retryable(
RunStatus::Failed {
reason: FailureReason::Cancelled,
},
&fixtures::RUN_1,
)
.unwrap();
}
}

105
lib/components/fabro-workflow/src/operations/rewind.rs
View file

@ -1,105 +0,0 @@
use fabro_store::Database;
use fabro_types::{Principal, RunId};
use tracing::error;
use super::archive;
use super::fork::{self, ForkOutcome, ForkRunInput, ResolvedForkTarget};
use super::timeline::ForkTarget;
use crate::error::Error;
use crate::event::{self, Event};
#[derive(Debug, Clone)]
pub struct RewindInput {
pub run_id: RunId,
pub target: Option<ForkTarget>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum RewindOutcome {
Full {
source_run_id: RunId,
new_run_id: RunId,
target: ResolvedForkTarget,
},
Partial {
source_run_id: RunId,
new_run_id: RunId,
target: ResolvedForkTarget,
archive_error: String,
},
}
pub async fn rewind(
store: &Database,
input: &RewindInput,
actor: Option<Principal>,
) -> Result<RewindOutcome, Error> {
let projection = store
.open_run(&input.run_id)
.await
.map_err(|err| Error::engine(err.to_string()))?
.state()
.await
.map_err(|err| Error::engine(err.to_string()))?;
let current = projection.status;
archive::ensure_not_archived(projection.archived_at.is_some(), &input.run_id)?;
if current.terminal_status().is_none() {
return Err(Error::Precondition(format!(
"run {} must be terminal (succeeded, failed, or dead) to rewind; current status is {current}",
input.run_id
)));
}
let forked = Box::pin(fork::fork_run(store, &ForkRunInput {
source_run_id: input.run_id,
target: input.target.clone(),
}))
.await?;
match archive::archive(store, &input.run_id, actor).await {
Ok(_) => {
append_superseded_event_best_effort(store, &forked).await;
Ok(RewindOutcome::Full {
source_run_id: forked.source_run_id,
new_run_id: forked.new_run_id,
target: forked.target,
})
}
Err(err) => Ok(RewindOutcome::Partial {
source_run_id: forked.source_run_id,
new_run_id: forked.new_run_id,
target: forked.target,
archive_error: err.to_string(),
}),
}
}
async fn append_superseded_event_best_effort(store: &Database, forked: &ForkOutcome) {
let run_store = match store.open_run(&forked.source_run_id).await {
Ok(run_store) => run_store,
Err(err) => {
error!(
source_run_id = %forked.source_run_id,
new_run_id = %forked.new_run_id,
error = %err,
"failed to open run for RunSupersededBy append after archive"
);
return;
}
};
let event = Event::RunSupersededBy {
new_run_id: forked.new_run_id,
target_checkpoint_ordinal: forked.target.checkpoint_ordinal,
target_node_id: forked.target.node_id.clone(),
target_visit: forked.target.visit,
};
if let Err(err) = event::append_event(&run_store, &forked.source_run_id, &event).await {
error!(
source_run_id = %forked.source_run_id,
new_run_id = %forked.new_run_id,
error = %err,
"failed to append RunSupersededBy after archive"
);
}
}

3358
lib/components/fabro-workflow/src/operations/start.rs
View file

File diff suppressed because it is too large Load diff

367
lib/components/fabro-workflow/src/operations/timeline.rs
View file

@ -1,367 +0,0 @@
use std::collections::HashMap;
use std::str::FromStr;
use anyhow::{Context, Result, bail};
use fabro_graphviz::graph::Graph;
use fabro_graphviz::parser;
use fabro_store::{Database, RunProjection};
use fabro_types::RunId;
use crate::error::Error;
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ForkTarget {
Ordinal(usize),
LatestVisit(String),
SpecificVisit(String, usize),
}
impl FromStr for ForkTarget {
type Err = anyhow::Error;
fn from_str(s: &str) -> Result<Self> {
if let Some(rest) = s.strip_prefix('@') {
let n: usize = rest
.parse()
.with_context(|| format!("invalid ordinal: @{rest}"))?;
if n == 0 {
bail!("ordinal must be >= 1");
}
return Ok(Self::Ordinal(n));
}
if let Some(at_pos) = s.rfind('@') {
let name = &s[..at_pos];
let visit_str = &s[at_pos + 1..];
if !name.is_empty() && !visit_str.is_empty() {
if let Ok(visit) = visit_str.parse::<usize>() {
if visit == 0 {
bail!("visit number must be >= 1");
}
return Ok(Self::SpecificVisit(name.to_string(), visit));
}
}
}
Ok(Self::LatestVisit(s.to_string()))
}
}
#[derive(Debug, Clone)]
pub struct TimelineEntry {
pub ordinal: usize,
pub node_name: String,
pub visit: usize,
pub checkpoint_seq: u32,
pub run_commit_sha: Option<String>,
}
#[derive(Debug, Clone)]
pub struct RunTimeline {
pub entries: Vec<TimelineEntry>,
pub parallel_map: HashMap<String, String>,
}
impl RunTimeline {
pub fn resolve(&self, target: &ForkTarget) -> Result<&TimelineEntry> {
match target {
ForkTarget::Ordinal(n) => {
self.entries
.iter()
.find(|e| e.ordinal == *n)
.ok_or_else(|| {
anyhow::anyhow!("ordinal @{n} out of range (max @{})", self.entries.len())
})
}
ForkTarget::LatestVisit(name) => {
let effective_name = self.parallel_map.get(name).unwrap_or(name);
self.entries
.iter()
.rev()
.find(|e| e.node_name == *effective_name)
.ok_or_else(|| {
if effective_name == name {
anyhow::anyhow!("no checkpoint found for node '{name}'")
} else {
anyhow::anyhow!(
"node '{name}' is inside parallel '{effective_name}'; \
no checkpoint found for '{effective_name}'"
)
}
})
}
ForkTarget::SpecificVisit(name, visit) => {
let effective_name = self.parallel_map.get(name).unwrap_or(name);
self.entries
.iter()
.find(|e| e.node_name == *effective_name && e.visit == *visit)
.ok_or_else(|| {
if effective_name == name {
anyhow::anyhow!("no visit {visit} found for node '{name}'")
} else {
anyhow::anyhow!(
"node '{name}' is inside parallel '{effective_name}'; \
no visit {visit} found for '{effective_name}'"
)
}
})
}
}
}
}
pub fn build_timeline(state: &RunProjection) -> Result<RunTimeline> {
let mut entries = Vec::new();
for record in &state.checkpoints {
let checkpoint = &record.checkpoint;
let ordinal = entries.len() + 1;
let visit = checkpoint
.node_visits
.get(&checkpoint.current_node)
.copied()
.unwrap_or(1);
entries.push(TimelineEntry {
ordinal,
node_name: checkpoint.current_node.clone(),
visit,
checkpoint_seq: record.seq,
run_commit_sha: checkpoint.git_commit_sha.clone(),
});
}
Ok(RunTimeline {
entries,
parallel_map: load_parallel_map(state),
})
}
pub async fn timeline(store: &Database, run_id: &RunId) -> Result<Vec<TimelineEntry>, Error> {
let run = store
.open_run(run_id)
.await
.map_err(|err| Error::engine(err.to_string()))?;
let state = run
.state()
.await
.map_err(|err| Error::engine(err.to_string()))?;
build_timeline(&state)
.map(|timeline| timeline.entries)
.map_err(|err| Error::engine(err.to_string()))
}
fn detect_parallel_interior(graph: &Graph) -> HashMap<String, String> {
let mut interior_map = HashMap::new();
for node in graph.nodes.values() {
if node.handler_type() != Some("parallel") {
continue;
}
let parallel_id = &node.id;
let mut queue: Vec<String> = graph
.outgoing_edges(parallel_id)
.iter()
.map(|e| e.to.clone())
.collect();
let mut visited = std::collections::HashSet::new();
while let Some(current) = queue.pop() {
if !visited.insert(current.clone()) {
continue;
}
if let Some(n) = graph.nodes.get(&current) {
if n.handler_type() == Some("parallel.fan_in") {
continue;
}
}
interior_map.insert(current.clone(), parallel_id.clone());
for edge in graph.outgoing_edges(&current) {
queue.push(edge.to.clone());
}
}
}
interior_map
}
fn load_parallel_map(state: &RunProjection) -> HashMap<String, String> {
let spec = &state.spec;
let map = detect_parallel_interior(&spec.graph);
if !map.is_empty() {
return map;
}
let Some(dot_source) = spec.graph_source.as_ref() else {
return HashMap::new();
};
let Ok(graph) = parser::parse(dot_source) else {
return HashMap::new();
};
detect_parallel_interior(&graph)
}
#[cfg(test)]
mod tests {
use std::collections::HashMap;
use chrono::Utc;
use fabro_types::{
Checkpoint, CheckpointRecord, Graph, PetriAdmission, RunDiff, RunSpec, WorkflowSettings,
fixtures, test_support,
};
use super::*;
fn checkpoint(
seq: u32,
current_node: &str,
visit: usize,
git_commit_sha: Option<&str>,
) -> CheckpointRecord {
let mut node_visits = HashMap::new();
node_visits.insert(current_node.to_string(), visit);
let checkpoint = Checkpoint {
timestamp: Utc::now(),
current_node: current_node.to_string(),
completed_nodes: Vec::new(),
node_retries: HashMap::new(),
context_values: HashMap::new(),
node_outcomes: HashMap::new(),
next_node_id: None,
git_commit_sha: git_commit_sha.map(ToOwned::to_owned),
loop_failure_signatures: HashMap::new(),
restart_failure_signatures: HashMap::new(),
node_visits,
};
CheckpointRecord {
seq,
checkpoint,
diff: RunDiff::default(),
}
}
fn test_projection() -> RunProjection {
RunProjection::new(
"Test run".to_string(),
RunSpec {
run_id: fixtures::RUN_1,
settings: WorkflowSettings::default(),
graph: Graph::new("test"),
graph_source: None,
workflow_slug: None,
workflow_version_id: None,
target: None,
automation: None,
source_directory: None,
labels: HashMap::new(),
provenance: test_support::test_run_provenance(),
definition_blob: None,
spec_blob: None,
git: None,
fork_source_ref: None,
admission: PetriAdmission::default(),
},
Utc::now(),
)
}
#[test]
fn parse_target_ordinal() {
assert_eq!("@4".parse::<ForkTarget>().unwrap(), ForkTarget::Ordinal(4));
}
#[test]
fn parse_target_latest_visit() {
assert_eq!(
"step2".parse::<ForkTarget>().unwrap(),
ForkTarget::LatestVisit("step2".to_string())
);
}
#[test]
fn build_timeline_simple() {
let mut state = test_projection();
state.checkpoints = vec![
checkpoint(7, "start", 1, Some("aaa")),
checkpoint(9, "build", 1, Some("bbb")),
];
let timeline = build_timeline(&state).unwrap();
assert_eq!(timeline.entries.len(), 2);
assert_eq!(timeline.entries[0].node_name, "start");
assert_eq!(timeline.entries[0].checkpoint_seq, 7);
assert_eq!(timeline.entries[1].node_name, "build");
}
#[test]
fn resolve_latest_visit() {
let timeline = RunTimeline {
entries: vec![
TimelineEntry {
ordinal: 1,
node_name: "start".to_string(),
visit: 1,
checkpoint_seq: 7,
run_commit_sha: Some("aaa".to_string()),
},
TimelineEntry {
ordinal: 2,
node_name: "build".to_string(),
visit: 1,
checkpoint_seq: 9,
run_commit_sha: Some("bbb".to_string()),
},
TimelineEntry {
ordinal: 3,
node_name: "build".to_string(),
visit: 2,
checkpoint_seq: 11,
run_commit_sha: Some("ccc".to_string()),
},
],
parallel_map: HashMap::new(),
};
let entry = timeline
.resolve(&ForkTarget::LatestVisit("build".to_string()))
.unwrap();
assert_eq!(entry.ordinal, 3);
}
#[test]
fn parallel_interior_detection() {
let mut graph = Graph::new("test");
let mut parallel_node = fabro_graphviz::graph::Node::new("parallel1");
parallel_node.attrs.insert(
"shape".to_string(),
fabro_graphviz::graph::AttrValue::String("component".to_string()),
);
graph.nodes.insert("parallel1".to_string(), parallel_node);
let mut fan_in = fabro_graphviz::graph::Node::new("fan_in1");
fan_in.attrs.insert(
"shape".to_string(),
fabro_graphviz::graph::AttrValue::String("tripleoctagon".to_string()),
);
graph.nodes.insert("fan_in1".to_string(), fan_in);
let mut a = fabro_graphviz::graph::Node::new("a");
a.attrs.insert(
"shape".to_string(),
fabro_graphviz::graph::AttrValue::String("box".to_string()),
);
graph.nodes.insert("a".to_string(), a);
graph.edges.push(fabro_graphviz::graph::Edge {
from: "parallel1".to_string(),
to: "a".to_string(),
attrs: HashMap::new(),
});
graph.edges.push(fabro_graphviz::graph::Edge {
from: "a".to_string(),
to: "fan_in1".to_string(),
attrs: HashMap::new(),
});
let map = detect_parallel_interior(&graph);
assert_eq!(map.get("a"), Some(&"parallel1".to_string()));
assert!(!map.contains_key("parallel1"));
}
}

108
lib/components/fabro-workflow/src/outcome.rs
View file

@ -1,111 +1,13 @@
pub use fabro_core::outcome::{
pub use fabro_types::ModelUsage;
pub use fabro_types::outcome::{
FailureCategory, FailureDetail, OutcomeMeta, StageOutcome, StageState,
};
pub use fabro_types::ModelUsage;
use crate::error::{FailureSignature, classify_failure_reason};
pub type Outcome = fabro_core::Outcome<Option<ModelUsage>>;
pub trait OutcomeExt: Sized {
fn fail_deterministic(reason: impl Into<String>) -> Self;
fn fail_classify(reason: impl Into<String>) -> Self;
fn retry_classify(reason: impl Into<String>) -> Self;
fn simulated(node_id: &str) -> Self;
#[must_use]
fn with_signature(self, sig: Option<impl Into<String>>) -> Self;
fn failure_reason(&self) -> Option<&str>;
fn failure_category(&self) -> Option<FailureCategory>;
fn classified_failure_category(&self) -> Option<FailureCategory>;
}
impl OutcomeExt for Outcome {
fn fail_deterministic(reason: impl Into<String>) -> Self {
Self {
status: StageOutcome::Failed {
retry_requested: false,
},
failure: Some(FailureDetail::new(reason, FailureCategory::Deterministic)),
..Self::default()
}
}
fn fail_classify(reason: impl Into<String>) -> Self {
let reason = reason.into();
let category = classify_failure_reason(&reason);
Self {
status: StageOutcome::Failed {
retry_requested: false,
},
failure: Some(FailureDetail::new(reason, category)),
..Self::default()
}
}
fn retry_classify(reason: impl Into<String>) -> Self {
let reason = reason.into();
let category = classify_failure_reason(&reason);
Self {
status: StageOutcome::Failed {
retry_requested: true,
},
failure: Some(FailureDetail::new(reason, category)),
..Self::default()
}
}
fn simulated(node_id: &str) -> Self {
Self {
notes: Some(format!("[Simulated] {node_id}")),
..Self::success()
}
}
fn with_signature(mut self, sig: Option<impl Into<String>>) -> Self {
if let Some(ref mut failure) = self.failure {
failure.signature = sig.map(|sig| FailureSignature(sig.into()));
}
self
}
fn failure_reason(&self) -> Option<&str> {
self.failure
.as_ref()
.map(|failure| failure.message.as_str())
}
fn failure_category(&self) -> Option<FailureCategory> {
self.failure.as_ref().map(|failure| failure.category)
}
fn classified_failure_category(&self) -> Option<FailureCategory> {
match self.status {
StageOutcome::Succeeded | StageOutcome::PartiallySucceeded | StageOutcome::Skipped => {
None
}
StageOutcome::Failed { .. } => self
.failure_category()
.or(Some(FailureCategory::Deterministic)),
}
}
}
/// A stage outcome carrying the model usage the stage reported.
pub type Outcome = fabro_types::Outcome<Option<ModelUsage>>;
/// Format a USD cost for display, to the cent.
#[must_use]
pub fn format_cost(cost: f64) -> String {
format!("${cost:.2}")
}
#[cfg(test)]
mod tests {
use super::OutcomeExt;
#[test]
fn retry_classify_marks_failed_outcome_with_retry_request() {
let outcome = crate::outcome::Outcome::retry_classify("timeout");
assert_eq!(outcome.status, crate::outcome::StageOutcome::Failed {
retry_requested: true,
});
assert!(outcome.status.retry_requested());
}
}

357
lib/components/fabro-workflow/src/pipeline/execute.rs
View file

@ -1,357 +0,0 @@
use std::sync::Arc;
use std::time::{Duration, Instant};
use fabro_core::executor::ExecutorBuilder;
use fabro_core::handler::NodeHandler;
use fabro_core::state::ExecutionState;
use tokio::sync::watch;
use tokio::task::JoinHandle;
use tokio::time::{Instant as TokioInstant, sleep_until};
use tokio_util::sync::CancellationToken;
use super::types::{Executed, Initialized};
use crate::artifact;
use crate::context::{self, Context};
use crate::error::Error;
use crate::event::{Emitter, Event};
use crate::graph::WorkflowGraph;
use crate::interview_runtime::InterviewBlockState;
use crate::lifecycle::WorkflowLifecycle;
use crate::node_handler::WorkflowNodeHandler;
use crate::outcome::Outcome;
use crate::records::Checkpoint;
fn seed_context_from_checkpoint(checkpoint: Option<&Checkpoint>) -> Context {
let context = Context::new();
if let Some(cp) = checkpoint {
for (k, v) in &cp.context_values {
context.set(k.clone(), v.clone());
}
}
context
}
/// Background watchdog that cancels a run which stops emitting events.
struct StallWatchdog {
/// Cancelled by the monitor once the run stalls. Handed to the executor.
stall_token: CancellationToken,
/// Cancelled by us to stop the monitor once the run finishes.
shutdown: CancellationToken,
task: JoinHandle<()>,
}
impl StallWatchdog {
fn spawn(
stall_timeout: Duration,
emitter: Arc<Emitter>,
interview_blocks: watch::Receiver<InterviewBlockState>,
) -> Self {
let stall_token = CancellationToken::new();
let shutdown = CancellationToken::new();
emitter.touch();
let task = tokio::spawn(monitor_for_stall(
stall_timeout,
stall_token.clone(),
shutdown.clone(),
emitter,
interview_blocks,
));
Self {
stall_token,
shutdown,
task,
}
}
fn stall_token(&self) -> CancellationToken {
self.stall_token.clone()
}
async fn stop(self) {
self.shutdown.cancel();
if let Err(error) = self.task.await {
tracing::error!(error = ?error, "stall watchdog task failed");
}
}
}
/// Cancels `stall_token` once the run goes `stall_timeout` without emitting an
/// event. Waiting on human input suspends the timer, and the first unblock
/// starts a fresh full deadline.
///
/// Ordinary activity does not wake this task — a busy run emits an event per
/// agent stream delta. The deadline instead re-reads `Emitter::last_activity()`
/// when it fires and re-arms if the run was active in the meantime.
async fn monitor_for_stall(
stall_timeout: Duration,
stall_token: CancellationToken,
shutdown: CancellationToken,
emitter: Arc<Emitter>,
mut interview_blocks: watch::Receiver<InterviewBlockState>,
) {
let mut deadline = emitter.last_activity() + stall_timeout;
loop {
let blocked = interview_blocks.borrow_and_update().is_run_blocked();
tokio::select! {
biased;
() = shutdown.cancelled() => return,
changed = interview_blocks.changed() => {
if changed.is_err() {
return;
}
// Blocking parks the timer; unblocking restarts the full budget.
deadline = TokioInstant::now() + stall_timeout;
}
() = sleep_until(deadline), if !blocked => {
let extended = emitter.last_activity() + stall_timeout;
if extended > deadline {
deadline = extended;
continue;
}
stall_token.cancel();
return;
}
}
}
}
/// EXECUTE phase: run the workflow graph.
///
/// Infallible at the function level — engine errors are captured in `outcome`.
pub async fn execute(init: Initialized) -> Executed {
let Initialized {
graph,
source: _,
run_options,
checkpoint,
seed_context,
on_node,
artifact_sink,
run_control,
engine,
model,
} = init;
let mut checkpoint = checkpoint;
if let Some(cp) = checkpoint.as_mut() {
artifact::normalize_checkpoint_for_resume(cp);
}
let start = Instant::now();
let graph_arc = Arc::new(graph.clone());
let wf_graph = WorkflowGraph(Arc::clone(&graph_arc));
let handler = Arc::new(WorkflowNodeHandler {
services: Arc::clone(&engine),
run_dir: run_options.run_dir.clone(),
graph: Arc::clone(&graph_arc),
});
let settings_arc = Arc::new(run_options.clone());
let lifecycle = WorkflowLifecycle::new(
&engine.run.emitter,
engine.run.hook_runner.clone(),
&engine.run.sandbox,
graph_arc,
&run_options.run_dir,
&engine.run.run_store,
artifact_sink,
&engine.run.locations,
&settings_arc,
Arc::clone(&engine.run.sandbox_git),
checkpoint.is_some(),
on_node,
run_control,
engine.run.stage_executions.clone(),
);
if let Some(ref cp) = checkpoint {
lifecycle.restore_circuit_breaker(
cp.loop_failure_signatures.clone(),
cp.restart_failure_signatures.clone(),
);
if cp.context_values.get(context::keys::INTERNAL_FIDELITY)
== Some(&serde_json::json!(
context::keys::Fidelity::Full.to_string()
))
{
lifecycle.set_degrade_fidelity_on_resume(true);
}
}
let state = if let Some(ref cp) = checkpoint {
match ExecutionState::new(&wf_graph).map_err(|e| Error::engine(e.to_string())) {
Ok(mut s) => {
for (k, v) in &cp.context_values {
s.context.set(k.clone(), v.clone());
}
s.completed_nodes.clone_from(&cp.completed_nodes);
s.node_retries.clone_from(&cp.node_retries);
if cp.node_visits.is_empty() {
for id in &cp.completed_nodes {
*s.node_visits.entry(id.clone()).or_insert(0) += 1;
}
} else {
s.node_visits.clone_from(&cp.node_visits);
}
for (k, v) in &cp.node_outcomes {
s.node_outcomes.insert(k.clone(), v.clone());
}
s.stage_index = cp.completed_nodes.len();
if let Some(ref next) = cp.next_node_id {
s.current_node_id.clone_from(next);
} else {
let edges = graph.outgoing_edges(&cp.current_node);
if let Some(edge) = edges.first() {
s.current_node_id.clone_from(&edge.to);
} else {
s.current_node_id.clone_from(&cp.current_node);
}
}
s
}
Err(err) => {
return Executed {
graph,
outcome: Err(err),
run_options,
wall_time_ms: crate::millis_u64(start.elapsed()),
final_context: seed_context_from_checkpoint(checkpoint.as_ref()),
engine,
model,
};
}
}
} else if let Some(seed) = seed_context {
match ExecutionState::new(&wf_graph).map_err(|e| Error::engine(e.to_string())) {
Ok(s) => {
for (k, v) in seed.snapshot() {
s.context.set(k, v);
}
s
}
Err(err) => {
return Executed {
graph,
outcome: Err(err),
run_options,
wall_time_ms: crate::millis_u64(start.elapsed()),
final_context: seed,
engine,
model,
};
}
}
} else {
match ExecutionState::new(&wf_graph).map_err(|e| Error::engine(e.to_string())) {
Ok(s) => s,
Err(err) => {
return Executed {
graph,
outcome: Err(err),
run_options,
wall_time_ms: crate::millis_u64(start.elapsed()),
final_context: Context::new(),
engine,
model,
};
}
}
};
let initial_context = state.context.clone();
let graph_max = graph.max_node_visits();
let max_node_visits = if graph_max > 0 {
Some(usize::try_from(graph_max).expect("positive max_node_visits should fit in usize"))
} else if run_options.dry_run_enabled() {
Some(10)
} else {
None
};
let stall_watchdog = graph.stall_timeout().map(|stall_timeout| {
StallWatchdog::spawn(
stall_timeout,
Arc::clone(&engine.run.emitter),
engine.run.interview_blocker.subscribe(),
)
});
let mut builder = ExecutorBuilder::new(handler as Arc<dyn NodeHandler<WorkflowGraph>>)
.lifecycle(Box::new(lifecycle));
builder = builder.cancel_token(run_options.cancel_token.clone());
if let Some(token) = stall_watchdog.as_ref().map(StallWatchdog::stall_token) {
builder = builder.stall_token(token);
}
if let Some(limit) = max_node_visits {
builder = builder.max_node_visits(limit);
}
let executor = builder.build();
let result = executor.run(&wf_graph, state).await;
if let Some(watchdog) = stall_watchdog {
watchdog.stop().await;
}
let (outcome, final_context) = match result {
Ok((core_outcome, final_state)) => {
let ctx = final_state.context.clone();
let result = if core_outcome.status.is_failure() {
core_outcome
} else {
let mut out = Outcome::success();
out.notes = Some("Pipeline completed".to_string());
out
};
(Ok(result), ctx)
}
Err(fabro_core::Error::StallTimeout { node_id }) => {
let stall_timeout = graph.stall_timeout().unwrap_or_default();
let idle_secs = stall_timeout.as_secs();
engine.run.emitter.emit(&Event::StallWatchdogTimeout {
node: node_id.clone(),
idle_seconds: idle_secs,
});
(
Err(Error::engine(format!(
"stall watchdog: node \"{node_id}\" had no activity for {idle_secs}s"
))),
initial_context,
)
}
Err(fabro_core::Error::Cancelled) => (Err(Error::Cancelled), initial_context),
Err(fabro_core::Error::Blocked { message }) => {
(Err(Error::engine(message)), initial_context)
}
Err(error @ fabro_core::Error::Context { .. }) => (
Err(Error::engine_with_source(
"Pipeline lifecycle operation failed",
error,
)),
initial_context,
),
Err(e) => (Err(Error::engine(e.to_string())), initial_context),
};
engine.registry.shutdown_all(&engine.run.emitter).await;
let wall_time_ms = crate::millis_u64(start.elapsed());
Executed {
graph,
outcome,
run_options,
wall_time_ms,
final_context,
engine,
model,
}
}
#[cfg(test)]
#[path = "execute/tests.rs"]
mod tests;

1649
lib/components/fabro-workflow/src/pipeline/execute/tests.rs
View file

File diff suppressed because it is too large Load diff

1301
lib/components/fabro-workflow/src/pipeline/finalize.rs
View file

File diff suppressed because it is too large Load diff

2001
lib/components/fabro-workflow/src/pipeline/initialize.rs
View file

File diff suppressed because it is too large Load diff

20
lib/components/fabro-workflow/src/pipeline/mod.rs
View file

@ -1,31 +1,13 @@
mod execute;
mod finalize;
mod initialize;
mod parse;
mod persist;
mod publish;
mod pull_request;
mod transform;
pub(crate) mod types;
mod validate;
pub use execute::execute;
pub(crate) use finalize::build_conclusion_from_store;
#[cfg(any(test, feature = "test-support"))]
pub(crate) use finalize::{build_terminal_event, usage_from_projection};
pub use finalize::{classify_engine_result, conclude, finalize};
pub use initialize::initialize;
pub use parse::parse;
pub(crate) use persist::persist;
pub use publish::publish;
pub use pull_request::{
AutoMergeOptions, CreatedPullRequest, OpenPullRequestRequest, PrContent, build_pr_content,
open_pull_request,
};
pub use transform::transform;
pub use types::{
Concluded, Executed, FinalizeOptions, Finalized, InitOptions, Initialized, LlmSpec, Parsed,
Persisted, PublishOptions, PublishOutcome, Published, ResumeState, SandboxEnvSpec,
TEMPLATE_UNDEFINED_VARIABLE_RULE, TransformOptions, Transformed, Validated,
Parsed, Persisted, TEMPLATE_UNDEFINED_VARIABLE_RULE, TransformOptions, Transformed, Validated,
};
pub use validate::validate;

291
lib/components/fabro-workflow/src/pipeline/persist.rs
View file

@ -1,9 +1,5 @@
use std::path::Path;
use super::types::{PersistOptions, Persisted, Validated};
use crate::error::Error;
use crate::records::RunSpec;
use crate::runtime_store::RunStoreHandle;
/// PERSIST phase: create the run directory and return durable metadata for
/// store persistence.
@ -30,86 +26,17 @@ pub(crate) fn persist(
))
}
pub(crate) async fn load_from_store(
run_store: &RunStoreHandle,
run_dir: &Path,
) -> Result<Persisted, Error> {
let state = run_store
.state()
.await
.map_err(|err| Error::engine(err.to_string()))?;
let run_spec = executable_run_spec(run_store, state.spec).await?;
let graph = run_spec.graph.clone();
let source = run_spec.graph_source.clone().unwrap_or_default();
Ok(Persisted::new(
graph,
source,
Vec::new(),
run_dir.to_path_buf(),
run_spec,
))
}
/// Replace the event-folded spec content with the exact bytes from the spec
/// blob. Stored events pass through secret redaction, so the folded spec is
/// display data; the blob written at creation is what execution must see.
/// Runs created before the blob existed fall back to the folded spec.
async fn executable_run_spec(
run_store: &RunStoreHandle,
folded: RunSpec,
) -> Result<RunSpec, Error> {
let Some(blob_id) = folded.spec_blob else {
return Ok(folded);
};
let bytes = run_store
.read_blob(&blob_id)
.await
.map_err(|err| Error::engine_with_anyhow("failed to read run spec blob", err))?
.ok_or_else(|| {
Error::engine(format!(
"run spec blob is missing from the run store: {blob_id}"
))
})?;
let mut spec: RunSpec = serde_json::from_slice(&bytes)
.map_err(|err| Error::engine_with_source("run spec blob was not valid JSON", err))?;
// The event stream stays authoritative for run identity, provenance, and
// blob ids. Prefer the unredacted graph source from the blob, with the
// folded source as a compatibility fallback.
spec.run_id = folded.run_id;
spec.provenance = folded.provenance;
spec.definition_blob = folded.definition_blob;
spec.spec_blob = folded.spec_blob;
spec.fork_source_ref = folded.fork_source_ref;
spec.graph_source = spec.graph_source.or(folded.graph_source);
Ok(spec)
}
#[cfg(test)]
#[expect(clippy::disallowed_methods, reason = "tests stage pipeline fixtures")]
mod tests {
use std::collections::HashMap;
use std::sync::Arc;
use std::time::Duration;
use fabro_graphviz::graph::{AttrValue, Edge, Graph, Node};
use fabro_store::{Database, RunDatabase};
use fabro_types::{PetriAdmission, fixtures, test_support};
use object_store::memory::InMemory;
use super::*;
use crate::event::{Event, append_event};
use crate::records::RunSpec;
fn memory_store() -> Arc<Database> {
Arc::new(fabro_store::test_support::test_database(
Arc::new(InMemory::new()),
"",
Duration::from_millis(1),
None,
))
}
fn graph_and_source() -> (Graph, String) {
let source = r#"digraph test {
graph [goal="Ship feature"];
@ -192,52 +119,6 @@ mod tests {
}
}
async fn seeded_store(record: &RunSpec, source: Option<&str>) -> RunDatabase {
seeded_store_with(record, source, Some(record)).await
}
async fn seeded_store_with(
record: &RunSpec,
source: Option<&str>,
blob_record: Option<&RunSpec>,
) -> RunDatabase {
let store = memory_store();
let run_store = store.create_run(&record.run_id).await.unwrap();
let spec_blob = match blob_record {
Some(blob_record) => Some(
run_store
.write_blob(&serde_json::to_vec(blob_record).unwrap())
.await
.unwrap(),
),
None => None,
};
append_event(&run_store, &record.run_id, &Event::RunCreated {
run_id: record.run_id,
title: None,
settings: serde_json::to_value(&record.settings).unwrap(),
graph: serde_json::to_value(&record.graph).unwrap(),
workflow_source: source.map(ToOwned::to_owned),
labels: record.labels.clone().into_iter().collect(),
source_directory: record.source_directory.clone(),
workflow_slug: record.workflow_slug.clone(),
workflow_version_id: None,
target: record.target.clone(),
automation: record.automation.clone(),
provenance: record.provenance.clone(),
spec_blob,
git: record.git.clone(),
fork_source_ref: record.fork_source_ref.clone(),
retried_from: None,
parent_id: None,
web_url: None,
admission: PetriAdmission::default(),
})
.await
.unwrap();
run_store
}
#[test]
fn persist_creates_run_dir_without_writing_legacy_files() {
let temp = tempfile::tempdir().unwrap();
@ -287,139 +168,6 @@ mod tests {
);
}
#[tokio::test]
async fn load_from_store_roundtrips_full_run_spec_fields() {
let temp = tempfile::tempdir().unwrap();
let run_dir = temp.path().join("run");
let (graph, source) = graph_and_source();
let mut expected = sample_record(different_graph());
expected.graph = graph.clone();
persist(
Validated::new(graph, source.clone(), vec![]),
PersistOptions {
run_dir: run_dir.clone(),
run_spec: expected.clone(),
},
)
.unwrap();
let run_store = seeded_store(&expected, Some(&source)).await;
let loaded = load_from_store(&run_store.clone().into(), &run_dir)
.await
.unwrap();
let loaded_record = loaded.run_spec();
assert_eq!(loaded_record.run_id, expected.run_id);
assert!(
(loaded_record.run_id.created_at().timestamp_millis()
- expected.run_id.created_at().timestamp_millis())
.abs()
<= 1
);
assert_eq!(loaded_record.settings, expected.settings);
assert_eq!(
serde_json::to_value(&loaded_record.graph).unwrap(),
serde_json::to_value(&expected.graph).unwrap()
);
assert_eq!(loaded_record.workflow_slug, expected.workflow_slug);
assert_eq!(loaded_record.source_directory, expected.source_directory);
assert_eq!(loaded_record.base_branch(), expected.base_branch());
assert_eq!(loaded_record.labels, expected.labels);
assert_eq!(loaded.source(), source);
assert!(loaded.diagnostics().is_empty());
}
#[tokio::test]
async fn load_from_store_preserves_high_entropy_dockerfile_content() {
// The spec the worker executes must survive the store byte-identical.
// Event redaction is a storage/display concern; when it reaches the
// spec that `load_from_store` rehydrates, the sandbox builds a
// corrupted Dockerfile: `ARG NAME=<hex>` pairs come back as
// `ARG REDACTED`, the build's `set -eu` step fails on the unset
// variable, and the environment's snapshot identity silently changes.
let temp = tempfile::tempdir().unwrap();
let run_dir = temp.path().join("run");
std::fs::create_dir_all(&run_dir).unwrap();
let (graph, source) = graph_and_source();
// Two shapes that must both survive: the hex pins that triggered the
// production failure, and a token high-entropy enough that any
// detector will keep flagging it in stored events. The second keeps
// this test red until execution stops reading redacted content,
// independent of how the entropy heuristic evolves.
let dockerfile = "FROM buildpack-deps:noble\n\
ARG DOCKER_INSTALL_COMMIT=5ce20f2eef3615d08fea941eda5a109e949e8ebf\n\
ARG DOCKER_INSTALL_SHA256=b991f2806186f7287bb9e53362060c382e906d154599b2fb0982f34246bacfd4\n\
ENV CACHE_SALT=xK9mZ2vL8nQ5rT1wY4bC7dF0gH3jE6p\n\
RUN install-docker \"${DOCKER_INSTALL_COMMIT}\" \"${DOCKER_INSTALL_SHA256}\"\n";
let mut record = sample_record(different_graph());
record.graph = graph;
record.settings.run.environment.image.dockerfile = Some(
fabro_types::settings::run::DockerfileSource::Inline(dockerfile.to_string()),
);
let run_store = seeded_store(&record, Some(&source)).await;
let loaded = load_from_store(&run_store.clone().into(), &run_dir)
.await
.unwrap();
assert_eq!(
loaded.run_spec().settings.run.environment.image.dockerfile,
Some(fabro_types::settings::run::DockerfileSource::Inline(
dockerfile.to_string()
)),
"the executable run spec must round-trip through the store unredacted"
);
}
#[tokio::test]
async fn load_from_store_falls_back_to_folded_spec_without_spec_blob() {
// Runs created before the spec blob existed carry no spec_blob on
// run.created; the folded spec is their only copy.
let temp = tempfile::tempdir().unwrap();
let run_dir = temp.path().join("run");
std::fs::create_dir_all(&run_dir).unwrap();
let (graph, source) = graph_and_source();
let mut record = sample_record(different_graph());
record.graph = graph;
let run_store = seeded_store_with(&record, Some(&source), None).await;
let loaded = load_from_store(&run_store.clone().into(), &run_dir)
.await
.unwrap();
assert_eq!(loaded.run_spec().settings, record.settings);
assert_eq!(loaded.run_spec().spec_blob, None);
}
#[tokio::test]
async fn load_from_store_uses_fork_reference_from_event_fold() {
let temp = tempfile::tempdir().unwrap();
let run_dir = temp.path().join("run");
std::fs::create_dir_all(&run_dir).unwrap();
let (graph, source) = graph_and_source();
let source_record = sample_record(graph.clone());
let mut fork_record = source_record.clone();
fork_record.run_id = fixtures::RUN_7;
fork_record.fork_source_ref = Some(fabro_types::ForkSourceRef {
source_run_id: source_record.run_id,
checkpoint_sha: "checkpoint-sha".to_string(),
});
let run_store = seeded_store_with(&fork_record, Some(&source), Some(&source_record)).await;
let loaded = load_from_store(&run_store.clone().into(), &run_dir)
.await
.unwrap();
assert_eq!(loaded.run_spec().run_id, fork_record.run_id);
assert_eq!(
loaded.run_spec().fork_source_ref,
fork_record.fork_source_ref
);
}
#[test]
fn persist_returns_error_on_io_failure() {
let temp = tempfile::tempdir().unwrap();
@ -435,43 +183,4 @@ mod tests {
assert!(matches!(err, Error::Io(_)));
}
#[tokio::test]
async fn load_from_store_uses_empty_source_when_graph_missing() {
let temp = tempfile::tempdir().unwrap();
let run_dir = temp.path().join("run");
std::fs::create_dir_all(&run_dir).unwrap();
let (graph, _source) = graph_and_source();
let mut record = sample_record(different_graph());
record.graph = graph;
let run_store = seeded_store(&record, None).await;
let loaded = load_from_store(&run_store.clone().into(), &run_dir)
.await
.unwrap();
assert!(loaded.source().is_empty());
}
#[tokio::test]
async fn load_from_store_reads_graph_from_run_spec_and_source_from_store() {
let temp = tempfile::tempdir().unwrap();
let run_dir = temp.path().join("run");
std::fs::create_dir_all(&run_dir).unwrap();
let (graph, source) = graph_and_source();
let mut record = sample_record(different_graph());
record.graph = graph.clone();
let run_store = seeded_store(&record, Some(&source)).await;
let loaded = load_from_store(&run_store.clone().into(), &run_dir)
.await
.unwrap();
assert_eq!(
serde_json::to_value(loaded.graph()).unwrap(),
serde_json::to_value(graph).unwrap()
);
assert_eq!(loaded.source(), source);
}
}

407
lib/components/fabro-workflow/src/pipeline/publish.rs
View file

@ -1,407 +0,0 @@
use std::fmt::Write as _;
use std::sync::Arc;
use fabro_types::ExecOutputTail;
use super::pull_request::{AutoMergeOptions, OpenPullRequestRequest, open_pull_request};
use super::types::{Concluded, PublishOptions, PublishOutcome, Published};
use crate::error::{Error, FailureCategory, classify_failure_reason};
use crate::event::Event;
use crate::lifecycle::git::push_run_branch;
/// PUBLISH phase: push the final run commit and, when configured, open a pull
/// request.
///
/// Publish is always present in the pipeline. It becomes a no-op when the run
/// did not succeed, is a dry run, or has no remote branch configured.
pub async fn publish(concluded: Concluded, options: &PublishOptions) -> Published {
let mut publish_outcome = PublishOutcome::default();
let publish_error = concluded.publish(options, &mut publish_outcome).await.err();
let Concluded {
outcome,
conclusion,
artifact_count,
graph: _,
run_options,
services,
} = concluded;
Published {
execution_outcome: outcome,
publish_outcome,
publish_error,
conclusion,
artifact_count,
run_options,
services,
}
}
/// Build the terminal publish error from a failed push operation.
///
/// Retries exhausted on transient classifications stay `TransientInfra`: a
/// mature-token 404 is not proof of permanent access loss — a service-side
/// failure presents the same surface — so `Deterministic` would need
/// independent evidence this path does not gather. Each attempt becomes one
/// bounded cause line in the failure detail; git output stays inside the
/// exec output tail.
fn publish_push_error(
run_branch: &str,
push_error: fabro_sandbox::Error,
exec_output_tail: Option<ExecOutputTail>,
attempts: &[fabro_sandbox::PushAttempt],
last_successful_push_at: Option<chrono::DateTime<chrono::Utc>>,
) -> Error {
let message = match last_successful_push_at {
Some(at) => format!(
"failed to push run branch '{run_branch}' (last successful push at {})",
at.to_rfc3339_opts(chrono::SecondsFormat::Millis, true)
),
None => format!("failed to push run branch '{run_branch}'"),
};
let failure_class = match attempts.last().and_then(|attempt| attempt.retry_reason) {
Some(_) => FailureCategory::TransientInfra,
None => classify_failure_reason(&format!(
"{message}: {}",
fabro_sandbox::display_for_log(&push_error)
)),
};
let causes = attempts.iter().map(push_attempt_cause).collect();
Error::publish_with_source_and_class(
message,
push_error,
failure_class,
exec_output_tail,
causes,
)
}
/// One bounded line per push attempt for the failure detail.
fn push_attempt_cause(attempt: &fabro_sandbox::PushAttempt) -> String {
let outcome = if attempt.success {
"succeeded".to_string()
} else {
attempt
.retry_reason
.map_or_else(|| "unclassified".to_string(), |reason| reason.to_string())
};
let mut line = format!(
"push attempt {} at {}: {outcome}",
attempt.attempt,
attempt
.started_at
.to_rfc3339_opts(chrono::SecondsFormat::Millis, true)
);
if let Some(age_ms) = attempt
.token
.and_then(|token| token.age_at(attempt.started_at))
.map(|age| u64::try_from(age.as_millis()).unwrap_or(u64::MAX))
{
let _ = write!(line, " (token age {age_ms}ms)");
}
line
}
impl Concluded {
/// Run the publish steps, recording each one into `outcome` as it lands.
///
/// `outcome` accumulates what actually happened, so a branch that reached
/// the remote is still reported when pull request creation later fails.
async fn publish(
&self,
options: &PublishOptions,
outcome: &mut PublishOutcome,
) -> Result<(), Error> {
// A run that did not succeed, or that never intended to touch the
// remote, has nothing to publish — even when a pull request was asked
// for. Only a run that got far enough to publish can fail publishing.
if !self
.outcome
.as_ref()
.is_ok_and(|o| o.status.is_successful())
|| self.run_options.dry_run_enabled()
{
return Ok(());
}
let pull_request_requested = options.pr_config.is_some();
let (origin_url, run_branch) = match self.publish_target(options) {
Ok(target) => target,
Err(_) if !pull_request_requested => return Ok(()),
Err(reason) => return Err(self.pull_request_error(reason)),
};
self.push_final_commit(run_branch).await?;
outcome.pushed_branch = Some(run_branch.to_string());
let Some(pr_config) = options.pr_config.as_ref() else {
return Ok(());
};
let diff = self.conclusion.diff.patch.as_deref().unwrap_or_default();
if diff.trim().is_empty() {
return Ok(());
}
// Only pull request creation needs the SHA, to check that the remote
// branch really carries this run's work. Pushing does not: the refspec
// sends whatever the branch points at.
let final_sha = self
.conclusion
.final_git_commit_sha
.as_deref()
.ok_or_else(|| {
self.pull_request_error("pull request creation requires the run's final commit SHA")
})?;
let base_branch = self.run_options.base_branch.as_deref().ok_or_else(|| {
self.pull_request_error("pull request creation requires a base branch")
})?;
let credentials = options.github_app.as_ref().ok_or_else(|| {
self.pull_request_error("pull request creation requires GitHub credentials")
})?;
let github_base_url = fabro_github::github_api_base_url();
let created = open_pull_request(OpenPullRequestRequest {
github: fabro_github::GitHubContext::new(credentials, &github_base_url),
origin_url,
base_branch,
head_branch: run_branch,
expected_head_sha: final_sha,
goal: self.graph.goal(),
diff,
model: &options.model,
draft: pr_config.draft,
auto_merge: pr_config.auto_merge.then_some(AutoMergeOptions {
merge_strategy: pr_config.merge_strategy,
}),
run_store: &self.services.run_store,
llm_source: Arc::clone(&self.services.llm_source),
catalog: Arc::clone(&self.services.catalog),
conclusion: Some(&self.conclusion),
run_state: None,
})
.await
.map_err(|error| {
self.services.emitter.emit(&Event::PullRequestFailed {
creation_id: None,
error: error.clone(),
});
Error::publish_with_source("failed to create pull request", anyhow::anyhow!(error))
})?;
self.services.emitter.emit(&Event::pull_request_created(
&created.link,
&created.base_branch,
&created.head_branch,
final_sha,
&created.title,
pr_config.draft,
));
outcome.pr_url = Some(created.link.html_url());
Ok(())
}
/// The origin and run branch to publish to.
///
/// `Err` carries why there is no target. That is only a failure when a
/// pull request was requested; otherwise publish just has nothing to do.
fn publish_target<'a>(
&'a self,
options: &'a PublishOptions,
) -> Result<(&'a str, &'a str), &'static str> {
let origin_url = options
.origin_url
.as_deref()
.filter(|origin| !origin.trim().is_empty())
.ok_or("pull request creation requires a GitHub origin URL")?;
let run_branch = self
.run_options
.run_branch()
.ok_or("pull request creation requires a run branch")?;
if !self.run_options.settings.run.run_branch.push {
return Err("pull request creation requires run branch pushing");
}
Ok((origin_url, run_branch))
}
async fn push_final_commit(&self, run_branch: &str) -> Result<(), Error> {
// The terminal push guards the whole run's value, so it gets a real
// retry budget; attempts are nearly free at this point.
let policy = fabro_sandbox::publish_push_policy();
match push_run_branch(self.services.sandbox.as_ref(), run_branch, &policy).await {
Ok(report) => {
self.services.sandbox_git.record_successful_push();
self.services.emitter.emit(&Event::GitPush {
branch: run_branch.to_string(),
success: true,
exec_output_tail: None,
attempts: report.attempts,
});
Ok(())
}
Err(push_error) => {
let fabro_sandbox::PushError { report, error } = push_error;
let exec_output_tail = fabro_sandbox::default_redacted_output_tail(&error);
let attempts = report.attempts;
self.services.emitter.emit(&Event::GitPush {
branch: run_branch.to_string(),
success: false,
exec_output_tail: exec_output_tail.clone(),
attempts: attempts.clone(),
});
Err(publish_push_error(
run_branch,
error,
exec_output_tail,
&attempts,
self.services.sandbox_git.last_successful_push_at(),
))
}
}
}
fn pull_request_error(&self, message: &str) -> Error {
self.services.emitter.emit(&Event::PullRequestFailed {
creation_id: None,
error: message.to_string(),
});
Error::publish(message)
}
}
#[cfg(test)]
mod tests {
use chrono::Utc;
use super::*;
use crate::error::FailureCategory;
fn push_attempt(
attempt: u32,
retry_reason: Option<fabro_sandbox::GitRetryReason>,
token_age_ms: Option<u64>,
) -> fabro_sandbox::PushAttempt {
let started_at = Utc::now();
fabro_sandbox::PushAttempt {
attempt,
started_at,
success: false,
retry_reason,
exec_output_tail: None,
token: token_age_ms.map(|age_ms| fabro_sandbox::TokenSnapshot {
generation: 14,
provenance: fabro_sandbox::TokenProvenance::Minted {
minted_at: started_at
- chrono::Duration::milliseconds(i64::try_from(age_ms).unwrap()),
expires_at: started_at + chrono::Duration::hours(1),
},
}),
}
}
fn push_attempts_with_reasons(
reasons: &[Option<fabro_sandbox::GitRetryReason>],
) -> Vec<fabro_sandbox::PushAttempt> {
reasons
.iter()
.enumerate()
.map(|(index, reason)| fabro_sandbox::PushAttempt {
attempt: u32::try_from(index).unwrap() + 1,
started_at: Utc::now(),
success: false,
retry_reason: *reason,
exec_output_tail: None,
token: None,
})
.collect()
}
fn push_source_error() -> fabro_sandbox::Error {
fabro_sandbox::Error::message("remote: Repository not found.")
}
/// Exhausted retries on a retryable classification are transient
/// infrastructure, not deterministic: the same push succeeded manually an
/// hour after run 01M0DH033P2XSTHAGVBHG6922F failed, with no
/// configuration change.
#[test]
fn exhausted_transient_retries_classify_as_transient_infra() {
let attempts = push_attempts_with_reasons(&[
Some(fabro_sandbox::GitRetryReason::TokenReplication),
Some(fabro_sandbox::GitRetryReason::TokenReplication),
]);
let error =
publish_push_error("fabro/run/test", push_source_error(), None, &attempts, None);
assert_eq!(error.failure_category(), FailureCategory::TransientInfra);
}
#[test]
fn permanently_classified_push_falls_back_to_message_sniffing() {
let attempts = push_attempts_with_reasons(&[None]);
let error =
publish_push_error("fabro/run/test", push_source_error(), None, &attempts, None);
// "Repository not found." carries no transient hint for the
// heuristic, so the fallback stays deterministic.
assert_eq!(error.failure_category(), FailureCategory::Deterministic);
}
#[test]
fn failure_detail_renders_one_cause_line_per_attempt() {
let attempts = vec![
push_attempt(
1,
Some(fabro_sandbox::GitRetryReason::TokenReplication),
Some(180),
),
push_attempt(
2,
Some(fabro_sandbox::GitRetryReason::TokenReplication),
Some(3320),
),
];
let last_push = Utc::now() - chrono::Duration::seconds(67);
let error = publish_push_error(
"fabro/run/test",
push_source_error(),
None,
&attempts,
Some(last_push),
);
let detail = error.to_failure_detail();
assert!(
detail.message.contains("last successful push at"),
"{}",
detail.message
);
let attempt_lines: Vec<&String> = detail
.causes
.iter()
.filter(|cause| cause.starts_with("push attempt"))
.collect();
assert_eq!(attempt_lines.len(), 2);
assert!(
attempt_lines[0].contains("token_replication"),
"{attempt_lines:?}"
);
assert!(
attempt_lines[0].contains("(token age 180ms)"),
"{attempt_lines:?}"
);
assert!(
attempt_lines[1].contains("(token age 3320ms)"),
"{attempt_lines:?}"
);
assert_eq!(
detail
.causes
.iter()
.filter(|cause| cause.as_str() == "remote: Repository not found.")
.count(),
1,
"the source chain must not repeat the inner push error"
);
}
}

2052
lib/components/fabro-workflow/src/pipeline/pull_request.rs
View file

File diff suppressed because it is too large Load diff

208
lib/components/fabro-workflow/src/pipeline/types.rs
View file

@ -1,40 +1,14 @@
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use fabro_graphviz::graph::Graph;
use fabro_interview::Interviewer;
use fabro_llm::lithos_catalog::Catalog;
use fabro_mcp::config::McpServerSettings;
use fabro_sandbox::SandboxSpec;
use fabro_template::TemplateContext;
use fabro_types::settings::run::{
PullRequestSettings, ResolvedGithubIntegration, RunModelControls,
};
use fabro_types::settings::server::ServerSandboxProvidersSettings;
use fabro_types::{ManifestPath, RunId, RunProjection};
use fabro_validate::{Diagnostic, Severity};
use fabro_vault::Vault;
use lithos_llm::catalog::ProviderId;
use tokio::sync::RwLock as AsyncRwLock;
use crate::artifact_upload::ArtifactSink;
use crate::context::Context;
use crate::error::Error;
use crate::event::Emitter;
use crate::file_resolver::FileResolver;
use crate::handler::HandlerRegistry;
use crate::model_fallback::ModelFallbackPolicy;
use crate::outcome::Outcome;
use crate::records::{Checkpoint, Conclusion, RunSpec};
use crate::run_control::RunControlState;
use crate::run_options::{GitCheckpointOptions, LifecycleOptions, RunOptions};
use crate::runtime_store::RunStoreHandle;
use crate::services::{EngineServices, FabroRunToolServices, RunServices};
use crate::stage_execution::StageExecutionSeed;
use crate::steering_hub::SteeringHub;
use crate::records::RunSpec;
use crate::transforms::{ModelResolutionTransform, RenderMode, Transform};
use crate::workflow_bundle::WorkflowBundle;
/// Output of the PARSE phase.
#[non_exhaustive]
@ -228,168 +202,6 @@ impl Persisted {
self.run_spec,
)
}
pub async fn load_from_store(
run_store: &RunStoreHandle,
run_dir: &Path,
) -> Result<Self, Error> {
super::persist::load_from_store(run_store, run_dir).await
}
}
#[derive(Clone)]
pub struct LlmSpec {
pub model: String,
pub provider_id: ProviderId,
pub fallbacks: ModelFallbackPolicy,
pub mcp_servers: Vec<McpServerSettings>,
pub model_controls: RunModelControls,
pub dry_run: bool,
}
#[derive(Clone)]
pub struct SandboxEnvSpec {
pub toml_env: HashMap<String, String>,
/// The resolved GitHub integration request (interpolated permissions
/// plus declared additional repositories). `None` when the run requests
/// no `GITHUB_TOKEN`.
pub github_integration: Option<ResolvedGithubIntegration>,
pub origin_url: Option<String>,
}
/// Opaque, internally consistent state needed to resume from the latest
/// checkpoint in a run projection.
pub struct ResumeState {
checkpoint: Checkpoint,
stage_executions: StageExecutionSeed,
}
impl ResumeState {
/// Build resume state from a projection's latest checkpoint and complete
/// stage history.
#[must_use]
pub fn from_projection(projection: &RunProjection) -> Option<Self> {
let checkpoint_record = projection.checkpoints.last()?;
Some(Self {
checkpoint: checkpoint_record.checkpoint.clone(),
stage_executions: StageExecutionSeed::from_projection(
projection,
checkpoint_record.seq,
),
})
}
pub(crate) fn into_parts(self) -> (Checkpoint, StageExecutionSeed) {
(self.checkpoint, self.stage_executions)
}
#[cfg(test)]
pub(crate) fn for_test(checkpoint: Checkpoint, stage_executions: StageExecutionSeed) -> Self {
Self {
checkpoint,
stage_executions,
}
}
}
pub struct InitOptions {
pub run_store: RunStoreHandle,
pub dry_run: bool,
pub emitter: Arc<Emitter>,
pub sandbox: SandboxSpec,
pub llm: LlmSpec,
pub interviewer: Arc<dyn Interviewer>,
pub steering_hub: Arc<SteeringHub>,
pub catalog: Arc<Catalog>,
pub lifecycle: LifecycleOptions,
pub run_options: RunOptions,
pub workflow_path: Option<ManifestPath>,
pub workflow_bundle: Option<Arc<WorkflowBundle>>,
pub hooks: fabro_hooks::HookSettings,
pub sandbox_env: SandboxEnvSpec,
pub vault: Arc<AsyncRwLock<Vault>>,
/// The server's sandbox provider settings, for reattaching a run's
/// sandbox on resume.
pub sandbox_providers: ServerSandboxProvidersSettings,
pub git: Option<GitCheckpointOptions>,
pub registry_override: Option<Arc<HandlerRegistry>>,
pub artifact_sink: Option<ArtifactSink>,
pub run_control: Option<Arc<RunControlState>>,
pub resume: Option<ResumeState>,
pub seed_context: Option<Context>,
pub fabro_run_tools: Option<FabroRunToolServices>,
}
/// Output of the INITIALIZE phase.
#[non_exhaustive]
pub struct Initialized {
pub graph: Graph,
pub source: String,
pub run_options: RunOptions,
pub(crate) checkpoint: Option<Checkpoint>,
pub(crate) seed_context: Option<Context>,
pub on_node: crate::OnNodeCallback,
pub artifact_sink: Option<ArtifactSink>,
pub run_control: Option<Arc<RunControlState>>,
pub engine: Arc<EngineServices>,
pub model: String,
}
/// Output of the EXECUTE phase.
#[non_exhaustive]
pub struct Executed {
pub graph: Graph,
pub outcome: Result<Outcome, Error>,
pub run_options: RunOptions,
/// Run wall-clock time in milliseconds from EXECUTE start to outcome.
pub wall_time_ms: u64,
pub final_context: Context,
pub engine: Arc<EngineServices>,
pub model: String,
}
/// Output of the CONCLUDE phase.
#[non_exhaustive]
pub struct Concluded {
pub outcome: Result<Outcome, Error>,
pub conclusion: Conclusion,
pub artifact_count: usize,
pub graph: Graph,
pub run_options: RunOptions,
pub services: Arc<RunServices>,
}
/// What the PUBLISH phase actually accomplished.
///
/// Recorded separately from the phase's error so a branch that reached the
/// remote is still reported when a later step, such as pull request creation,
/// fails. An all-`None` value means publish had nothing to do.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct PublishOutcome {
pub pushed_branch: Option<String>,
pub pr_url: Option<String>,
}
/// Output of the PUBLISH phase.
#[non_exhaustive]
pub struct Published {
pub execution_outcome: Result<Outcome, Error>,
pub publish_outcome: PublishOutcome,
pub publish_error: Option<Error>,
pub conclusion: Conclusion,
pub artifact_count: usize,
pub run_options: RunOptions,
pub services: Arc<RunServices>,
}
/// Output of the FINALIZE phase.
#[non_exhaustive]
pub struct Finalized {
pub run_id: RunId,
pub outcome: Result<Outcome, Error>,
pub conclusion: Conclusion,
pub pushed_branch: Option<String>,
pub pr_url: Option<String>,
}
/// Options for the TRANSFORM phase.
@ -404,21 +216,3 @@ pub struct TransformOptions {
/// model and provider selectors for catalog-free structural validation.
pub model_resolution: Option<ModelResolutionTransform>,
}
/// Options for the FINALIZE phase.
pub struct FinalizeOptions {
pub run_dir: PathBuf,
pub run_id: RunId,
pub workflow_name: String,
pub preserve_sandbox: bool,
pub stop_on_terminal: bool,
pub last_git_sha: Option<String>,
}
/// Options for the PUBLISH phase.
pub struct PublishOptions {
pub pr_config: Option<PullRequestSettings>,
pub github_app: Option<fabro_github::GitHubCredentials>,
pub origin_url: Option<String>,
pub model: String,
}

0
lib/components/fabro-workflow/src/pipeline/prompts/pr_body.md → lib/components/fabro-workflow/src/prompts/pr_body.md
View file

2056
lib/components/fabro-workflow/src/pull_request.rs
View file

File diff suppressed because it is too large Load diff

52
lib/components/fabro-workflow/src/records/checkpoint.rs
View file

@ -1,52 +0,0 @@
use std::collections::HashMap;
pub use fabro_types::checkpoint::Checkpoint;
use fabro_types::failure_signature::FailureSignature;
use crate::artifact;
use crate::context::Context;
use crate::outcome::Outcome;
pub trait CheckpointExt {
fn from_context(
context: &Context,
current_node: &str,
completed_nodes: Vec<String>,
node_retries: HashMap<String, u32>,
node_outcomes: HashMap<String, Outcome>,
next_node_id: Option<String>,
loop_failure_signatures: HashMap<FailureSignature, usize>,
restart_failure_signatures: HashMap<FailureSignature, usize>,
node_visits: HashMap<String, usize>,
) -> Self;
}
impl CheckpointExt for Checkpoint {
fn from_context(
context: &Context,
current_node: &str,
completed_nodes: Vec<String>,
node_retries: HashMap<String, u32>,
mut node_outcomes: HashMap<String, Outcome>,
next_node_id: Option<String>,
loop_failure_signatures: HashMap<FailureSignature, usize>,
restart_failure_signatures: HashMap<FailureSignature, usize>,
node_visits: HashMap<String, usize>,
) -> Self {
artifact::normalize_durable_outcomes(&mut node_outcomes);
Self {
timestamp: chrono::Utc::now(),
current_node: current_node.to_string(),
completed_nodes,
node_retries,
context_values: artifact::durable_context_snapshot(context),
node_outcomes,
next_node_id,
git_commit_sha: None,
loop_failure_signatures,
restart_failure_signatures,
node_visits,
}
}
}

3
lib/components/fabro-workflow/src/records/mod.rs
View file

@ -1,9 +1,8 @@
mod checkpoint;
mod conclusion;
mod run;
mod start;
pub use checkpoint::{Checkpoint, CheckpointExt};
pub use conclusion::{Conclusion, StageSummary};
pub use fabro_types::checkpoint::Checkpoint;
pub use run::RunSpec;
pub use start::StartRecord;

190
lib/components/fabro-workflow/src/retry.rs
View file

@ -1,190 +0,0 @@
use std::time::Duration;
use fabro_core::retry::{BackoffPolicy, RetryPolicy};
use fabro_graphviz::graph::types::{Graph as GvGraph, Node as GvNode};
const DEFAULT_BACKOFF: BackoffPolicy = BackoffPolicy {
initial_delay: Duration::from_secs(5),
factor: 2.0,
max_delay: Duration::from_mins(1),
jitter: true,
};
/// Build a retry policy from node and graph attributes.
/// If the node has a `retry_policy` attribute naming a preset, use that.
/// Otherwise, fall back to `max_retries` / graph default.
pub(crate) fn build_retry_policy(node: &GvNode, graph: &GvGraph) -> RetryPolicy {
if let Some(preset) = node.retry_policy() {
if let Some(policy) = preset_retry_policy(preset) {
return policy;
}
}
let max_retries = node
.max_retries()
.unwrap_or_else(|| graph.default_max_retries());
let max_attempts = u32::try_from(max_retries + 1).unwrap_or(1).max(1);
RetryPolicy {
max_attempts,
backoff: DEFAULT_BACKOFF,
}
}
fn preset_retry_policy(preset: &str) -> Option<RetryPolicy> {
match preset {
"none" => Some(RetryPolicy {
max_attempts: 1,
backoff: DEFAULT_BACKOFF,
}),
"standard" => Some(RetryPolicy {
max_attempts: 5,
backoff: DEFAULT_BACKOFF,
}),
"aggressive" => Some(RetryPolicy {
max_attempts: 5,
backoff: BackoffPolicy {
initial_delay: Duration::from_millis(500),
..DEFAULT_BACKOFF
},
}),
"linear" => Some(RetryPolicy {
max_attempts: 3,
backoff: BackoffPolicy {
initial_delay: Duration::from_millis(500),
factor: 1.0,
..DEFAULT_BACKOFF
},
}),
"patient" => Some(RetryPolicy {
max_attempts: 3,
backoff: BackoffPolicy {
initial_delay: Duration::from_secs(2),
factor: 3.0,
..DEFAULT_BACKOFF
},
}),
_ => None,
}
}
#[cfg(test)]
mod tests {
use std::time::Duration;
use fabro_graphviz::graph::{AttrValue, Graph, Node};
use super::*;
#[test]
fn build_retry_policy_from_node() {
let mut node = Node::new("n");
node.attrs
.insert("max_retries".to_string(), AttrValue::Integer(3));
let graph = Graph::new("test");
let policy = build_retry_policy(&node, &graph);
assert_eq!(policy.max_attempts, 4);
}
#[test]
fn build_retry_policy_from_graph_default() {
let node = Node::new("n");
let mut graph = Graph::new("test");
graph
.attrs
.insert("default_max_retries".to_string(), AttrValue::Integer(2));
let policy = build_retry_policy(&node, &graph);
assert_eq!(policy.max_attempts, 3);
}
#[test]
fn build_retry_policy_no_attrs_uses_graph_default_0() {
let node = Node::new("n");
let graph = Graph::new("test");
let policy = build_retry_policy(&node, &graph);
assert_eq!(policy.max_attempts, 1);
}
#[test]
fn build_retry_policy_from_retry_policy_attr() {
let mut node = Node::new("n");
node.attrs.insert(
"retry_policy".to_string(),
AttrValue::String("aggressive".to_string()),
);
let graph = Graph::new("test");
let policy = build_retry_policy(&node, &graph);
assert_eq!(policy.max_attempts, 5);
assert_eq!(policy.backoff.initial_delay, Duration::from_millis(500));
}
#[test]
fn build_retry_policy_fallback_when_no_retry_policy_attr() {
let mut node = Node::new("n");
node.attrs
.insert("max_retries".to_string(), AttrValue::Integer(3));
let graph = Graph::new("test");
let policy = build_retry_policy(&node, &graph);
assert_eq!(policy.max_attempts, 4);
assert_eq!(policy.backoff.initial_delay, Duration::from_secs(5));
}
#[test]
fn build_retry_policy_all_presets() {
let presets = [
("none", 1u32),
("standard", 5),
("aggressive", 5),
("linear", 3),
("patient", 3),
];
let graph = Graph::new("test");
let (name, expected) = presets[0];
let mut node = Node::new("n");
node.attrs.insert(
"retry_policy".to_string(),
AttrValue::String(name.to_string()),
);
assert_eq!(build_retry_policy(&node, &graph).max_attempts, expected);
let (name, expected) = presets[1];
node.attrs.insert(
"retry_policy".to_string(),
AttrValue::String(name.to_string()),
);
assert_eq!(build_retry_policy(&node, &graph).max_attempts, expected);
let (name, expected) = presets[2];
node.attrs.insert(
"retry_policy".to_string(),
AttrValue::String(name.to_string()),
);
assert_eq!(build_retry_policy(&node, &graph).max_attempts, expected);
let (name, expected) = presets[3];
node.attrs.insert(
"retry_policy".to_string(),
AttrValue::String(name.to_string()),
);
assert_eq!(build_retry_policy(&node, &graph).max_attempts, expected);
let (name, expected) = presets[4];
node.attrs.insert(
"retry_policy".to_string(),
AttrValue::String(name.to_string()),
);
assert_eq!(build_retry_policy(&node, &graph).max_attempts, expected);
}
#[test]
fn build_retry_policy_unknown_preset_falls_back() {
let mut node = Node::new("n");
node.attrs.insert(
"retry_policy".to_string(),
AttrValue::String("unknown_preset".to_string()),
);
let graph = Graph::new("test");
let policy = build_retry_policy(&node, &graph);
assert_eq!(policy.max_attempts, 1);
}
}

45
lib/components/fabro-workflow/src/run_control.rs
View file

@ -1,45 +0,0 @@
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use tokio::sync::Notify;
use crate::event::{Emitter, Event};
#[derive(Default)]
pub struct RunControlState {
pause_requested: AtomicBool,
notify: Notify,
}
impl RunControlState {
#[must_use]
pub fn new() -> Arc<Self> {
Arc::new(Self::default())
}
pub fn request_pause(&self) {
self.pause_requested.store(true, Ordering::Relaxed);
self.notify.notify_waiters();
}
pub fn request_unpause(&self) {
self.pause_requested.store(false, Ordering::Relaxed);
self.notify.notify_waiters();
}
pub fn pause_requested(&self) -> bool {
self.pause_requested.load(Ordering::Relaxed)
}
pub async fn wait_if_paused(&self, emitter: &Emitter) {
if !self.pause_requested() {
return;
}
emitter.emit(&Event::RunPaused);
while self.pause_requested() {
self.notify.notified().await;
}
emitter.emit(&Event::RunUnpaused);
}
}

31
lib/components/fabro-workflow/src/run_dir.rs
View file

@ -1,31 +0,0 @@
use crate::context::Context;
/// Read the workflow visit ordinal from context.
///
/// The raw context value is `0` when unset; workflow execution code treats
/// missing counts as the first visit for stage/log naming.
pub(crate) fn visit_from_context(context: &Context) -> usize {
context.node_visit_count().max(1)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::context::Context;
#[test]
fn visit_from_context_defaults_to_first_visit() {
let ctx = Context::new();
assert_eq!(visit_from_context(&ctx), 1);
}
#[test]
fn visit_from_context_preserves_stored_visit() {
let ctx = Context::new();
ctx.set(
crate::context::keys::INTERNAL_NODE_VISIT_COUNT,
serde_json::json!(3),
);
assert_eq!(visit_from_context(&ctx), 3);
}
}

8
lib/components/fabro-workflow/src/run_materialization.rs
View file

@ -64,7 +64,13 @@ fn materialize_run_with_eligible_providers(
settings.run.model.name = Some(resolved_model);
settings.run.model.provider = Some(resolved_provider.into_string());
materialize_goal_and_pull_request(&mut settings, graph);
Ok(settings)
}
/// The graph's goal becomes the run's inline goal (none when the graph has
/// none), and a pull request block the settings disable is dropped.
pub fn materialize_goal_and_pull_request(settings: &mut WorkflowSettings, graph: &Graph) {
let goal = graph.goal().to_string();
settings.run.goal = if goal.is_empty() {
None
@ -80,8 +86,6 @@ fn materialize_run_with_eligible_providers(
{
settings.run.pull_request = None;
}
Ok(settings)
}
pub(crate) fn resolve_run_model(

Some files were not shown because too many files have changed in this diff Show more