From 2d8cf4b4eff4fdc203b6e5686eaaf5b84d9f2c6b Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Tue, 24 Mar 2026 15:15:46 -0400 Subject: [PATCH] Implement all sub-lifecycles fully per plan specification MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fill in the previously stubbed ArtifactLifecycle and GitLifecycle, and complete FidelityLifecycle and CircuitBreakerLifecycle with their full behavior. Wire the orchestrator with context seeding, shared state, and all callback orderings matching the plan. FidelityLifecycle: use resolve_fidelity/resolve_thread_id for full resolution chains, add preamble building via build_preamble, set thread.{tid}.current_node key, store raw Edge for proper resolution. CircuitBreakerLifecycle: add on_edge_selected with TransientInfra guard and restart_failure_signatures tracking for loop_restart edges. EventLifecycle: add Skipped guard in after_node (engine.rs:2080 parity), read GitCheckpointResult for GitCommit/GitPush events in on_checkpoint, read artifact_store count and last_git_sha in on_run_end. HookLifecycle: add Skipped guard in after_node, add on_checkpoint for CheckpointSaved hook. DiskLifecycle: add on_run_start with write_manifest + write_run_status, use write_node_status with visit-based directory naming. GitLifecycle: full implementation — on_run_start resets last_git_sha and inits metadata branch; on_checkpoint does shadow commit, run branch commit, checkpoint re-save with SHA, push, and diff.patch; on_run_end writes final.patch. ArtifactLifecycle: full implementation — on_run_start swaps fresh store, before_attempt records epoch, after_attempt collects assets and emits AssetsCaptured, after_node offloads large values and syncs to sandbox. Orchestrator: context seeding (mirror_graph_attributes, INTERNAL_RUN_ID, INTERNAL_WORK_DIR) with is_initial_resume gating, shared state for checkpoint_git_result/last_git_sha/artifact_store, full callback wiring. Promote write_manifest, write_node_status, git_diff to pub(crate). Add Clone to RunConfig. Constructor takes Arc + is_resume. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../src/core_adapter/lifecycle/artifact.rs | 157 +++++++++- .../core_adapter/lifecycle/circuit_breaker.rs | 56 +++- .../src/core_adapter/lifecycle/disk.rs | 28 +- .../src/core_adapter/lifecycle/event.rs | 42 ++- .../src/core_adapter/lifecycle/fidelity.rs | 118 +++++--- .../src/core_adapter/lifecycle/git.rs | 275 +++++++++++++++++- .../src/core_adapter/lifecycle/hook.rs | 20 +- .../src/core_adapter/lifecycle/mod.rs | 149 ++++++++-- lib/crates/fabro-workflows/src/engine.rs | 30 +- 9 files changed, 775 insertions(+), 100 deletions(-) diff --git a/lib/crates/fabro-workflows/src/core_adapter/lifecycle/artifact.rs b/lib/crates/fabro-workflows/src/core_adapter/lifecycle/artifact.rs index 8471ddde6..0c6a7abe7 100644 --- a/lib/crates/fabro-workflows/src/core_adapter/lifecycle/artifact.rs +++ b/lib/crates/fabro-workflows/src/core_adapter/lifecycle/artifact.rs @@ -1,11 +1,160 @@ +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; + use async_trait::async_trait; -use fabro_core::lifecycle::RunLifecycle; +use fabro_core::graph::NodeSpec; +use fabro_core::lifecycle::{AttemptContext, AttemptResultContext, RunLifecycle}; +use fabro_core::outcome::NodeResult; +use fabro_core::state::RunState; use super::super::graph::WorkflowGraph; +use super::super::WorkflowNode; +use crate::artifact::{offload_large_values, sync_artifacts_to_env, ArtifactStore}; +use crate::engine; +use crate::event::{EventEmitter, RunNoticeLevel, WorkflowRunEvent}; +use crate::outcome::StageUsage; +use fabro_core::lifecycle::NodeDecision; + +type WfRunState = RunState>; +type WfNodeResult = NodeResult>; +type WfNodeDecision = NodeDecision>; + /// Sub-lifecycle responsible for artifact collection, offloading, and syncing. -/// Currently a stub — artifact operations are not yet wired through the core adapter. -pub struct ArtifactLifecycle; +pub struct ArtifactLifecycle { + pub sandbox: Arc, + pub artifact_store: Arc>, + pub artifact_base_dir: Option, + pub emitter: Arc, + pub run_dir: PathBuf, + pub asset_globs: Vec, + /// Per-attempt state: epoch seconds when the attempt started. + attempt_start_epoch: Mutex>, +} + +impl ArtifactLifecycle { + #[allow(clippy::too_many_arguments)] + pub fn new( + sandbox: Arc, + artifact_store: Arc>, + artifact_base_dir: Option, + emitter: Arc, + run_dir: PathBuf, + asset_globs: Vec, + ) -> Self { + Self { + sandbox, + artifact_store, + artifact_base_dir, + emitter, + run_dir, + asset_globs, + attempt_start_epoch: Mutex::new(None), + } + } +} #[async_trait] -impl RunLifecycle for ArtifactLifecycle {} +impl RunLifecycle for ArtifactLifecycle { + async fn on_run_start( + &self, + _graph: &WorkflowGraph, + _state: &WfRunState, + ) -> fabro_core::error::Result<()> { + // Swap in a fresh artifact store on restart (don't call clear() — preserves files on disk) + let mut store = self.artifact_store.lock().unwrap(); + *store = ArtifactStore::new(self.artifact_base_dir.clone()); + Ok(()) + } + + async fn before_attempt( + &self, + _ctx: &AttemptContext<'_, WorkflowGraph>, + _state: &WfRunState, + ) -> fabro_core::error::Result { + // Record epoch seconds (floored to integer for macOS stat mtime parity) + let epoch = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs() as f64) + .unwrap_or(0.0); + *self.attempt_start_epoch.lock().unwrap() = Some(epoch); + Ok(NodeDecision::Continue) + } + + async fn after_attempt( + &self, + ctx: &AttemptResultContext<'_, WorkflowGraph>, + state: &WfRunState, + ) -> fabro_core::error::Result<()> { + if self.asset_globs.is_empty() { + return Ok(()); + } + let epoch = self.attempt_start_epoch.lock().unwrap().unwrap_or(0.0); + let node_id = ctx.node.id(); + let visit = state.node_visits.get(node_id).copied().unwrap_or(1); + let stage_dir = engine::node_dir(&self.run_dir, node_id, visit); + let _ = std::fs::create_dir_all(&stage_dir); + + match crate::asset_snapshot::collect_assets( + &*self.sandbox, + &stage_dir, + &self.asset_globs, + epoch, + ) + .await + { + Ok(summary) if summary.files_copied > 0 => { + self.emitter.emit(&WorkflowRunEvent::AssetsCaptured { + node_id: node_id.to_string(), + files_copied: summary.files_copied, + total_bytes: summary.total_bytes, + files_skipped: summary.files_skipped, + }); + } + Ok(_) => {} // no files collected + Err(e) => { + self.emitter.emit(&WorkflowRunEvent::RunNotice { + level: RunNoticeLevel::Warn, + code: "asset_collection_failed".to_string(), + message: format!("[node: {node_id}] asset collection failed: {e}"), + }); + } + } + + Ok(()) + } + + async fn after_node( + &self, + node: &WorkflowNode, + result: &mut WfNodeResult, + _state: &WfRunState, + ) -> fabro_core::error::Result<()> { + let node_id = node.id(); + + // Offload large context_updates values to artifact store + { + let store = self.artifact_store.lock().unwrap(); + if let Err(e) = offload_large_values(&mut result.outcome.context_updates, &store) { + self.emitter.emit(&WorkflowRunEvent::RunNotice { + level: RunNoticeLevel::Warn, + code: "artifact_offload_failed".to_string(), + message: format!("[node: {node_id}] artifact offload failed: {e}"), + }); + } + } + + // Sync file-backed artifacts to sandbox environment + if let Err(e) = + sync_artifacts_to_env(&mut result.outcome.context_updates, &*self.sandbox).await + { + self.emitter.emit(&WorkflowRunEvent::RunNotice { + level: RunNoticeLevel::Warn, + code: "artifact_sync_failed".to_string(), + message: format!("[node: {node_id}] artifact sync failed: {e}"), + }); + } + + Ok(()) + } +} diff --git a/lib/crates/fabro-workflows/src/core_adapter/lifecycle/circuit_breaker.rs b/lib/crates/fabro-workflows/src/core_adapter/lifecycle/circuit_breaker.rs index 5050f7fbd..f2ee38238 100644 --- a/lib/crates/fabro-workflows/src/core_adapter/lifecycle/circuit_breaker.rs +++ b/lib/crates/fabro-workflows/src/core_adapter/lifecycle/circuit_breaker.rs @@ -4,14 +4,14 @@ use std::sync::Mutex; use async_trait::async_trait; use fabro_core::error::{CoreError, Result as CoreResult}; -use fabro_core::lifecycle::RunLifecycle; +use fabro_core::lifecycle::{EdgeContext, EdgeDecision, RunLifecycle}; use fabro_core::outcome::NodeResult; use fabro_core::state::RunState; use super::super::graph::WorkflowGraph; use super::super::WorkflowNode; -use crate::error::FailureSignature; -use crate::outcome::{StageStatus, StageUsage}; +use crate::error::{FailureCategory, FailureSignature}; +use crate::outcome::{OutcomeExt, StageStatus, StageUsage}; type WfRunState = RunState>; type WfNodeResult = NodeResult>; @@ -99,4 +99,54 @@ impl RunLifecycle for CircuitBreakerLifecycle { Ok(()) } + + async fn on_edge_selected( + &self, + ctx: &EdgeContext<'_, WorkflowGraph>, + _state: &WfRunState, + ) -> CoreResult { + // Only guard loop_restart edges + let Some(ref edge) = ctx.edge else { + return Ok(EdgeDecision::Continue); + }; + if !edge.inner().loop_restart() { + return Ok(EdgeDecision::Continue); + } + + let outcome = ctx.outcome; + + // Guard: only TransientInfra failures may trigger loop_restart + let failure_class = outcome.failure_category(); + if let Some(fc) = failure_class { + if fc != FailureCategory::TransientInfra { + return Err(CoreError::blocked(format!( + "loop_restart blocked: failure_class={fc} (requires transient_infra), failure_reason={}", + outcome.failure_reason().unwrap_or("none"), + ))); + } + } + + // Circuit breaker: check restart failure signatures + if let Some(ref failure) = outcome.failure { + let sig = FailureSignature::new( + ctx.from, + failure.category, + failure.signature.as_deref(), + Some(failure.message.as_str()), + ); + if failure.category.is_signature_tracked() { + let mut sigs = self.restart_failure_signatures.lock().unwrap(); + let count = sigs.entry(sig.clone()).or_insert(0); + *count += 1; + let limit = self.loop_restart_signature_limit; + if *count >= limit { + return Err(CoreError::blocked(format!( + "loop_restart circuit breaker: signature {sig} repeated {count} times (limit {limit})" + ))); + } + } + } + + Ok(EdgeDecision::Continue) + } } diff --git a/lib/crates/fabro-workflows/src/core_adapter/lifecycle/disk.rs b/lib/crates/fabro-workflows/src/core_adapter/lifecycle/disk.rs index 07e5b092b..e585c27a6 100644 --- a/lib/crates/fabro-workflows/src/core_adapter/lifecycle/disk.rs +++ b/lib/crates/fabro-workflows/src/core_adapter/lifecycle/disk.rs @@ -12,6 +12,7 @@ use super::super::graph::WorkflowGraph; use super::super::WorkflowNode; use super::circuit_breaker::CircuitBreakerLifecycle; use crate::checkpoint::Checkpoint; +use crate::engine::{self, RunConfig}; use crate::event::{EventEmitter, RunNoticeLevel, WorkflowRunEvent}; use crate::outcome::StageUsage; @@ -22,6 +23,8 @@ type WfNodeResult = NodeResult>; pub struct DiskLifecycle { pub run_dir: PathBuf, pub run_id: String, + pub graph: Arc, + pub config: Arc, pub emitter: Arc, pub circuit_breaker: Arc, pub checkpoint_enabled: bool, @@ -29,18 +32,31 @@ pub struct DiskLifecycle { #[async_trait] impl RunLifecycle for DiskLifecycle { + async fn on_run_start( + &self, + _graph: &WorkflowGraph, + _state: &WfRunState, + ) -> fabro_core::error::Result<()> { + // Write manifest.json + engine::write_manifest(&self.run_dir, &self.graph, &self.config); + // Write run status as Running + crate::run_status::write_run_status( + &self.run_dir, + crate::run_status::RunStatus::Running, + None, + ); + Ok(()) + } + async fn after_node( &self, node: &WorkflowNode, result: &mut WfNodeResult, - _state: &WfRunState, + state: &WfRunState, ) -> fabro_core::error::Result<()> { let gv = node.inner(); - let outcome = &result.outcome; - let status_dir = self.run_dir.join("stages").join(&gv.id); - let _ = std::fs::create_dir_all(&status_dir); - let status_path = status_dir.join("status.json"); - let _ = crate::save_json(outcome, &status_path, "node_status"); + let visit = state.node_visits.get(gv.id.as_str()).copied().unwrap_or(1); + engine::write_node_status(&self.run_dir, &gv.id, visit, &result.outcome); Ok(()) } diff --git a/lib/crates/fabro-workflows/src/core_adapter/lifecycle/event.rs b/lib/crates/fabro-workflows/src/core_adapter/lifecycle/event.rs index a56ddaa8d..1f5e0067e 100644 --- a/lib/crates/fabro-workflows/src/core_adapter/lifecycle/event.rs +++ b/lib/crates/fabro-workflows/src/core_adapter/lifecycle/event.rs @@ -10,6 +10,8 @@ use fabro_core::state::RunState; use super::super::graph::WorkflowGraph; use super::super::WorkflowNode; +use super::git::GitCheckpointResult; +use crate::artifact::ArtifactStore; use crate::event::{EventEmitter, WorkflowRunEvent}; use crate::outcome::{FailureCategory, FailureDetail, Outcome, StageStatus, StageUsage}; @@ -29,6 +31,11 @@ pub struct EventLifecycle { pub run_branch: Option, pub worktree_dir: Option, pub goal: Option, + // Shared swappable handle (same instance as orchestrator) + pub artifact_store: Arc>, + // Cross-lifecycle data + pub checkpoint_git_result: Arc>>, + pub last_git_sha: Arc>>, } #[async_trait] @@ -157,7 +164,7 @@ impl RunLifecycle for EventLifecycle { state: &WfRunState, ) -> fabro_core::error::Result<()> { let outcome = &result.outcome; - // Skip events for Skipped nodes + // Skipped nodes had no StageStarted, so skip completion events (engine.rs:2080) if outcome.status == StageStatus::Skipped { return Ok(()); } @@ -231,11 +238,34 @@ impl RunLifecycle for EventLifecycle { _state: &WfRunState, ) -> fabro_core::error::Result<()> { let status = result.outcome.status.to_string(); + + // Read git checkpoint result (set by GitLifecycle) + let git_result = self.checkpoint_git_result.lock().unwrap().take(); + + let git_sha = git_result.as_ref().and_then(|r| r.commit_sha.clone()); + self.emitter.emit(&WorkflowRunEvent::CheckpointCompleted { node_id: node.id().to_string(), status, - git_commit_sha: None, + git_commit_sha: git_sha.clone(), }); + + // Emit GitCommit + GitPush events if git produced results + if let Some(ref result) = git_result { + if let Some(ref sha) = result.commit_sha { + self.emitter.emit(&WorkflowRunEvent::GitCommit { + node_id: Some(node.id().to_string()), + sha: sha.clone(), + }); + } + for (branch, success) in &result.push_results { + self.emitter.emit(&WorkflowRunEvent::GitPush { + branch: branch.clone(), + success: *success, + }); + } + } + Ok(()) } @@ -244,14 +274,16 @@ impl RunLifecycle for EventLifecycle { return; } let duration_ms = self.run_start.lock().unwrap().elapsed().as_millis() as u64; + let artifact_count = self.artifact_store.lock().unwrap().list().len(); + let last_sha = self.last_git_sha.lock().unwrap().clone(); if outcome.status == StageStatus::Success || outcome.status == StageStatus::PartialSuccess { self.emitter.emit(&WorkflowRunEvent::WorkflowRunCompleted { duration_ms, - artifact_count: 0, + artifact_count, status: outcome.status.to_string(), total_cost: None, - final_git_commit_sha: None, + final_git_commit_sha: last_sha, usage: None, }); } else { @@ -263,7 +295,7 @@ impl RunLifecycle for EventLifecycle { self.emitter.emit(&WorkflowRunEvent::WorkflowRunFailed { error: crate::error::FabroError::engine(error_msg), duration_ms, - git_commit_sha: None, + git_commit_sha: last_sha, }); } } diff --git a/lib/crates/fabro-workflows/src/core_adapter/lifecycle/fidelity.rs b/lib/crates/fabro-workflows/src/core_adapter/lifecycle/fidelity.rs index f801470bb..8f2c2d7a6 100644 --- a/lib/crates/fabro-workflows/src/core_adapter/lifecycle/fidelity.rs +++ b/lib/crates/fabro-workflows/src/core_adapter/lifecycle/fidelity.rs @@ -9,16 +9,18 @@ use fabro_core::state::RunState; use super::super::graph::WorkflowGraph; use super::super::WorkflowNode; use crate::context::keys; +use crate::engine; use crate::outcome::StageUsage; +use crate::preamble::build_preamble; type WfRunState = RunState>; type WfNodeDecision = NodeDecision>; -/// Data captured from an edge selection to pass to the next node's before_node. +/// Graphviz edge captured from edge selection, passed to the next node's before_node +/// for fidelity/thread resolution. #[derive(Debug, Clone)] struct IncomingEdgeData { - fidelity: Option, - thread_id: Option, + edge: Arc, } /// Sub-lifecycle responsible for fidelity/thread resolution and context key setup. @@ -63,7 +65,71 @@ impl RunLifecycle for FidelityLifecycle { let incoming = self.incoming_edge_data.lock().unwrap().take(); let gv_node = node.inner(); - // Set context keys for the current node + // 1. Fidelity resolution via resolve_fidelity: edge → node → graph default → Compact + let incoming_edge_ref = incoming.as_ref().map(|d| d.edge.as_ref()); + let fidelity = engine::resolve_fidelity(incoming_edge_ref, gv_node, &self.graph); + + // 2. Fidelity degradation on resume (full → summary:high) + let fidelity = { + let mut degrade = self.degrade_fidelity_on_resume.lock().unwrap(); + if *degrade { + *degrade = false; + fidelity.degraded() + } else { + fidelity + } + }; + + // 3. Set INTERNAL_FIDELITY + state.context.set( + keys::INTERNAL_FIDELITY, + serde_json::json!(fidelity.to_string()), + ); + + // 4. Preamble building: if Full, empty preamble; otherwise build from context + let preamble = { + let wf_context = crate::context::Context::from_values(state.context.snapshot()); + build_preamble( + fidelity, + &wf_context, + &self.graph, + &state.completed_nodes, + &state.node_outcomes, + ) + }; + state + .context + .set(keys::CURRENT_PREAMBLE, serde_json::json!(preamble)); + + // 5. Thread ID resolution via resolve_thread_id: edge → node → graph default → class → previous + let thread_id = engine::resolve_thread_id( + incoming_edge_ref, + gv_node, + &self.graph, + state.previous_node_id.as_deref(), + ); + + // 6. Set thread.{tid}.current_node + if let Some(ref tid) = thread_id { + let key = format!("thread.{tid}.current_node"); + state.context.set(key, serde_json::json!(node.id())); + } + + // 7. Set INTERNAL_THREAD_ID (or null) + match thread_id { + Some(tid) => { + state + .context + .set(keys::INTERNAL_THREAD_ID, serde_json::json!(tid)); + } + None => { + state + .context + .set(keys::INTERNAL_THREAD_ID, serde_json::Value::Null); + } + } + + // 8. Set INTERNAL_NODE_VISIT_COUNT and CURRENT_NODE let visits = state.node_visits.get(node.id()).copied().unwrap_or(0); state .context @@ -72,47 +138,6 @@ impl RunLifecycle for FidelityLifecycle { .context .set(keys::INTERNAL_NODE_VISIT_COUNT, serde_json::json!(visits)); - // Fidelity resolution: edge → node → graph default → compact - let fidelity = if let Some(ref edge_data) = incoming { - edge_data - .fidelity - .as_deref() - .or(gv_node.fidelity()) - .unwrap_or("compact") - .to_string() - } else { - gv_node.fidelity().unwrap_or("compact").to_string() - }; - - // Fidelity degradation on resume - let fidelity = { - let mut degrade = self.degrade_fidelity_on_resume.lock().unwrap(); - if *degrade { - *degrade = false; - let parsed: keys::Fidelity = fidelity.parse().unwrap_or_default(); - parsed.degraded().to_string() - } else { - fidelity - } - }; - - state - .context - .set(keys::INTERNAL_FIDELITY, serde_json::json!(fidelity)); - - // Thread ID resolution: edge → node → graph default → previous node - if let Some(ref edge_data) = incoming { - if let Some(ref tid) = edge_data.thread_id { - state - .context - .set(keys::INTERNAL_THREAD_ID, serde_json::json!(tid)); - } - } else if let Some(tid) = gv_node.thread_id() { - state - .context - .set(keys::INTERNAL_THREAD_ID, serde_json::json!(tid)); - } - Ok(NodeDecision::Continue) } @@ -125,8 +150,7 @@ impl RunLifecycle for FidelityLifecycle { if let Some(ref edge) = ctx.edge { let gv_edge = edge.inner(); let edge_data = IncomingEdgeData { - fidelity: gv_edge.fidelity().map(String::from), - thread_id: gv_edge.thread_id().map(String::from), + edge: Arc::new(gv_edge.clone()), }; *self.incoming_edge_data.lock().unwrap() = Some(edge_data); } diff --git a/lib/crates/fabro-workflows/src/core_adapter/lifecycle/git.rs b/lib/crates/fabro-workflows/src/core_adapter/lifecycle/git.rs index a49fdda78..ccdc99a46 100644 --- a/lib/crates/fabro-workflows/src/core_adapter/lifecycle/git.rs +++ b/lib/crates/fabro-workflows/src/core_adapter/lifecycle/git.rs @@ -1,11 +1,280 @@ +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; + use async_trait::async_trait; +use fabro_core::error::CoreError; +use fabro_core::graph::NodeSpec; use fabro_core::lifecycle::RunLifecycle; +use fabro_core::outcome::NodeResult; +use fabro_core::state::RunState; use super::super::graph::WorkflowGraph; +use super::super::WorkflowNode; +use crate::artifact::ArtifactStore; +use crate::engine::{self, RunConfig}; +use crate::event::{EventEmitter, RunNoticeLevel, WorkflowRunEvent}; +use crate::outcome::{Outcome, StageStatus, StageUsage}; + +type WfRunState = RunState>; +type WfNodeResult = NodeResult>; + +/// Result of a git checkpoint operation, shared with EventLifecycle. +#[derive(Debug, Clone)] +pub struct GitCheckpointResult { + pub commit_sha: Option, + pub push_results: Vec<(String, bool)>, +} + /// Sub-lifecycle responsible for git operations (checkpoint commits, pushes, diffs). -/// Currently a stub — git operations are not yet wired through the core adapter. -pub struct GitLifecycle; +pub struct GitLifecycle { + pub sandbox: Arc, + pub artifact_store: Arc>, + pub emitter: Arc, + pub run_dir: PathBuf, + pub run_id: String, + pub config: Arc, + pub start_node_id: Option, + // Cross-lifecycle data (shared with EventLifecycle) + pub checkpoint_git_result: Arc>>, + pub last_git_sha: Arc>>, +} #[async_trait] -impl RunLifecycle for GitLifecycle {} +impl RunLifecycle for GitLifecycle { + async fn on_run_start( + &self, + _graph: &WorkflowGraph, + _state: &WfRunState, + ) -> fabro_core::error::Result<()> { + // Reset last_git_sha (diff base parity) + *self.last_git_sha.lock().unwrap() = None; + + // Init metadata branch (best-effort) + if let (Some(_), Some(ref repo_path)) = + (&self.config.meta_branch, &self.config.host_repo_path) + { + let store = crate::git::MetadataStore::new(repo_path, &self.config.git_author); + let manifest_bytes = { + let manifest_path = self.run_dir.join("manifest.json"); + std::fs::read(&manifest_path).unwrap_or_default() + }; + let dot_source = std::fs::read(self.run_dir.join("graph.fabro")) + .or_else(|_| std::fs::read(self.run_dir.join("graph.dot"))) + .unwrap_or_default(); + let sandbox_json = std::fs::read(self.run_dir.join("sandbox.json")).ok(); + let mut extra_files: Vec<(&str, &[u8])> = Vec::new(); + if let Some(ref data) = sandbox_json { + extra_files.push(("sandbox.json", data)); + } + if let Err(e) = store.init_run(&self.run_id, &manifest_bytes, &dot_source, &extra_files) + { + tracing::warn!( + run_id = %self.run_id, + error = %e, + "Metadata branch init failed" + ); + } + } + + Ok(()) + } + + async fn on_checkpoint( + &self, + node: &WorkflowNode, + result: &WfNodeResult, + _next_node_id: Option<&str>, + state: &WfRunState, + ) -> fabro_core::error::Result<()> { + let node_id = node.id(); + + // Skip git checkpoint for the start node (always empty) or if git disabled + if self.start_node_id.as_deref() == Some(node_id) || !self.config.git_checkpoint_enabled { + return Ok(()); + } + + // Shadow commit (best-effort, metadata branch) + let shadow_sha: Option = if let (Some(_), Some(ref repo_path)) = + (&self.config.meta_branch, &self.config.host_repo_path) + { + let store = crate::git::MetadataStore::new(repo_path, &self.config.git_author); + // Build checkpoint JSON for shadow branch + let checkpoint_path = self.run_dir.join("checkpoint.json"); + std::fs::read(&checkpoint_path).ok().and_then(|cp_json| { + let artifact_store = self.artifact_store.lock().unwrap(); + let mut extra_entries: Vec<(String, Vec)> = artifact_store + .list() + .iter() + .filter_map(|info| { + info.file_path.as_ref().and_then(|path| { + std::fs::read(path) + .ok() + .map(|data| (format!("artifacts/{}.json", info.id), data)) + }) + }) + .collect(); + extra_entries.extend(crate::git::scan_node_files(&self.run_dir)); + let extra_refs: Vec<(&str, &[u8])> = extra_entries + .iter() + .map(|(k, v)| (k.as_str(), v.as_slice())) + .collect(); + match store.write_checkpoint(&self.run_id, &cp_json, &extra_refs) { + Ok(sha) => Some(sha), + Err(e) => { + self.emitter.emit(&WorkflowRunEvent::RunNotice { + level: RunNoticeLevel::Warn, + code: "checkpoint_metadata_write_failed".to_string(), + message: format!( + "[node: {node_id}] metadata checkpoint write failed: {e}" + ), + }); + None + } + } + }) + } else { + None + }; + + // Run branch commit via sandbox + let completed_count = state.completed_nodes.len(); + let commit_result = engine::git_checkpoint( + &*self.sandbox, + &self.run_id, + node_id, + &result.outcome.status.to_string(), + completed_count, + shadow_sha, + &self.config.checkpoint_exclude_globs, + &self.config.git_author, + ) + .await; + + match commit_result { + Ok(sha) => { + let mut git_result = GitCheckpointResult { + commit_sha: Some(sha.clone()), + push_results: Vec::new(), + }; + + // Re-save checkpoint.json with SHA + let checkpoint_path = self.run_dir.join("checkpoint.json"); + if let Ok(mut cp) = crate::checkpoint::Checkpoint::load(&checkpoint_path) { + cp.git_commit_sha = Some(sha.clone()); + if let Err(e) = cp.save(&checkpoint_path) { + self.emitter.emit(&WorkflowRunEvent::RunNotice { + level: RunNoticeLevel::Warn, + code: "checkpoint_resave_failed".to_string(), + message: format!( + "[node: {node_id}] checkpoint re-save with SHA failed: {e}" + ), + }); + } + } + + // Push run branch (skip in dry-run mode) + if !self.config.dry_run { + if let Some(ref branch) = self.config.run_branch { + let push_ok = if self.sandbox.git_push_branch(branch).await { + true + } else if let Some(ref repo_path) = self.config.host_repo_path { + let refspec = format!("refs/heads/{branch}"); + engine::git_push_host( + repo_path, + &refspec, + &self.config.github_app, + "run branch", + ) + .await + } else { + false + }; + git_result.push_results.push((branch.clone(), push_ok)); + } + // Push metadata branch (always from host) + if let (Some(ref meta_branch), Some(ref repo_path)) = + (&self.config.meta_branch, &self.config.host_repo_path) + { + let refspec = format!("refs/heads/{meta_branch}"); + let meta_push_ok = engine::git_push_host( + repo_path, + &refspec, + &self.config.github_app, + "metadata branch", + ) + .await; + git_result + .push_results + .push((meta_branch.clone(), meta_push_ok)); + } + } + + // Save diff.patch + let visit = state.node_visits.get(node_id).copied().unwrap_or(1); + let prev = self + .last_git_sha + .lock() + .unwrap() + .clone() + .or_else(|| self.config.base_sha.clone()) + .unwrap_or_else(|| sha.clone()); + let diff_dest = engine::node_dir(&self.run_dir, node_id, visit).join("diff.patch"); + + match engine::git_diff(&*self.sandbox, &prev).await { + Ok(patch) if !patch.is_empty() => { + let _ = std::fs::write(&diff_dest, patch); + } + Ok(_) => {} + Err(err) => { + self.emitter.emit(&WorkflowRunEvent::RunNotice { + level: RunNoticeLevel::Warn, + code: "git_diff_failed".to_string(), + message: format!("[node: {node_id}] git diff failed: {err}"), + }); + } + } + + // Update shared state + *self.last_git_sha.lock().unwrap() = Some(sha); + *self.checkpoint_git_result.lock().unwrap() = Some(git_result); + } + Err(e) => { + // Emit CheckpointFailed and return error + self.emitter.emit(&WorkflowRunEvent::CheckpointFailed { + node_id: node_id.to_string(), + error: e.clone(), + }); + return Err(CoreError::Other(format!( + "git checkpoint commit failed for node '{node_id}': {e}" + ))); + } + } + + Ok(()) + } + + async fn on_run_end(&self, outcome: &Outcome, _state: &WfRunState) { + // Write final.patch on success + if (outcome.status == StageStatus::Success || outcome.status == StageStatus::PartialSuccess) + && self.config.git_checkpoint_enabled + { + if let Some(ref base_sha) = self.config.base_sha { + let diff_dest = self.run_dir.join("final.patch"); + match engine::git_diff(&*self.sandbox, base_sha).await { + Ok(patch) if !patch.is_empty() => { + let _ = std::fs::write(&diff_dest, patch); + } + Ok(_) => {} + Err(err) => { + self.emitter.emit(&WorkflowRunEvent::RunNotice { + level: RunNoticeLevel::Warn, + code: "git_diff_failed".to_string(), + message: format!("final diff failed: {err}"), + }); + } + } + } + } + } +} diff --git a/lib/crates/fabro-workflows/src/core_adapter/lifecycle/hook.rs b/lib/crates/fabro-workflows/src/core_adapter/lifecycle/hook.rs index be1cd842f..98f7f66ab 100644 --- a/lib/crates/fabro-workflows/src/core_adapter/lifecycle/hook.rs +++ b/lib/crates/fabro-workflows/src/core_adapter/lifecycle/hook.rs @@ -92,11 +92,10 @@ impl RunLifecycle for HookLifecycle { _state: &WfRunState, ) -> CoreResult<()> { let outcome = &result.outcome; - // Skip hooks for Skipped nodes + // Skipped nodes had no StageStarted, so skip hooks (engine.rs:2080) if outcome.status == StageStatus::Skipped { return Ok(()); } - let hook_event = if outcome.status == StageStatus::Fail { HookEvent::StageFailed } else { @@ -132,6 +131,23 @@ impl RunLifecycle for HookLifecycle { } } + async fn on_checkpoint( + &self, + node: &WorkflowNode, + _result: &WfNodeResult, + _next_node_id: Option<&str>, + _state: &WfRunState, + ) -> CoreResult<()> { + let mut hook_ctx = HookContext::new( + HookEvent::CheckpointSaved, + self.run_id.clone(), + self.graph_name.clone(), + ); + hook_ctx.node_id = Some(node.inner().id.clone()); + let _ = self.run_hook(&hook_ctx).await; + Ok(()) + } + async fn on_run_end(&self, outcome: &Outcome, state: &WfRunState) { if state.cancelled { return; diff --git a/lib/crates/fabro-workflows/src/core_adapter/lifecycle/mod.rs b/lib/crates/fabro-workflows/src/core_adapter/lifecycle/mod.rs index a8a73f0d8..1fda1c1c6 100644 --- a/lib/crates/fabro-workflows/src/core_adapter/lifecycle/mod.rs +++ b/lib/crates/fabro-workflows/src/core_adapter/lifecycle/mod.rs @@ -9,6 +9,7 @@ pub mod hook; use std::collections::HashMap; use std::path::PathBuf; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use std::time::Instant; @@ -23,6 +24,9 @@ use fabro_core::state::RunState; use super::graph::WorkflowGraph; use super::WorkflowNode; +use crate::artifact::ArtifactStore; +use crate::context; +use crate::engine::RunConfig; use crate::event::EventEmitter; use crate::outcome::{Outcome, StageUsage}; use fabro_hooks::HookRunner; @@ -34,7 +38,7 @@ use self::circuit_breaker::CircuitBreakerLifecycle; use self::disk::DiskLifecycle; use self::event::EventLifecycle; use self::fidelity::FidelityLifecycle; -use self::git::GitLifecycle; +use self::git::{GitCheckpointResult, GitLifecycle}; use self::hook::HookLifecycle; type WfRunState = RunState>; @@ -50,12 +54,19 @@ pub struct WorkflowLifecycle { auto_status: AutoStatusLifecycle, circuit_breaker: Arc, disk: DiskLifecycle, - #[allow(dead_code)] // stub — will be wired when git operations move to core adapter git: GitLifecycle, - #[allow(dead_code)] // stub — will be wired when artifact operations move to core adapter artifact: ArtifactLifecycle, /// Set in on_edge_selected when loop_restart approved; read+cleared by EventLifecycle::on_run_start restarted_from: Arc>>, + /// Shared git checkpoint result (written by git, read by event) + checkpoint_git_result: Arc>>, + /// True when constructed with a checkpoint; cleared after first on_run_start. + /// Gates mirror_graph_attributes on initial resume. + is_initial_resume: AtomicBool, + // Config needed for context seeding + graph: Arc, + run_id: String, + working_directory: Option, } impl WorkflowLifecycle { @@ -66,32 +77,46 @@ impl WorkflowLifecycle { sandbox: Arc, graph: Arc, run_dir: PathBuf, - run_id: String, - _dry_run: bool, - _labels: HashMap, + config: Arc, + is_resume: bool, ) -> Self { let restarted_from: Arc>> = Arc::new(Mutex::new(None)); let loop_restart_signature_limit = graph.loop_restart_signature_limit(); + let checkpoint_git_result: Arc>> = + Arc::new(Mutex::new(None)); + let last_git_sha: Arc>> = Arc::new(Mutex::new(None)); + let artifact_store = Arc::new(Mutex::new(ArtifactStore::new(Some(run_dir.clone())))); let circuit_breaker = Arc::new(CircuitBreakerLifecycle::new(loop_restart_signature_limit)); + let local_git_checkpoint = + config.git_checkpoint_enabled && sandbox.host_git_dir().is_some(); + let working_directory = if local_git_checkpoint { + Some(sandbox.working_directory().to_string()) + } else { + None + }; + let event = EventLifecycle { emitter: Arc::clone(&emitter), graph_name: graph.name.clone(), - run_id: run_id.clone(), + run_id: config.run_id.clone(), run_start: Mutex::new(Instant::now()), restarted_from: Arc::clone(&restarted_from), - base_sha: None, - run_branch: None, - worktree_dir: None, - goal: None, + base_sha: config.base_sha.clone(), + run_branch: config.run_branch.clone(), + worktree_dir: working_directory.clone(), + goal: (!graph.goal().is_empty()).then(|| graph.goal().to_string()), + artifact_store: Arc::clone(&artifact_store), + last_git_sha: Arc::clone(&last_git_sha), + checkpoint_git_result: Arc::clone(&checkpoint_git_result), }; let hook = HookLifecycle { hook_runner, sandbox: Arc::clone(&sandbox), run_dir: run_dir.clone(), - run_id: run_id.clone(), + run_id: config.run_id.clone(), graph_name: graph.name.clone(), }; @@ -99,12 +124,37 @@ impl WorkflowLifecycle { let disk = DiskLifecycle { run_dir: run_dir.clone(), - run_id: run_id.clone(), + run_id: config.run_id.clone(), + graph: Arc::clone(&graph), + config: Arc::clone(&config), emitter: Arc::clone(&emitter), circuit_breaker: Arc::clone(&circuit_breaker), checkpoint_enabled: true, }; + let start_node_id = graph.find_start_node().map(|n| n.id.clone()); + + let git = GitLifecycle { + sandbox: Arc::clone(&sandbox), + artifact_store: Arc::clone(&artifact_store), + emitter: Arc::clone(&emitter), + run_dir: run_dir.clone(), + run_id: config.run_id.clone(), + config: Arc::clone(&config), + start_node_id, + checkpoint_git_result: Arc::clone(&checkpoint_git_result), + last_git_sha: Arc::clone(&last_git_sha), + }; + + let artifact = ArtifactLifecycle::new( + Arc::clone(&sandbox), + Arc::clone(&artifact_store), + Some(run_dir.clone()), + Arc::clone(&emitter), + run_dir, + config.asset_globs.clone(), + ); + Self { event, hook, @@ -112,9 +162,14 @@ impl WorkflowLifecycle { auto_status: AutoStatusLifecycle, circuit_breaker, disk, - git: GitLifecycle, - artifact: ArtifactLifecycle, + git, + artifact, restarted_from, + checkpoint_git_result, + is_initial_resume: AtomicBool::new(is_resume), + graph, + run_id: config.run_id.clone(), + working_directory, } } @@ -136,11 +191,44 @@ impl WorkflowLifecycle { #[async_trait] impl RunLifecycle for WorkflowLifecycle { async fn on_run_start(&self, graph: &WorkflowGraph, state: &WfRunState) -> CoreResult<()> { + // Re-seed context keys (fires on initial start AND after every loop restart). + // mirror_graph_attributes: skip on initial checkpoint resume (context already has them) + if self.is_initial_resume.swap(false, Ordering::Relaxed) { + // First on_run_start after checkpoint resume — skip mirror_graph_attributes + } else { + // Mirror graph-level attributes into the core context + if !self.graph.goal().is_empty() { + state.context.set( + context::keys::GRAPH_GOAL, + serde_json::json!(self.graph.goal()), + ); + } + for (key, val) in &self.graph.attrs { + state.context.set( + context::keys::graph_attr_key(key), + serde_json::json!(val.to_string_value()), + ); + } + } + // Always set run_id and work_dir (idempotent) + state.context.set( + context::keys::INTERNAL_RUN_ID, + serde_json::json!(self.run_id), + ); + if let Some(ref wd) = self.working_directory { + state + .context + .set(context::keys::INTERNAL_WORK_DIR, serde_json::json!(wd)); + } + // Reset restart-scoped state self.fidelity.on_run_start(graph, state).await?; + self.artifact.on_run_start(graph, state).await?; // Observable callbacks self.event.on_run_start(graph, state).await?; self.hook.on_run_start(graph, state).await?; + self.disk.on_run_start(graph, state).await?; + self.git.on_run_start(graph, state).await?; Ok(()) } @@ -175,6 +263,8 @@ impl RunLifecycle for WorkflowLifecycle { } // Event emission self.event.before_attempt(ctx, state).await?; + // Record epoch AFTER hook+event (engine.rs:968→1006) + self.artifact.before_attempt(ctx, state).await?; Ok(NodeDecision::Continue) } @@ -183,6 +273,7 @@ impl RunLifecycle for WorkflowLifecycle { ctx: &AttemptResultContext<'_, WorkflowGraph>, state: &WfRunState, ) -> CoreResult<()> { + self.artifact.after_attempt(ctx, state).await?; self.event.after_attempt(ctx, state).await?; Ok(()) } @@ -198,6 +289,7 @@ impl RunLifecycle for WorkflowLifecycle { self.event.after_node(node, result, state).await?; self.hook.after_node(node, result, state).await?; self.disk.after_node(node, result, state).await?; + self.artifact.after_node(node, result, state).await?; Ok(()) } @@ -213,16 +305,20 @@ impl RunLifecycle for WorkflowLifecycle { // Hook can override/block match self.hook.on_edge_selected(ctx, state).await? { EdgeDecision::Continue => { - // If loop_restart edge approved by hook, mark for LoopRestart emission - if let Some(ref edge) = ctx.edge { - if edge.inner().loop_restart() { - *self.restarted_from.lock().unwrap() = - Some((ctx.from.to_string(), ctx.to.to_string())); + // Edge unchanged — check circuit breaker for loop_restart + let decision = self.circuit_breaker.on_edge_selected(ctx, state).await?; + // If loop_restart edge approved by both hook and circuit breaker, mark for LoopRestart emission + if matches!(decision, EdgeDecision::Continue) { + if let Some(ref edge) = ctx.edge { + if edge.inner().loop_restart() { + *self.restarted_from.lock().unwrap() = + Some((ctx.from.to_string(), ctx.to.to_string())); + } } } - Ok(EdgeDecision::Continue) + Ok(decision) } - decision => Ok(decision), + decision => Ok(decision), // Override/Block — skip circuit breaker } } @@ -236,9 +332,17 @@ impl RunLifecycle for WorkflowLifecycle { self.disk .on_checkpoint(node, result, next_node_id, state) .await?; + self.git + .on_checkpoint(node, result, next_node_id, state) + .await?; self.event .on_checkpoint(node, result, next_node_id, state) .await?; + self.hook + .on_checkpoint(node, result, next_node_id, state) + .await?; + // Clear checkpoint result for next checkpoint + *self.checkpoint_git_result.lock().unwrap() = None; Ok(()) } @@ -248,5 +352,6 @@ impl RunLifecycle for WorkflowLifecycle { } self.event.on_run_end(outcome, state).await; self.hook.on_run_end(outcome, state).await; + self.git.on_run_end(outcome, state).await; } } diff --git a/lib/crates/fabro-workflows/src/engine.rs b/lib/crates/fabro-workflows/src/engine.rs index 8f03063bc..a8273f90b 100644 --- a/lib/crates/fabro-workflows/src/engine.rs +++ b/lib/crates/fabro-workflows/src/engine.rs @@ -241,7 +241,11 @@ pub fn resolve_thread_id( // --- Run directory helpers (spec 5.6) --- /// Write manifest.json at the start of a workflow run. Returns the manifest. -fn write_manifest(run_dir: &Path, graph: &Graph, config: &RunConfig) -> crate::manifest::Manifest { +pub(crate) fn write_manifest( + run_dir: &Path, + graph: &Graph, + config: &RunConfig, +) -> crate::manifest::Manifest { let workflow_name = if graph.name.is_empty() { "unnamed".to_string() } else { @@ -289,7 +293,7 @@ pub fn visit_from_context(context: &Context) -> usize { } /// Write status.json for a completed node into {`run_dir}/nodes/{node_id}/status.json`. -fn write_node_status(run_dir: &Path, node_id: &str, visit: usize, outcome: &Outcome) { +pub(crate) fn write_node_status(run_dir: &Path, node_id: &str, visit: usize, outcome: &Outcome) { let node_dir = node_dir(run_dir, node_id, visit); let _ = std::fs::create_dir_all(&node_dir); let status = serde_json::json!({ @@ -723,7 +727,10 @@ pub async fn git_push_host( } /// Run a git diff via the sandbox. -async fn git_diff(sandbox: &dyn Sandbox, base: &str) -> std::result::Result { +pub(crate) async fn git_diff( + sandbox: &dyn Sandbox, + base: &str, +) -> std::result::Result { let cmd = format!("{GIT_REMOTE} diff {base} HEAD"); match sandbox.exec_command(&cmd, 30_000, None, None, None).await { Ok(r) if r.exit_code == 0 => Ok(r.stdout), @@ -777,6 +784,7 @@ pub async fn git_replace_worktree(sandbox: &dyn Sandbox, path: &str, branch: &st } /// Configuration for a workflow run. +#[derive(Clone)] pub struct RunConfig { pub run_dir: PathBuf, pub cancel_token: Option>, @@ -932,7 +940,7 @@ impl WorkflowRunEngine { } /// Mirror graph-level attributes into the context. - fn mirror_graph_attributes(graph: &Graph, context: &Context) { + pub(crate) fn mirror_graph_attributes(graph: &Graph, context: &Context) { if !graph.goal().is_empty() { context.set(context::keys::GRAPH_GOAL, serde_json::json!(graph.goal())); } @@ -1467,23 +1475,29 @@ impl WorkflowRunEngine { }); // Build lifecycle + let config_arc = std::sync::Arc::new(config.clone()); let lifecycle = crate::core_adapter::WorkflowLifecycle::new( self.services.emitter.clone(), self.services.hook_runner.clone(), self.services.sandbox.clone(), graph_arc, config.run_dir.clone(), - config.run_id.clone(), - config.dry_run, - config.labels.clone(), + config_arc, + resume_checkpoint.is_some(), ); - // Restore circuit breaker state from checkpoint + // Restore state from checkpoint if let Some(cp) = resume_checkpoint { lifecycle.restore_circuit_breaker( cp.loop_failure_signatures.clone(), cp.restart_failure_signatures.clone(), ); + // Degrade fidelity on the first resumed node when prior fidelity was Full + if cp.context_values.get(context::keys::INTERNAL_FIDELITY) + == Some(&serde_json::json!(context::keys::Fidelity::Full.to_string())) + { + lifecycle.set_degrade_fidelity_on_resume(true); + } } // Build RunState