diff --git a/lib/crates/fabro-workflows/src/core_adapter/lifecycle/artifact.rs b/lib/crates/fabro-workflows/src/core_adapter/lifecycle/artifact.rs index 8471ddde6..0c6a7abe7 100644 --- a/lib/crates/fabro-workflows/src/core_adapter/lifecycle/artifact.rs +++ b/lib/crates/fabro-workflows/src/core_adapter/lifecycle/artifact.rs @@ -1,11 +1,160 @@ +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; + use async_trait::async_trait; -use fabro_core::lifecycle::RunLifecycle; +use fabro_core::graph::NodeSpec; +use fabro_core::lifecycle::{AttemptContext, AttemptResultContext, RunLifecycle}; +use fabro_core::outcome::NodeResult; +use fabro_core::state::RunState; use super::super::graph::WorkflowGraph; +use super::super::WorkflowNode; +use crate::artifact::{offload_large_values, sync_artifacts_to_env, ArtifactStore}; +use crate::engine; +use crate::event::{EventEmitter, RunNoticeLevel, WorkflowRunEvent}; +use crate::outcome::StageUsage; +use fabro_core::lifecycle::NodeDecision; + +type WfRunState = RunState>; +type WfNodeResult = NodeResult>; +type WfNodeDecision = NodeDecision>; + /// Sub-lifecycle responsible for artifact collection, offloading, and syncing. -/// Currently a stub — artifact operations are not yet wired through the core adapter. -pub struct ArtifactLifecycle; +pub struct ArtifactLifecycle { + pub sandbox: Arc, + pub artifact_store: Arc>, + pub artifact_base_dir: Option, + pub emitter: Arc, + pub run_dir: PathBuf, + pub asset_globs: Vec, + /// Per-attempt state: epoch seconds when the attempt started. + attempt_start_epoch: Mutex>, +} + +impl ArtifactLifecycle { + #[allow(clippy::too_many_arguments)] + pub fn new( + sandbox: Arc, + artifact_store: Arc>, + artifact_base_dir: Option, + emitter: Arc, + run_dir: PathBuf, + asset_globs: Vec, + ) -> Self { + Self { + sandbox, + artifact_store, + artifact_base_dir, + emitter, + run_dir, + asset_globs, + attempt_start_epoch: Mutex::new(None), + } + } +} #[async_trait] -impl RunLifecycle for ArtifactLifecycle {} +impl RunLifecycle for ArtifactLifecycle { + async fn on_run_start( + &self, + _graph: &WorkflowGraph, + _state: &WfRunState, + ) -> fabro_core::error::Result<()> { + // Swap in a fresh artifact store on restart (don't call clear() — preserves files on disk) + let mut store = self.artifact_store.lock().unwrap(); + *store = ArtifactStore::new(self.artifact_base_dir.clone()); + Ok(()) + } + + async fn before_attempt( + &self, + _ctx: &AttemptContext<'_, WorkflowGraph>, + _state: &WfRunState, + ) -> fabro_core::error::Result { + // Record epoch seconds (floored to integer for macOS stat mtime parity) + let epoch = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs() as f64) + .unwrap_or(0.0); + *self.attempt_start_epoch.lock().unwrap() = Some(epoch); + Ok(NodeDecision::Continue) + } + + async fn after_attempt( + &self, + ctx: &AttemptResultContext<'_, WorkflowGraph>, + state: &WfRunState, + ) -> fabro_core::error::Result<()> { + if self.asset_globs.is_empty() { + return Ok(()); + } + let epoch = self.attempt_start_epoch.lock().unwrap().unwrap_or(0.0); + let node_id = ctx.node.id(); + let visit = state.node_visits.get(node_id).copied().unwrap_or(1); + let stage_dir = engine::node_dir(&self.run_dir, node_id, visit); + let _ = std::fs::create_dir_all(&stage_dir); + + match crate::asset_snapshot::collect_assets( + &*self.sandbox, + &stage_dir, + &self.asset_globs, + epoch, + ) + .await + { + Ok(summary) if summary.files_copied > 0 => { + self.emitter.emit(&WorkflowRunEvent::AssetsCaptured { + node_id: node_id.to_string(), + files_copied: summary.files_copied, + total_bytes: summary.total_bytes, + files_skipped: summary.files_skipped, + }); + } + Ok(_) => {} // no files collected + Err(e) => { + self.emitter.emit(&WorkflowRunEvent::RunNotice { + level: RunNoticeLevel::Warn, + code: "asset_collection_failed".to_string(), + message: format!("[node: {node_id}] asset collection failed: {e}"), + }); + } + } + + Ok(()) + } + + async fn after_node( + &self, + node: &WorkflowNode, + result: &mut WfNodeResult, + _state: &WfRunState, + ) -> fabro_core::error::Result<()> { + let node_id = node.id(); + + // Offload large context_updates values to artifact store + { + let store = self.artifact_store.lock().unwrap(); + if let Err(e) = offload_large_values(&mut result.outcome.context_updates, &store) { + self.emitter.emit(&WorkflowRunEvent::RunNotice { + level: RunNoticeLevel::Warn, + code: "artifact_offload_failed".to_string(), + message: format!("[node: {node_id}] artifact offload failed: {e}"), + }); + } + } + + // Sync file-backed artifacts to sandbox environment + if let Err(e) = + sync_artifacts_to_env(&mut result.outcome.context_updates, &*self.sandbox).await + { + self.emitter.emit(&WorkflowRunEvent::RunNotice { + level: RunNoticeLevel::Warn, + code: "artifact_sync_failed".to_string(), + message: format!("[node: {node_id}] artifact sync failed: {e}"), + }); + } + + Ok(()) + } +} diff --git a/lib/crates/fabro-workflows/src/core_adapter/lifecycle/circuit_breaker.rs b/lib/crates/fabro-workflows/src/core_adapter/lifecycle/circuit_breaker.rs index 5050f7fbd..f2ee38238 100644 --- a/lib/crates/fabro-workflows/src/core_adapter/lifecycle/circuit_breaker.rs +++ b/lib/crates/fabro-workflows/src/core_adapter/lifecycle/circuit_breaker.rs @@ -4,14 +4,14 @@ use std::sync::Mutex; use async_trait::async_trait; use fabro_core::error::{CoreError, Result as CoreResult}; -use fabro_core::lifecycle::RunLifecycle; +use fabro_core::lifecycle::{EdgeContext, EdgeDecision, RunLifecycle}; use fabro_core::outcome::NodeResult; use fabro_core::state::RunState; use super::super::graph::WorkflowGraph; use super::super::WorkflowNode; -use crate::error::FailureSignature; -use crate::outcome::{StageStatus, StageUsage}; +use crate::error::{FailureCategory, FailureSignature}; +use crate::outcome::{OutcomeExt, StageStatus, StageUsage}; type WfRunState = RunState>; type WfNodeResult = NodeResult>; @@ -99,4 +99,54 @@ impl RunLifecycle for CircuitBreakerLifecycle { Ok(()) } + + async fn on_edge_selected( + &self, + ctx: &EdgeContext<'_, WorkflowGraph>, + _state: &WfRunState, + ) -> CoreResult { + // Only guard loop_restart edges + let Some(ref edge) = ctx.edge else { + return Ok(EdgeDecision::Continue); + }; + if !edge.inner().loop_restart() { + return Ok(EdgeDecision::Continue); + } + + let outcome = ctx.outcome; + + // Guard: only TransientInfra failures may trigger loop_restart + let failure_class = outcome.failure_category(); + if let Some(fc) = failure_class { + if fc != FailureCategory::TransientInfra { + return Err(CoreError::blocked(format!( + "loop_restart blocked: failure_class={fc} (requires transient_infra), failure_reason={}", + outcome.failure_reason().unwrap_or("none"), + ))); + } + } + + // Circuit breaker: check restart failure signatures + if let Some(ref failure) = outcome.failure { + let sig = FailureSignature::new( + ctx.from, + failure.category, + failure.signature.as_deref(), + Some(failure.message.as_str()), + ); + if failure.category.is_signature_tracked() { + let mut sigs = self.restart_failure_signatures.lock().unwrap(); + let count = sigs.entry(sig.clone()).or_insert(0); + *count += 1; + let limit = self.loop_restart_signature_limit; + if *count >= limit { + return Err(CoreError::blocked(format!( + "loop_restart circuit breaker: signature {sig} repeated {count} times (limit {limit})" + ))); + } + } + } + + Ok(EdgeDecision::Continue) + } } diff --git a/lib/crates/fabro-workflows/src/core_adapter/lifecycle/disk.rs b/lib/crates/fabro-workflows/src/core_adapter/lifecycle/disk.rs index 07e5b092b..e585c27a6 100644 --- a/lib/crates/fabro-workflows/src/core_adapter/lifecycle/disk.rs +++ b/lib/crates/fabro-workflows/src/core_adapter/lifecycle/disk.rs @@ -12,6 +12,7 @@ use super::super::graph::WorkflowGraph; use super::super::WorkflowNode; use super::circuit_breaker::CircuitBreakerLifecycle; use crate::checkpoint::Checkpoint; +use crate::engine::{self, RunConfig}; use crate::event::{EventEmitter, RunNoticeLevel, WorkflowRunEvent}; use crate::outcome::StageUsage; @@ -22,6 +23,8 @@ type WfNodeResult = NodeResult>; pub struct DiskLifecycle { pub run_dir: PathBuf, pub run_id: String, + pub graph: Arc, + pub config: Arc, pub emitter: Arc, pub circuit_breaker: Arc, pub checkpoint_enabled: bool, @@ -29,18 +32,31 @@ pub struct DiskLifecycle { #[async_trait] impl RunLifecycle for DiskLifecycle { + async fn on_run_start( + &self, + _graph: &WorkflowGraph, + _state: &WfRunState, + ) -> fabro_core::error::Result<()> { + // Write manifest.json + engine::write_manifest(&self.run_dir, &self.graph, &self.config); + // Write run status as Running + crate::run_status::write_run_status( + &self.run_dir, + crate::run_status::RunStatus::Running, + None, + ); + Ok(()) + } + async fn after_node( &self, node: &WorkflowNode, result: &mut WfNodeResult, - _state: &WfRunState, + state: &WfRunState, ) -> fabro_core::error::Result<()> { let gv = node.inner(); - let outcome = &result.outcome; - let status_dir = self.run_dir.join("stages").join(&gv.id); - let _ = std::fs::create_dir_all(&status_dir); - let status_path = status_dir.join("status.json"); - let _ = crate::save_json(outcome, &status_path, "node_status"); + let visit = state.node_visits.get(gv.id.as_str()).copied().unwrap_or(1); + engine::write_node_status(&self.run_dir, &gv.id, visit, &result.outcome); Ok(()) } diff --git a/lib/crates/fabro-workflows/src/core_adapter/lifecycle/event.rs b/lib/crates/fabro-workflows/src/core_adapter/lifecycle/event.rs index a56ddaa8d..1f5e0067e 100644 --- a/lib/crates/fabro-workflows/src/core_adapter/lifecycle/event.rs +++ b/lib/crates/fabro-workflows/src/core_adapter/lifecycle/event.rs @@ -10,6 +10,8 @@ use fabro_core::state::RunState; use super::super::graph::WorkflowGraph; use super::super::WorkflowNode; +use super::git::GitCheckpointResult; +use crate::artifact::ArtifactStore; use crate::event::{EventEmitter, WorkflowRunEvent}; use crate::outcome::{FailureCategory, FailureDetail, Outcome, StageStatus, StageUsage}; @@ -29,6 +31,11 @@ pub struct EventLifecycle { pub run_branch: Option, pub worktree_dir: Option, pub goal: Option, + // Shared swappable handle (same instance as orchestrator) + pub artifact_store: Arc>, + // Cross-lifecycle data + pub checkpoint_git_result: Arc>>, + pub last_git_sha: Arc>>, } #[async_trait] @@ -157,7 +164,7 @@ impl RunLifecycle for EventLifecycle { state: &WfRunState, ) -> fabro_core::error::Result<()> { let outcome = &result.outcome; - // Skip events for Skipped nodes + // Skipped nodes had no StageStarted, so skip completion events (engine.rs:2080) if outcome.status == StageStatus::Skipped { return Ok(()); } @@ -231,11 +238,34 @@ impl RunLifecycle for EventLifecycle { _state: &WfRunState, ) -> fabro_core::error::Result<()> { let status = result.outcome.status.to_string(); + + // Read git checkpoint result (set by GitLifecycle) + let git_result = self.checkpoint_git_result.lock().unwrap().take(); + + let git_sha = git_result.as_ref().and_then(|r| r.commit_sha.clone()); + self.emitter.emit(&WorkflowRunEvent::CheckpointCompleted { node_id: node.id().to_string(), status, - git_commit_sha: None, + git_commit_sha: git_sha.clone(), }); + + // Emit GitCommit + GitPush events if git produced results + if let Some(ref result) = git_result { + if let Some(ref sha) = result.commit_sha { + self.emitter.emit(&WorkflowRunEvent::GitCommit { + node_id: Some(node.id().to_string()), + sha: sha.clone(), + }); + } + for (branch, success) in &result.push_results { + self.emitter.emit(&WorkflowRunEvent::GitPush { + branch: branch.clone(), + success: *success, + }); + } + } + Ok(()) } @@ -244,14 +274,16 @@ impl RunLifecycle for EventLifecycle { return; } let duration_ms = self.run_start.lock().unwrap().elapsed().as_millis() as u64; + let artifact_count = self.artifact_store.lock().unwrap().list().len(); + let last_sha = self.last_git_sha.lock().unwrap().clone(); if outcome.status == StageStatus::Success || outcome.status == StageStatus::PartialSuccess { self.emitter.emit(&WorkflowRunEvent::WorkflowRunCompleted { duration_ms, - artifact_count: 0, + artifact_count, status: outcome.status.to_string(), total_cost: None, - final_git_commit_sha: None, + final_git_commit_sha: last_sha, usage: None, }); } else { @@ -263,7 +295,7 @@ impl RunLifecycle for EventLifecycle { self.emitter.emit(&WorkflowRunEvent::WorkflowRunFailed { error: crate::error::FabroError::engine(error_msg), duration_ms, - git_commit_sha: None, + git_commit_sha: last_sha, }); } } diff --git a/lib/crates/fabro-workflows/src/core_adapter/lifecycle/fidelity.rs b/lib/crates/fabro-workflows/src/core_adapter/lifecycle/fidelity.rs index f801470bb..8f2c2d7a6 100644 --- a/lib/crates/fabro-workflows/src/core_adapter/lifecycle/fidelity.rs +++ b/lib/crates/fabro-workflows/src/core_adapter/lifecycle/fidelity.rs @@ -9,16 +9,18 @@ use fabro_core::state::RunState; use super::super::graph::WorkflowGraph; use super::super::WorkflowNode; use crate::context::keys; +use crate::engine; use crate::outcome::StageUsage; +use crate::preamble::build_preamble; type WfRunState = RunState>; type WfNodeDecision = NodeDecision>; -/// Data captured from an edge selection to pass to the next node's before_node. +/// Graphviz edge captured from edge selection, passed to the next node's before_node +/// for fidelity/thread resolution. #[derive(Debug, Clone)] struct IncomingEdgeData { - fidelity: Option, - thread_id: Option, + edge: Arc, } /// Sub-lifecycle responsible for fidelity/thread resolution and context key setup. @@ -63,7 +65,71 @@ impl RunLifecycle for FidelityLifecycle { let incoming = self.incoming_edge_data.lock().unwrap().take(); let gv_node = node.inner(); - // Set context keys for the current node + // 1. Fidelity resolution via resolve_fidelity: edge → node → graph default → Compact + let incoming_edge_ref = incoming.as_ref().map(|d| d.edge.as_ref()); + let fidelity = engine::resolve_fidelity(incoming_edge_ref, gv_node, &self.graph); + + // 2. Fidelity degradation on resume (full → summary:high) + let fidelity = { + let mut degrade = self.degrade_fidelity_on_resume.lock().unwrap(); + if *degrade { + *degrade = false; + fidelity.degraded() + } else { + fidelity + } + }; + + // 3. Set INTERNAL_FIDELITY + state.context.set( + keys::INTERNAL_FIDELITY, + serde_json::json!(fidelity.to_string()), + ); + + // 4. Preamble building: if Full, empty preamble; otherwise build from context + let preamble = { + let wf_context = crate::context::Context::from_values(state.context.snapshot()); + build_preamble( + fidelity, + &wf_context, + &self.graph, + &state.completed_nodes, + &state.node_outcomes, + ) + }; + state + .context + .set(keys::CURRENT_PREAMBLE, serde_json::json!(preamble)); + + // 5. Thread ID resolution via resolve_thread_id: edge → node → graph default → class → previous + let thread_id = engine::resolve_thread_id( + incoming_edge_ref, + gv_node, + &self.graph, + state.previous_node_id.as_deref(), + ); + + // 6. Set thread.{tid}.current_node + if let Some(ref tid) = thread_id { + let key = format!("thread.{tid}.current_node"); + state.context.set(key, serde_json::json!(node.id())); + } + + // 7. Set INTERNAL_THREAD_ID (or null) + match thread_id { + Some(tid) => { + state + .context + .set(keys::INTERNAL_THREAD_ID, serde_json::json!(tid)); + } + None => { + state + .context + .set(keys::INTERNAL_THREAD_ID, serde_json::Value::Null); + } + } + + // 8. Set INTERNAL_NODE_VISIT_COUNT and CURRENT_NODE let visits = state.node_visits.get(node.id()).copied().unwrap_or(0); state .context @@ -72,47 +138,6 @@ impl RunLifecycle for FidelityLifecycle { .context .set(keys::INTERNAL_NODE_VISIT_COUNT, serde_json::json!(visits)); - // Fidelity resolution: edge → node → graph default → compact - let fidelity = if let Some(ref edge_data) = incoming { - edge_data - .fidelity - .as_deref() - .or(gv_node.fidelity()) - .unwrap_or("compact") - .to_string() - } else { - gv_node.fidelity().unwrap_or("compact").to_string() - }; - - // Fidelity degradation on resume - let fidelity = { - let mut degrade = self.degrade_fidelity_on_resume.lock().unwrap(); - if *degrade { - *degrade = false; - let parsed: keys::Fidelity = fidelity.parse().unwrap_or_default(); - parsed.degraded().to_string() - } else { - fidelity - } - }; - - state - .context - .set(keys::INTERNAL_FIDELITY, serde_json::json!(fidelity)); - - // Thread ID resolution: edge → node → graph default → previous node - if let Some(ref edge_data) = incoming { - if let Some(ref tid) = edge_data.thread_id { - state - .context - .set(keys::INTERNAL_THREAD_ID, serde_json::json!(tid)); - } - } else if let Some(tid) = gv_node.thread_id() { - state - .context - .set(keys::INTERNAL_THREAD_ID, serde_json::json!(tid)); - } - Ok(NodeDecision::Continue) } @@ -125,8 +150,7 @@ impl RunLifecycle for FidelityLifecycle { if let Some(ref edge) = ctx.edge { let gv_edge = edge.inner(); let edge_data = IncomingEdgeData { - fidelity: gv_edge.fidelity().map(String::from), - thread_id: gv_edge.thread_id().map(String::from), + edge: Arc::new(gv_edge.clone()), }; *self.incoming_edge_data.lock().unwrap() = Some(edge_data); } diff --git a/lib/crates/fabro-workflows/src/core_adapter/lifecycle/git.rs b/lib/crates/fabro-workflows/src/core_adapter/lifecycle/git.rs index a49fdda78..ccdc99a46 100644 --- a/lib/crates/fabro-workflows/src/core_adapter/lifecycle/git.rs +++ b/lib/crates/fabro-workflows/src/core_adapter/lifecycle/git.rs @@ -1,11 +1,280 @@ +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; + use async_trait::async_trait; +use fabro_core::error::CoreError; +use fabro_core::graph::NodeSpec; use fabro_core::lifecycle::RunLifecycle; +use fabro_core::outcome::NodeResult; +use fabro_core::state::RunState; use super::super::graph::WorkflowGraph; +use super::super::WorkflowNode; +use crate::artifact::ArtifactStore; +use crate::engine::{self, RunConfig}; +use crate::event::{EventEmitter, RunNoticeLevel, WorkflowRunEvent}; +use crate::outcome::{Outcome, StageStatus, StageUsage}; + +type WfRunState = RunState>; +type WfNodeResult = NodeResult>; + +/// Result of a git checkpoint operation, shared with EventLifecycle. +#[derive(Debug, Clone)] +pub struct GitCheckpointResult { + pub commit_sha: Option, + pub push_results: Vec<(String, bool)>, +} + /// Sub-lifecycle responsible for git operations (checkpoint commits, pushes, diffs). -/// Currently a stub — git operations are not yet wired through the core adapter. -pub struct GitLifecycle; +pub struct GitLifecycle { + pub sandbox: Arc, + pub artifact_store: Arc>, + pub emitter: Arc, + pub run_dir: PathBuf, + pub run_id: String, + pub config: Arc, + pub start_node_id: Option, + // Cross-lifecycle data (shared with EventLifecycle) + pub checkpoint_git_result: Arc>>, + pub last_git_sha: Arc>>, +} #[async_trait] -impl RunLifecycle for GitLifecycle {} +impl RunLifecycle for GitLifecycle { + async fn on_run_start( + &self, + _graph: &WorkflowGraph, + _state: &WfRunState, + ) -> fabro_core::error::Result<()> { + // Reset last_git_sha (diff base parity) + *self.last_git_sha.lock().unwrap() = None; + + // Init metadata branch (best-effort) + if let (Some(_), Some(ref repo_path)) = + (&self.config.meta_branch, &self.config.host_repo_path) + { + let store = crate::git::MetadataStore::new(repo_path, &self.config.git_author); + let manifest_bytes = { + let manifest_path = self.run_dir.join("manifest.json"); + std::fs::read(&manifest_path).unwrap_or_default() + }; + let dot_source = std::fs::read(self.run_dir.join("graph.fabro")) + .or_else(|_| std::fs::read(self.run_dir.join("graph.dot"))) + .unwrap_or_default(); + let sandbox_json = std::fs::read(self.run_dir.join("sandbox.json")).ok(); + let mut extra_files: Vec<(&str, &[u8])> = Vec::new(); + if let Some(ref data) = sandbox_json { + extra_files.push(("sandbox.json", data)); + } + if let Err(e) = store.init_run(&self.run_id, &manifest_bytes, &dot_source, &extra_files) + { + tracing::warn!( + run_id = %self.run_id, + error = %e, + "Metadata branch init failed" + ); + } + } + + Ok(()) + } + + async fn on_checkpoint( + &self, + node: &WorkflowNode, + result: &WfNodeResult, + _next_node_id: Option<&str>, + state: &WfRunState, + ) -> fabro_core::error::Result<()> { + let node_id = node.id(); + + // Skip git checkpoint for the start node (always empty) or if git disabled + if self.start_node_id.as_deref() == Some(node_id) || !self.config.git_checkpoint_enabled { + return Ok(()); + } + + // Shadow commit (best-effort, metadata branch) + let shadow_sha: Option = if let (Some(_), Some(ref repo_path)) = + (&self.config.meta_branch, &self.config.host_repo_path) + { + let store = crate::git::MetadataStore::new(repo_path, &self.config.git_author); + // Build checkpoint JSON for shadow branch + let checkpoint_path = self.run_dir.join("checkpoint.json"); + std::fs::read(&checkpoint_path).ok().and_then(|cp_json| { + let artifact_store = self.artifact_store.lock().unwrap(); + let mut extra_entries: Vec<(String, Vec)> = artifact_store + .list() + .iter() + .filter_map(|info| { + info.file_path.as_ref().and_then(|path| { + std::fs::read(path) + .ok() + .map(|data| (format!("artifacts/{}.json", info.id), data)) + }) + }) + .collect(); + extra_entries.extend(crate::git::scan_node_files(&self.run_dir)); + let extra_refs: Vec<(&str, &[u8])> = extra_entries + .iter() + .map(|(k, v)| (k.as_str(), v.as_slice())) + .collect(); + match store.write_checkpoint(&self.run_id, &cp_json, &extra_refs) { + Ok(sha) => Some(sha), + Err(e) => { + self.emitter.emit(&WorkflowRunEvent::RunNotice { + level: RunNoticeLevel::Warn, + code: "checkpoint_metadata_write_failed".to_string(), + message: format!( + "[node: {node_id}] metadata checkpoint write failed: {e}" + ), + }); + None + } + } + }) + } else { + None + }; + + // Run branch commit via sandbox + let completed_count = state.completed_nodes.len(); + let commit_result = engine::git_checkpoint( + &*self.sandbox, + &self.run_id, + node_id, + &result.outcome.status.to_string(), + completed_count, + shadow_sha, + &self.config.checkpoint_exclude_globs, + &self.config.git_author, + ) + .await; + + match commit_result { + Ok(sha) => { + let mut git_result = GitCheckpointResult { + commit_sha: Some(sha.clone()), + push_results: Vec::new(), + }; + + // Re-save checkpoint.json with SHA + let checkpoint_path = self.run_dir.join("checkpoint.json"); + if let Ok(mut cp) = crate::checkpoint::Checkpoint::load(&checkpoint_path) { + cp.git_commit_sha = Some(sha.clone()); + if let Err(e) = cp.save(&checkpoint_path) { + self.emitter.emit(&WorkflowRunEvent::RunNotice { + level: RunNoticeLevel::Warn, + code: "checkpoint_resave_failed".to_string(), + message: format!( + "[node: {node_id}] checkpoint re-save with SHA failed: {e}" + ), + }); + } + } + + // Push run branch (skip in dry-run mode) + if !self.config.dry_run { + if let Some(ref branch) = self.config.run_branch { + let push_ok = if self.sandbox.git_push_branch(branch).await { + true + } else if let Some(ref repo_path) = self.config.host_repo_path { + let refspec = format!("refs/heads/{branch}"); + engine::git_push_host( + repo_path, + &refspec, + &self.config.github_app, + "run branch", + ) + .await + } else { + false + }; + git_result.push_results.push((branch.clone(), push_ok)); + } + // Push metadata branch (always from host) + if let (Some(ref meta_branch), Some(ref repo_path)) = + (&self.config.meta_branch, &self.config.host_repo_path) + { + let refspec = format!("refs/heads/{meta_branch}"); + let meta_push_ok = engine::git_push_host( + repo_path, + &refspec, + &self.config.github_app, + "metadata branch", + ) + .await; + git_result + .push_results + .push((meta_branch.clone(), meta_push_ok)); + } + } + + // Save diff.patch + let visit = state.node_visits.get(node_id).copied().unwrap_or(1); + let prev = self + .last_git_sha + .lock() + .unwrap() + .clone() + .or_else(|| self.config.base_sha.clone()) + .unwrap_or_else(|| sha.clone()); + let diff_dest = engine::node_dir(&self.run_dir, node_id, visit).join("diff.patch"); + + match engine::git_diff(&*self.sandbox, &prev).await { + Ok(patch) if !patch.is_empty() => { + let _ = std::fs::write(&diff_dest, patch); + } + Ok(_) => {} + Err(err) => { + self.emitter.emit(&WorkflowRunEvent::RunNotice { + level: RunNoticeLevel::Warn, + code: "git_diff_failed".to_string(), + message: format!("[node: {node_id}] git diff failed: {err}"), + }); + } + } + + // Update shared state + *self.last_git_sha.lock().unwrap() = Some(sha); + *self.checkpoint_git_result.lock().unwrap() = Some(git_result); + } + Err(e) => { + // Emit CheckpointFailed and return error + self.emitter.emit(&WorkflowRunEvent::CheckpointFailed { + node_id: node_id.to_string(), + error: e.clone(), + }); + return Err(CoreError::Other(format!( + "git checkpoint commit failed for node '{node_id}': {e}" + ))); + } + } + + Ok(()) + } + + async fn on_run_end(&self, outcome: &Outcome, _state: &WfRunState) { + // Write final.patch on success + if (outcome.status == StageStatus::Success || outcome.status == StageStatus::PartialSuccess) + && self.config.git_checkpoint_enabled + { + if let Some(ref base_sha) = self.config.base_sha { + let diff_dest = self.run_dir.join("final.patch"); + match engine::git_diff(&*self.sandbox, base_sha).await { + Ok(patch) if !patch.is_empty() => { + let _ = std::fs::write(&diff_dest, patch); + } + Ok(_) => {} + Err(err) => { + self.emitter.emit(&WorkflowRunEvent::RunNotice { + level: RunNoticeLevel::Warn, + code: "git_diff_failed".to_string(), + message: format!("final diff failed: {err}"), + }); + } + } + } + } + } +} diff --git a/lib/crates/fabro-workflows/src/core_adapter/lifecycle/hook.rs b/lib/crates/fabro-workflows/src/core_adapter/lifecycle/hook.rs index be1cd842f..98f7f66ab 100644 --- a/lib/crates/fabro-workflows/src/core_adapter/lifecycle/hook.rs +++ b/lib/crates/fabro-workflows/src/core_adapter/lifecycle/hook.rs @@ -92,11 +92,10 @@ impl RunLifecycle for HookLifecycle { _state: &WfRunState, ) -> CoreResult<()> { let outcome = &result.outcome; - // Skip hooks for Skipped nodes + // Skipped nodes had no StageStarted, so skip hooks (engine.rs:2080) if outcome.status == StageStatus::Skipped { return Ok(()); } - let hook_event = if outcome.status == StageStatus::Fail { HookEvent::StageFailed } else { @@ -132,6 +131,23 @@ impl RunLifecycle for HookLifecycle { } } + async fn on_checkpoint( + &self, + node: &WorkflowNode, + _result: &WfNodeResult, + _next_node_id: Option<&str>, + _state: &WfRunState, + ) -> CoreResult<()> { + let mut hook_ctx = HookContext::new( + HookEvent::CheckpointSaved, + self.run_id.clone(), + self.graph_name.clone(), + ); + hook_ctx.node_id = Some(node.inner().id.clone()); + let _ = self.run_hook(&hook_ctx).await; + Ok(()) + } + async fn on_run_end(&self, outcome: &Outcome, state: &WfRunState) { if state.cancelled { return; diff --git a/lib/crates/fabro-workflows/src/core_adapter/lifecycle/mod.rs b/lib/crates/fabro-workflows/src/core_adapter/lifecycle/mod.rs index a8a73f0d8..1fda1c1c6 100644 --- a/lib/crates/fabro-workflows/src/core_adapter/lifecycle/mod.rs +++ b/lib/crates/fabro-workflows/src/core_adapter/lifecycle/mod.rs @@ -9,6 +9,7 @@ pub mod hook; use std::collections::HashMap; use std::path::PathBuf; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use std::time::Instant; @@ -23,6 +24,9 @@ use fabro_core::state::RunState; use super::graph::WorkflowGraph; use super::WorkflowNode; +use crate::artifact::ArtifactStore; +use crate::context; +use crate::engine::RunConfig; use crate::event::EventEmitter; use crate::outcome::{Outcome, StageUsage}; use fabro_hooks::HookRunner; @@ -34,7 +38,7 @@ use self::circuit_breaker::CircuitBreakerLifecycle; use self::disk::DiskLifecycle; use self::event::EventLifecycle; use self::fidelity::FidelityLifecycle; -use self::git::GitLifecycle; +use self::git::{GitCheckpointResult, GitLifecycle}; use self::hook::HookLifecycle; type WfRunState = RunState>; @@ -50,12 +54,19 @@ pub struct WorkflowLifecycle { auto_status: AutoStatusLifecycle, circuit_breaker: Arc, disk: DiskLifecycle, - #[allow(dead_code)] // stub — will be wired when git operations move to core adapter git: GitLifecycle, - #[allow(dead_code)] // stub — will be wired when artifact operations move to core adapter artifact: ArtifactLifecycle, /// Set in on_edge_selected when loop_restart approved; read+cleared by EventLifecycle::on_run_start restarted_from: Arc>>, + /// Shared git checkpoint result (written by git, read by event) + checkpoint_git_result: Arc>>, + /// True when constructed with a checkpoint; cleared after first on_run_start. + /// Gates mirror_graph_attributes on initial resume. + is_initial_resume: AtomicBool, + // Config needed for context seeding + graph: Arc, + run_id: String, + working_directory: Option, } impl WorkflowLifecycle { @@ -66,32 +77,46 @@ impl WorkflowLifecycle { sandbox: Arc, graph: Arc, run_dir: PathBuf, - run_id: String, - _dry_run: bool, - _labels: HashMap, + config: Arc, + is_resume: bool, ) -> Self { let restarted_from: Arc>> = Arc::new(Mutex::new(None)); let loop_restart_signature_limit = graph.loop_restart_signature_limit(); + let checkpoint_git_result: Arc>> = + Arc::new(Mutex::new(None)); + let last_git_sha: Arc>> = Arc::new(Mutex::new(None)); + let artifact_store = Arc::new(Mutex::new(ArtifactStore::new(Some(run_dir.clone())))); let circuit_breaker = Arc::new(CircuitBreakerLifecycle::new(loop_restart_signature_limit)); + let local_git_checkpoint = + config.git_checkpoint_enabled && sandbox.host_git_dir().is_some(); + let working_directory = if local_git_checkpoint { + Some(sandbox.working_directory().to_string()) + } else { + None + }; + let event = EventLifecycle { emitter: Arc::clone(&emitter), graph_name: graph.name.clone(), - run_id: run_id.clone(), + run_id: config.run_id.clone(), run_start: Mutex::new(Instant::now()), restarted_from: Arc::clone(&restarted_from), - base_sha: None, - run_branch: None, - worktree_dir: None, - goal: None, + base_sha: config.base_sha.clone(), + run_branch: config.run_branch.clone(), + worktree_dir: working_directory.clone(), + goal: (!graph.goal().is_empty()).then(|| graph.goal().to_string()), + artifact_store: Arc::clone(&artifact_store), + last_git_sha: Arc::clone(&last_git_sha), + checkpoint_git_result: Arc::clone(&checkpoint_git_result), }; let hook = HookLifecycle { hook_runner, sandbox: Arc::clone(&sandbox), run_dir: run_dir.clone(), - run_id: run_id.clone(), + run_id: config.run_id.clone(), graph_name: graph.name.clone(), }; @@ -99,12 +124,37 @@ impl WorkflowLifecycle { let disk = DiskLifecycle { run_dir: run_dir.clone(), - run_id: run_id.clone(), + run_id: config.run_id.clone(), + graph: Arc::clone(&graph), + config: Arc::clone(&config), emitter: Arc::clone(&emitter), circuit_breaker: Arc::clone(&circuit_breaker), checkpoint_enabled: true, }; + let start_node_id = graph.find_start_node().map(|n| n.id.clone()); + + let git = GitLifecycle { + sandbox: Arc::clone(&sandbox), + artifact_store: Arc::clone(&artifact_store), + emitter: Arc::clone(&emitter), + run_dir: run_dir.clone(), + run_id: config.run_id.clone(), + config: Arc::clone(&config), + start_node_id, + checkpoint_git_result: Arc::clone(&checkpoint_git_result), + last_git_sha: Arc::clone(&last_git_sha), + }; + + let artifact = ArtifactLifecycle::new( + Arc::clone(&sandbox), + Arc::clone(&artifact_store), + Some(run_dir.clone()), + Arc::clone(&emitter), + run_dir, + config.asset_globs.clone(), + ); + Self { event, hook, @@ -112,9 +162,14 @@ impl WorkflowLifecycle { auto_status: AutoStatusLifecycle, circuit_breaker, disk, - git: GitLifecycle, - artifact: ArtifactLifecycle, + git, + artifact, restarted_from, + checkpoint_git_result, + is_initial_resume: AtomicBool::new(is_resume), + graph, + run_id: config.run_id.clone(), + working_directory, } } @@ -136,11 +191,44 @@ impl WorkflowLifecycle { #[async_trait] impl RunLifecycle for WorkflowLifecycle { async fn on_run_start(&self, graph: &WorkflowGraph, state: &WfRunState) -> CoreResult<()> { + // Re-seed context keys (fires on initial start AND after every loop restart). + // mirror_graph_attributes: skip on initial checkpoint resume (context already has them) + if self.is_initial_resume.swap(false, Ordering::Relaxed) { + // First on_run_start after checkpoint resume — skip mirror_graph_attributes + } else { + // Mirror graph-level attributes into the core context + if !self.graph.goal().is_empty() { + state.context.set( + context::keys::GRAPH_GOAL, + serde_json::json!(self.graph.goal()), + ); + } + for (key, val) in &self.graph.attrs { + state.context.set( + context::keys::graph_attr_key(key), + serde_json::json!(val.to_string_value()), + ); + } + } + // Always set run_id and work_dir (idempotent) + state.context.set( + context::keys::INTERNAL_RUN_ID, + serde_json::json!(self.run_id), + ); + if let Some(ref wd) = self.working_directory { + state + .context + .set(context::keys::INTERNAL_WORK_DIR, serde_json::json!(wd)); + } + // Reset restart-scoped state self.fidelity.on_run_start(graph, state).await?; + self.artifact.on_run_start(graph, state).await?; // Observable callbacks self.event.on_run_start(graph, state).await?; self.hook.on_run_start(graph, state).await?; + self.disk.on_run_start(graph, state).await?; + self.git.on_run_start(graph, state).await?; Ok(()) } @@ -175,6 +263,8 @@ impl RunLifecycle for WorkflowLifecycle { } // Event emission self.event.before_attempt(ctx, state).await?; + // Record epoch AFTER hook+event (engine.rs:968→1006) + self.artifact.before_attempt(ctx, state).await?; Ok(NodeDecision::Continue) } @@ -183,6 +273,7 @@ impl RunLifecycle for WorkflowLifecycle { ctx: &AttemptResultContext<'_, WorkflowGraph>, state: &WfRunState, ) -> CoreResult<()> { + self.artifact.after_attempt(ctx, state).await?; self.event.after_attempt(ctx, state).await?; Ok(()) } @@ -198,6 +289,7 @@ impl RunLifecycle for WorkflowLifecycle { self.event.after_node(node, result, state).await?; self.hook.after_node(node, result, state).await?; self.disk.after_node(node, result, state).await?; + self.artifact.after_node(node, result, state).await?; Ok(()) } @@ -213,16 +305,20 @@ impl RunLifecycle for WorkflowLifecycle { // Hook can override/block match self.hook.on_edge_selected(ctx, state).await? { EdgeDecision::Continue => { - // If loop_restart edge approved by hook, mark for LoopRestart emission - if let Some(ref edge) = ctx.edge { - if edge.inner().loop_restart() { - *self.restarted_from.lock().unwrap() = - Some((ctx.from.to_string(), ctx.to.to_string())); + // Edge unchanged — check circuit breaker for loop_restart + let decision = self.circuit_breaker.on_edge_selected(ctx, state).await?; + // If loop_restart edge approved by both hook and circuit breaker, mark for LoopRestart emission + if matches!(decision, EdgeDecision::Continue) { + if let Some(ref edge) = ctx.edge { + if edge.inner().loop_restart() { + *self.restarted_from.lock().unwrap() = + Some((ctx.from.to_string(), ctx.to.to_string())); + } } } - Ok(EdgeDecision::Continue) + Ok(decision) } - decision => Ok(decision), + decision => Ok(decision), // Override/Block — skip circuit breaker } } @@ -236,9 +332,17 @@ impl RunLifecycle for WorkflowLifecycle { self.disk .on_checkpoint(node, result, next_node_id, state) .await?; + self.git + .on_checkpoint(node, result, next_node_id, state) + .await?; self.event .on_checkpoint(node, result, next_node_id, state) .await?; + self.hook + .on_checkpoint(node, result, next_node_id, state) + .await?; + // Clear checkpoint result for next checkpoint + *self.checkpoint_git_result.lock().unwrap() = None; Ok(()) } @@ -248,5 +352,6 @@ impl RunLifecycle for WorkflowLifecycle { } self.event.on_run_end(outcome, state).await; self.hook.on_run_end(outcome, state).await; + self.git.on_run_end(outcome, state).await; } } diff --git a/lib/crates/fabro-workflows/src/engine.rs b/lib/crates/fabro-workflows/src/engine.rs index 8f03063bc..a8273f90b 100644 --- a/lib/crates/fabro-workflows/src/engine.rs +++ b/lib/crates/fabro-workflows/src/engine.rs @@ -241,7 +241,11 @@ pub fn resolve_thread_id( // --- Run directory helpers (spec 5.6) --- /// Write manifest.json at the start of a workflow run. Returns the manifest. -fn write_manifest(run_dir: &Path, graph: &Graph, config: &RunConfig) -> crate::manifest::Manifest { +pub(crate) fn write_manifest( + run_dir: &Path, + graph: &Graph, + config: &RunConfig, +) -> crate::manifest::Manifest { let workflow_name = if graph.name.is_empty() { "unnamed".to_string() } else { @@ -289,7 +293,7 @@ pub fn visit_from_context(context: &Context) -> usize { } /// Write status.json for a completed node into {`run_dir}/nodes/{node_id}/status.json`. -fn write_node_status(run_dir: &Path, node_id: &str, visit: usize, outcome: &Outcome) { +pub(crate) fn write_node_status(run_dir: &Path, node_id: &str, visit: usize, outcome: &Outcome) { let node_dir = node_dir(run_dir, node_id, visit); let _ = std::fs::create_dir_all(&node_dir); let status = serde_json::json!({ @@ -723,7 +727,10 @@ pub async fn git_push_host( } /// Run a git diff via the sandbox. -async fn git_diff(sandbox: &dyn Sandbox, base: &str) -> std::result::Result { +pub(crate) async fn git_diff( + sandbox: &dyn Sandbox, + base: &str, +) -> std::result::Result { let cmd = format!("{GIT_REMOTE} diff {base} HEAD"); match sandbox.exec_command(&cmd, 30_000, None, None, None).await { Ok(r) if r.exit_code == 0 => Ok(r.stdout), @@ -777,6 +784,7 @@ pub async fn git_replace_worktree(sandbox: &dyn Sandbox, path: &str, branch: &st } /// Configuration for a workflow run. +#[derive(Clone)] pub struct RunConfig { pub run_dir: PathBuf, pub cancel_token: Option>, @@ -932,7 +940,7 @@ impl WorkflowRunEngine { } /// Mirror graph-level attributes into the context. - fn mirror_graph_attributes(graph: &Graph, context: &Context) { + pub(crate) fn mirror_graph_attributes(graph: &Graph, context: &Context) { if !graph.goal().is_empty() { context.set(context::keys::GRAPH_GOAL, serde_json::json!(graph.goal())); } @@ -1467,23 +1475,29 @@ impl WorkflowRunEngine { }); // Build lifecycle + let config_arc = std::sync::Arc::new(config.clone()); let lifecycle = crate::core_adapter::WorkflowLifecycle::new( self.services.emitter.clone(), self.services.hook_runner.clone(), self.services.sandbox.clone(), graph_arc, config.run_dir.clone(), - config.run_id.clone(), - config.dry_run, - config.labels.clone(), + config_arc, + resume_checkpoint.is_some(), ); - // Restore circuit breaker state from checkpoint + // Restore state from checkpoint if let Some(cp) = resume_checkpoint { lifecycle.restore_circuit_breaker( cp.loop_failure_signatures.clone(), cp.restart_failure_signatures.clone(), ); + // Degrade fidelity on the first resumed node when prior fidelity was Full + if cp.context_values.get(context::keys::INTERNAL_FIDELITY) + == Some(&serde_json::json!(context::keys::Fidelity::Full.to_string())) + { + lifecycle.set_degrade_fidelity_on_resume(true); + } } // Build RunState