mirror of
https://github.com/fabro-sh/fabro.git
synced 2026-09-05 08:10:39 +00:00
Implement all sub-lifecycles fully per plan specification
Fill in the previously stubbed ArtifactLifecycle and GitLifecycle, and
complete FidelityLifecycle and CircuitBreakerLifecycle with their full
behavior. Wire the orchestrator with context seeding, shared state, and
all callback orderings matching the plan.
FidelityLifecycle: use resolve_fidelity/resolve_thread_id for full
resolution chains, add preamble building via build_preamble, set
thread.{tid}.current_node key, store raw Edge for proper resolution.
CircuitBreakerLifecycle: add on_edge_selected with TransientInfra guard
and restart_failure_signatures tracking for loop_restart edges.
EventLifecycle: add Skipped guard in after_node (engine.rs:2080 parity),
read GitCheckpointResult for GitCommit/GitPush events in on_checkpoint,
read artifact_store count and last_git_sha in on_run_end.
HookLifecycle: add Skipped guard in after_node, add on_checkpoint for
CheckpointSaved hook.
DiskLifecycle: add on_run_start with write_manifest + write_run_status,
use write_node_status with visit-based directory naming.
GitLifecycle: full implementation — on_run_start resets last_git_sha and
inits metadata branch; on_checkpoint does shadow commit, run branch
commit, checkpoint re-save with SHA, push, and diff.patch; on_run_end
writes final.patch.
ArtifactLifecycle: full implementation — on_run_start swaps fresh store,
before_attempt records epoch, after_attempt collects assets and emits
AssetsCaptured, after_node offloads large values and syncs to sandbox.
Orchestrator: context seeding (mirror_graph_attributes, INTERNAL_RUN_ID,
INTERNAL_WORK_DIR) with is_initial_resume gating, shared state for
checkpoint_git_result/last_git_sha/artifact_store, full callback wiring.
Promote write_manifest, write_node_status, git_diff to pub(crate).
Add Clone to RunConfig. Constructor takes Arc<RunConfig> + is_resume.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
1c64d9f516
commit
2d8cf4b4ef
9 changed files with 775 additions and 100 deletions
|
|
@ -1,11 +1,160 @@
|
|||
use std::path::PathBuf;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use async_trait::async_trait;
|
||||
|
||||
use fabro_core::lifecycle::RunLifecycle;
|
||||
use fabro_core::graph::NodeSpec;
|
||||
use fabro_core::lifecycle::{AttemptContext, AttemptResultContext, RunLifecycle};
|
||||
use fabro_core::outcome::NodeResult;
|
||||
use fabro_core::state::RunState;
|
||||
|
||||
use super::super::graph::WorkflowGraph;
|
||||
use super::super::WorkflowNode;
|
||||
use crate::artifact::{offload_large_values, sync_artifacts_to_env, ArtifactStore};
|
||||
use crate::engine;
|
||||
use crate::event::{EventEmitter, RunNoticeLevel, WorkflowRunEvent};
|
||||
use crate::outcome::StageUsage;
|
||||
use fabro_core::lifecycle::NodeDecision;
|
||||
|
||||
type WfRunState = RunState<Option<StageUsage>>;
|
||||
type WfNodeResult = NodeResult<Option<StageUsage>>;
|
||||
type WfNodeDecision = NodeDecision<Option<StageUsage>>;
|
||||
|
||||
/// Sub-lifecycle responsible for artifact collection, offloading, and syncing.
|
||||
/// Currently a stub — artifact operations are not yet wired through the core adapter.
|
||||
pub struct ArtifactLifecycle;
|
||||
pub struct ArtifactLifecycle {
|
||||
pub sandbox: Arc<dyn fabro_sandbox::Sandbox>,
|
||||
pub artifact_store: Arc<Mutex<ArtifactStore>>,
|
||||
pub artifact_base_dir: Option<PathBuf>,
|
||||
pub emitter: Arc<EventEmitter>,
|
||||
pub run_dir: PathBuf,
|
||||
pub asset_globs: Vec<String>,
|
||||
/// Per-attempt state: epoch seconds when the attempt started.
|
||||
attempt_start_epoch: Mutex<Option<f64>>,
|
||||
}
|
||||
|
||||
impl ArtifactLifecycle {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn new(
|
||||
sandbox: Arc<dyn fabro_sandbox::Sandbox>,
|
||||
artifact_store: Arc<Mutex<ArtifactStore>>,
|
||||
artifact_base_dir: Option<PathBuf>,
|
||||
emitter: Arc<EventEmitter>,
|
||||
run_dir: PathBuf,
|
||||
asset_globs: Vec<String>,
|
||||
) -> Self {
|
||||
Self {
|
||||
sandbox,
|
||||
artifact_store,
|
||||
artifact_base_dir,
|
||||
emitter,
|
||||
run_dir,
|
||||
asset_globs,
|
||||
attempt_start_epoch: Mutex::new(None),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl RunLifecycle<WorkflowGraph> for ArtifactLifecycle {}
|
||||
impl RunLifecycle<WorkflowGraph> for ArtifactLifecycle {
|
||||
async fn on_run_start(
|
||||
&self,
|
||||
_graph: &WorkflowGraph,
|
||||
_state: &WfRunState,
|
||||
) -> fabro_core::error::Result<()> {
|
||||
// Swap in a fresh artifact store on restart (don't call clear() — preserves files on disk)
|
||||
let mut store = self.artifact_store.lock().unwrap();
|
||||
*store = ArtifactStore::new(self.artifact_base_dir.clone());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn before_attempt(
|
||||
&self,
|
||||
_ctx: &AttemptContext<'_, WorkflowGraph>,
|
||||
_state: &WfRunState,
|
||||
) -> fabro_core::error::Result<WfNodeDecision> {
|
||||
// Record epoch seconds (floored to integer for macOS stat mtime parity)
|
||||
let epoch = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_secs() as f64)
|
||||
.unwrap_or(0.0);
|
||||
*self.attempt_start_epoch.lock().unwrap() = Some(epoch);
|
||||
Ok(NodeDecision::Continue)
|
||||
}
|
||||
|
||||
async fn after_attempt(
|
||||
&self,
|
||||
ctx: &AttemptResultContext<'_, WorkflowGraph>,
|
||||
state: &WfRunState,
|
||||
) -> fabro_core::error::Result<()> {
|
||||
if self.asset_globs.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let epoch = self.attempt_start_epoch.lock().unwrap().unwrap_or(0.0);
|
||||
let node_id = ctx.node.id();
|
||||
let visit = state.node_visits.get(node_id).copied().unwrap_or(1);
|
||||
let stage_dir = engine::node_dir(&self.run_dir, node_id, visit);
|
||||
let _ = std::fs::create_dir_all(&stage_dir);
|
||||
|
||||
match crate::asset_snapshot::collect_assets(
|
||||
&*self.sandbox,
|
||||
&stage_dir,
|
||||
&self.asset_globs,
|
||||
epoch,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(summary) if summary.files_copied > 0 => {
|
||||
self.emitter.emit(&WorkflowRunEvent::AssetsCaptured {
|
||||
node_id: node_id.to_string(),
|
||||
files_copied: summary.files_copied,
|
||||
total_bytes: summary.total_bytes,
|
||||
files_skipped: summary.files_skipped,
|
||||
});
|
||||
}
|
||||
Ok(_) => {} // no files collected
|
||||
Err(e) => {
|
||||
self.emitter.emit(&WorkflowRunEvent::RunNotice {
|
||||
level: RunNoticeLevel::Warn,
|
||||
code: "asset_collection_failed".to_string(),
|
||||
message: format!("[node: {node_id}] asset collection failed: {e}"),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn after_node(
|
||||
&self,
|
||||
node: &WorkflowNode,
|
||||
result: &mut WfNodeResult,
|
||||
_state: &WfRunState,
|
||||
) -> fabro_core::error::Result<()> {
|
||||
let node_id = node.id();
|
||||
|
||||
// Offload large context_updates values to artifact store
|
||||
{
|
||||
let store = self.artifact_store.lock().unwrap();
|
||||
if let Err(e) = offload_large_values(&mut result.outcome.context_updates, &store) {
|
||||
self.emitter.emit(&WorkflowRunEvent::RunNotice {
|
||||
level: RunNoticeLevel::Warn,
|
||||
code: "artifact_offload_failed".to_string(),
|
||||
message: format!("[node: {node_id}] artifact offload failed: {e}"),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Sync file-backed artifacts to sandbox environment
|
||||
if let Err(e) =
|
||||
sync_artifacts_to_env(&mut result.outcome.context_updates, &*self.sandbox).await
|
||||
{
|
||||
self.emitter.emit(&WorkflowRunEvent::RunNotice {
|
||||
level: RunNoticeLevel::Warn,
|
||||
code: "artifact_sync_failed".to_string(),
|
||||
message: format!("[node: {node_id}] artifact sync failed: {e}"),
|
||||
});
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,14 +4,14 @@ use std::sync::Mutex;
|
|||
use async_trait::async_trait;
|
||||
|
||||
use fabro_core::error::{CoreError, Result as CoreResult};
|
||||
use fabro_core::lifecycle::RunLifecycle;
|
||||
use fabro_core::lifecycle::{EdgeContext, EdgeDecision, RunLifecycle};
|
||||
use fabro_core::outcome::NodeResult;
|
||||
use fabro_core::state::RunState;
|
||||
|
||||
use super::super::graph::WorkflowGraph;
|
||||
use super::super::WorkflowNode;
|
||||
use crate::error::FailureSignature;
|
||||
use crate::outcome::{StageStatus, StageUsage};
|
||||
use crate::error::{FailureCategory, FailureSignature};
|
||||
use crate::outcome::{OutcomeExt, StageStatus, StageUsage};
|
||||
|
||||
type WfRunState = RunState<Option<StageUsage>>;
|
||||
type WfNodeResult = NodeResult<Option<StageUsage>>;
|
||||
|
|
@ -99,4 +99,54 @@ impl RunLifecycle<WorkflowGraph> for CircuitBreakerLifecycle {
|
|||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn on_edge_selected(
|
||||
&self,
|
||||
ctx: &EdgeContext<'_, WorkflowGraph>,
|
||||
_state: &WfRunState,
|
||||
) -> CoreResult<EdgeDecision> {
|
||||
// Only guard loop_restart edges
|
||||
let Some(ref edge) = ctx.edge else {
|
||||
return Ok(EdgeDecision::Continue);
|
||||
};
|
||||
if !edge.inner().loop_restart() {
|
||||
return Ok(EdgeDecision::Continue);
|
||||
}
|
||||
|
||||
let outcome = ctx.outcome;
|
||||
|
||||
// Guard: only TransientInfra failures may trigger loop_restart
|
||||
let failure_class = outcome.failure_category();
|
||||
if let Some(fc) = failure_class {
|
||||
if fc != FailureCategory::TransientInfra {
|
||||
return Err(CoreError::blocked(format!(
|
||||
"loop_restart blocked: failure_class={fc} (requires transient_infra), failure_reason={}",
|
||||
outcome.failure_reason().unwrap_or("none"),
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
// Circuit breaker: check restart failure signatures
|
||||
if let Some(ref failure) = outcome.failure {
|
||||
let sig = FailureSignature::new(
|
||||
ctx.from,
|
||||
failure.category,
|
||||
failure.signature.as_deref(),
|
||||
Some(failure.message.as_str()),
|
||||
);
|
||||
if failure.category.is_signature_tracked() {
|
||||
let mut sigs = self.restart_failure_signatures.lock().unwrap();
|
||||
let count = sigs.entry(sig.clone()).or_insert(0);
|
||||
*count += 1;
|
||||
let limit = self.loop_restart_signature_limit;
|
||||
if *count >= limit {
|
||||
return Err(CoreError::blocked(format!(
|
||||
"loop_restart circuit breaker: signature {sig} repeated {count} times (limit {limit})"
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(EdgeDecision::Continue)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ use super::super::graph::WorkflowGraph;
|
|||
use super::super::WorkflowNode;
|
||||
use super::circuit_breaker::CircuitBreakerLifecycle;
|
||||
use crate::checkpoint::Checkpoint;
|
||||
use crate::engine::{self, RunConfig};
|
||||
use crate::event::{EventEmitter, RunNoticeLevel, WorkflowRunEvent};
|
||||
use crate::outcome::StageUsage;
|
||||
|
||||
|
|
@ -22,6 +23,8 @@ type WfNodeResult = NodeResult<Option<StageUsage>>;
|
|||
pub struct DiskLifecycle {
|
||||
pub run_dir: PathBuf,
|
||||
pub run_id: String,
|
||||
pub graph: Arc<fabro_graphviz::graph::types::Graph>,
|
||||
pub config: Arc<RunConfig>,
|
||||
pub emitter: Arc<EventEmitter>,
|
||||
pub circuit_breaker: Arc<CircuitBreakerLifecycle>,
|
||||
pub checkpoint_enabled: bool,
|
||||
|
|
@ -29,18 +32,31 @@ pub struct DiskLifecycle {
|
|||
|
||||
#[async_trait]
|
||||
impl RunLifecycle<WorkflowGraph> for DiskLifecycle {
|
||||
async fn on_run_start(
|
||||
&self,
|
||||
_graph: &WorkflowGraph,
|
||||
_state: &WfRunState,
|
||||
) -> fabro_core::error::Result<()> {
|
||||
// Write manifest.json
|
||||
engine::write_manifest(&self.run_dir, &self.graph, &self.config);
|
||||
// Write run status as Running
|
||||
crate::run_status::write_run_status(
|
||||
&self.run_dir,
|
||||
crate::run_status::RunStatus::Running,
|
||||
None,
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn after_node(
|
||||
&self,
|
||||
node: &WorkflowNode,
|
||||
result: &mut WfNodeResult,
|
||||
_state: &WfRunState,
|
||||
state: &WfRunState,
|
||||
) -> fabro_core::error::Result<()> {
|
||||
let gv = node.inner();
|
||||
let outcome = &result.outcome;
|
||||
let status_dir = self.run_dir.join("stages").join(&gv.id);
|
||||
let _ = std::fs::create_dir_all(&status_dir);
|
||||
let status_path = status_dir.join("status.json");
|
||||
let _ = crate::save_json(outcome, &status_path, "node_status");
|
||||
let visit = state.node_visits.get(gv.id.as_str()).copied().unwrap_or(1);
|
||||
engine::write_node_status(&self.run_dir, &gv.id, visit, &result.outcome);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -10,6 +10,8 @@ use fabro_core::state::RunState;
|
|||
|
||||
use super::super::graph::WorkflowGraph;
|
||||
use super::super::WorkflowNode;
|
||||
use super::git::GitCheckpointResult;
|
||||
use crate::artifact::ArtifactStore;
|
||||
use crate::event::{EventEmitter, WorkflowRunEvent};
|
||||
use crate::outcome::{FailureCategory, FailureDetail, Outcome, StageStatus, StageUsage};
|
||||
|
||||
|
|
@ -29,6 +31,11 @@ pub struct EventLifecycle {
|
|||
pub run_branch: Option<String>,
|
||||
pub worktree_dir: Option<String>,
|
||||
pub goal: Option<String>,
|
||||
// Shared swappable handle (same instance as orchestrator)
|
||||
pub artifact_store: Arc<Mutex<ArtifactStore>>,
|
||||
// Cross-lifecycle data
|
||||
pub checkpoint_git_result: Arc<Mutex<Option<GitCheckpointResult>>>,
|
||||
pub last_git_sha: Arc<Mutex<Option<String>>>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
|
|
@ -157,7 +164,7 @@ impl RunLifecycle<WorkflowGraph> for EventLifecycle {
|
|||
state: &WfRunState,
|
||||
) -> fabro_core::error::Result<()> {
|
||||
let outcome = &result.outcome;
|
||||
// Skip events for Skipped nodes
|
||||
// Skipped nodes had no StageStarted, so skip completion events (engine.rs:2080)
|
||||
if outcome.status == StageStatus::Skipped {
|
||||
return Ok(());
|
||||
}
|
||||
|
|
@ -231,11 +238,34 @@ impl RunLifecycle<WorkflowGraph> for EventLifecycle {
|
|||
_state: &WfRunState,
|
||||
) -> fabro_core::error::Result<()> {
|
||||
let status = result.outcome.status.to_string();
|
||||
|
||||
// Read git checkpoint result (set by GitLifecycle)
|
||||
let git_result = self.checkpoint_git_result.lock().unwrap().take();
|
||||
|
||||
let git_sha = git_result.as_ref().and_then(|r| r.commit_sha.clone());
|
||||
|
||||
self.emitter.emit(&WorkflowRunEvent::CheckpointCompleted {
|
||||
node_id: node.id().to_string(),
|
||||
status,
|
||||
git_commit_sha: None,
|
||||
git_commit_sha: git_sha.clone(),
|
||||
});
|
||||
|
||||
// Emit GitCommit + GitPush events if git produced results
|
||||
if let Some(ref result) = git_result {
|
||||
if let Some(ref sha) = result.commit_sha {
|
||||
self.emitter.emit(&WorkflowRunEvent::GitCommit {
|
||||
node_id: Some(node.id().to_string()),
|
||||
sha: sha.clone(),
|
||||
});
|
||||
}
|
||||
for (branch, success) in &result.push_results {
|
||||
self.emitter.emit(&WorkflowRunEvent::GitPush {
|
||||
branch: branch.clone(),
|
||||
success: *success,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
@ -244,14 +274,16 @@ impl RunLifecycle<WorkflowGraph> for EventLifecycle {
|
|||
return;
|
||||
}
|
||||
let duration_ms = self.run_start.lock().unwrap().elapsed().as_millis() as u64;
|
||||
let artifact_count = self.artifact_store.lock().unwrap().list().len();
|
||||
let last_sha = self.last_git_sha.lock().unwrap().clone();
|
||||
|
||||
if outcome.status == StageStatus::Success || outcome.status == StageStatus::PartialSuccess {
|
||||
self.emitter.emit(&WorkflowRunEvent::WorkflowRunCompleted {
|
||||
duration_ms,
|
||||
artifact_count: 0,
|
||||
artifact_count,
|
||||
status: outcome.status.to_string(),
|
||||
total_cost: None,
|
||||
final_git_commit_sha: None,
|
||||
final_git_commit_sha: last_sha,
|
||||
usage: None,
|
||||
});
|
||||
} else {
|
||||
|
|
@ -263,7 +295,7 @@ impl RunLifecycle<WorkflowGraph> for EventLifecycle {
|
|||
self.emitter.emit(&WorkflowRunEvent::WorkflowRunFailed {
|
||||
error: crate::error::FabroError::engine(error_msg),
|
||||
duration_ms,
|
||||
git_commit_sha: None,
|
||||
git_commit_sha: last_sha,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,16 +9,18 @@ use fabro_core::state::RunState;
|
|||
use super::super::graph::WorkflowGraph;
|
||||
use super::super::WorkflowNode;
|
||||
use crate::context::keys;
|
||||
use crate::engine;
|
||||
use crate::outcome::StageUsage;
|
||||
use crate::preamble::build_preamble;
|
||||
|
||||
type WfRunState = RunState<Option<StageUsage>>;
|
||||
type WfNodeDecision = NodeDecision<Option<StageUsage>>;
|
||||
|
||||
/// Data captured from an edge selection to pass to the next node's before_node.
|
||||
/// Graphviz edge captured from edge selection, passed to the next node's before_node
|
||||
/// for fidelity/thread resolution.
|
||||
#[derive(Debug, Clone)]
|
||||
struct IncomingEdgeData {
|
||||
fidelity: Option<String>,
|
||||
thread_id: Option<String>,
|
||||
edge: Arc<fabro_graphviz::graph::types::Edge>,
|
||||
}
|
||||
|
||||
/// Sub-lifecycle responsible for fidelity/thread resolution and context key setup.
|
||||
|
|
@ -63,7 +65,71 @@ impl RunLifecycle<WorkflowGraph> for FidelityLifecycle {
|
|||
let incoming = self.incoming_edge_data.lock().unwrap().take();
|
||||
let gv_node = node.inner();
|
||||
|
||||
// Set context keys for the current node
|
||||
// 1. Fidelity resolution via resolve_fidelity: edge → node → graph default → Compact
|
||||
let incoming_edge_ref = incoming.as_ref().map(|d| d.edge.as_ref());
|
||||
let fidelity = engine::resolve_fidelity(incoming_edge_ref, gv_node, &self.graph);
|
||||
|
||||
// 2. Fidelity degradation on resume (full → summary:high)
|
||||
let fidelity = {
|
||||
let mut degrade = self.degrade_fidelity_on_resume.lock().unwrap();
|
||||
if *degrade {
|
||||
*degrade = false;
|
||||
fidelity.degraded()
|
||||
} else {
|
||||
fidelity
|
||||
}
|
||||
};
|
||||
|
||||
// 3. Set INTERNAL_FIDELITY
|
||||
state.context.set(
|
||||
keys::INTERNAL_FIDELITY,
|
||||
serde_json::json!(fidelity.to_string()),
|
||||
);
|
||||
|
||||
// 4. Preamble building: if Full, empty preamble; otherwise build from context
|
||||
let preamble = {
|
||||
let wf_context = crate::context::Context::from_values(state.context.snapshot());
|
||||
build_preamble(
|
||||
fidelity,
|
||||
&wf_context,
|
||||
&self.graph,
|
||||
&state.completed_nodes,
|
||||
&state.node_outcomes,
|
||||
)
|
||||
};
|
||||
state
|
||||
.context
|
||||
.set(keys::CURRENT_PREAMBLE, serde_json::json!(preamble));
|
||||
|
||||
// 5. Thread ID resolution via resolve_thread_id: edge → node → graph default → class → previous
|
||||
let thread_id = engine::resolve_thread_id(
|
||||
incoming_edge_ref,
|
||||
gv_node,
|
||||
&self.graph,
|
||||
state.previous_node_id.as_deref(),
|
||||
);
|
||||
|
||||
// 6. Set thread.{tid}.current_node
|
||||
if let Some(ref tid) = thread_id {
|
||||
let key = format!("thread.{tid}.current_node");
|
||||
state.context.set(key, serde_json::json!(node.id()));
|
||||
}
|
||||
|
||||
// 7. Set INTERNAL_THREAD_ID (or null)
|
||||
match thread_id {
|
||||
Some(tid) => {
|
||||
state
|
||||
.context
|
||||
.set(keys::INTERNAL_THREAD_ID, serde_json::json!(tid));
|
||||
}
|
||||
None => {
|
||||
state
|
||||
.context
|
||||
.set(keys::INTERNAL_THREAD_ID, serde_json::Value::Null);
|
||||
}
|
||||
}
|
||||
|
||||
// 8. Set INTERNAL_NODE_VISIT_COUNT and CURRENT_NODE
|
||||
let visits = state.node_visits.get(node.id()).copied().unwrap_or(0);
|
||||
state
|
||||
.context
|
||||
|
|
@ -72,47 +138,6 @@ impl RunLifecycle<WorkflowGraph> for FidelityLifecycle {
|
|||
.context
|
||||
.set(keys::INTERNAL_NODE_VISIT_COUNT, serde_json::json!(visits));
|
||||
|
||||
// Fidelity resolution: edge → node → graph default → compact
|
||||
let fidelity = if let Some(ref edge_data) = incoming {
|
||||
edge_data
|
||||
.fidelity
|
||||
.as_deref()
|
||||
.or(gv_node.fidelity())
|
||||
.unwrap_or("compact")
|
||||
.to_string()
|
||||
} else {
|
||||
gv_node.fidelity().unwrap_or("compact").to_string()
|
||||
};
|
||||
|
||||
// Fidelity degradation on resume
|
||||
let fidelity = {
|
||||
let mut degrade = self.degrade_fidelity_on_resume.lock().unwrap();
|
||||
if *degrade {
|
||||
*degrade = false;
|
||||
let parsed: keys::Fidelity = fidelity.parse().unwrap_or_default();
|
||||
parsed.degraded().to_string()
|
||||
} else {
|
||||
fidelity
|
||||
}
|
||||
};
|
||||
|
||||
state
|
||||
.context
|
||||
.set(keys::INTERNAL_FIDELITY, serde_json::json!(fidelity));
|
||||
|
||||
// Thread ID resolution: edge → node → graph default → previous node
|
||||
if let Some(ref edge_data) = incoming {
|
||||
if let Some(ref tid) = edge_data.thread_id {
|
||||
state
|
||||
.context
|
||||
.set(keys::INTERNAL_THREAD_ID, serde_json::json!(tid));
|
||||
}
|
||||
} else if let Some(tid) = gv_node.thread_id() {
|
||||
state
|
||||
.context
|
||||
.set(keys::INTERNAL_THREAD_ID, serde_json::json!(tid));
|
||||
}
|
||||
|
||||
Ok(NodeDecision::Continue)
|
||||
}
|
||||
|
||||
|
|
@ -125,8 +150,7 @@ impl RunLifecycle<WorkflowGraph> for FidelityLifecycle {
|
|||
if let Some(ref edge) = ctx.edge {
|
||||
let gv_edge = edge.inner();
|
||||
let edge_data = IncomingEdgeData {
|
||||
fidelity: gv_edge.fidelity().map(String::from),
|
||||
thread_id: gv_edge.thread_id().map(String::from),
|
||||
edge: Arc::new(gv_edge.clone()),
|
||||
};
|
||||
*self.incoming_edge_data.lock().unwrap() = Some(edge_data);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,11 +1,280 @@
|
|||
use std::path::PathBuf;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use async_trait::async_trait;
|
||||
|
||||
use fabro_core::error::CoreError;
|
||||
use fabro_core::graph::NodeSpec;
|
||||
use fabro_core::lifecycle::RunLifecycle;
|
||||
use fabro_core::outcome::NodeResult;
|
||||
use fabro_core::state::RunState;
|
||||
|
||||
use super::super::graph::WorkflowGraph;
|
||||
use super::super::WorkflowNode;
|
||||
use crate::artifact::ArtifactStore;
|
||||
use crate::engine::{self, RunConfig};
|
||||
use crate::event::{EventEmitter, RunNoticeLevel, WorkflowRunEvent};
|
||||
use crate::outcome::{Outcome, StageStatus, StageUsage};
|
||||
|
||||
type WfRunState = RunState<Option<StageUsage>>;
|
||||
type WfNodeResult = NodeResult<Option<StageUsage>>;
|
||||
|
||||
/// Result of a git checkpoint operation, shared with EventLifecycle.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct GitCheckpointResult {
|
||||
pub commit_sha: Option<String>,
|
||||
pub push_results: Vec<(String, bool)>,
|
||||
}
|
||||
|
||||
/// Sub-lifecycle responsible for git operations (checkpoint commits, pushes, diffs).
|
||||
/// Currently a stub — git operations are not yet wired through the core adapter.
|
||||
pub struct GitLifecycle;
|
||||
pub struct GitLifecycle {
|
||||
pub sandbox: Arc<dyn fabro_sandbox::Sandbox>,
|
||||
pub artifact_store: Arc<Mutex<ArtifactStore>>,
|
||||
pub emitter: Arc<EventEmitter>,
|
||||
pub run_dir: PathBuf,
|
||||
pub run_id: String,
|
||||
pub config: Arc<RunConfig>,
|
||||
pub start_node_id: Option<String>,
|
||||
// Cross-lifecycle data (shared with EventLifecycle)
|
||||
pub checkpoint_git_result: Arc<Mutex<Option<GitCheckpointResult>>>,
|
||||
pub last_git_sha: Arc<Mutex<Option<String>>>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl RunLifecycle<WorkflowGraph> for GitLifecycle {}
|
||||
impl RunLifecycle<WorkflowGraph> for GitLifecycle {
|
||||
async fn on_run_start(
|
||||
&self,
|
||||
_graph: &WorkflowGraph,
|
||||
_state: &WfRunState,
|
||||
) -> fabro_core::error::Result<()> {
|
||||
// Reset last_git_sha (diff base parity)
|
||||
*self.last_git_sha.lock().unwrap() = None;
|
||||
|
||||
// Init metadata branch (best-effort)
|
||||
if let (Some(_), Some(ref repo_path)) =
|
||||
(&self.config.meta_branch, &self.config.host_repo_path)
|
||||
{
|
||||
let store = crate::git::MetadataStore::new(repo_path, &self.config.git_author);
|
||||
let manifest_bytes = {
|
||||
let manifest_path = self.run_dir.join("manifest.json");
|
||||
std::fs::read(&manifest_path).unwrap_or_default()
|
||||
};
|
||||
let dot_source = std::fs::read(self.run_dir.join("graph.fabro"))
|
||||
.or_else(|_| std::fs::read(self.run_dir.join("graph.dot")))
|
||||
.unwrap_or_default();
|
||||
let sandbox_json = std::fs::read(self.run_dir.join("sandbox.json")).ok();
|
||||
let mut extra_files: Vec<(&str, &[u8])> = Vec::new();
|
||||
if let Some(ref data) = sandbox_json {
|
||||
extra_files.push(("sandbox.json", data));
|
||||
}
|
||||
if let Err(e) = store.init_run(&self.run_id, &manifest_bytes, &dot_source, &extra_files)
|
||||
{
|
||||
tracing::warn!(
|
||||
run_id = %self.run_id,
|
||||
error = %e,
|
||||
"Metadata branch init failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn on_checkpoint(
|
||||
&self,
|
||||
node: &WorkflowNode,
|
||||
result: &WfNodeResult,
|
||||
_next_node_id: Option<&str>,
|
||||
state: &WfRunState,
|
||||
) -> fabro_core::error::Result<()> {
|
||||
let node_id = node.id();
|
||||
|
||||
// Skip git checkpoint for the start node (always empty) or if git disabled
|
||||
if self.start_node_id.as_deref() == Some(node_id) || !self.config.git_checkpoint_enabled {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Shadow commit (best-effort, metadata branch)
|
||||
let shadow_sha: Option<String> = if let (Some(_), Some(ref repo_path)) =
|
||||
(&self.config.meta_branch, &self.config.host_repo_path)
|
||||
{
|
||||
let store = crate::git::MetadataStore::new(repo_path, &self.config.git_author);
|
||||
// Build checkpoint JSON for shadow branch
|
||||
let checkpoint_path = self.run_dir.join("checkpoint.json");
|
||||
std::fs::read(&checkpoint_path).ok().and_then(|cp_json| {
|
||||
let artifact_store = self.artifact_store.lock().unwrap();
|
||||
let mut extra_entries: Vec<(String, Vec<u8>)> = artifact_store
|
||||
.list()
|
||||
.iter()
|
||||
.filter_map(|info| {
|
||||
info.file_path.as_ref().and_then(|path| {
|
||||
std::fs::read(path)
|
||||
.ok()
|
||||
.map(|data| (format!("artifacts/{}.json", info.id), data))
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
extra_entries.extend(crate::git::scan_node_files(&self.run_dir));
|
||||
let extra_refs: Vec<(&str, &[u8])> = extra_entries
|
||||
.iter()
|
||||
.map(|(k, v)| (k.as_str(), v.as_slice()))
|
||||
.collect();
|
||||
match store.write_checkpoint(&self.run_id, &cp_json, &extra_refs) {
|
||||
Ok(sha) => Some(sha),
|
||||
Err(e) => {
|
||||
self.emitter.emit(&WorkflowRunEvent::RunNotice {
|
||||
level: RunNoticeLevel::Warn,
|
||||
code: "checkpoint_metadata_write_failed".to_string(),
|
||||
message: format!(
|
||||
"[node: {node_id}] metadata checkpoint write failed: {e}"
|
||||
),
|
||||
});
|
||||
None
|
||||
}
|
||||
}
|
||||
})
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// Run branch commit via sandbox
|
||||
let completed_count = state.completed_nodes.len();
|
||||
let commit_result = engine::git_checkpoint(
|
||||
&*self.sandbox,
|
||||
&self.run_id,
|
||||
node_id,
|
||||
&result.outcome.status.to_string(),
|
||||
completed_count,
|
||||
shadow_sha,
|
||||
&self.config.checkpoint_exclude_globs,
|
||||
&self.config.git_author,
|
||||
)
|
||||
.await;
|
||||
|
||||
match commit_result {
|
||||
Ok(sha) => {
|
||||
let mut git_result = GitCheckpointResult {
|
||||
commit_sha: Some(sha.clone()),
|
||||
push_results: Vec::new(),
|
||||
};
|
||||
|
||||
// Re-save checkpoint.json with SHA
|
||||
let checkpoint_path = self.run_dir.join("checkpoint.json");
|
||||
if let Ok(mut cp) = crate::checkpoint::Checkpoint::load(&checkpoint_path) {
|
||||
cp.git_commit_sha = Some(sha.clone());
|
||||
if let Err(e) = cp.save(&checkpoint_path) {
|
||||
self.emitter.emit(&WorkflowRunEvent::RunNotice {
|
||||
level: RunNoticeLevel::Warn,
|
||||
code: "checkpoint_resave_failed".to_string(),
|
||||
message: format!(
|
||||
"[node: {node_id}] checkpoint re-save with SHA failed: {e}"
|
||||
),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Push run branch (skip in dry-run mode)
|
||||
if !self.config.dry_run {
|
||||
if let Some(ref branch) = self.config.run_branch {
|
||||
let push_ok = if self.sandbox.git_push_branch(branch).await {
|
||||
true
|
||||
} else if let Some(ref repo_path) = self.config.host_repo_path {
|
||||
let refspec = format!("refs/heads/{branch}");
|
||||
engine::git_push_host(
|
||||
repo_path,
|
||||
&refspec,
|
||||
&self.config.github_app,
|
||||
"run branch",
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
false
|
||||
};
|
||||
git_result.push_results.push((branch.clone(), push_ok));
|
||||
}
|
||||
// Push metadata branch (always from host)
|
||||
if let (Some(ref meta_branch), Some(ref repo_path)) =
|
||||
(&self.config.meta_branch, &self.config.host_repo_path)
|
||||
{
|
||||
let refspec = format!("refs/heads/{meta_branch}");
|
||||
let meta_push_ok = engine::git_push_host(
|
||||
repo_path,
|
||||
&refspec,
|
||||
&self.config.github_app,
|
||||
"metadata branch",
|
||||
)
|
||||
.await;
|
||||
git_result
|
||||
.push_results
|
||||
.push((meta_branch.clone(), meta_push_ok));
|
||||
}
|
||||
}
|
||||
|
||||
// Save diff.patch
|
||||
let visit = state.node_visits.get(node_id).copied().unwrap_or(1);
|
||||
let prev = self
|
||||
.last_git_sha
|
||||
.lock()
|
||||
.unwrap()
|
||||
.clone()
|
||||
.or_else(|| self.config.base_sha.clone())
|
||||
.unwrap_or_else(|| sha.clone());
|
||||
let diff_dest = engine::node_dir(&self.run_dir, node_id, visit).join("diff.patch");
|
||||
|
||||
match engine::git_diff(&*self.sandbox, &prev).await {
|
||||
Ok(patch) if !patch.is_empty() => {
|
||||
let _ = std::fs::write(&diff_dest, patch);
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(err) => {
|
||||
self.emitter.emit(&WorkflowRunEvent::RunNotice {
|
||||
level: RunNoticeLevel::Warn,
|
||||
code: "git_diff_failed".to_string(),
|
||||
message: format!("[node: {node_id}] git diff failed: {err}"),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Update shared state
|
||||
*self.last_git_sha.lock().unwrap() = Some(sha);
|
||||
*self.checkpoint_git_result.lock().unwrap() = Some(git_result);
|
||||
}
|
||||
Err(e) => {
|
||||
// Emit CheckpointFailed and return error
|
||||
self.emitter.emit(&WorkflowRunEvent::CheckpointFailed {
|
||||
node_id: node_id.to_string(),
|
||||
error: e.clone(),
|
||||
});
|
||||
return Err(CoreError::Other(format!(
|
||||
"git checkpoint commit failed for node '{node_id}': {e}"
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn on_run_end(&self, outcome: &Outcome, _state: &WfRunState) {
|
||||
// Write final.patch on success
|
||||
if (outcome.status == StageStatus::Success || outcome.status == StageStatus::PartialSuccess)
|
||||
&& self.config.git_checkpoint_enabled
|
||||
{
|
||||
if let Some(ref base_sha) = self.config.base_sha {
|
||||
let diff_dest = self.run_dir.join("final.patch");
|
||||
match engine::git_diff(&*self.sandbox, base_sha).await {
|
||||
Ok(patch) if !patch.is_empty() => {
|
||||
let _ = std::fs::write(&diff_dest, patch);
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(err) => {
|
||||
self.emitter.emit(&WorkflowRunEvent::RunNotice {
|
||||
level: RunNoticeLevel::Warn,
|
||||
code: "git_diff_failed".to_string(),
|
||||
message: format!("final diff failed: {err}"),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -92,11 +92,10 @@ impl RunLifecycle<WorkflowGraph> for HookLifecycle {
|
|||
_state: &WfRunState,
|
||||
) -> CoreResult<()> {
|
||||
let outcome = &result.outcome;
|
||||
// Skip hooks for Skipped nodes
|
||||
// Skipped nodes had no StageStarted, so skip hooks (engine.rs:2080)
|
||||
if outcome.status == StageStatus::Skipped {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let hook_event = if outcome.status == StageStatus::Fail {
|
||||
HookEvent::StageFailed
|
||||
} else {
|
||||
|
|
@ -132,6 +131,23 @@ impl RunLifecycle<WorkflowGraph> for HookLifecycle {
|
|||
}
|
||||
}
|
||||
|
||||
async fn on_checkpoint(
|
||||
&self,
|
||||
node: &WorkflowNode,
|
||||
_result: &WfNodeResult,
|
||||
_next_node_id: Option<&str>,
|
||||
_state: &WfRunState,
|
||||
) -> CoreResult<()> {
|
||||
let mut hook_ctx = HookContext::new(
|
||||
HookEvent::CheckpointSaved,
|
||||
self.run_id.clone(),
|
||||
self.graph_name.clone(),
|
||||
);
|
||||
hook_ctx.node_id = Some(node.inner().id.clone());
|
||||
let _ = self.run_hook(&hook_ctx).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn on_run_end(&self, outcome: &Outcome, state: &WfRunState) {
|
||||
if state.cancelled {
|
||||
return;
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ pub mod hook;
|
|||
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Instant;
|
||||
|
||||
|
|
@ -23,6 +24,9 @@ use fabro_core::state::RunState;
|
|||
|
||||
use super::graph::WorkflowGraph;
|
||||
use super::WorkflowNode;
|
||||
use crate::artifact::ArtifactStore;
|
||||
use crate::context;
|
||||
use crate::engine::RunConfig;
|
||||
use crate::event::EventEmitter;
|
||||
use crate::outcome::{Outcome, StageUsage};
|
||||
use fabro_hooks::HookRunner;
|
||||
|
|
@ -34,7 +38,7 @@ use self::circuit_breaker::CircuitBreakerLifecycle;
|
|||
use self::disk::DiskLifecycle;
|
||||
use self::event::EventLifecycle;
|
||||
use self::fidelity::FidelityLifecycle;
|
||||
use self::git::GitLifecycle;
|
||||
use self::git::{GitCheckpointResult, GitLifecycle};
|
||||
use self::hook::HookLifecycle;
|
||||
|
||||
type WfRunState = RunState<Option<StageUsage>>;
|
||||
|
|
@ -50,12 +54,19 @@ pub struct WorkflowLifecycle {
|
|||
auto_status: AutoStatusLifecycle,
|
||||
circuit_breaker: Arc<CircuitBreakerLifecycle>,
|
||||
disk: DiskLifecycle,
|
||||
#[allow(dead_code)] // stub — will be wired when git operations move to core adapter
|
||||
git: GitLifecycle,
|
||||
#[allow(dead_code)] // stub — will be wired when artifact operations move to core adapter
|
||||
artifact: ArtifactLifecycle,
|
||||
/// Set in on_edge_selected when loop_restart approved; read+cleared by EventLifecycle::on_run_start
|
||||
restarted_from: Arc<Mutex<Option<(String, String)>>>,
|
||||
/// Shared git checkpoint result (written by git, read by event)
|
||||
checkpoint_git_result: Arc<Mutex<Option<GitCheckpointResult>>>,
|
||||
/// True when constructed with a checkpoint; cleared after first on_run_start.
|
||||
/// Gates mirror_graph_attributes on initial resume.
|
||||
is_initial_resume: AtomicBool,
|
||||
// Config needed for context seeding
|
||||
graph: Arc<fabro_graphviz::graph::types::Graph>,
|
||||
run_id: String,
|
||||
working_directory: Option<String>,
|
||||
}
|
||||
|
||||
impl WorkflowLifecycle {
|
||||
|
|
@ -66,32 +77,46 @@ impl WorkflowLifecycle {
|
|||
sandbox: Arc<dyn Sandbox>,
|
||||
graph: Arc<fabro_graphviz::graph::types::Graph>,
|
||||
run_dir: PathBuf,
|
||||
run_id: String,
|
||||
_dry_run: bool,
|
||||
_labels: HashMap<String, String>,
|
||||
config: Arc<RunConfig>,
|
||||
is_resume: bool,
|
||||
) -> Self {
|
||||
let restarted_from: Arc<Mutex<Option<(String, String)>>> = Arc::new(Mutex::new(None));
|
||||
let loop_restart_signature_limit = graph.loop_restart_signature_limit();
|
||||
let checkpoint_git_result: Arc<Mutex<Option<GitCheckpointResult>>> =
|
||||
Arc::new(Mutex::new(None));
|
||||
let last_git_sha: Arc<Mutex<Option<String>>> = Arc::new(Mutex::new(None));
|
||||
let artifact_store = Arc::new(Mutex::new(ArtifactStore::new(Some(run_dir.clone()))));
|
||||
|
||||
let circuit_breaker = Arc::new(CircuitBreakerLifecycle::new(loop_restart_signature_limit));
|
||||
|
||||
let local_git_checkpoint =
|
||||
config.git_checkpoint_enabled && sandbox.host_git_dir().is_some();
|
||||
let working_directory = if local_git_checkpoint {
|
||||
Some(sandbox.working_directory().to_string())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let event = EventLifecycle {
|
||||
emitter: Arc::clone(&emitter),
|
||||
graph_name: graph.name.clone(),
|
||||
run_id: run_id.clone(),
|
||||
run_id: config.run_id.clone(),
|
||||
run_start: Mutex::new(Instant::now()),
|
||||
restarted_from: Arc::clone(&restarted_from),
|
||||
base_sha: None,
|
||||
run_branch: None,
|
||||
worktree_dir: None,
|
||||
goal: None,
|
||||
base_sha: config.base_sha.clone(),
|
||||
run_branch: config.run_branch.clone(),
|
||||
worktree_dir: working_directory.clone(),
|
||||
goal: (!graph.goal().is_empty()).then(|| graph.goal().to_string()),
|
||||
artifact_store: Arc::clone(&artifact_store),
|
||||
last_git_sha: Arc::clone(&last_git_sha),
|
||||
checkpoint_git_result: Arc::clone(&checkpoint_git_result),
|
||||
};
|
||||
|
||||
let hook = HookLifecycle {
|
||||
hook_runner,
|
||||
sandbox: Arc::clone(&sandbox),
|
||||
run_dir: run_dir.clone(),
|
||||
run_id: run_id.clone(),
|
||||
run_id: config.run_id.clone(),
|
||||
graph_name: graph.name.clone(),
|
||||
};
|
||||
|
||||
|
|
@ -99,12 +124,37 @@ impl WorkflowLifecycle {
|
|||
|
||||
let disk = DiskLifecycle {
|
||||
run_dir: run_dir.clone(),
|
||||
run_id: run_id.clone(),
|
||||
run_id: config.run_id.clone(),
|
||||
graph: Arc::clone(&graph),
|
||||
config: Arc::clone(&config),
|
||||
emitter: Arc::clone(&emitter),
|
||||
circuit_breaker: Arc::clone(&circuit_breaker),
|
||||
checkpoint_enabled: true,
|
||||
};
|
||||
|
||||
let start_node_id = graph.find_start_node().map(|n| n.id.clone());
|
||||
|
||||
let git = GitLifecycle {
|
||||
sandbox: Arc::clone(&sandbox),
|
||||
artifact_store: Arc::clone(&artifact_store),
|
||||
emitter: Arc::clone(&emitter),
|
||||
run_dir: run_dir.clone(),
|
||||
run_id: config.run_id.clone(),
|
||||
config: Arc::clone(&config),
|
||||
start_node_id,
|
||||
checkpoint_git_result: Arc::clone(&checkpoint_git_result),
|
||||
last_git_sha: Arc::clone(&last_git_sha),
|
||||
};
|
||||
|
||||
let artifact = ArtifactLifecycle::new(
|
||||
Arc::clone(&sandbox),
|
||||
Arc::clone(&artifact_store),
|
||||
Some(run_dir.clone()),
|
||||
Arc::clone(&emitter),
|
||||
run_dir,
|
||||
config.asset_globs.clone(),
|
||||
);
|
||||
|
||||
Self {
|
||||
event,
|
||||
hook,
|
||||
|
|
@ -112,9 +162,14 @@ impl WorkflowLifecycle {
|
|||
auto_status: AutoStatusLifecycle,
|
||||
circuit_breaker,
|
||||
disk,
|
||||
git: GitLifecycle,
|
||||
artifact: ArtifactLifecycle,
|
||||
git,
|
||||
artifact,
|
||||
restarted_from,
|
||||
checkpoint_git_result,
|
||||
is_initial_resume: AtomicBool::new(is_resume),
|
||||
graph,
|
||||
run_id: config.run_id.clone(),
|
||||
working_directory,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -136,11 +191,44 @@ impl WorkflowLifecycle {
|
|||
#[async_trait]
|
||||
impl RunLifecycle<WorkflowGraph> for WorkflowLifecycle {
|
||||
async fn on_run_start(&self, graph: &WorkflowGraph, state: &WfRunState) -> CoreResult<()> {
|
||||
// Re-seed context keys (fires on initial start AND after every loop restart).
|
||||
// mirror_graph_attributes: skip on initial checkpoint resume (context already has them)
|
||||
if self.is_initial_resume.swap(false, Ordering::Relaxed) {
|
||||
// First on_run_start after checkpoint resume — skip mirror_graph_attributes
|
||||
} else {
|
||||
// Mirror graph-level attributes into the core context
|
||||
if !self.graph.goal().is_empty() {
|
||||
state.context.set(
|
||||
context::keys::GRAPH_GOAL,
|
||||
serde_json::json!(self.graph.goal()),
|
||||
);
|
||||
}
|
||||
for (key, val) in &self.graph.attrs {
|
||||
state.context.set(
|
||||
context::keys::graph_attr_key(key),
|
||||
serde_json::json!(val.to_string_value()),
|
||||
);
|
||||
}
|
||||
}
|
||||
// Always set run_id and work_dir (idempotent)
|
||||
state.context.set(
|
||||
context::keys::INTERNAL_RUN_ID,
|
||||
serde_json::json!(self.run_id),
|
||||
);
|
||||
if let Some(ref wd) = self.working_directory {
|
||||
state
|
||||
.context
|
||||
.set(context::keys::INTERNAL_WORK_DIR, serde_json::json!(wd));
|
||||
}
|
||||
|
||||
// Reset restart-scoped state
|
||||
self.fidelity.on_run_start(graph, state).await?;
|
||||
self.artifact.on_run_start(graph, state).await?;
|
||||
// Observable callbacks
|
||||
self.event.on_run_start(graph, state).await?;
|
||||
self.hook.on_run_start(graph, state).await?;
|
||||
self.disk.on_run_start(graph, state).await?;
|
||||
self.git.on_run_start(graph, state).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
@ -175,6 +263,8 @@ impl RunLifecycle<WorkflowGraph> for WorkflowLifecycle {
|
|||
}
|
||||
// Event emission
|
||||
self.event.before_attempt(ctx, state).await?;
|
||||
// Record epoch AFTER hook+event (engine.rs:968→1006)
|
||||
self.artifact.before_attempt(ctx, state).await?;
|
||||
Ok(NodeDecision::Continue)
|
||||
}
|
||||
|
||||
|
|
@ -183,6 +273,7 @@ impl RunLifecycle<WorkflowGraph> for WorkflowLifecycle {
|
|||
ctx: &AttemptResultContext<'_, WorkflowGraph>,
|
||||
state: &WfRunState,
|
||||
) -> CoreResult<()> {
|
||||
self.artifact.after_attempt(ctx, state).await?;
|
||||
self.event.after_attempt(ctx, state).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -198,6 +289,7 @@ impl RunLifecycle<WorkflowGraph> for WorkflowLifecycle {
|
|||
self.event.after_node(node, result, state).await?;
|
||||
self.hook.after_node(node, result, state).await?;
|
||||
self.disk.after_node(node, result, state).await?;
|
||||
self.artifact.after_node(node, result, state).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
@ -213,16 +305,20 @@ impl RunLifecycle<WorkflowGraph> for WorkflowLifecycle {
|
|||
// Hook can override/block
|
||||
match self.hook.on_edge_selected(ctx, state).await? {
|
||||
EdgeDecision::Continue => {
|
||||
// If loop_restart edge approved by hook, mark for LoopRestart emission
|
||||
if let Some(ref edge) = ctx.edge {
|
||||
if edge.inner().loop_restart() {
|
||||
*self.restarted_from.lock().unwrap() =
|
||||
Some((ctx.from.to_string(), ctx.to.to_string()));
|
||||
// Edge unchanged — check circuit breaker for loop_restart
|
||||
let decision = self.circuit_breaker.on_edge_selected(ctx, state).await?;
|
||||
// If loop_restart edge approved by both hook and circuit breaker, mark for LoopRestart emission
|
||||
if matches!(decision, EdgeDecision::Continue) {
|
||||
if let Some(ref edge) = ctx.edge {
|
||||
if edge.inner().loop_restart() {
|
||||
*self.restarted_from.lock().unwrap() =
|
||||
Some((ctx.from.to_string(), ctx.to.to_string()));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(EdgeDecision::Continue)
|
||||
Ok(decision)
|
||||
}
|
||||
decision => Ok(decision),
|
||||
decision => Ok(decision), // Override/Block — skip circuit breaker
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -236,9 +332,17 @@ impl RunLifecycle<WorkflowGraph> for WorkflowLifecycle {
|
|||
self.disk
|
||||
.on_checkpoint(node, result, next_node_id, state)
|
||||
.await?;
|
||||
self.git
|
||||
.on_checkpoint(node, result, next_node_id, state)
|
||||
.await?;
|
||||
self.event
|
||||
.on_checkpoint(node, result, next_node_id, state)
|
||||
.await?;
|
||||
self.hook
|
||||
.on_checkpoint(node, result, next_node_id, state)
|
||||
.await?;
|
||||
// Clear checkpoint result for next checkpoint
|
||||
*self.checkpoint_git_result.lock().unwrap() = None;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
@ -248,5 +352,6 @@ impl RunLifecycle<WorkflowGraph> for WorkflowLifecycle {
|
|||
}
|
||||
self.event.on_run_end(outcome, state).await;
|
||||
self.hook.on_run_end(outcome, state).await;
|
||||
self.git.on_run_end(outcome, state).await;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -241,7 +241,11 @@ pub fn resolve_thread_id(
|
|||
// --- Run directory helpers (spec 5.6) ---
|
||||
|
||||
/// Write manifest.json at the start of a workflow run. Returns the manifest.
|
||||
fn write_manifest(run_dir: &Path, graph: &Graph, config: &RunConfig) -> crate::manifest::Manifest {
|
||||
pub(crate) fn write_manifest(
|
||||
run_dir: &Path,
|
||||
graph: &Graph,
|
||||
config: &RunConfig,
|
||||
) -> crate::manifest::Manifest {
|
||||
let workflow_name = if graph.name.is_empty() {
|
||||
"unnamed".to_string()
|
||||
} else {
|
||||
|
|
@ -289,7 +293,7 @@ pub fn visit_from_context(context: &Context) -> usize {
|
|||
}
|
||||
|
||||
/// Write status.json for a completed node into {`run_dir}/nodes/{node_id}/status.json`.
|
||||
fn write_node_status(run_dir: &Path, node_id: &str, visit: usize, outcome: &Outcome) {
|
||||
pub(crate) fn write_node_status(run_dir: &Path, node_id: &str, visit: usize, outcome: &Outcome) {
|
||||
let node_dir = node_dir(run_dir, node_id, visit);
|
||||
let _ = std::fs::create_dir_all(&node_dir);
|
||||
let status = serde_json::json!({
|
||||
|
|
@ -723,7 +727,10 @@ pub async fn git_push_host(
|
|||
}
|
||||
|
||||
/// Run a git diff via the sandbox.
|
||||
async fn git_diff(sandbox: &dyn Sandbox, base: &str) -> std::result::Result<String, String> {
|
||||
pub(crate) async fn git_diff(
|
||||
sandbox: &dyn Sandbox,
|
||||
base: &str,
|
||||
) -> std::result::Result<String, String> {
|
||||
let cmd = format!("{GIT_REMOTE} diff {base} HEAD");
|
||||
match sandbox.exec_command(&cmd, 30_000, None, None, None).await {
|
||||
Ok(r) if r.exit_code == 0 => Ok(r.stdout),
|
||||
|
|
@ -777,6 +784,7 @@ pub async fn git_replace_worktree(sandbox: &dyn Sandbox, path: &str, branch: &st
|
|||
}
|
||||
|
||||
/// Configuration for a workflow run.
|
||||
#[derive(Clone)]
|
||||
pub struct RunConfig {
|
||||
pub run_dir: PathBuf,
|
||||
pub cancel_token: Option<Arc<AtomicBool>>,
|
||||
|
|
@ -932,7 +940,7 @@ impl WorkflowRunEngine {
|
|||
}
|
||||
|
||||
/// Mirror graph-level attributes into the context.
|
||||
fn mirror_graph_attributes(graph: &Graph, context: &Context) {
|
||||
pub(crate) fn mirror_graph_attributes(graph: &Graph, context: &Context) {
|
||||
if !graph.goal().is_empty() {
|
||||
context.set(context::keys::GRAPH_GOAL, serde_json::json!(graph.goal()));
|
||||
}
|
||||
|
|
@ -1467,23 +1475,29 @@ impl WorkflowRunEngine {
|
|||
});
|
||||
|
||||
// Build lifecycle
|
||||
let config_arc = std::sync::Arc::new(config.clone());
|
||||
let lifecycle = crate::core_adapter::WorkflowLifecycle::new(
|
||||
self.services.emitter.clone(),
|
||||
self.services.hook_runner.clone(),
|
||||
self.services.sandbox.clone(),
|
||||
graph_arc,
|
||||
config.run_dir.clone(),
|
||||
config.run_id.clone(),
|
||||
config.dry_run,
|
||||
config.labels.clone(),
|
||||
config_arc,
|
||||
resume_checkpoint.is_some(),
|
||||
);
|
||||
|
||||
// Restore circuit breaker state from checkpoint
|
||||
// Restore state from checkpoint
|
||||
if let Some(cp) = resume_checkpoint {
|
||||
lifecycle.restore_circuit_breaker(
|
||||
cp.loop_failure_signatures.clone(),
|
||||
cp.restart_failure_signatures.clone(),
|
||||
);
|
||||
// Degrade fidelity on the first resumed node when prior fidelity was Full
|
||||
if cp.context_values.get(context::keys::INTERNAL_FIDELITY)
|
||||
== Some(&serde_json::json!(context::keys::Fidelity::Full.to_string()))
|
||||
{
|
||||
lifecycle.set_degrade_fidelity_on_resume(true);
|
||||
}
|
||||
}
|
||||
|
||||
// Build RunState
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue