mirror of
https://github.com/fabro-sh/fabro.git
synced 2026-09-05 08:10:39 +00:00
Refactor fabro-workflows graph ops modules
This commit is contained in:
parent
2203bb8d9e
commit
e189ea8bf6
13 changed files with 743 additions and 804 deletions
|
|
@ -1,12 +1,13 @@
|
|||
mod routing;
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use fabro_core::error::{CoreError, Result as CoreResult};
|
||||
use fabro_core::graph::{EdgeSelection, EdgeSpec, Graph, NodeSpec};
|
||||
use fabro_core::graph::{EdgeSelection as CoreEdgeSelection, EdgeSpec, Graph, NodeSpec};
|
||||
use fabro_graphviz::graph::types::{Edge as GvEdge, Graph as GvGraph, Node as GvNode};
|
||||
|
||||
use crate::context::Context;
|
||||
use crate::graph_ops;
|
||||
use crate::outcome::{Outcome, StageUsage};
|
||||
|
||||
// ---- WorkflowNode ----
|
||||
|
|
@ -26,7 +27,7 @@ impl NodeSpec for WorkflowNode {
|
|||
}
|
||||
|
||||
fn is_terminal(&self) -> bool {
|
||||
graph_ops::is_terminal(&self.0)
|
||||
routing::is_terminal(&self.0)
|
||||
}
|
||||
|
||||
fn max_visits(&self) -> Option<usize> {
|
||||
|
|
@ -102,15 +103,15 @@ impl Graph for WorkflowGraph {
|
|||
node: &Self::Node,
|
||||
outcome: &Outcome,
|
||||
context: &Context,
|
||||
) -> Option<EdgeSelection<Self>> {
|
||||
let selection = graph_ops::select_edge(
|
||||
) -> Option<CoreEdgeSelection<Self>> {
|
||||
let selection = routing::select_edge(
|
||||
node.inner(),
|
||||
outcome,
|
||||
context,
|
||||
self.inner(),
|
||||
node.inner().selection(),
|
||||
);
|
||||
selection.map(|sel| EdgeSelection {
|
||||
selection.map(|sel| CoreEdgeSelection {
|
||||
edge: WorkflowEdge(Arc::new(sel.edge.clone())),
|
||||
reason: sel.reason,
|
||||
})
|
||||
|
|
@ -120,10 +121,10 @@ impl Graph for WorkflowGraph {
|
|||
&self,
|
||||
outcomes: &HashMap<String, Outcome>,
|
||||
) -> std::result::Result<(), String> {
|
||||
graph_ops::check_goal_gates(self.inner(), outcomes)
|
||||
routing::check_goal_gates(self.inner(), outcomes)
|
||||
}
|
||||
|
||||
fn get_retry_target(&self, failed_node_id: &str) -> Option<String> {
|
||||
graph_ops::get_retry_target(failed_node_id, self.inner())
|
||||
routing::get_retry_target(failed_node_id, self.inner())
|
||||
}
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -10,7 +10,7 @@ use crate::context::keys;
|
|||
use crate::context::{Context, WorkflowContext};
|
||||
use crate::error::FabroError;
|
||||
use crate::event::WorkflowRunEvent;
|
||||
use crate::graph_ops::set_hook_node;
|
||||
use crate::hook_context::set_hook_node;
|
||||
use crate::millis_u64;
|
||||
use crate::outcome::{Outcome, OutcomeExt, StageStatus};
|
||||
use fabro_graphviz::graph::{Graph, Node};
|
||||
|
|
|
|||
35
lib/crates/fabro-workflows/src/hook_context.rs
Normal file
35
lib/crates/fabro-workflows/src/hook_context.rs
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
use fabro_graphviz::graph::types::Node as GvNode;
|
||||
use fabro_hooks::HookContext;
|
||||
|
||||
/// Populate node-related fields on a `HookContext` from a graph node.
|
||||
pub(crate) fn set_hook_node(ctx: &mut HookContext, node: &GvNode) {
|
||||
ctx.node_id = Some(node.id.clone());
|
||||
ctx.node_label = Some(node.label().to_string());
|
||||
ctx.handler_type = node.handler_type().map(String::from);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use fabro_graphviz::graph::{AttrValue, Node};
|
||||
use fabro_hooks::HookEvent;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn set_hook_node_populates_hook_context_fields() {
|
||||
let mut node = Node::new("approve");
|
||||
node.attrs.insert(
|
||||
"label".to_string(),
|
||||
AttrValue::String("Approve PR".to_string()),
|
||||
);
|
||||
node.attrs
|
||||
.insert("type".to_string(), AttrValue::String("human".to_string()));
|
||||
|
||||
let mut ctx = HookContext::new(HookEvent::StageStart, "run-1".into(), "graph".into());
|
||||
set_hook_node(&mut ctx, &node);
|
||||
|
||||
assert_eq!(ctx.node_id.as_deref(), Some("approve"));
|
||||
assert_eq!(ctx.node_label.as_deref(), Some("Approve PR"));
|
||||
assert_eq!(ctx.handler_type.as_deref(), Some("human"));
|
||||
}
|
||||
}
|
||||
|
|
@ -98,8 +98,8 @@ pub mod error;
|
|||
pub mod event;
|
||||
pub mod git;
|
||||
pub mod graph;
|
||||
pub mod graph_ops;
|
||||
pub mod handler;
|
||||
mod hook_context;
|
||||
pub mod lifecycle;
|
||||
pub mod node_handler;
|
||||
pub mod operations;
|
||||
|
|
@ -107,6 +107,7 @@ pub mod outcome;
|
|||
pub mod pipeline;
|
||||
pub mod pull_request;
|
||||
pub mod records;
|
||||
mod retry;
|
||||
pub mod run_dir;
|
||||
pub mod run_lookup;
|
||||
pub mod run_settings;
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@ use fabro_core::state::RunState;
|
|||
use crate::error::{FailureCategory, FailureSignature};
|
||||
use crate::graph::WorkflowGraph;
|
||||
use crate::graph::WorkflowNode;
|
||||
use crate::graph_ops::classify_outcome;
|
||||
use crate::outcome::{OutcomeExt, StageStatus, StageUsage};
|
||||
|
||||
type WfRunState = RunState<Option<StageUsage>>;
|
||||
|
|
@ -69,7 +68,7 @@ impl RunLifecycle<WorkflowGraph> for CircuitBreakerLifecycle {
|
|||
let outcome = &result.outcome;
|
||||
|
||||
let outcome_failure_category = if outcome.status == StageStatus::Fail {
|
||||
classify_outcome(outcome)
|
||||
outcome.classified_failure_category()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
|
@ -117,7 +116,7 @@ impl RunLifecycle<WorkflowGraph> for CircuitBreakerLifecycle {
|
|||
let outcome = ctx.outcome;
|
||||
|
||||
// Guard: only TransientInfra failures may trigger loop_restart
|
||||
let failure_class = classify_outcome(outcome);
|
||||
let failure_class = outcome.classified_failure_category();
|
||||
if let Some(fc) = failure_class {
|
||||
if fc != FailureCategory::TransientInfra {
|
||||
return Ok(EdgeDecision::Block(format!(
|
||||
|
|
|
|||
|
|
@ -13,12 +13,20 @@ use crate::artifact::ArtifactStore;
|
|||
use crate::event::{EventEmitter, WorkflowRunEvent};
|
||||
use crate::graph::WorkflowGraph;
|
||||
use crate::graph::WorkflowNode;
|
||||
use crate::graph_ops::node_script;
|
||||
use crate::outcome::{FailureCategory, FailureDetail, Outcome, StageStatus, StageUsage};
|
||||
use fabro_graphviz::graph::types::Node as GvNode;
|
||||
|
||||
type WfRunState = RunState<Option<StageUsage>>;
|
||||
type WfNodeResult = NodeResult<Option<StageUsage>>;
|
||||
|
||||
fn node_script(node: &GvNode) -> Option<String> {
|
||||
node.attrs
|
||||
.get("script")
|
||||
.or_else(|| node.attrs.get("tool_command"))
|
||||
.and_then(|v| v.as_str())
|
||||
.map(String::from)
|
||||
}
|
||||
|
||||
/// Sub-lifecycle responsible for emitting workflow run events.
|
||||
pub struct EventLifecycle {
|
||||
pub emitter: Arc<EventEmitter>,
|
||||
|
|
|
|||
|
|
@ -5,11 +5,11 @@ use async_trait::async_trait;
|
|||
use fabro_core::graph::NodeSpec;
|
||||
use fabro_core::lifecycle::{EdgeContext, NodeDecision, RunLifecycle};
|
||||
use fabro_core::state::RunState;
|
||||
use fabro_graphviz::graph::types::{Edge as GvEdge, Graph as GvGraph, Node as GvNode};
|
||||
|
||||
use crate::context::keys;
|
||||
use crate::graph::WorkflowGraph;
|
||||
use crate::graph::WorkflowNode;
|
||||
use crate::graph_ops::{resolve_fidelity, resolve_thread_id};
|
||||
use crate::handler::llm::preamble::build_preamble;
|
||||
use crate::outcome::StageUsage;
|
||||
|
||||
|
|
@ -20,19 +20,19 @@ type WfNodeDecision = NodeDecision<Option<StageUsage>>;
|
|||
/// for fidelity/thread resolution.
|
||||
#[derive(Debug, Clone)]
|
||||
struct IncomingEdgeData {
|
||||
edge: Arc<fabro_graphviz::graph::types::Edge>,
|
||||
edge: Arc<GvEdge>,
|
||||
}
|
||||
|
||||
/// Sub-lifecycle responsible for fidelity/thread resolution and context key setup.
|
||||
pub struct FidelityLifecycle {
|
||||
pub graph: Arc<fabro_graphviz::graph::types::Graph>,
|
||||
pub graph: Arc<GvGraph>,
|
||||
incoming_edge_data: Mutex<Option<IncomingEdgeData>>,
|
||||
/// True on the first node after checkpoint resume when prior fidelity was Full.
|
||||
degrade_fidelity_on_resume: Mutex<bool>,
|
||||
}
|
||||
|
||||
impl FidelityLifecycle {
|
||||
pub fn new(graph: Arc<fabro_graphviz::graph::types::Graph>) -> Self {
|
||||
pub fn new(graph: Arc<GvGraph>) -> Self {
|
||||
Self {
|
||||
graph,
|
||||
incoming_edge_data: Mutex::new(None),
|
||||
|
|
@ -154,3 +154,265 @@ impl RunLifecycle<WorkflowGraph> for FidelityLifecycle {
|
|||
Ok(fabro_core::lifecycle::EdgeDecision::Continue)
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve the context fidelity for a node, following the precedence:
|
||||
/// 1. Incoming edge `fidelity` attribute
|
||||
/// 2. Target node `fidelity` attribute
|
||||
/// 3. Graph `default_fidelity` attribute
|
||||
/// 4. Default: Compact
|
||||
fn resolve_fidelity(
|
||||
incoming_edge: Option<&GvEdge>,
|
||||
node: &GvNode,
|
||||
graph: &GvGraph,
|
||||
) -> keys::Fidelity {
|
||||
let (resolved, source) = if let Some(f) = incoming_edge
|
||||
.and_then(|e| e.fidelity())
|
||||
.and_then(|s| s.parse().ok())
|
||||
{
|
||||
(f, "edge")
|
||||
} else if let Some(f) = node.fidelity().and_then(|s| s.parse().ok()) {
|
||||
(f, "node")
|
||||
} else if let Some(f) = graph.default_fidelity().and_then(|s| s.parse().ok()) {
|
||||
(f, "graph")
|
||||
} else {
|
||||
(keys::Fidelity::default(), "default")
|
||||
};
|
||||
|
||||
tracing::debug!(
|
||||
node = %node.id,
|
||||
fidelity = %resolved,
|
||||
source = source,
|
||||
"Fidelity resolved"
|
||||
);
|
||||
|
||||
resolved
|
||||
}
|
||||
|
||||
/// Resolve the thread ID for a node, following the precedence:
|
||||
/// 1. Incoming edge `thread_id` attribute
|
||||
/// 2. Target node `thread_id` attribute
|
||||
/// 3. Graph-level default thread
|
||||
/// 4. Derived class from enclosing subgraph (first class from the node's classes list)
|
||||
/// 5. Fallback to previous node ID
|
||||
fn resolve_thread_id(
|
||||
incoming_edge: Option<&GvEdge>,
|
||||
node: &GvNode,
|
||||
graph: &GvGraph,
|
||||
previous_node_id: Option<&str>,
|
||||
) -> Option<String> {
|
||||
if let Some(edge) = incoming_edge {
|
||||
if let Some(tid) = edge.thread_id() {
|
||||
return Some(tid.to_string());
|
||||
}
|
||||
}
|
||||
if let Some(tid) = node.thread_id() {
|
||||
return Some(tid.to_string());
|
||||
}
|
||||
if let Some(tid) = graph.default_thread() {
|
||||
return Some(tid.to_string());
|
||||
}
|
||||
if let Some(first_class) = node.classes.first() {
|
||||
return Some(first_class.clone());
|
||||
}
|
||||
previous_node_id.map(String::from)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use fabro_graphviz::graph::{AttrValue, Edge, Graph, Node};
|
||||
|
||||
use super::*;
|
||||
use crate::context::keys::Fidelity;
|
||||
|
||||
#[test]
|
||||
fn fidelity_defaults_to_compact() {
|
||||
let node = Node::new("work");
|
||||
let graph = Graph::new("test");
|
||||
assert_eq!(resolve_fidelity(None, &node, &graph), Fidelity::Compact);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fidelity_from_graph_default() {
|
||||
let node = Node::new("work");
|
||||
let mut graph = Graph::new("test");
|
||||
graph.attrs.insert(
|
||||
"default_fidelity".to_string(),
|
||||
AttrValue::String("truncate".to_string()),
|
||||
);
|
||||
assert_eq!(resolve_fidelity(None, &node, &graph), Fidelity::Truncate);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fidelity_from_node_overrides_graph() {
|
||||
let mut node = Node::new("work");
|
||||
node.attrs.insert(
|
||||
"fidelity".to_string(),
|
||||
AttrValue::String("full".to_string()),
|
||||
);
|
||||
let mut graph = Graph::new("test");
|
||||
graph.attrs.insert(
|
||||
"default_fidelity".to_string(),
|
||||
AttrValue::String("truncate".to_string()),
|
||||
);
|
||||
assert_eq!(resolve_fidelity(None, &node, &graph), Fidelity::Full);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fidelity_from_edge_overrides_node() {
|
||||
let mut node = Node::new("work");
|
||||
node.attrs.insert(
|
||||
"fidelity".to_string(),
|
||||
AttrValue::String("full".to_string()),
|
||||
);
|
||||
let mut edge = Edge::new("a", "work");
|
||||
edge.attrs.insert(
|
||||
"fidelity".to_string(),
|
||||
AttrValue::String("summary:high".to_string()),
|
||||
);
|
||||
let graph = Graph::new("test");
|
||||
assert_eq!(
|
||||
resolve_fidelity(Some(&edge), &node, &graph),
|
||||
Fidelity::SummaryHigh
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn thread_id_from_node_attribute() {
|
||||
let mut node = Node::new("work");
|
||||
node.attrs.insert(
|
||||
"thread_id".to_string(),
|
||||
AttrValue::String("main-thread".to_string()),
|
||||
);
|
||||
let graph = Graph::new("test");
|
||||
assert_eq!(
|
||||
resolve_thread_id(None, &node, &graph, Some("prev")),
|
||||
Some("main-thread".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn thread_id_from_edge_attribute() {
|
||||
let node = Node::new("work");
|
||||
let mut edge = Edge::new("prev", "work");
|
||||
edge.attrs.insert(
|
||||
"thread_id".to_string(),
|
||||
AttrValue::String("edge-thread".to_string()),
|
||||
);
|
||||
let graph = Graph::new("test");
|
||||
assert_eq!(
|
||||
resolve_thread_id(Some(&edge), &node, &graph, Some("prev")),
|
||||
Some("edge-thread".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn thread_id_node_used_when_no_edge_thread() {
|
||||
let mut node = Node::new("work");
|
||||
node.attrs.insert(
|
||||
"thread_id".to_string(),
|
||||
AttrValue::String("node-thread".to_string()),
|
||||
);
|
||||
let edge = Edge::new("prev", "work");
|
||||
let graph = Graph::new("test");
|
||||
assert_eq!(
|
||||
resolve_thread_id(Some(&edge), &node, &graph, Some("prev")),
|
||||
Some("node-thread".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn thread_id_edge_overrides_node() {
|
||||
let mut node = Node::new("work");
|
||||
node.attrs.insert(
|
||||
"thread_id".to_string(),
|
||||
AttrValue::String("node-thread".to_string()),
|
||||
);
|
||||
let mut edge = Edge::new("prev", "work");
|
||||
edge.attrs.insert(
|
||||
"thread_id".to_string(),
|
||||
AttrValue::String("edge-thread".to_string()),
|
||||
);
|
||||
let graph = Graph::new("test");
|
||||
assert_eq!(
|
||||
resolve_thread_id(Some(&edge), &node, &graph, Some("prev")),
|
||||
Some("edge-thread".to_string()),
|
||||
"edge thread_id should override node thread_id"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn thread_id_from_graph_default_thread() {
|
||||
let node = Node::new("work");
|
||||
let mut graph = Graph::new("test");
|
||||
graph.attrs.insert(
|
||||
"default_thread".to_string(),
|
||||
AttrValue::String("shared-thread".to_string()),
|
||||
);
|
||||
assert_eq!(
|
||||
resolve_thread_id(None, &node, &graph, Some("prev")),
|
||||
Some("shared-thread".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn thread_id_edge_overrides_graph_default() {
|
||||
let node = Node::new("work");
|
||||
let mut edge = Edge::new("prev", "work");
|
||||
edge.attrs.insert(
|
||||
"thread_id".to_string(),
|
||||
AttrValue::String("edge-thread".to_string()),
|
||||
);
|
||||
let mut graph = Graph::new("test");
|
||||
graph.attrs.insert(
|
||||
"default_thread".to_string(),
|
||||
AttrValue::String("shared-thread".to_string()),
|
||||
);
|
||||
assert_eq!(
|
||||
resolve_thread_id(Some(&edge), &node, &graph, Some("prev")),
|
||||
Some("edge-thread".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn thread_id_graph_default_overrides_class() {
|
||||
let mut node = Node::new("work");
|
||||
node.classes = vec!["planning".to_string()];
|
||||
let mut graph = Graph::new("test");
|
||||
graph.attrs.insert(
|
||||
"default_thread".to_string(),
|
||||
AttrValue::String("shared-thread".to_string()),
|
||||
);
|
||||
assert_eq!(
|
||||
resolve_thread_id(None, &node, &graph, Some("prev")),
|
||||
Some("shared-thread".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn thread_id_from_node_class() {
|
||||
let mut node = Node::new("work");
|
||||
node.classes = vec!["planning".to_string(), "review".to_string()];
|
||||
let graph = Graph::new("test");
|
||||
assert_eq!(
|
||||
resolve_thread_id(None, &node, &graph, Some("prev")),
|
||||
Some("planning".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn thread_id_fallback_to_previous_node() {
|
||||
let node = Node::new("work");
|
||||
let graph = Graph::new("test");
|
||||
assert_eq!(
|
||||
resolve_thread_id(None, &node, &graph, Some("prev_node")),
|
||||
Some("prev_node".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn thread_id_none_when_no_sources() {
|
||||
let node = Node::new("start");
|
||||
let graph = Graph::new("test");
|
||||
assert_eq!(resolve_thread_id(None, &node, &graph, None), None);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ use fabro_core::state::RunState;
|
|||
|
||||
use crate::graph::WorkflowGraph;
|
||||
use crate::graph::WorkflowNode;
|
||||
use crate::graph_ops::set_hook_node;
|
||||
use crate::hook_context::set_hook_node;
|
||||
use crate::outcome::{Outcome, OutcomeExt, StageStatus, StageUsage};
|
||||
use fabro_hooks::{HookContext, HookDecision, HookEvent, HookRunner};
|
||||
use fabro_sandbox::Sandbox;
|
||||
|
|
|
|||
|
|
@ -307,7 +307,7 @@ impl RunLifecycle<WorkflowGraph> for WorkflowLifecycle {
|
|||
) -> CoreResult<()> {
|
||||
let outcome = &result.outcome;
|
||||
let retry_count = state.node_retries.get(node.id()).copied().unwrap_or(0);
|
||||
let failure_class = crate::graph_ops::classify_outcome(outcome);
|
||||
let failure_class = outcome.classified_failure_category();
|
||||
let failure_signature = failure_class
|
||||
.map(|category| {
|
||||
let signature_hint = outcome
|
||||
|
|
|
|||
|
|
@ -16,7 +16,8 @@ use crate::graph::WorkflowGraph;
|
|||
use crate::graph::WorkflowNode;
|
||||
use crate::handler::{format_panic_message, EngineServices};
|
||||
use crate::outcome::{Outcome, StageStatus};
|
||||
use crate::{graph_ops, run_dir};
|
||||
use crate::retry::build_retry_policy;
|
||||
use crate::run_dir;
|
||||
|
||||
/// Production node handler that bridges fabro-core's NodeHandler to the
|
||||
/// existing fabro-workflows Handler trait via EngineServices.
|
||||
|
|
@ -113,11 +114,7 @@ impl NodeHandler<WorkflowGraph> for WorkflowNodeHandler {
|
|||
|
||||
fn retry_policy(&self, node: &WorkflowNode, _graph: &WorkflowGraph) -> CoreRetryPolicy {
|
||||
let gv_node = node.inner();
|
||||
let wf_policy = graph_ops::build_retry_policy(gv_node, &self.graph);
|
||||
CoreRetryPolicy {
|
||||
max_attempts: wf_policy.max_attempts,
|
||||
backoff: wf_policy.backoff,
|
||||
}
|
||||
build_retry_policy(gv_node, &self.graph)
|
||||
}
|
||||
|
||||
fn on_retries_exhausted(&self, node: &WorkflowNode, last_outcome: Outcome) -> Outcome {
|
||||
|
|
|
|||
|
|
@ -62,6 +62,13 @@ pub trait OutcomeExt: Sized {
|
|||
|
||||
/// Get the failure category, if this is a failed outcome.
|
||||
fn failure_category(&self) -> Option<FailureCategory>;
|
||||
|
||||
/// Resolve the effective failure category for this outcome.
|
||||
///
|
||||
/// Returns `None` for success, partial success, and skipped outcomes.
|
||||
/// Failed and retry outcomes default to `Deterministic` when no
|
||||
/// structured failure category is present.
|
||||
fn classified_failure_category(&self) -> Option<FailureCategory>;
|
||||
}
|
||||
|
||||
impl OutcomeExt for Outcome {
|
||||
|
|
@ -114,6 +121,15 @@ impl OutcomeExt for Outcome {
|
|||
fn failure_category(&self) -> Option<FailureCategory> {
|
||||
self.failure.as_ref().map(|f| f.category)
|
||||
}
|
||||
|
||||
fn classified_failure_category(&self) -> Option<FailureCategory> {
|
||||
match self.status {
|
||||
StageStatus::Success | StageStatus::PartialSuccess | StageStatus::Skipped => None,
|
||||
StageStatus::Fail | StageStatus::Retry => self
|
||||
.failure_category()
|
||||
.or(Some(FailureCategory::Deterministic)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute the dollar cost for a stage's token usage, if pricing is available.
|
||||
|
|
@ -274,6 +290,75 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classified_failure_category_returns_none_for_success() {
|
||||
assert!(Outcome::success().classified_failure_category().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classified_failure_category_returns_none_for_skipped() {
|
||||
assert!(Outcome::skipped("").classified_failure_category().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classified_failure_category_returns_none_for_partial_success() {
|
||||
let outcome = Outcome {
|
||||
status: StageStatus::PartialSuccess,
|
||||
..Outcome::success()
|
||||
};
|
||||
assert!(outcome.classified_failure_category().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classified_failure_category_reads_failure_detail() {
|
||||
let mut outcome = Outcome::fail_classify("some error");
|
||||
outcome.failure.as_mut().unwrap().category = FailureCategory::BudgetExhausted;
|
||||
assert_eq!(
|
||||
outcome.classified_failure_category(),
|
||||
Some(FailureCategory::BudgetExhausted)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classified_failure_category_uses_failure_reason_heuristics() {
|
||||
let outcome = Outcome::fail_classify("rate limited by provider");
|
||||
assert_eq!(
|
||||
outcome.classified_failure_category(),
|
||||
Some(FailureCategory::TransientInfra)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classified_failure_category_defaults_to_deterministic() {
|
||||
let outcome = Outcome::fail_classify("something went wrong");
|
||||
assert_eq!(
|
||||
outcome.classified_failure_category(),
|
||||
Some(FailureCategory::Deterministic)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classified_failure_category_fail_no_reason_is_deterministic() {
|
||||
let outcome = Outcome {
|
||||
status: StageStatus::Fail,
|
||||
failure: None,
|
||||
..Outcome::success()
|
||||
};
|
||||
assert_eq!(
|
||||
outcome.classified_failure_category(),
|
||||
Some(FailureCategory::Deterministic)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classified_failure_category_retry_status_uses_heuristics() {
|
||||
let outcome = Outcome::retry_classify("connection refused");
|
||||
assert_eq!(
|
||||
outcome.classified_failure_category(),
|
||||
Some(FailureCategory::TransientInfra)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stage_usage_serialization_with_cache_and_reasoning() {
|
||||
let usage = StageUsage {
|
||||
|
|
|
|||
190
lib/crates/fabro-workflows/src/retry.rs
Normal file
190
lib/crates/fabro-workflows/src/retry.rs
Normal file
|
|
@ -0,0 +1,190 @@
|
|||
use std::time::Duration;
|
||||
|
||||
use fabro_core::retry::{BackoffPolicy, RetryPolicy};
|
||||
use fabro_graphviz::graph::types::{Graph as GvGraph, Node as GvNode};
|
||||
|
||||
const DEFAULT_BACKOFF: BackoffPolicy = BackoffPolicy {
|
||||
initial_delay: Duration::from_millis(5_000),
|
||||
factor: 2.0,
|
||||
max_delay: Duration::from_millis(60_000),
|
||||
jitter: true,
|
||||
};
|
||||
|
||||
/// Build a retry policy from node and graph attributes.
|
||||
/// If the node has a `retry_policy` attribute naming a preset, use that.
|
||||
/// Otherwise, fall back to `max_retries` / graph default.
|
||||
pub(crate) fn build_retry_policy(node: &GvNode, graph: &GvGraph) -> RetryPolicy {
|
||||
if let Some(preset) = node.retry_policy() {
|
||||
if let Some(policy) = preset_retry_policy(preset) {
|
||||
return policy;
|
||||
}
|
||||
}
|
||||
|
||||
let max_retries = node
|
||||
.max_retries()
|
||||
.unwrap_or_else(|| graph.default_max_retries());
|
||||
let max_attempts = u32::try_from(max_retries + 1).unwrap_or(1).max(1);
|
||||
|
||||
RetryPolicy {
|
||||
max_attempts,
|
||||
backoff: DEFAULT_BACKOFF,
|
||||
}
|
||||
}
|
||||
|
||||
fn preset_retry_policy(preset: &str) -> Option<RetryPolicy> {
|
||||
match preset {
|
||||
"none" => Some(RetryPolicy {
|
||||
max_attempts: 1,
|
||||
backoff: DEFAULT_BACKOFF,
|
||||
}),
|
||||
"standard" => Some(RetryPolicy {
|
||||
max_attempts: 5,
|
||||
backoff: DEFAULT_BACKOFF,
|
||||
}),
|
||||
"aggressive" => Some(RetryPolicy {
|
||||
max_attempts: 5,
|
||||
backoff: BackoffPolicy {
|
||||
initial_delay: Duration::from_millis(500),
|
||||
..DEFAULT_BACKOFF
|
||||
},
|
||||
}),
|
||||
"linear" => Some(RetryPolicy {
|
||||
max_attempts: 3,
|
||||
backoff: BackoffPolicy {
|
||||
initial_delay: Duration::from_millis(500),
|
||||
factor: 1.0,
|
||||
..DEFAULT_BACKOFF
|
||||
},
|
||||
}),
|
||||
"patient" => Some(RetryPolicy {
|
||||
max_attempts: 3,
|
||||
backoff: BackoffPolicy {
|
||||
initial_delay: Duration::from_millis(2_000),
|
||||
factor: 3.0,
|
||||
..DEFAULT_BACKOFF
|
||||
},
|
||||
}),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::time::Duration;
|
||||
|
||||
use fabro_graphviz::graph::{AttrValue, Graph, Node};
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn build_retry_policy_from_node() {
|
||||
let mut node = Node::new("n");
|
||||
node.attrs
|
||||
.insert("max_retries".to_string(), AttrValue::Integer(3));
|
||||
let graph = Graph::new("test");
|
||||
let policy = build_retry_policy(&node, &graph);
|
||||
assert_eq!(policy.max_attempts, 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_retry_policy_from_graph_default() {
|
||||
let node = Node::new("n");
|
||||
let mut graph = Graph::new("test");
|
||||
graph
|
||||
.attrs
|
||||
.insert("default_max_retries".to_string(), AttrValue::Integer(2));
|
||||
let policy = build_retry_policy(&node, &graph);
|
||||
assert_eq!(policy.max_attempts, 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_retry_policy_no_attrs_uses_graph_default_0() {
|
||||
let node = Node::new("n");
|
||||
let graph = Graph::new("test");
|
||||
let policy = build_retry_policy(&node, &graph);
|
||||
assert_eq!(policy.max_attempts, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_retry_policy_from_retry_policy_attr() {
|
||||
let mut node = Node::new("n");
|
||||
node.attrs.insert(
|
||||
"retry_policy".to_string(),
|
||||
AttrValue::String("aggressive".to_string()),
|
||||
);
|
||||
let graph = Graph::new("test");
|
||||
let policy = build_retry_policy(&node, &graph);
|
||||
assert_eq!(policy.max_attempts, 5);
|
||||
assert_eq!(policy.backoff.initial_delay, Duration::from_millis(500));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_retry_policy_fallback_when_no_retry_policy_attr() {
|
||||
let mut node = Node::new("n");
|
||||
node.attrs
|
||||
.insert("max_retries".to_string(), AttrValue::Integer(3));
|
||||
let graph = Graph::new("test");
|
||||
let policy = build_retry_policy(&node, &graph);
|
||||
assert_eq!(policy.max_attempts, 4);
|
||||
assert_eq!(policy.backoff.initial_delay, Duration::from_millis(5_000));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_retry_policy_all_presets() {
|
||||
let presets = [
|
||||
("none", 1u32),
|
||||
("standard", 5),
|
||||
("aggressive", 5),
|
||||
("linear", 3),
|
||||
("patient", 3),
|
||||
];
|
||||
let graph = Graph::new("test");
|
||||
let (name, expected) = presets[0];
|
||||
let mut node = Node::new("n");
|
||||
node.attrs.insert(
|
||||
"retry_policy".to_string(),
|
||||
AttrValue::String(name.to_string()),
|
||||
);
|
||||
assert_eq!(build_retry_policy(&node, &graph).max_attempts, expected);
|
||||
|
||||
let (name, expected) = presets[1];
|
||||
node.attrs.insert(
|
||||
"retry_policy".to_string(),
|
||||
AttrValue::String(name.to_string()),
|
||||
);
|
||||
assert_eq!(build_retry_policy(&node, &graph).max_attempts, expected);
|
||||
|
||||
let (name, expected) = presets[2];
|
||||
node.attrs.insert(
|
||||
"retry_policy".to_string(),
|
||||
AttrValue::String(name.to_string()),
|
||||
);
|
||||
assert_eq!(build_retry_policy(&node, &graph).max_attempts, expected);
|
||||
|
||||
let (name, expected) = presets[3];
|
||||
node.attrs.insert(
|
||||
"retry_policy".to_string(),
|
||||
AttrValue::String(name.to_string()),
|
||||
);
|
||||
assert_eq!(build_retry_policy(&node, &graph).max_attempts, expected);
|
||||
|
||||
let (name, expected) = presets[4];
|
||||
node.attrs.insert(
|
||||
"retry_policy".to_string(),
|
||||
AttrValue::String(name.to_string()),
|
||||
);
|
||||
assert_eq!(build_retry_policy(&node, &graph).max_attempts, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_retry_policy_unknown_preset_falls_back() {
|
||||
let mut node = Node::new("n");
|
||||
node.attrs.insert(
|
||||
"retry_policy".to_string(),
|
||||
AttrValue::String("unknown_preset".to_string()),
|
||||
);
|
||||
let graph = Graph::new("test");
|
||||
let policy = build_retry_policy(&node, &graph);
|
||||
assert_eq!(policy.max_attempts, 1);
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue