fabro/lib/components/fabro-workflow/src/error.rs
Bryan Helmkamp 82bcafcfca
Port fabro-workflow, hooks, validate, and store to the lithos types
Workflow LLM handlers build lithos requests, bill from lithos usage and
cost, and classify failures from lithos `ErrorKind`. Model resolution and
fallback use the fabro-llm selection and catalog helpers. Validation
rules read the lithos catalog, and store fixtures use the new
`BilledModelUsage` shape.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-09 17:26:57 -06:00

2266 lines
67 KiB
Rust

use std::fmt;
use std::sync::{Arc, LazyLock};
use fabro_graphviz::Error as GraphvizError;
use fabro_llm::{ErrorFacts, ErrorKind, LlmError, ModelSelectionError};
use fabro_template::TemplateError;
pub use fabro_types::failure_signature::FailureSignature;
pub use fabro_types::outcome::FailureCategory;
use fabro_types::settings::{AmbiguousModelRef, ResolveError};
use fabro_types::{ExecOutputTail, FailureReason, RunFailure};
use fabro_util::error::{SharedError, collect_causes, collect_chain, render_with_causes};
use fabro_validate::Diagnostic;
use regex::Regex;
use thiserror::Error as ThisError;
use crate::event::RunEventPersistenceError;
use crate::outcome::{FailureDetail, Outcome, StageOutcome};
/// Classify an LLM error into a `FailureCategory` based on its structure.
#[must_use]
pub fn classify_sdk_error<E: ErrorFacts + ?Sized>(err: &E) -> FailureCategory {
match err.kind() {
ErrorKind::RateLimit
| ErrorKind::Server
| ErrorKind::Network
| ErrorKind::Timeout
| ErrorKind::StreamDecode => FailureCategory::TransientInfra,
ErrorKind::ContextLength | ErrorKind::QuotaExceeded => FailureCategory::BudgetExhausted,
ErrorKind::Cancelled => FailureCategory::Canceled,
// Configuration, model selection, auth, access, not-found, invalid
// request, content filter, provider, decode, resource limit, and
// middleware failures are deterministic. `ErrorKind` is
// non-exhaustive: a category added by a newer lithos never enables
// automatic retry either.
_ => FailureCategory::Deterministic,
}
}
const TRANSIENT_INFRA_HINTS: &[&str] = &[
"timeout",
"timed out",
"rate limit",
"rate limited",
"connection refused",
"connection reset",
"500",
"502",
"503",
"504",
"context deadline exceeded",
"could not resolve host",
"could not resolve hostname",
"temporary failure",
"network is unreachable",
"broken pipe",
"tls handshake timeout",
"i/o timeout",
"no route to host",
"temporarily unavailable",
"try again",
"too many requests",
"service unavailable",
"gateway timeout",
"econnrefused",
"econnreset",
"dial tcp",
"transport is closing",
"stream disconnected",
"stream closed before",
"index.crates.io",
"download of config.json failed",
"toolchain_or_dependency_registry_unavailable",
"toolchain dependency resolution blocked by network",
"toolchain_workspace_io",
"cross-device link",
"invalid cross-device link",
"os error 18",
"state change in progress",
"sandbox stop still in progress",
];
const BUDGET_EXHAUSTED_HINTS: &[&str] = &[
"turn limit",
"token limit",
"context length",
"budget",
"quota exceeded",
"max_tokens",
"max tokens",
"context window exceeded",
"budget exhausted",
"token limit exceeded",
];
const STRUCTURAL_HINTS: &[&str] = &[
"write_scope_violation",
"write scope violation",
"scope violation",
];
#[derive(Debug, Clone)]
pub struct SharedTemplateError(Arc<TemplateError>);
impl SharedTemplateError {
#[must_use]
pub fn new(error: TemplateError) -> Self {
Self(Arc::new(error))
}
#[must_use]
pub fn inner(&self) -> &TemplateError {
&self.0
}
}
impl fmt::Display for SharedTemplateError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Display::fmt(&self.0, formatter)
}
}
impl std::error::Error for SharedTemplateError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
self.0.source()
}
}
impl miette::Diagnostic for SharedTemplateError {
fn code<'a>(&'a self) -> Option<Box<dyn fmt::Display + 'a>> {
miette::Diagnostic::code(self.inner())
}
fn help<'a>(&'a self) -> Option<Box<dyn fmt::Display + 'a>> {
miette::Diagnostic::help(self.inner())
}
fn source_code(&self) -> Option<&dyn miette::SourceCode> {
miette::Diagnostic::source_code(self.inner())
}
fn labels(&self) -> Option<Box<dyn Iterator<Item = miette::LabeledSpan> + '_>> {
miette::Diagnostic::labels(self.inner())
}
fn diagnostic_source(&self) -> Option<&dyn miette::Diagnostic> {
miette::Diagnostic::diagnostic_source(self.inner())
}
}
/// Matches git SHAs and other long hex blobs.
static HEX_RE: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"\b[0-9a-f]{7,64}\b").expect("hardcoded regex should compile"));
/// Classify a failure reason string using heuristics.
///
/// This is the fallback when structured error information is not available
/// (e.g. for `Handler(String)` or `Engine(String)` errors).
#[must_use]
pub fn classify_failure_reason(reason: &str) -> FailureCategory {
// Mask commit SHAs first. They are hex, so one contains "500" or "503"
// often enough to matter, which would read as a transient infra hint. The
// bare status codes those hints look for are too short to be masked.
let lowered = reason.to_lowercase();
let lower = HEX_RE.replace_all(&lowered, "<hex>");
if lower.contains("interrupt")
|| (lower.contains("cancel")
&& !lower.contains("cancelling due to test failure")
&& !lower.contains("canceling due to test failure"))
{
return FailureCategory::Canceled;
}
if TRANSIENT_INFRA_HINTS
.iter()
.any(|hint| lower.contains(hint))
{
return FailureCategory::TransientInfra;
}
if BUDGET_EXHAUSTED_HINTS
.iter()
.any(|hint| lower.contains(hint))
{
return FailureCategory::BudgetExhausted;
}
if STRUCTURAL_HINTS.iter().any(|hint| lower.contains(hint)) {
return FailureCategory::Structural;
}
FailureCategory::Deterministic
}
/// Normalize a failure reason for stable signature grouping.
///
/// Replaces variable data (hex strings, digits) with placeholders so that
/// semantically identical errors produce the same signature regardless of
/// line numbers, commit hashes, or timestamps.
pub fn normalize_failure_reason(reason: &str) -> String {
static DIGITS_RE: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"\b\d+\b").expect("hardcoded regex should compile"));
static COMMA_SPACE_RE: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r",\s+").expect("hardcoded regex should compile"));
static WHITESPACE_RE: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"\s+").expect("hardcoded regex should compile"));
let s = reason.trim().to_lowercase();
if s.is_empty() {
return String::new();
}
let s = HEX_RE.replace_all(&s, "<hex>");
let s = DIGITS_RE.replace_all(&s, "<n>");
let s = COMMA_SPACE_RE.replace_all(&s, ",");
let s = WHITESPACE_RE.replace_all(&s, " ");
let s = s.trim();
if s.len() > 240 {
s[..s.floor_char_boundary(240)].to_string()
} else {
s.to_string()
}
}
pub trait FailureSignatureExt {
fn new(
node_id: &str,
failure_class: FailureCategory,
signature_hint: Option<&str>,
failure_reason: Option<&str>,
) -> Self;
}
impl FailureSignatureExt for FailureSignature {
fn new(
node_id: &str,
failure_class: FailureCategory,
signature_hint: Option<&str>,
failure_reason: Option<&str>,
) -> Self {
let reason = signature_hint
.map(normalize_failure_reason)
.filter(|s| !s.is_empty())
.or_else(|| failure_reason.map(normalize_failure_reason))
.filter(|s| !s.is_empty())
.unwrap_or_else(|| "unknown".to_string());
Self(format!("{}|{}|{}", node_id.trim(), failure_class, reason))
}
}
/// Pipeline stage that produced an [`Error::Stage`].
///
/// The three stages share a failure shape — a message, an eagerly classified
/// [`FailureCategory`], an optional command output tail, and an optional
/// source — and differ only in where they run and whether a retry is possible.
#[derive(Debug, Clone, Copy, PartialEq, Eq, strum::Display)]
pub enum ErrorStage {
/// A node handler failed. Retryable: the engine can re-run the node.
Handler,
/// The engine itself failed while driving the graph. Retryable.
Engine,
/// The publish stage failed. Terminal: publish runs once, after execution.
Publish,
}
#[derive(ThisError, Debug, Clone)]
pub enum Error {
#[error("Parse error: {0}")]
Parse(String),
#[error("Validation error: {0}")]
Validation(String),
#[error("Validation failed")]
ValidationFailed { diagnostics: Vec<Diagnostic> },
#[error("Validation error: script interpolation failed in {owner}: {source} ({fix})")]
ScriptInterpolation {
owner: String,
fix: String,
#[source]
source: ResolveError,
},
#[error("Model selection failed: {0}")]
ModelSelection(#[from] ModelSelectionError),
#[error("Model reference failed: {0}")]
ModelReference(#[from] AmbiguousModelRef),
#[error("{message}")]
Template {
message: String,
#[source]
source: SharedTemplateError,
},
#[error("{stage} error: {message}")]
Stage {
stage: ErrorStage,
message: String,
failure_class: FailureCategory,
exec_output_tail: Option<ExecOutputTail>,
/// Structured context lines appended after the source chain in
/// `causes()` — e.g. one line per push attempt on a publish push
/// failure.
extra_causes: Vec<String>,
#[source]
source: Option<SharedError>,
},
#[error("LLM error: {0}")]
Llm(LlmError),
#[error("Checkpoint error: {0}")]
Checkpoint(String),
#[error("Stylesheet error: {0}")]
Stylesheet(String),
#[error("I/O error: {0}")]
Io(String),
#[error("Precondition failed: {0}")]
Precondition(String),
#[error("Run not found: {0}")]
RunNotFound(String),
#[error("Unsupported operation: {0}")]
Unsupported(String),
#[error("{0}")]
OutputSchemaValidation(String),
#[error("Pipeline cancelled")]
Cancelled,
}
impl Error {
/// Smart constructor for Handler errors. Classifies the failure reason
/// eagerly.
/// Build a stage error, classifying the message eagerly.
fn stage(
stage: ErrorStage,
message: impl Into<String>,
exec_output_tail: Option<ExecOutputTail>,
) -> Self {
let message = message.into();
let failure_class = classify_failure_reason(&message);
Self::Stage {
stage,
message,
failure_class,
exec_output_tail,
extra_causes: Vec::new(),
source: None,
}
}
/// Build a stage error from a source, classifying the rendered chain so
/// hints buried in the causes still reach [`Self::failure_category`].
fn stage_with_source(
stage: ErrorStage,
message: impl Into<String>,
source: impl Into<anyhow::Error>,
exec_output_tail: Option<ExecOutputTail>,
) -> Self {
Self::stage_with_source_details(stage, message, source, None, exec_output_tail, Vec::new())
}
fn stage_with_source_details(
stage: ErrorStage,
message: impl Into<String>,
source: impl Into<anyhow::Error>,
failure_class: Option<FailureCategory>,
exec_output_tail: Option<ExecOutputTail>,
extra_causes: Vec<String>,
) -> Self {
let message = message.into();
let source = SharedError::new(source.into());
let failure_class = failure_class.unwrap_or_else(|| {
classify_failure_reason(&render_with_causes(&message, &collect_chain(&source)))
});
Self::Stage {
stage,
message,
failure_class,
exec_output_tail,
extra_causes,
source: Some(source),
}
}
pub fn handler(message: impl Into<String>) -> Self {
Self::stage(ErrorStage::Handler, message, None)
}
pub fn template(message: impl Into<String>, source: TemplateError) -> Self {
Self::Template {
message: message.into(),
source: SharedTemplateError::new(source),
}
}
pub fn handler_with_exec_output_tail(
message: impl Into<String>,
exec_output_tail: Option<ExecOutputTail>,
) -> Self {
Self::stage(ErrorStage::Handler, message, exec_output_tail)
}
pub fn handler_with_source(
message: impl Into<String>,
source: impl Into<anyhow::Error>,
) -> Self {
Self::handler_with_source_and_exec_output_tail(message, source, None)
}
pub fn handler_with_source_and_exec_output_tail(
message: impl Into<String>,
source: impl Into<anyhow::Error>,
exec_output_tail: Option<ExecOutputTail>,
) -> Self {
Self::stage_with_source(ErrorStage::Handler, message, source, exec_output_tail)
}
pub fn handler_with_anyhow(message: impl Into<String>, source: anyhow::Error) -> Self {
Self::handler_with_source(message, source)
}
pub fn engine(message: impl Into<String>) -> Self {
Self::stage(ErrorStage::Engine, message, None)
}
pub fn engine_with_source(
message: impl Into<String>,
source: impl Into<anyhow::Error>,
) -> Self {
Self::stage_with_source(ErrorStage::Engine, message, source, None)
}
pub fn engine_with_anyhow(message: impl Into<String>, source: anyhow::Error) -> Self {
Self::engine_with_source(message, source)
}
/// Build an error for the required publish stage.
pub fn publish(message: impl Into<String>) -> Self {
Self::stage(ErrorStage::Publish, message, None)
}
pub fn publish_with_source(
message: impl Into<String>,
source: impl Into<anyhow::Error>,
) -> Self {
Self::publish_with_source_and_exec_output_tail(message, source, None)
}
pub fn publish_with_source_and_exec_output_tail(
message: impl Into<String>,
source: impl Into<anyhow::Error>,
exec_output_tail: Option<ExecOutputTail>,
) -> Self {
Self::stage_with_source(ErrorStage::Publish, message, source, exec_output_tail)
}
/// Build a publish error with an explicitly determined failure category,
/// for callers that know more than message sniffing can recover — e.g.
/// exhausted push retries whose attempts all classified as transient.
/// `extra_causes` lines land after the source chain in the failure
/// detail (one line per push attempt).
pub fn publish_with_source_and_class(
message: impl Into<String>,
source: impl Into<anyhow::Error>,
failure_class: FailureCategory,
exec_output_tail: Option<ExecOutputTail>,
extra_causes: Vec<String>,
) -> Self {
Self::stage_with_source_details(
ErrorStage::Publish,
message,
source,
Some(failure_class),
exec_output_tail,
extra_causes,
)
}
#[must_use]
pub fn causes(&self) -> Vec<String> {
match self {
Self::Stage {
source,
extra_causes,
..
} => {
let mut causes = source
.as_ref()
.map_or_else(Vec::new, |source| collect_chain(source));
causes.extend(extra_causes.iter().cloned());
causes
}
Self::Template { source, .. } => collect_chain(source),
Self::ScriptInterpolation { source, .. } => collect_chain(source),
Self::Llm(err) => collect_causes(err),
_ => Vec::new(),
}
}
#[must_use]
pub fn display_with_causes(&self) -> String {
render_with_causes(&self.to_string(), &self.causes())
}
/// Whether this error category is retryable (transient) or terminal.
///
/// Retryable: Handler and Engine stages (the engine can re-run the node),
/// I/O, and LLM errors the SDK marks retryable. Terminal: the Publish
/// stage (it runs once, after execution), Parse, Validation,
/// OutputSchemaValidation, Stylesheet, Checkpoint, and Cancelled.
#[must_use]
pub fn is_retryable(&self) -> bool {
match self {
Self::Io(_) => true,
Self::Stage { stage, .. } => {
matches!(stage, ErrorStage::Handler | ErrorStage::Engine)
}
Self::Llm(sdk_err) => sdk_err.is_retryable(),
Self::Parse(_)
| Self::Validation(_)
| Self::ValidationFailed { .. }
| Self::ScriptInterpolation { .. }
| Self::ModelSelection(_)
| Self::ModelReference(_)
| Self::Template { .. }
| Self::Stylesheet(_)
| Self::Checkpoint(_)
| Self::Precondition(_)
| Self::RunNotFound(_)
| Self::Unsupported(_)
| Self::OutputSchemaValidation(_)
| Self::Cancelled => false,
}
}
/// Classify this error into a `FailureCategory`.
#[must_use]
pub fn failure_category(&self) -> FailureCategory {
match self {
Self::Cancelled => FailureCategory::Canceled,
Self::Llm(sdk_err) => classify_sdk_error(sdk_err),
Self::Io(_) => FailureCategory::TransientInfra,
Self::Parse(_)
| Self::Validation(_)
| Self::ValidationFailed { .. }
| Self::ScriptInterpolation { .. }
| Self::ModelSelection(_)
| Self::ModelReference(_)
| Self::Template { .. }
| Self::Stylesheet(_)
| Self::Checkpoint(_)
| Self::Unsupported(_)
| Self::OutputSchemaValidation(_) => FailureCategory::Deterministic,
Self::Precondition(_) | Self::RunNotFound(_) => FailureCategory::Structural,
Self::Stage { failure_class, .. } => *failure_class,
}
}
/// The terminal [`FailureReason`] this error maps to on a run.
#[must_use]
pub fn failure_reason(&self) -> FailureReason {
match self {
Self::Cancelled => FailureReason::Cancelled,
Self::Stage {
stage: ErrorStage::Publish,
..
} => FailureReason::PublishFailed,
_ => FailureReason::WorkflowError,
}
}
/// Return a stable failure signature hint when structured error info is
/// available.
#[must_use]
pub fn failure_signature_hint(&self) -> Option<FailureSignature> {
match self {
Self::Llm(sdk_err) => Some(FailureSignature(sdk_err.failure_signature_hint())),
_ => None,
}
}
#[must_use]
pub fn to_failure_detail(&self) -> FailureDetail {
let (message, explicit_exec_output_tail) = match self {
Self::Stage {
message,
exec_output_tail,
..
} => (message.clone(), exec_output_tail.clone()),
_ => (self.to_string(), None),
};
FailureDetail {
message,
causes: self.causes(),
category: self.failure_category(),
system_actor: None,
signature: self.failure_signature_hint(),
exec_output_tail: explicit_exec_output_tail
.or_else(|| fabro_sandbox::default_redacted_output_tail(self)),
}
}
/// Build a fail `Outcome` with structured `FailureDetail`.
pub fn to_fail_outcome(&self) -> Outcome {
let failure = self.to_failure_detail();
Outcome {
status: StageOutcome::Failed {
retry_requested: false,
},
failure: Some(failure),
..Outcome::success()
}
}
}
impl miette::Diagnostic for Error {
fn code<'a>(&'a self) -> Option<Box<dyn fmt::Display + 'a>> {
match self {
Self::Template { source, .. } => miette::Diagnostic::code(source),
_ => None,
}
}
fn help<'a>(&'a self) -> Option<Box<dyn fmt::Display + 'a>> {
match self {
Self::Template { source, .. } => miette::Diagnostic::help(source),
_ => None,
}
}
fn source_code(&self) -> Option<&dyn miette::SourceCode> {
match self {
Self::Template { source, .. } => miette::Diagnostic::source_code(source),
_ => None,
}
}
fn labels(&self) -> Option<Box<dyn Iterator<Item = miette::LabeledSpan> + '_>> {
match self {
Self::Template { source, .. } => miette::Diagnostic::labels(source),
_ => None,
}
}
fn diagnostic_source(&self) -> Option<&dyn miette::Diagnostic> {
match self {
Self::Template { source, .. } => Some(source),
_ => None,
}
}
}
#[must_use]
pub fn run_failure_from_error(error: &Error, reason: FailureReason) -> RunFailure {
RunFailure {
reason,
detail: error.to_failure_detail(),
}
}
#[must_use]
pub fn run_failure_from_outcome_failure(
failure: &FailureDetail,
reason: FailureReason,
) -> RunFailure {
RunFailure {
reason,
detail: failure.clone(),
}
}
impl From<std::io::Error> for Error {
fn from(err: std::io::Error) -> Self {
Self::Io(err.to_string())
}
}
impl From<LlmError> for Error {
fn from(err: LlmError) -> Self {
Self::Llm(err)
}
}
impl From<fabro_llm::Error> for Error {
fn from(err: fabro_llm::Error) -> Self {
Self::Llm(LlmError::from(err))
}
}
impl From<GraphvizError> for Error {
fn from(e: GraphvizError) -> Self {
match e {
GraphvizError::Parse(msg) => Self::Parse(msg),
GraphvizError::Stylesheet(msg) => Self::Stylesheet(msg),
}
}
}
impl From<fabro_template::TemplateError> for Error {
fn from(err: fabro_template::TemplateError) -> Self {
let rendered = collect_chain(&err).join(": ");
Self::template(format!("template expansion failed: {rendered}"), err)
}
}
impl From<fabro_validate::ValidationError> for Error {
fn from(e: fabro_validate::ValidationError) -> Self {
Self::Validation(e.0)
}
}
impl From<RunEventPersistenceError> for Error {
fn from(err: RunEventPersistenceError) -> Self {
Self::engine_with_source("run event persistence failed", err)
}
}
impl From<fabro_checkpoint::MetadataError> for Error {
fn from(err: fabro_checkpoint::MetadataError) -> Self {
match err {
err @ fabro_checkpoint::MetadataError::Deserialize {
entity: "checkpoint",
..
} => Self::Checkpoint(err.to_string()),
err => {
let message = err.to_string();
Self::engine_with_source(message, err)
}
}
}
}
pub type Result<T> = std::result::Result<T, Error>;
#[cfg(test)]
mod tests {
use fabro_checkpoint::MetadataError;
use fabro_llm::RetryClassification;
use super::*;
/// A stored LLM error of `kind` from the `openai` provider.
fn sdk_error(kind: ErrorKind, message: &str) -> LlmError {
LlmError::from(
fabro_llm::Error::new(kind, message).with_provider(fabro_types::provider_ids::openai()),
)
}
/// A transient failure the provider may be asked to repeat.
fn transient_error(kind: ErrorKind, message: &str) -> LlmError {
LlmError::from(
fabro_llm::Error::new(kind, message)
.with_provider(fabro_types::provider_ids::openai())
.with_retry(RetryClassification::Safe),
)
}
use crate::outcome::OutcomeExt;
#[derive(Debug)]
struct TestCause(&'static str);
impl std::fmt::Display for TestCause {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(self.0)
}
}
impl std::error::Error for TestCause {}
#[derive(Debug)]
struct TestOuterError {
message: &'static str,
source: TestCause,
}
impl std::fmt::Display for TestOuterError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(self.message)
}
}
impl std::error::Error for TestOuterError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
Some(&self.source)
}
}
#[test]
fn parse_error_display() {
let err = Error::Parse("unexpected token".to_string());
assert_eq!(err.to_string(), "Parse error: unexpected token");
}
#[test]
fn validation_error_display() {
let err = Error::Validation("missing start node".to_string());
assert_eq!(err.to_string(), "Validation error: missing start node");
}
#[test]
fn validation_failed_display() {
let err = Error::ValidationFailed {
diagnostics: vec![Diagnostic {
rule: "test".to_string(),
severity: fabro_validate::Severity::Error,
message: "missing start node".to_string(),
node_id: None,
edge: None,
fix: None,
..Diagnostic::default()
}],
};
assert_eq!(err.to_string(), "Validation failed");
}
#[test]
fn template_error_variant_preserves_source_chain() {
let template_err = fabro_template::render_named(
"workflow.fabro",
"{{ inputs.missing }}",
&fabro_template::TemplateContext::new(),
)
.unwrap_err();
let err = Error::template("template expansion failed", template_err);
let chain = collect_chain(&err);
assert!(
chain
.iter()
.any(|part| part.contains("template expansion failed"))
);
assert!(
chain
.iter()
.any(|part| part.contains("undefined template variable"))
);
}
#[test]
fn engine_error_display() {
let err = Error::engine("no outgoing edge");
assert_eq!(err.to_string(), "Engine error: no outgoing edge");
}
#[test]
fn engine_error_with_source_preserves_cause_chain() {
let source = TestOuterError {
message: "Failed to pull Docker image buildpack-deps:noble",
source: TestCause("connection refused"),
};
let err = Error::engine_with_source("Failed to initialize sandbox", source);
assert_eq!(
err.to_string(),
"Engine error: Failed to initialize sandbox"
);
assert_eq!(err.causes(), vec![
"Failed to pull Docker image buildpack-deps:noble".to_string(),
"connection refused".to_string(),
]);
assert_eq!(
err.display_with_causes(),
"Engine error: Failed to initialize sandbox\n caused by: Failed to pull Docker image buildpack-deps:noble\n caused by: connection refused"
);
assert_eq!(err.failure_category(), FailureCategory::TransientInfra);
}
#[test]
fn engine_error_with_sandbox_state_change_cause_classifies_transient() {
let source = TestOuterError {
message: "Failed to start Daytona sandbox",
source: TestCause("Sandbox state change in progress"),
};
let err = Error::engine_with_source("Pipeline lifecycle operation failed", source);
assert_eq!(err.failure_category(), FailureCategory::TransientInfra);
assert!(err.is_retryable());
}
#[test]
fn handler_error_display() {
let err = Error::handler("LLM call failed");
assert_eq!(err.to_string(), "Handler error: LLM call failed");
}
#[test]
fn checkpoint_error_display() {
let err = Error::Checkpoint("file not found".to_string());
assert_eq!(err.to_string(), "Checkpoint error: file not found");
}
#[test]
fn io_error_display() {
let err = Error::Io("permission denied".to_string());
assert_eq!(err.to_string(), "I/O error: permission denied");
}
#[test]
fn io_error_from_std() {
let io_err = std::io::Error::new(std::io::ErrorKind::NotFound, "not found");
let err = Error::from(io_err);
assert!(matches!(err, Error::Io(_)));
assert!(err.to_string().contains("not found"));
}
#[test]
fn result_type_alias_works() {
let ok: Result<i32> = Ok(42);
assert!(ok.is_ok());
let err: Result<i32> = Err(Error::Parse("bad".to_string()));
assert!(err.is_err());
}
#[test]
fn metadata_checkpoint_deserialize_error_preserves_source_detail() {
let source = serde_json::from_str::<serde_json::Value>("not json").unwrap_err();
let source_message = source.to_string();
let fabro_error = Error::from(MetadataError::Deserialize {
entity: "checkpoint",
branch: "fabro/meta/run-1".to_string(),
source,
});
assert!(matches!(fabro_error, Error::Checkpoint(_)));
let message = fabro_error.to_string();
assert!(message.contains("deserialize checkpoint on branch fabro/meta/run-1"));
assert!(message.contains(&source_message));
}
#[test]
fn metadata_non_checkpoint_deserialize_error_maps_to_engine_with_source_detail() {
let source = serde_json::from_str::<serde_json::Value>("not json").unwrap_err();
let source_message = source.to_string();
let fabro_error = Error::from(MetadataError::Deserialize {
entity: "run spec",
branch: "fabro/meta/run-1".to_string(),
source,
});
assert!(matches!(fabro_error, Error::Stage {
stage: ErrorStage::Engine,
..
}));
let message = fabro_error.to_string();
assert!(message.contains("deserialize run spec on branch fabro/meta/run-1"));
assert!(message.contains(&source_message));
}
#[test]
fn cancelled_error_display() {
let err = Error::Cancelled;
assert_eq!(err.to_string(), "Pipeline cancelled");
}
#[test]
fn cancelled_is_not_retryable() {
assert!(!Error::Cancelled.is_retryable());
}
#[test]
fn is_retryable_terminal_errors() {
assert!(!Error::Parse("bad".to_string()).is_retryable());
assert!(!Error::Validation("bad".to_string()).is_retryable());
assert!(
!Error::ValidationFailed {
diagnostics: vec![],
}
.is_retryable()
);
assert!(!Error::Stylesheet("bad".to_string()).is_retryable());
assert!(!Error::Checkpoint("bad".to_string()).is_retryable());
}
#[test]
fn is_retryable_transient_errors() {
assert!(Error::handler("timeout").is_retryable());
assert!(Error::engine("transient").is_retryable());
assert!(Error::Io("connection reset".to_string()).is_retryable());
}
// --- FailureCategory Display/FromStr/serde tests ---
#[test]
fn failure_class_display_all_values() {
assert_eq!(
FailureCategory::TransientInfra.to_string(),
"transient_infra"
);
assert_eq!(FailureCategory::Deterministic.to_string(), "deterministic");
assert_eq!(
FailureCategory::BudgetExhausted.to_string(),
"budget_exhausted"
);
assert_eq!(
FailureCategory::CompilationLoop.to_string(),
"compilation_loop"
);
assert_eq!(FailureCategory::Canceled.to_string(), "canceled");
assert_eq!(FailureCategory::Structural.to_string(), "structural");
}
#[test]
fn failure_class_from_str_all_values() {
assert_eq!(
"transient_infra".parse::<FailureCategory>().unwrap(),
FailureCategory::TransientInfra
);
assert_eq!(
"deterministic".parse::<FailureCategory>().unwrap(),
FailureCategory::Deterministic
);
assert_eq!(
"budget_exhausted".parse::<FailureCategory>().unwrap(),
FailureCategory::BudgetExhausted
);
assert_eq!(
"compilation_loop".parse::<FailureCategory>().unwrap(),
FailureCategory::CompilationLoop
);
assert_eq!(
"canceled".parse::<FailureCategory>().unwrap(),
FailureCategory::Canceled
);
assert_eq!(
"structural".parse::<FailureCategory>().unwrap(),
FailureCategory::Structural
);
}
#[test]
fn failure_class_from_str_invalid() {
assert_eq!(
"unknown".parse::<FailureCategory>().unwrap(),
FailureCategory::Deterministic
);
}
#[test]
fn failure_class_from_str_alias_retryable() {
assert_eq!(
"retryable".parse::<FailureCategory>().unwrap(),
FailureCategory::TransientInfra
);
}
#[test]
fn failure_class_from_str_alias_transient() {
assert_eq!(
"transient".parse::<FailureCategory>().unwrap(),
FailureCategory::TransientInfra
);
}
#[test]
fn failure_class_from_str_alias_permanent() {
assert_eq!(
"permanent".parse::<FailureCategory>().unwrap(),
FailureCategory::Deterministic
);
}
#[test]
fn failure_class_from_str_alias_cancelled_british() {
assert_eq!(
"cancelled".parse::<FailureCategory>().unwrap(),
FailureCategory::Canceled
);
}
#[test]
fn failure_class_from_str_alias_budget() {
assert_eq!(
"budget".parse::<FailureCategory>().unwrap(),
FailureCategory::BudgetExhausted
);
}
#[test]
fn failure_class_from_str_alias_compile_loop() {
assert_eq!(
"compile_loop".parse::<FailureCategory>().unwrap(),
FailureCategory::CompilationLoop
);
}
#[test]
fn failure_class_from_str_alias_scope_violation() {
assert_eq!(
"scope_violation".parse::<FailureCategory>().unwrap(),
FailureCategory::Structural
);
}
#[test]
fn failure_class_from_str_unknown_defaults_deterministic() {
assert_eq!(
"garbage_xyz".parse::<FailureCategory>().unwrap(),
FailureCategory::Deterministic
);
}
#[test]
fn failure_class_from_str_case_insensitive() {
assert_eq!(
"TRANSIENT_INFRA".parse::<FailureCategory>().unwrap(),
FailureCategory::TransientInfra
);
}
#[test]
fn failure_class_from_str_trims_whitespace() {
assert_eq!(
" transient_infra ".parse::<FailureCategory>().unwrap(),
FailureCategory::TransientInfra
);
}
#[test]
fn failure_class_from_str_empty_defaults_deterministic() {
assert_eq!(
"".parse::<FailureCategory>().unwrap(),
FailureCategory::Deterministic
);
}
#[test]
fn failure_class_serde_roundtrip() {
let values = [
FailureCategory::TransientInfra,
FailureCategory::Deterministic,
FailureCategory::BudgetExhausted,
FailureCategory::CompilationLoop,
FailureCategory::Canceled,
FailureCategory::Structural,
];
for fc in values {
let json = serde_json::to_string(&fc).unwrap();
let parsed: FailureCategory = serde_json::from_str(&json).unwrap();
assert_eq!(parsed, fc);
}
}
// --- Llm variant tests ---
#[test]
fn llm_error_display() {
let sdk_err = transient_error(ErrorKind::Network, "connection refused");
let err = Error::Llm(sdk_err);
assert_eq!(err.to_string(), "LLM error: connection refused");
}
#[test]
fn llm_error_retryable_delegates_to_sdk() {
let retryable = Error::Llm(transient_error(ErrorKind::Network, "timeout"));
assert!(retryable.is_retryable());
let non_retryable = Error::Llm(sdk_error(ErrorKind::Configuration, "bad config"));
assert!(!non_retryable.is_retryable());
}
#[test]
fn llm_error_from_sdk_error() {
let sdk_err = transient_error(ErrorKind::StreamDecode, "broken pipe");
let err = Error::from(sdk_err);
assert!(matches!(err, Error::Llm(_)));
}
// --- failure_class() method tests ---
#[test]
fn failure_class_cancelled() {
assert_eq!(
Error::Cancelled.failure_category(),
FailureCategory::Canceled
);
}
#[test]
fn failure_class_io() {
assert_eq!(
Error::Io("disk full".into()).failure_category(),
FailureCategory::TransientInfra
);
}
#[test]
fn failure_class_parse() {
assert_eq!(
Error::Parse("bad syntax".into()).failure_category(),
FailureCategory::Deterministic
);
}
#[test]
fn failure_class_handler_with_timeout() {
assert_eq!(
Error::handler("request timed out").failure_category(),
FailureCategory::TransientInfra
);
}
#[test]
fn failure_class_handler_deterministic() {
assert_eq!(
Error::handler("invalid configuration").failure_category(),
FailureCategory::Deterministic
);
}
#[test]
fn failure_class_llm_rate_limit() {
let err = Error::Llm(transient_error(ErrorKind::RateLimit, "too fast"));
assert_eq!(err.failure_category(), FailureCategory::TransientInfra);
}
#[test]
fn failure_class_llm_context_length() {
let err = Error::Llm(sdk_error(ErrorKind::ContextLength, "too long"));
assert_eq!(err.failure_category(), FailureCategory::BudgetExhausted);
}
#[test]
fn failure_class_llm_auth() {
let err = Error::Llm(sdk_error(ErrorKind::Authentication, "bad key"));
assert_eq!(err.failure_category(), FailureCategory::Deterministic);
}
#[test]
fn failure_class_llm_abort() {
let err = Error::Llm(sdk_error(ErrorKind::Cancelled, "user cancelled"));
assert_eq!(err.failure_category(), FailureCategory::Canceled);
}
#[test]
fn failure_class_llm_timeout() {
let err = Error::Llm(transient_error(ErrorKind::Timeout, "timed out"));
assert_eq!(err.failure_category(), FailureCategory::TransientInfra);
}
// --- classify_sdk_error tests ---
#[test]
fn classify_sdk_rate_limit() {
let err = transient_error(ErrorKind::RateLimit, "too fast");
assert_eq!(classify_sdk_error(&err), FailureCategory::TransientInfra);
}
#[test]
fn classify_sdk_server() {
let err = transient_error(ErrorKind::Server, "500");
assert_eq!(classify_sdk_error(&err), FailureCategory::TransientInfra);
}
#[test]
fn classify_sdk_context_length() {
let err = sdk_error(ErrorKind::ContextLength, "too long");
assert_eq!(classify_sdk_error(&err), FailureCategory::BudgetExhausted);
}
#[test]
fn classify_sdk_quota_exceeded() {
let err = sdk_error(ErrorKind::QuotaExceeded, "out of quota");
assert_eq!(classify_sdk_error(&err), FailureCategory::BudgetExhausted);
}
#[test]
fn classify_sdk_auth() {
let err = sdk_error(ErrorKind::Authentication, "bad key");
assert_eq!(classify_sdk_error(&err), FailureCategory::Deterministic);
}
#[test]
fn classify_sdk_request_timeout() {
let err = transient_error(ErrorKind::Timeout, "timed out");
assert_eq!(classify_sdk_error(&err), FailureCategory::TransientInfra);
}
#[test]
fn classify_sdk_abort() {
let err = sdk_error(ErrorKind::Cancelled, "cancelled");
assert_eq!(classify_sdk_error(&err), FailureCategory::Canceled);
}
#[test]
fn classify_sdk_invalid_tool_call() {
let err = sdk_error(ErrorKind::InvalidRequest, "bad tool");
assert_eq!(classify_sdk_error(&err), FailureCategory::Deterministic);
}
#[test]
fn classify_sdk_invalid_request() {
let err = sdk_error(ErrorKind::InvalidRequest, "unsupported reasoning effort");
assert_eq!(classify_sdk_error(&err), FailureCategory::Deterministic);
}
// --- hints count guards ---
#[test]
fn transient_infra_hints_count() {
assert_eq!(TRANSIENT_INFRA_HINTS.len(), 40);
}
#[test]
fn budget_exhausted_hints_count() {
assert_eq!(BUDGET_EXHAUSTED_HINTS.len(), 10);
}
#[test]
fn structural_hints_count() {
assert_eq!(STRUCTURAL_HINTS.len(), 3);
}
// --- classify_failure_reason regression tests ---
// Canceled
#[test]
fn classify_reason_cancel() {
assert_eq!(
classify_failure_reason("operation cancelled by user"),
FailureCategory::Canceled
);
}
#[test]
fn classify_reason_nextest_canceling_due_to_test_failure_is_deterministic() {
assert_eq!(
classify_failure_reason(
"Script failed with exit code: 100\n\nCancelling due to test failure: 7 tests still running"
),
FailureCategory::Deterministic
);
}
#[test]
fn classify_reason_abort() {
assert_eq!(
classify_failure_reason("interrupted by signal"),
FailureCategory::Canceled
);
}
// Budget exhausted
#[test]
fn classify_reason_turn_limit() {
assert_eq!(
classify_failure_reason("exceeded turn limit of 10"),
FailureCategory::BudgetExhausted
);
}
#[test]
fn classify_reason_token_limit() {
assert_eq!(
classify_failure_reason("token limit reached"),
FailureCategory::BudgetExhausted
);
}
#[test]
fn classify_reason_context_length() {
assert_eq!(
classify_failure_reason("context length exceeded"),
FailureCategory::BudgetExhausted
);
}
#[test]
fn classify_reason_budget() {
assert_eq!(
classify_failure_reason("budget exceeded for run"),
FailureCategory::BudgetExhausted
);
}
#[test]
fn classify_reason_quota_exceeded() {
assert_eq!(
classify_failure_reason("quota exceeded"),
FailureCategory::BudgetExhausted
);
}
#[test]
fn classify_reason_max_tokens() {
assert_eq!(
classify_failure_reason("max_tokens exceeded"),
FailureCategory::BudgetExhausted
);
}
#[test]
fn classify_reason_max_tokens_space() {
assert_eq!(
classify_failure_reason("max tokens reached"),
FailureCategory::BudgetExhausted
);
}
#[test]
fn classify_reason_context_window_exceeded() {
assert_eq!(
classify_failure_reason("context window exceeded"),
FailureCategory::BudgetExhausted
);
}
#[test]
fn classify_reason_budget_exhausted() {
assert_eq!(
classify_failure_reason("budget exhausted for this session"),
FailureCategory::BudgetExhausted
);
}
#[test]
fn classify_reason_token_limit_exceeded() {
assert_eq!(
classify_failure_reason("token limit exceeded"),
FailureCategory::BudgetExhausted
);
}
// Structural
#[test]
fn classify_reason_scope_violation() {
assert_eq!(
classify_failure_reason("scope violation detected"),
FailureCategory::Structural
);
}
// Transient infra
#[test]
fn classify_reason_timeout() {
assert_eq!(
classify_failure_reason("request timed out after 30s"),
FailureCategory::TransientInfra
);
}
#[test]
fn classify_reason_rate_limit() {
assert_eq!(
classify_failure_reason("rate limited by provider"),
FailureCategory::TransientInfra
);
}
#[test]
fn classify_reason_connection_refused() {
assert_eq!(
classify_failure_reason("connection refused"),
FailureCategory::TransientInfra
);
}
#[test]
fn classify_reason_connection_reset() {
assert_eq!(
classify_failure_reason("connection reset by peer"),
FailureCategory::TransientInfra
);
}
#[test]
fn classify_reason_sandbox_state_change_in_progress() {
assert_eq!(
classify_failure_reason(
"Pipeline lifecycle operation failed: failed to activate sandbox after node \
attempt survey: Failed to start Daytona sandbox: Sandbox state change in progress"
),
FailureCategory::TransientInfra
);
}
#[test]
fn classify_reason_sandbox_stop_still_in_progress() {
assert_eq!(
classify_failure_reason("Daytona sandbox stop still in progress after 120s"),
FailureCategory::TransientInfra
);
}
#[test]
fn classify_reason_500() {
assert_eq!(
classify_failure_reason("HTTP 500 Internal Server Error"),
FailureCategory::TransientInfra
);
}
#[test]
fn classify_reason_502() {
assert_eq!(
classify_failure_reason("HTTP 502 Bad Gateway"),
FailureCategory::TransientInfra
);
}
#[test]
fn classify_reason_503() {
assert_eq!(
classify_failure_reason("HTTP 503 Service Unavailable"),
FailureCategory::TransientInfra
);
}
#[test]
fn classify_reason_504() {
assert_eq!(
classify_failure_reason("HTTP 504 Gateway Timeout"),
FailureCategory::TransientInfra
);
}
#[test]
fn classify_reason_context_deadline_exceeded() {
assert_eq!(
classify_failure_reason("context deadline exceeded"),
FailureCategory::TransientInfra
);
}
#[test]
fn classify_reason_could_not_resolve_host() {
assert_eq!(
classify_failure_reason("could not resolve host api.example.com"),
FailureCategory::TransientInfra
);
}
#[test]
fn classify_reason_could_not_resolve_hostname() {
assert_eq!(
classify_failure_reason("could not resolve hostname"),
FailureCategory::TransientInfra
);
}
#[test]
fn classify_reason_temporary_failure() {
assert_eq!(
classify_failure_reason("temporary failure"),
FailureCategory::TransientInfra
);
}
#[test]
fn classify_reason_temporary_failure_in_name_resolution() {
assert_eq!(
classify_failure_reason("temporary failure in name resolution"),
FailureCategory::TransientInfra
);
}
#[test]
fn classify_reason_network_is_unreachable() {
assert_eq!(
classify_failure_reason("network is unreachable"),
FailureCategory::TransientInfra
);
}
#[test]
fn classify_reason_broken_pipe() {
assert_eq!(
classify_failure_reason("broken pipe"),
FailureCategory::TransientInfra
);
}
#[test]
fn classify_reason_tls_handshake_timeout() {
assert_eq!(
classify_failure_reason("tls handshake timeout"),
FailureCategory::TransientInfra
);
}
#[test]
fn classify_reason_io_timeout() {
assert_eq!(
classify_failure_reason("i/o timeout"),
FailureCategory::TransientInfra
);
}
#[test]
fn classify_reason_no_route_to_host() {
assert_eq!(
classify_failure_reason("no route to host"),
FailureCategory::TransientInfra
);
}
#[test]
fn classify_reason_temporarily_unavailable() {
assert_eq!(
classify_failure_reason("resource temporarily unavailable"),
FailureCategory::TransientInfra
);
}
#[test]
fn classify_reason_try_again() {
assert_eq!(
classify_failure_reason("try again later"),
FailureCategory::TransientInfra
);
}
#[test]
fn classify_reason_too_many_requests() {
assert_eq!(
classify_failure_reason("too many requests"),
FailureCategory::TransientInfra
);
}
#[test]
fn classify_reason_service_unavailable() {
assert_eq!(
classify_failure_reason("service unavailable"),
FailureCategory::TransientInfra
);
}
#[test]
fn classify_reason_gateway_timeout() {
assert_eq!(
classify_failure_reason("gateway timeout"),
FailureCategory::TransientInfra
);
}
#[test]
fn classify_reason_econnrefused() {
assert_eq!(
classify_failure_reason("ECONNREFUSED"),
FailureCategory::TransientInfra
);
}
#[test]
fn classify_reason_econnreset() {
assert_eq!(
classify_failure_reason("ECONNRESET"),
FailureCategory::TransientInfra
);
}
#[test]
fn classify_reason_dial_tcp() {
assert_eq!(
classify_failure_reason("dial tcp 10.0.0.1:443: connect: connection refused"),
FailureCategory::TransientInfra
);
}
#[test]
fn classify_reason_transport_is_closing() {
assert_eq!(
classify_failure_reason("transport is closing"),
FailureCategory::TransientInfra
);
}
#[test]
fn classify_reason_stream_disconnected() {
assert_eq!(
classify_failure_reason("stream disconnected"),
FailureCategory::TransientInfra
);
}
#[test]
fn classify_reason_stream_closed_before() {
assert_eq!(
classify_failure_reason("stream closed before completion"),
FailureCategory::TransientInfra
);
}
#[test]
fn classify_reason_index_crates_io() {
assert_eq!(
classify_failure_reason("failed to fetch index.crates.io"),
FailureCategory::TransientInfra
);
}
#[test]
fn classify_reason_download_config_json_failed() {
assert_eq!(
classify_failure_reason("download of config.json failed"),
FailureCategory::TransientInfra
);
}
#[test]
fn classify_reason_toolchain_registry_unavailable() {
assert_eq!(
classify_failure_reason("toolchain_or_dependency_registry_unavailable"),
FailureCategory::TransientInfra
);
}
#[test]
fn classify_reason_toolchain_dependency_network() {
assert_eq!(
classify_failure_reason("toolchain dependency resolution blocked by network"),
FailureCategory::TransientInfra
);
}
#[test]
fn classify_reason_toolchain_workspace_io() {
assert_eq!(
classify_failure_reason("toolchain_workspace_io"),
FailureCategory::TransientInfra
);
}
#[test]
fn classify_reason_cross_device_link() {
assert_eq!(
classify_failure_reason("cross-device link"),
FailureCategory::TransientInfra
);
}
#[test]
fn classify_reason_invalid_cross_device_link() {
assert_eq!(
classify_failure_reason("invalid cross-device link"),
FailureCategory::TransientInfra
);
}
#[test]
fn classify_reason_os_error_18() {
assert_eq!(
classify_failure_reason("os error 18"),
FailureCategory::TransientInfra
);
}
// Structural
#[test]
fn classify_reason_write_scope_violation_underscore() {
assert_eq!(
classify_failure_reason("write_scope_violation detected"),
FailureCategory::Structural
);
}
#[test]
fn classify_reason_write_scope_violation_space() {
assert_eq!(
classify_failure_reason("write scope violation detected"),
FailureCategory::Structural
);
}
// Default deterministic
#[test]
fn classify_reason_default_deterministic() {
assert_eq!(
classify_failure_reason("invalid configuration parameter"),
FailureCategory::Deterministic
);
}
// --- normalize_failure_reason tests ---
#[test]
fn normalize_empty_and_whitespace_returns_empty() {
assert_eq!(normalize_failure_reason(""), "");
assert_eq!(normalize_failure_reason(" "), "");
assert_eq!(normalize_failure_reason("\n\t"), "");
}
#[test]
fn normalize_lowercases_and_trims() {
assert_eq!(normalize_failure_reason(" Hello World "), "hello world");
}
#[test]
fn normalize_replaces_hex_strings() {
assert_eq!(
normalize_failure_reason("commit abc123def0"),
"commit <hex>"
);
// Short hex (< 7 chars) not replaced
assert_eq!(normalize_failure_reason("value abcdef"), "value abcdef");
}
#[test]
fn normalize_replaces_digit_sequences() {
assert_eq!(normalize_failure_reason("line 42"), "line <n>");
assert_eq!(normalize_failure_reason("error 0"), "error <n>");
}
#[test]
fn normalize_collapses_comma_space_and_whitespace() {
assert_eq!(normalize_failure_reason("a, b, c"), "a,b,c");
assert_eq!(normalize_failure_reason("a b"), "a b");
}
#[test]
fn normalize_truncates_to_240_chars() {
let long = "a".repeat(300);
let result = normalize_failure_reason(&long);
assert_eq!(result.len(), 240);
}
#[test]
fn normalize_truncation_respects_utf8_boundaries() {
// Build a string of 2-byte chars ("é" is 2 bytes in UTF-8) that crosses
// the 240 byte boundary mid-character.
let input = "é".repeat(200); // 400 bytes, each char is 2 bytes
let result = normalize_failure_reason(&input);
assert!(result.len() <= 240);
// Must be valid UTF-8 (String guarantees this, but verify length is even
// since every char is 2 bytes)
assert_eq!(result.len() % 2, 0);
// Also test with a mix: 239 ASCII bytes + a 2-byte char
let input2 = format!("{}{}", "a".repeat(239), "é");
let result2 = normalize_failure_reason(&input2);
assert!(result2.len() <= 240);
// Should truncate to 239 (dropping the 2-byte char that would push to 241)
assert_eq!(result2.len(), 239);
}
#[test]
fn normalize_combined_example() {
assert_eq!(
normalize_failure_reason("Error at line 42 in abc123def"),
"error at line <n> in <hex>"
);
}
// --- FailureSignature tests ---
#[test]
fn failure_signature_format() {
let sig = FailureSignature::new(
"verify",
FailureCategory::Deterministic,
None,
Some("test failed"),
);
assert_eq!(sig.to_string(), "verify|deterministic|test failed");
}
#[test]
fn failure_signature_display() {
let sig = FailureSignature::new(
"build",
FailureCategory::Structural,
None,
Some("scope violation"),
);
assert_eq!(format!("{sig}"), "build|structural|scope violation");
}
#[test]
fn failure_signature_hint_takes_priority() {
let sig = FailureSignature::new(
"verify",
FailureCategory::Deterministic,
Some("custom hint"),
Some("raw reason"),
);
assert_eq!(sig.to_string(), "verify|deterministic|custom hint");
}
#[test]
fn failure_signature_missing_reason_falls_back_to_unknown() {
let sig = FailureSignature::new("node", FailureCategory::Deterministic, None, None);
assert_eq!(sig.to_string(), "node|deterministic|unknown");
}
#[test]
fn failure_signature_equality_and_hash() {
let sig1 = FailureSignature::new(
"verify",
FailureCategory::Deterministic,
None,
Some("test failed"),
);
let sig2 = FailureSignature::new(
"verify",
FailureCategory::Deterministic,
None,
Some("test failed"),
);
assert_eq!(sig1, sig2);
let mut map = std::collections::HashMap::new();
map.insert(sig1.clone(), 1);
assert_eq!(map.get(&sig2), Some(&1));
}
// --- is_signature_tracked tests ---
#[test]
fn is_signature_tracked_deterministic_and_structural() {
assert!(FailureCategory::Deterministic.is_signature_tracked());
assert!(FailureCategory::Structural.is_signature_tracked());
}
#[test]
fn is_signature_tracked_false_for_others() {
assert!(!FailureCategory::TransientInfra.is_signature_tracked());
assert!(!FailureCategory::BudgetExhausted.is_signature_tracked());
assert!(!FailureCategory::Canceled.is_signature_tracked());
assert!(!FailureCategory::CompilationLoop.is_signature_tracked());
}
// --- failure_signature_hint tests ---
#[test]
fn failure_signature_hint_llm_returns_some() {
let err = Error::Llm(sdk_error(ErrorKind::Authentication, "bad key"));
assert_eq!(
err.failure_signature_hint(),
Some(FailureSignature(
"api_deterministic|openai|authentication".to_string()
))
);
}
#[test]
fn failure_signature_hint_handler_returns_none() {
let err = Error::handler("something failed");
assert_eq!(err.failure_signature_hint(), None);
}
#[test]
fn failure_signature_hint_engine_returns_none() {
let err = Error::engine("engine error");
assert_eq!(err.failure_signature_hint(), None);
}
// --- to_fail_outcome tests ---
#[test]
fn to_fail_outcome_llm_has_class_and_signature() {
let err = Error::Llm(sdk_error(ErrorKind::Authentication, "bad key"));
let outcome = err.to_fail_outcome();
assert_eq!(outcome.status, crate::outcome::StageOutcome::Failed {
retry_requested: false,
});
let failure = outcome.failure.as_ref().unwrap();
assert_eq!(failure.category, FailureCategory::Deterministic);
assert_eq!(
failure.signature.as_deref(),
Some("api_deterministic|openai|authentication")
);
}
#[test]
fn to_fail_outcome_handler_has_class_but_no_signature() {
let err = Error::handler("connection refused");
let outcome = err.to_fail_outcome();
assert_eq!(outcome.status, crate::outcome::StageOutcome::Failed {
retry_requested: false,
});
let failure = outcome.failure.as_ref().unwrap();
assert_eq!(failure.category, FailureCategory::TransientInfra);
assert!(failure.signature.is_none());
}
#[test]
fn to_fail_outcome_includes_error_message_as_reason() {
let err = Error::Llm(transient_error(ErrorKind::Network, "connection refused"));
let outcome = err.to_fail_outcome();
assert!(
outcome
.failure_reason()
.unwrap()
.contains("connection refused")
);
}
#[test]
fn to_fail_outcome_no_context_updates() {
let err = Error::Llm(transient_error(ErrorKind::Network, "refused"));
let outcome = err.to_fail_outcome();
assert!(outcome.context_updates.is_empty());
}
// --- Phase 2: Eager classification tests ---
#[test]
fn handler_eager_classification() {
let err = Error::handler("connection refused");
assert_eq!(err.failure_category(), FailureCategory::TransientInfra);
}
#[test]
fn handler_eager_classification_survives_clone() {
let err = Error::handler("connection refused");
let cloned = err.clone();
assert_eq!(cloned.failure_category(), FailureCategory::TransientInfra);
}
#[test]
fn handler_smart_constructor_preserves_message() {
let err = Error::handler("some error");
assert!(err.to_string().contains("some error"));
}
#[test]
fn engine_eager_classification() {
let err = Error::engine("rate limit exceeded");
assert_eq!(err.failure_category(), FailureCategory::TransientInfra);
}
#[test]
fn error_clone_preserves_display_for_all_variants() {
let errors: Vec<Error> = vec![
Error::Parse("bad".into()),
Error::Validation("bad".into()),
Error::ValidationFailed {
diagnostics: vec![Diagnostic {
rule: "test".into(),
severity: fabro_validate::Severity::Error,
message: "bad".into(),
node_id: None,
edge: None,
fix: None,
..Diagnostic::default()
}],
},
Error::engine("engine err"),
Error::publish("publish err"),
Error::handler("handler err"),
Error::Llm(transient_error(ErrorKind::Network, "refused")),
Error::Checkpoint("cp err".into()),
Error::Stylesheet("style err".into()),
Error::Io("io err".into()),
Error::Cancelled,
];
for err in errors {
assert_eq!(err.to_string(), err.clone().to_string());
}
}
#[test]
fn handler_display_unchanged() {
assert_eq!(
Error::handler("LLM call failed").to_string(),
"Handler error: LLM call failed"
);
}
#[test]
fn engine_display_unchanged() {
assert_eq!(
Error::engine("no outgoing edge").to_string(),
"Engine error: no outgoing edge"
);
}
/// Publish runs once, after execution, so no caller can retry it — even
/// when the message looks transient. The failure category is still
/// classified for reporting.
#[test]
fn publish_errors_are_terminal() {
assert!(!Error::publish("connection timed out").is_retryable());
assert!(!Error::publish("permission denied").is_retryable());
assert_eq!(
Error::publish("connection timed out").failure_category(),
FailureCategory::TransientInfra
);
}
#[test]
fn failure_reason_distinguishes_publish_and_cancelled() {
assert_eq!(
Error::publish("nope").failure_reason(),
FailureReason::PublishFailed
);
assert_eq!(Error::Cancelled.failure_reason(), FailureReason::Cancelled);
assert_eq!(
Error::engine("boom").failure_reason(),
FailureReason::WorkflowError
);
}
#[test]
fn failure_class_stability() {
let messages = [
"connection refused",
"timeout",
"rate limit",
"context length exceeded",
"cancel",
"invalid configuration",
"write_scope_violation",
];
for msg in messages {
assert_eq!(
Error::handler(msg).failure_category(),
classify_failure_reason(msg),
"mismatch for message: {msg}"
);
}
}
/// Commit SHAs are hex, so they contain digit runs like "503" often enough
/// to matter. Masking them keeps a deterministic failure from being
/// reported as transient just because of the SHA it names.
#[test]
fn commit_shas_do_not_trip_transient_infra_hints() {
let sha = "a503b1c9d4e2f7a8b6c3d0e1f2a3b4c5d6e7f8a9";
assert_eq!(
classify_failure_reason(&format!("failed to push final commit {sha} to branch 'x'")),
FailureCategory::Deterministic
);
// A real status code is still a transient hint.
assert_eq!(
classify_failure_reason("push rejected with 503"),
FailureCategory::TransientInfra
);
}
#[test]
fn to_fail_outcome_preserves_class() {
let err = Error::handler("timeout");
let outcome = err.to_fail_outcome();
assert_eq!(
outcome.failure_category(),
Some(FailureCategory::TransientInfra)
);
}
// --- E2E error pipeline tests ---
#[test]
fn e2e_llm_error_to_outcome_to_event_preserves_classification() {
use crate::event::Event;
// 1. Create SdkError → Error
let sdk_err = transient_error(ErrorKind::RateLimit, "too fast");
let arc_err = Error::Llm(sdk_err);
assert_eq!(arc_err.failure_category(), FailureCategory::TransientInfra);
// 2. Error → Outcome
let outcome = arc_err.to_fail_outcome();
assert_eq!(
outcome.failure_category(),
Some(FailureCategory::TransientInfra)
);
// 3. Outcome → StageFailed event
let failure = outcome.failure.clone().unwrap();
let event = Event::StageFailed {
node_id: "code".into(),
name: "code".into(),
index: 0,
failure: failure.clone(),
will_retry: false,
timing: fabro_types::StageTiming::wall_only(0),
billing: None,
actor: None,
};
// 4. Verify classification survived all the way through
match &event {
Event::StageFailed { failure, .. } => {
assert_eq!(failure.category, FailureCategory::TransientInfra);
}
_ => panic!("expected StageFailed"),
}
}
#[test]
fn e2e_handler_error_classified_at_edge() {
// handler smart constructor classifies eagerly
let err = Error::handler("connection refused");
assert_eq!(err.failure_category(), FailureCategory::TransientInfra);
// to_fail_outcome preserves
let outcome = err.to_fail_outcome();
assert_eq!(
outcome.failure_category(),
Some(FailureCategory::TransientInfra)
);
// event preserves
let failure = outcome.failure.unwrap();
assert_eq!(failure.category, FailureCategory::TransientInfra);
}
#[test]
fn e2e_handler_retryable_checks() {
assert!(Error::handler("timeout").is_retryable());
assert!(Error::handler("auth error").is_retryable());
}
#[test]
fn e2e_run_failure_projection_uses_handler_error_shape() {
let err = Error::handler("connection refused");
let failure = run_failure_from_error(&err, FailureReason::WorkflowError);
assert_eq!(failure.detail.message, "connection refused");
assert_eq!(failure.detail.causes, Vec::<String>::new());
assert_eq!(failure.reason, FailureReason::WorkflowError);
assert_eq!(failure.detail.category, FailureCategory::TransientInfra);
}
#[test]
fn e2e_serde_stability_agent_error() {
use fabro_agent::Error as AgentError;
let err = AgentError::Llm(transient_error(ErrorKind::RateLimit, "too fast"));
let json = serde_json::to_string(&err).unwrap();
let v: serde_json::Value = serde_json::from_str(&json).unwrap();
assert_eq!(v["type"], "llm");
let deserialized: AgentError = serde_json::from_str(&json).unwrap();
assert_eq!(err.to_string(), deserialized.to_string());
}
#[test]
fn e2e_failure_detail_in_outcome_serde_roundtrip() {
use crate::outcome::Outcome;
let outcome = Outcome::fail_classify("rate limit exceeded")
.with_signature(Some("api_transient|openai|rate_limited"));
let json = serde_json::to_string(&outcome).unwrap();
let deserialized: Outcome = serde_json::from_str(&json).unwrap();
let failure = deserialized.failure.unwrap();
assert_eq!(failure.message, "rate limit exceeded");
assert_eq!(failure.category, FailureCategory::TransientInfra);
assert_eq!(
failure.signature.as_deref(),
Some("api_transient|openai|rate_limited")
);
}
}