From 5ea508a5ca0812ec44c3a8e75cdeb3ff4c7c4672 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Sun, 1 Mar 2026 19:12:29 -0500 Subject: [PATCH] Add stable failure signature hints for model errors SdkError now produces hand-crafted signature hints (e.g. "api_deterministic|openai|authentication") that are identical regardless of error message wording, replacing fragile regex-normalized signatures for API errors. ArcError.to_fail_outcome() centralizes fail outcome construction with failure_class and failure_signature context_updates. Co-Authored-By: Claude Opus 4.6 --- crates/arc-llm/src/error.rs | 212 +++++++++++++++++++ crates/arc-workflows/src/engine.rs | 7 +- crates/arc-workflows/src/error.rs | 97 +++++++++ crates/arc-workflows/src/handler/codergen.rs | 2 +- crates/arc-workflows/src/handler/parallel.rs | 2 +- 5 files changed, 312 insertions(+), 8 deletions(-) diff --git a/crates/arc-llm/src/error.rs b/crates/arc-llm/src/error.rs index 4ecde283a..02b576465 100644 --- a/crates/arc-llm/src/error.rs +++ b/crates/arc-llm/src/error.rs @@ -131,6 +131,54 @@ impl SdkError { _ => None, } } + + #[must_use] + pub fn provider_name(&self) -> &str { + match self { + Self::Provider { detail, .. } => &detail.provider, + _ => "unknown", + } + } + + #[must_use] + pub fn failure_signature_hint(&self) -> String { + let provider = self.provider_name(); + match self { + Self::Provider { kind, .. } => { + let category = if self.retryable() { + "api_transient" + } else { + "api_deterministic" + }; + let detail = match kind { + ProviderErrorKind::RateLimit => "rate_limited", + ProviderErrorKind::Server => "server_error", + ProviderErrorKind::ContextLength => "context_length", + ProviderErrorKind::QuotaExceeded => "quota_exceeded", + ProviderErrorKind::Authentication => "authentication", + ProviderErrorKind::AccessDenied => "access_denied", + ProviderErrorKind::NotFound => "not_found", + ProviderErrorKind::InvalidRequest => "invalid_request", + ProviderErrorKind::ContentFilter => "content_filter", + }; + format!("{category}|{provider}|{detail}") + } + Self::RequestTimeout { .. } => format!("api_transient|{provider}|timeout"), + Self::Network { .. } => format!("api_transient|{provider}|network"), + Self::Stream { .. } => format!("api_transient|{provider}|stream"), + Self::Abort { .. } => format!("api_canceled|{provider}|abort"), + Self::Configuration { .. } => format!("api_deterministic|{provider}|configuration"), + Self::InvalidToolCall { .. } => { + format!("api_deterministic|{provider}|invalid_tool_call") + } + Self::NoObjectGenerated { .. } => { + format!("api_deterministic|{provider}|no_object") + } + Self::UnsupportedToolChoice { .. } => { + format!("api_deterministic|{provider}|unsupported_tool_choice") + } + } + } } /// HTTP status code to error type mapping (Section 6.4). @@ -721,4 +769,168 @@ mod tests { }; assert_eq!(err.status_code(), None); } + + #[test] + fn provider_name_from_provider_variant() { + let err = SdkError::Provider { + kind: ProviderErrorKind::Authentication, + detail: Box::new(ProviderErrorDetail::new("bad key", "openai")), + }; + assert_eq!(err.provider_name(), "openai"); + } + + #[test] + fn provider_name_defaults_to_unknown() { + let err = SdkError::Network { + message: "refused".into(), + }; + assert_eq!(err.provider_name(), "unknown"); + } + + #[test] + fn failure_signature_hint_provider_transient() { + let err = SdkError::Provider { + kind: ProviderErrorKind::RateLimit, + detail: Box::new(ProviderErrorDetail::new("too fast", "openai")), + }; + assert_eq!( + err.failure_signature_hint(), + "api_transient|openai|rate_limited" + ); + + let err = SdkError::Provider { + kind: ProviderErrorKind::Server, + detail: Box::new(ProviderErrorDetail::new("500", "anthropic")), + }; + assert_eq!( + err.failure_signature_hint(), + "api_transient|anthropic|server_error" + ); + } + + #[test] + fn failure_signature_hint_provider_deterministic() { + let err = SdkError::Provider { + kind: ProviderErrorKind::Authentication, + detail: Box::new(ProviderErrorDetail::new("bad key", "openai")), + }; + assert_eq!( + err.failure_signature_hint(), + "api_deterministic|openai|authentication" + ); + + let err = SdkError::Provider { + kind: ProviderErrorKind::AccessDenied, + detail: Box::new(ProviderErrorDetail::new("denied", "anthropic")), + }; + assert_eq!( + err.failure_signature_hint(), + "api_deterministic|anthropic|access_denied" + ); + + let err = SdkError::Provider { + kind: ProviderErrorKind::NotFound, + detail: Box::new(ProviderErrorDetail::new("missing", "openai")), + }; + assert_eq!( + err.failure_signature_hint(), + "api_deterministic|openai|not_found" + ); + + let err = SdkError::Provider { + kind: ProviderErrorKind::InvalidRequest, + detail: Box::new(ProviderErrorDetail::new("bad", "openai")), + }; + assert_eq!( + err.failure_signature_hint(), + "api_deterministic|openai|invalid_request" + ); + + let err = SdkError::Provider { + kind: ProviderErrorKind::ContentFilter, + detail: Box::new(ProviderErrorDetail::new("blocked", "openai")), + }; + assert_eq!( + err.failure_signature_hint(), + "api_deterministic|openai|content_filter" + ); + + let err = SdkError::Provider { + kind: ProviderErrorKind::ContextLength, + detail: Box::new(ProviderErrorDetail::new("too long", "openai")), + }; + assert_eq!( + err.failure_signature_hint(), + "api_deterministic|openai|context_length" + ); + + let err = SdkError::Provider { + kind: ProviderErrorKind::QuotaExceeded, + detail: Box::new(ProviderErrorDetail::new("out of quota", "openai")), + }; + assert_eq!( + err.failure_signature_hint(), + "api_deterministic|openai|quota_exceeded" + ); + } + + #[test] + fn failure_signature_hint_non_provider_variants() { + assert_eq!( + SdkError::RequestTimeout { + message: "timed out".into() + } + .failure_signature_hint(), + "api_transient|unknown|timeout" + ); + assert_eq!( + SdkError::Network { + message: "refused".into() + } + .failure_signature_hint(), + "api_transient|unknown|network" + ); + assert_eq!( + SdkError::Stream { + message: "broken".into() + } + .failure_signature_hint(), + "api_transient|unknown|stream" + ); + assert_eq!( + SdkError::Abort { + message: "cancelled".into() + } + .failure_signature_hint(), + "api_canceled|unknown|abort" + ); + assert_eq!( + SdkError::Configuration { + message: "bad".into() + } + .failure_signature_hint(), + "api_deterministic|unknown|configuration" + ); + assert_eq!( + SdkError::InvalidToolCall { + message: "bad".into() + } + .failure_signature_hint(), + "api_deterministic|unknown|invalid_tool_call" + ); + assert_eq!( + SdkError::NoObjectGenerated { + message: "none".into() + } + .failure_signature_hint(), + "api_deterministic|unknown|no_object" + ); + assert_eq!( + SdkError::UnsupportedToolChoice { + message: "nope".into() + } + .failure_signature_hint(), + "api_deterministic|unknown|unsupported_tool_choice" + ); + } } diff --git a/crates/arc-workflows/src/engine.rs b/crates/arc-workflows/src/engine.rs index 0b4375449..4f3ed4645 100644 --- a/crates/arc-workflows/src/engine.rs +++ b/crates/arc-workflows/src/engine.rs @@ -876,12 +876,7 @@ impl PipelineEngine { tokio::time::sleep(delay).await; continue; } - let mut fail_outcome = Outcome::fail(e.to_string()); - fail_outcome.context_updates.insert( - "failure_class".to_string(), - serde_json::json!(e.failure_class().to_string()), - ); - return Ok((fail_outcome, attempt)); + return Ok((e.to_fail_outcome(), attempt)); } }; diff --git a/crates/arc-workflows/src/error.rs b/crates/arc-workflows/src/error.rs index b80e0a6a7..db089eccc 100644 --- a/crates/arc-workflows/src/error.rs +++ b/crates/arc-workflows/src/error.rs @@ -351,6 +351,32 @@ impl ArcError { Self::Handler(msg) | Self::Engine(msg) => classify_failure_reason(msg), } } + + /// Return a stable failure signature hint when structured error info is available. + #[must_use] + pub fn failure_signature_hint(&self) -> Option { + match self { + Self::Llm(sdk_err) => Some(sdk_err.failure_signature_hint()), + _ => None, + } + } + + /// Build an `Outcome::fail` with `failure_class` and optional `failure_signature` + /// populated in `context_updates`. + pub fn to_fail_outcome(&self) -> crate::outcome::Outcome { + let mut outcome = crate::outcome::Outcome::fail(self.to_string()); + outcome.context_updates.insert( + "failure_class".to_string(), + serde_json::json!(self.failure_class().to_string()), + ); + if let Some(sig) = self.failure_signature_hint() { + outcome.context_updates.insert( + "failure_signature".to_string(), + serde_json::json!(sig), + ); + } + outcome + } } impl From for ArcError { @@ -1417,4 +1443,75 @@ mod tests { assert!(!FailureClass::Canceled.is_signature_tracked()); assert!(!FailureClass::CompilationLoop.is_signature_tracked()); } + + // --- failure_signature_hint tests --- + + #[test] + fn failure_signature_hint_llm_returns_some() { + let err = ArcError::Llm(SdkError::Provider { + kind: ProviderErrorKind::Authentication, + detail: Box::new(ProviderErrorDetail::new("bad key", "openai")), + }); + assert_eq!( + err.failure_signature_hint(), + Some("api_deterministic|openai|authentication".to_string()) + ); + } + + #[test] + fn failure_signature_hint_handler_returns_none() { + let err = ArcError::Handler("something failed".to_string()); + assert_eq!(err.failure_signature_hint(), None); + } + + #[test] + fn failure_signature_hint_engine_returns_none() { + let err = ArcError::Engine("engine error".to_string()); + assert_eq!(err.failure_signature_hint(), None); + } + + // --- to_fail_outcome tests --- + + #[test] + fn to_fail_outcome_llm_has_class_and_signature() { + let err = ArcError::Llm(SdkError::Provider { + kind: ProviderErrorKind::Authentication, + detail: Box::new(ProviderErrorDetail::new("bad key", "openai")), + }); + let outcome = err.to_fail_outcome(); + assert_eq!(outcome.status, crate::outcome::StageStatus::Fail); + assert_eq!( + outcome.context_updates.get("failure_class"), + Some(&serde_json::json!("deterministic")) + ); + assert_eq!( + outcome.context_updates.get("failure_signature"), + Some(&serde_json::json!("api_deterministic|openai|authentication")) + ); + } + + #[test] + fn to_fail_outcome_handler_has_class_but_no_signature() { + let err = ArcError::Handler("connection refused".to_string()); + let outcome = err.to_fail_outcome(); + assert_eq!(outcome.status, crate::outcome::StageStatus::Fail); + assert_eq!( + outcome.context_updates.get("failure_class"), + Some(&serde_json::json!("transient_infra")) + ); + assert!(!outcome.context_updates.contains_key("failure_signature")); + } + + #[test] + fn to_fail_outcome_includes_error_message_as_reason() { + let err = ArcError::Llm(SdkError::Network { + message: "connection refused".into(), + }); + let outcome = err.to_fail_outcome(); + assert!(outcome + .failure_reason + .as_ref() + .unwrap() + .contains("connection refused")); + } } diff --git a/crates/arc-workflows/src/handler/codergen.rs b/crates/arc-workflows/src/handler/codergen.rs index 18d081426..78e9603d1 100644 --- a/crates/arc-workflows/src/handler/codergen.rs +++ b/crates/arc-workflows/src/handler/codergen.rs @@ -271,7 +271,7 @@ impl Handler for CodergenHandler { return Err(e); } Err(e) => { - return Ok(Outcome::fail(e.to_string())); + return Ok(e.to_fail_outcome()); } } } else { diff --git a/crates/arc-workflows/src/handler/parallel.rs b/crates/arc-workflows/src/handler/parallel.rs index cc98e7b66..a9e6bd923 100644 --- a/crates/arc-workflows/src/handler/parallel.rs +++ b/crates/arc-workflows/src/handler/parallel.rs @@ -503,7 +503,7 @@ impl Handler for ParallelHandler { Ok(Err(e)) => { let result = BranchResult { id: String::new(), - outcome: Outcome::fail(e.to_string()), + outcome: e.to_fail_outcome(), head_sha: None, worktree_path: None, };