Add stable failure signature hints for model errors

SdkError now produces hand-crafted signature hints (e.g.
"api_deterministic|openai|authentication") that are identical regardless
of error message wording, replacing fragile regex-normalized signatures
for API errors. ArcError.to_fail_outcome() centralizes fail outcome
construction with failure_class and failure_signature context_updates.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Bryan Helmkamp 2026-03-01 19:12:29 -05:00
parent 3b8a0c3af2
commit 5ea508a5ca
5 changed files with 312 additions and 8 deletions

View file

@ -131,6 +131,54 @@ impl SdkError {
_ => None,
}
}
#[must_use]
pub fn provider_name(&self) -> &str {
match self {
Self::Provider { detail, .. } => &detail.provider,
_ => "unknown",
}
}
#[must_use]
pub fn failure_signature_hint(&self) -> String {
let provider = self.provider_name();
match self {
Self::Provider { kind, .. } => {
let category = if self.retryable() {
"api_transient"
} else {
"api_deterministic"
};
let detail = match kind {
ProviderErrorKind::RateLimit => "rate_limited",
ProviderErrorKind::Server => "server_error",
ProviderErrorKind::ContextLength => "context_length",
ProviderErrorKind::QuotaExceeded => "quota_exceeded",
ProviderErrorKind::Authentication => "authentication",
ProviderErrorKind::AccessDenied => "access_denied",
ProviderErrorKind::NotFound => "not_found",
ProviderErrorKind::InvalidRequest => "invalid_request",
ProviderErrorKind::ContentFilter => "content_filter",
};
format!("{category}|{provider}|{detail}")
}
Self::RequestTimeout { .. } => format!("api_transient|{provider}|timeout"),
Self::Network { .. } => format!("api_transient|{provider}|network"),
Self::Stream { .. } => format!("api_transient|{provider}|stream"),
Self::Abort { .. } => format!("api_canceled|{provider}|abort"),
Self::Configuration { .. } => format!("api_deterministic|{provider}|configuration"),
Self::InvalidToolCall { .. } => {
format!("api_deterministic|{provider}|invalid_tool_call")
}
Self::NoObjectGenerated { .. } => {
format!("api_deterministic|{provider}|no_object")
}
Self::UnsupportedToolChoice { .. } => {
format!("api_deterministic|{provider}|unsupported_tool_choice")
}
}
}
}
/// HTTP status code to error type mapping (Section 6.4).
@ -721,4 +769,168 @@ mod tests {
};
assert_eq!(err.status_code(), None);
}
#[test]
fn provider_name_from_provider_variant() {
let err = SdkError::Provider {
kind: ProviderErrorKind::Authentication,
detail: Box::new(ProviderErrorDetail::new("bad key", "openai")),
};
assert_eq!(err.provider_name(), "openai");
}
#[test]
fn provider_name_defaults_to_unknown() {
let err = SdkError::Network {
message: "refused".into(),
};
assert_eq!(err.provider_name(), "unknown");
}
#[test]
fn failure_signature_hint_provider_transient() {
let err = SdkError::Provider {
kind: ProviderErrorKind::RateLimit,
detail: Box::new(ProviderErrorDetail::new("too fast", "openai")),
};
assert_eq!(
err.failure_signature_hint(),
"api_transient|openai|rate_limited"
);
let err = SdkError::Provider {
kind: ProviderErrorKind::Server,
detail: Box::new(ProviderErrorDetail::new("500", "anthropic")),
};
assert_eq!(
err.failure_signature_hint(),
"api_transient|anthropic|server_error"
);
}
#[test]
fn failure_signature_hint_provider_deterministic() {
let err = SdkError::Provider {
kind: ProviderErrorKind::Authentication,
detail: Box::new(ProviderErrorDetail::new("bad key", "openai")),
};
assert_eq!(
err.failure_signature_hint(),
"api_deterministic|openai|authentication"
);
let err = SdkError::Provider {
kind: ProviderErrorKind::AccessDenied,
detail: Box::new(ProviderErrorDetail::new("denied", "anthropic")),
};
assert_eq!(
err.failure_signature_hint(),
"api_deterministic|anthropic|access_denied"
);
let err = SdkError::Provider {
kind: ProviderErrorKind::NotFound,
detail: Box::new(ProviderErrorDetail::new("missing", "openai")),
};
assert_eq!(
err.failure_signature_hint(),
"api_deterministic|openai|not_found"
);
let err = SdkError::Provider {
kind: ProviderErrorKind::InvalidRequest,
detail: Box::new(ProviderErrorDetail::new("bad", "openai")),
};
assert_eq!(
err.failure_signature_hint(),
"api_deterministic|openai|invalid_request"
);
let err = SdkError::Provider {
kind: ProviderErrorKind::ContentFilter,
detail: Box::new(ProviderErrorDetail::new("blocked", "openai")),
};
assert_eq!(
err.failure_signature_hint(),
"api_deterministic|openai|content_filter"
);
let err = SdkError::Provider {
kind: ProviderErrorKind::ContextLength,
detail: Box::new(ProviderErrorDetail::new("too long", "openai")),
};
assert_eq!(
err.failure_signature_hint(),
"api_deterministic|openai|context_length"
);
let err = SdkError::Provider {
kind: ProviderErrorKind::QuotaExceeded,
detail: Box::new(ProviderErrorDetail::new("out of quota", "openai")),
};
assert_eq!(
err.failure_signature_hint(),
"api_deterministic|openai|quota_exceeded"
);
}
#[test]
fn failure_signature_hint_non_provider_variants() {
assert_eq!(
SdkError::RequestTimeout {
message: "timed out".into()
}
.failure_signature_hint(),
"api_transient|unknown|timeout"
);
assert_eq!(
SdkError::Network {
message: "refused".into()
}
.failure_signature_hint(),
"api_transient|unknown|network"
);
assert_eq!(
SdkError::Stream {
message: "broken".into()
}
.failure_signature_hint(),
"api_transient|unknown|stream"
);
assert_eq!(
SdkError::Abort {
message: "cancelled".into()
}
.failure_signature_hint(),
"api_canceled|unknown|abort"
);
assert_eq!(
SdkError::Configuration {
message: "bad".into()
}
.failure_signature_hint(),
"api_deterministic|unknown|configuration"
);
assert_eq!(
SdkError::InvalidToolCall {
message: "bad".into()
}
.failure_signature_hint(),
"api_deterministic|unknown|invalid_tool_call"
);
assert_eq!(
SdkError::NoObjectGenerated {
message: "none".into()
}
.failure_signature_hint(),
"api_deterministic|unknown|no_object"
);
assert_eq!(
SdkError::UnsupportedToolChoice {
message: "nope".into()
}
.failure_signature_hint(),
"api_deterministic|unknown|unsupported_tool_choice"
);
}
}

View file

@ -876,12 +876,7 @@ impl PipelineEngine {
tokio::time::sleep(delay).await;
continue;
}
let mut fail_outcome = Outcome::fail(e.to_string());
fail_outcome.context_updates.insert(
"failure_class".to_string(),
serde_json::json!(e.failure_class().to_string()),
);
return Ok((fail_outcome, attempt));
return Ok((e.to_fail_outcome(), attempt));
}
};

View file

@ -351,6 +351,32 @@ impl ArcError {
Self::Handler(msg) | Self::Engine(msg) => classify_failure_reason(msg),
}
}
/// Return a stable failure signature hint when structured error info is available.
#[must_use]
pub fn failure_signature_hint(&self) -> Option<String> {
match self {
Self::Llm(sdk_err) => Some(sdk_err.failure_signature_hint()),
_ => None,
}
}
/// Build an `Outcome::fail` with `failure_class` and optional `failure_signature`
/// populated in `context_updates`.
pub fn to_fail_outcome(&self) -> crate::outcome::Outcome {
let mut outcome = crate::outcome::Outcome::fail(self.to_string());
outcome.context_updates.insert(
"failure_class".to_string(),
serde_json::json!(self.failure_class().to_string()),
);
if let Some(sig) = self.failure_signature_hint() {
outcome.context_updates.insert(
"failure_signature".to_string(),
serde_json::json!(sig),
);
}
outcome
}
}
impl From<std::io::Error> for ArcError {
@ -1417,4 +1443,75 @@ mod tests {
assert!(!FailureClass::Canceled.is_signature_tracked());
assert!(!FailureClass::CompilationLoop.is_signature_tracked());
}
// --- failure_signature_hint tests ---
#[test]
fn failure_signature_hint_llm_returns_some() {
let err = ArcError::Llm(SdkError::Provider {
kind: ProviderErrorKind::Authentication,
detail: Box::new(ProviderErrorDetail::new("bad key", "openai")),
});
assert_eq!(
err.failure_signature_hint(),
Some("api_deterministic|openai|authentication".to_string())
);
}
#[test]
fn failure_signature_hint_handler_returns_none() {
let err = ArcError::Handler("something failed".to_string());
assert_eq!(err.failure_signature_hint(), None);
}
#[test]
fn failure_signature_hint_engine_returns_none() {
let err = ArcError::Engine("engine error".to_string());
assert_eq!(err.failure_signature_hint(), None);
}
// --- to_fail_outcome tests ---
#[test]
fn to_fail_outcome_llm_has_class_and_signature() {
let err = ArcError::Llm(SdkError::Provider {
kind: ProviderErrorKind::Authentication,
detail: Box::new(ProviderErrorDetail::new("bad key", "openai")),
});
let outcome = err.to_fail_outcome();
assert_eq!(outcome.status, crate::outcome::StageStatus::Fail);
assert_eq!(
outcome.context_updates.get("failure_class"),
Some(&serde_json::json!("deterministic"))
);
assert_eq!(
outcome.context_updates.get("failure_signature"),
Some(&serde_json::json!("api_deterministic|openai|authentication"))
);
}
#[test]
fn to_fail_outcome_handler_has_class_but_no_signature() {
let err = ArcError::Handler("connection refused".to_string());
let outcome = err.to_fail_outcome();
assert_eq!(outcome.status, crate::outcome::StageStatus::Fail);
assert_eq!(
outcome.context_updates.get("failure_class"),
Some(&serde_json::json!("transient_infra"))
);
assert!(!outcome.context_updates.contains_key("failure_signature"));
}
#[test]
fn to_fail_outcome_includes_error_message_as_reason() {
let err = ArcError::Llm(SdkError::Network {
message: "connection refused".into(),
});
let outcome = err.to_fail_outcome();
assert!(outcome
.failure_reason
.as_ref()
.unwrap()
.contains("connection refused"));
}
}

View file

@ -271,7 +271,7 @@ impl Handler for CodergenHandler {
return Err(e);
}
Err(e) => {
return Ok(Outcome::fail(e.to_string()));
return Ok(e.to_fail_outcome());
}
}
} else {

View file

@ -503,7 +503,7 @@ impl Handler for ParallelHandler {
Ok(Err(e)) => {
let result = BranchResult {
id: String::new(),
outcome: Outcome::fail(e.to_string()),
outcome: e.to_fail_outcome(),
head_sha: None,
worktree_path: None,
};