mirror of
https://github.com/fabro-sh/fabro.git
synced 2026-09-06 08:18:58 +00:00
Add stable failure signature hints for model errors
SdkError now produces hand-crafted signature hints (e.g. "api_deterministic|openai|authentication") that are identical regardless of error message wording, replacing fragile regex-normalized signatures for API errors. ArcError.to_fail_outcome() centralizes fail outcome construction with failure_class and failure_signature context_updates. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
3b8a0c3af2
commit
5ea508a5ca
5 changed files with 312 additions and 8 deletions
|
|
@ -131,6 +131,54 @@ impl SdkError {
|
|||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn provider_name(&self) -> &str {
|
||||
match self {
|
||||
Self::Provider { detail, .. } => &detail.provider,
|
||||
_ => "unknown",
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn failure_signature_hint(&self) -> String {
|
||||
let provider = self.provider_name();
|
||||
match self {
|
||||
Self::Provider { kind, .. } => {
|
||||
let category = if self.retryable() {
|
||||
"api_transient"
|
||||
} else {
|
||||
"api_deterministic"
|
||||
};
|
||||
let detail = match kind {
|
||||
ProviderErrorKind::RateLimit => "rate_limited",
|
||||
ProviderErrorKind::Server => "server_error",
|
||||
ProviderErrorKind::ContextLength => "context_length",
|
||||
ProviderErrorKind::QuotaExceeded => "quota_exceeded",
|
||||
ProviderErrorKind::Authentication => "authentication",
|
||||
ProviderErrorKind::AccessDenied => "access_denied",
|
||||
ProviderErrorKind::NotFound => "not_found",
|
||||
ProviderErrorKind::InvalidRequest => "invalid_request",
|
||||
ProviderErrorKind::ContentFilter => "content_filter",
|
||||
};
|
||||
format!("{category}|{provider}|{detail}")
|
||||
}
|
||||
Self::RequestTimeout { .. } => format!("api_transient|{provider}|timeout"),
|
||||
Self::Network { .. } => format!("api_transient|{provider}|network"),
|
||||
Self::Stream { .. } => format!("api_transient|{provider}|stream"),
|
||||
Self::Abort { .. } => format!("api_canceled|{provider}|abort"),
|
||||
Self::Configuration { .. } => format!("api_deterministic|{provider}|configuration"),
|
||||
Self::InvalidToolCall { .. } => {
|
||||
format!("api_deterministic|{provider}|invalid_tool_call")
|
||||
}
|
||||
Self::NoObjectGenerated { .. } => {
|
||||
format!("api_deterministic|{provider}|no_object")
|
||||
}
|
||||
Self::UnsupportedToolChoice { .. } => {
|
||||
format!("api_deterministic|{provider}|unsupported_tool_choice")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// HTTP status code to error type mapping (Section 6.4).
|
||||
|
|
@ -721,4 +769,168 @@ mod tests {
|
|||
};
|
||||
assert_eq!(err.status_code(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_name_from_provider_variant() {
|
||||
let err = SdkError::Provider {
|
||||
kind: ProviderErrorKind::Authentication,
|
||||
detail: Box::new(ProviderErrorDetail::new("bad key", "openai")),
|
||||
};
|
||||
assert_eq!(err.provider_name(), "openai");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_name_defaults_to_unknown() {
|
||||
let err = SdkError::Network {
|
||||
message: "refused".into(),
|
||||
};
|
||||
assert_eq!(err.provider_name(), "unknown");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failure_signature_hint_provider_transient() {
|
||||
let err = SdkError::Provider {
|
||||
kind: ProviderErrorKind::RateLimit,
|
||||
detail: Box::new(ProviderErrorDetail::new("too fast", "openai")),
|
||||
};
|
||||
assert_eq!(
|
||||
err.failure_signature_hint(),
|
||||
"api_transient|openai|rate_limited"
|
||||
);
|
||||
|
||||
let err = SdkError::Provider {
|
||||
kind: ProviderErrorKind::Server,
|
||||
detail: Box::new(ProviderErrorDetail::new("500", "anthropic")),
|
||||
};
|
||||
assert_eq!(
|
||||
err.failure_signature_hint(),
|
||||
"api_transient|anthropic|server_error"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failure_signature_hint_provider_deterministic() {
|
||||
let err = SdkError::Provider {
|
||||
kind: ProviderErrorKind::Authentication,
|
||||
detail: Box::new(ProviderErrorDetail::new("bad key", "openai")),
|
||||
};
|
||||
assert_eq!(
|
||||
err.failure_signature_hint(),
|
||||
"api_deterministic|openai|authentication"
|
||||
);
|
||||
|
||||
let err = SdkError::Provider {
|
||||
kind: ProviderErrorKind::AccessDenied,
|
||||
detail: Box::new(ProviderErrorDetail::new("denied", "anthropic")),
|
||||
};
|
||||
assert_eq!(
|
||||
err.failure_signature_hint(),
|
||||
"api_deterministic|anthropic|access_denied"
|
||||
);
|
||||
|
||||
let err = SdkError::Provider {
|
||||
kind: ProviderErrorKind::NotFound,
|
||||
detail: Box::new(ProviderErrorDetail::new("missing", "openai")),
|
||||
};
|
||||
assert_eq!(
|
||||
err.failure_signature_hint(),
|
||||
"api_deterministic|openai|not_found"
|
||||
);
|
||||
|
||||
let err = SdkError::Provider {
|
||||
kind: ProviderErrorKind::InvalidRequest,
|
||||
detail: Box::new(ProviderErrorDetail::new("bad", "openai")),
|
||||
};
|
||||
assert_eq!(
|
||||
err.failure_signature_hint(),
|
||||
"api_deterministic|openai|invalid_request"
|
||||
);
|
||||
|
||||
let err = SdkError::Provider {
|
||||
kind: ProviderErrorKind::ContentFilter,
|
||||
detail: Box::new(ProviderErrorDetail::new("blocked", "openai")),
|
||||
};
|
||||
assert_eq!(
|
||||
err.failure_signature_hint(),
|
||||
"api_deterministic|openai|content_filter"
|
||||
);
|
||||
|
||||
let err = SdkError::Provider {
|
||||
kind: ProviderErrorKind::ContextLength,
|
||||
detail: Box::new(ProviderErrorDetail::new("too long", "openai")),
|
||||
};
|
||||
assert_eq!(
|
||||
err.failure_signature_hint(),
|
||||
"api_deterministic|openai|context_length"
|
||||
);
|
||||
|
||||
let err = SdkError::Provider {
|
||||
kind: ProviderErrorKind::QuotaExceeded,
|
||||
detail: Box::new(ProviderErrorDetail::new("out of quota", "openai")),
|
||||
};
|
||||
assert_eq!(
|
||||
err.failure_signature_hint(),
|
||||
"api_deterministic|openai|quota_exceeded"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failure_signature_hint_non_provider_variants() {
|
||||
assert_eq!(
|
||||
SdkError::RequestTimeout {
|
||||
message: "timed out".into()
|
||||
}
|
||||
.failure_signature_hint(),
|
||||
"api_transient|unknown|timeout"
|
||||
);
|
||||
assert_eq!(
|
||||
SdkError::Network {
|
||||
message: "refused".into()
|
||||
}
|
||||
.failure_signature_hint(),
|
||||
"api_transient|unknown|network"
|
||||
);
|
||||
assert_eq!(
|
||||
SdkError::Stream {
|
||||
message: "broken".into()
|
||||
}
|
||||
.failure_signature_hint(),
|
||||
"api_transient|unknown|stream"
|
||||
);
|
||||
assert_eq!(
|
||||
SdkError::Abort {
|
||||
message: "cancelled".into()
|
||||
}
|
||||
.failure_signature_hint(),
|
||||
"api_canceled|unknown|abort"
|
||||
);
|
||||
assert_eq!(
|
||||
SdkError::Configuration {
|
||||
message: "bad".into()
|
||||
}
|
||||
.failure_signature_hint(),
|
||||
"api_deterministic|unknown|configuration"
|
||||
);
|
||||
assert_eq!(
|
||||
SdkError::InvalidToolCall {
|
||||
message: "bad".into()
|
||||
}
|
||||
.failure_signature_hint(),
|
||||
"api_deterministic|unknown|invalid_tool_call"
|
||||
);
|
||||
assert_eq!(
|
||||
SdkError::NoObjectGenerated {
|
||||
message: "none".into()
|
||||
}
|
||||
.failure_signature_hint(),
|
||||
"api_deterministic|unknown|no_object"
|
||||
);
|
||||
assert_eq!(
|
||||
SdkError::UnsupportedToolChoice {
|
||||
message: "nope".into()
|
||||
}
|
||||
.failure_signature_hint(),
|
||||
"api_deterministic|unknown|unsupported_tool_choice"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -876,12 +876,7 @@ impl PipelineEngine {
|
|||
tokio::time::sleep(delay).await;
|
||||
continue;
|
||||
}
|
||||
let mut fail_outcome = Outcome::fail(e.to_string());
|
||||
fail_outcome.context_updates.insert(
|
||||
"failure_class".to_string(),
|
||||
serde_json::json!(e.failure_class().to_string()),
|
||||
);
|
||||
return Ok((fail_outcome, attempt));
|
||||
return Ok((e.to_fail_outcome(), attempt));
|
||||
}
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -351,6 +351,32 @@ impl ArcError {
|
|||
Self::Handler(msg) | Self::Engine(msg) => classify_failure_reason(msg),
|
||||
}
|
||||
}
|
||||
|
||||
/// Return a stable failure signature hint when structured error info is available.
|
||||
#[must_use]
|
||||
pub fn failure_signature_hint(&self) -> Option<String> {
|
||||
match self {
|
||||
Self::Llm(sdk_err) => Some(sdk_err.failure_signature_hint()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Build an `Outcome::fail` with `failure_class` and optional `failure_signature`
|
||||
/// populated in `context_updates`.
|
||||
pub fn to_fail_outcome(&self) -> crate::outcome::Outcome {
|
||||
let mut outcome = crate::outcome::Outcome::fail(self.to_string());
|
||||
outcome.context_updates.insert(
|
||||
"failure_class".to_string(),
|
||||
serde_json::json!(self.failure_class().to_string()),
|
||||
);
|
||||
if let Some(sig) = self.failure_signature_hint() {
|
||||
outcome.context_updates.insert(
|
||||
"failure_signature".to_string(),
|
||||
serde_json::json!(sig),
|
||||
);
|
||||
}
|
||||
outcome
|
||||
}
|
||||
}
|
||||
|
||||
impl From<std::io::Error> for ArcError {
|
||||
|
|
@ -1417,4 +1443,75 @@ mod tests {
|
|||
assert!(!FailureClass::Canceled.is_signature_tracked());
|
||||
assert!(!FailureClass::CompilationLoop.is_signature_tracked());
|
||||
}
|
||||
|
||||
// --- failure_signature_hint tests ---
|
||||
|
||||
#[test]
|
||||
fn failure_signature_hint_llm_returns_some() {
|
||||
let err = ArcError::Llm(SdkError::Provider {
|
||||
kind: ProviderErrorKind::Authentication,
|
||||
detail: Box::new(ProviderErrorDetail::new("bad key", "openai")),
|
||||
});
|
||||
assert_eq!(
|
||||
err.failure_signature_hint(),
|
||||
Some("api_deterministic|openai|authentication".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failure_signature_hint_handler_returns_none() {
|
||||
let err = ArcError::Handler("something failed".to_string());
|
||||
assert_eq!(err.failure_signature_hint(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failure_signature_hint_engine_returns_none() {
|
||||
let err = ArcError::Engine("engine error".to_string());
|
||||
assert_eq!(err.failure_signature_hint(), None);
|
||||
}
|
||||
|
||||
// --- to_fail_outcome tests ---
|
||||
|
||||
#[test]
|
||||
fn to_fail_outcome_llm_has_class_and_signature() {
|
||||
let err = ArcError::Llm(SdkError::Provider {
|
||||
kind: ProviderErrorKind::Authentication,
|
||||
detail: Box::new(ProviderErrorDetail::new("bad key", "openai")),
|
||||
});
|
||||
let outcome = err.to_fail_outcome();
|
||||
assert_eq!(outcome.status, crate::outcome::StageStatus::Fail);
|
||||
assert_eq!(
|
||||
outcome.context_updates.get("failure_class"),
|
||||
Some(&serde_json::json!("deterministic"))
|
||||
);
|
||||
assert_eq!(
|
||||
outcome.context_updates.get("failure_signature"),
|
||||
Some(&serde_json::json!("api_deterministic|openai|authentication"))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn to_fail_outcome_handler_has_class_but_no_signature() {
|
||||
let err = ArcError::Handler("connection refused".to_string());
|
||||
let outcome = err.to_fail_outcome();
|
||||
assert_eq!(outcome.status, crate::outcome::StageStatus::Fail);
|
||||
assert_eq!(
|
||||
outcome.context_updates.get("failure_class"),
|
||||
Some(&serde_json::json!("transient_infra"))
|
||||
);
|
||||
assert!(!outcome.context_updates.contains_key("failure_signature"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn to_fail_outcome_includes_error_message_as_reason() {
|
||||
let err = ArcError::Llm(SdkError::Network {
|
||||
message: "connection refused".into(),
|
||||
});
|
||||
let outcome = err.to_fail_outcome();
|
||||
assert!(outcome
|
||||
.failure_reason
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.contains("connection refused"));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -271,7 +271,7 @@ impl Handler for CodergenHandler {
|
|||
return Err(e);
|
||||
}
|
||||
Err(e) => {
|
||||
return Ok(Outcome::fail(e.to_string()));
|
||||
return Ok(e.to_fail_outcome());
|
||||
}
|
||||
}
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -503,7 +503,7 @@ impl Handler for ParallelHandler {
|
|||
Ok(Err(e)) => {
|
||||
let result = BranchResult {
|
||||
id: String::new(),
|
||||
outcome: Outcome::fail(e.to_string()),
|
||||
outcome: e.to_fail_outcome(),
|
||||
head_sha: None,
|
||||
worktree_path: None,
|
||||
};
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue