diff --git a/crates/arc-api/src/serve.rs b/crates/arc-api/src/serve.rs index 185f4b167..cc8590219 100644 --- a/crates/arc-api/src/serve.rs +++ b/crates/arc-api/src/serve.rs @@ -113,6 +113,7 @@ pub async fn serve_command(args: ServeArgs, styles: &'static Styles) -> anyhow:: Some(Box::new(AgentApiBackend::new( model.clone(), provider_enum, + Vec::new(), ))) } }) diff --git a/crates/arc-llm/src/catalog.rs b/crates/arc-llm/src/catalog.rs index 9a9cdbfa9..b6f79118b 100644 --- a/crates/arc-llm/src/catalog.rs +++ b/crates/arc-llm/src/catalog.rs @@ -1,6 +1,8 @@ -use crate::types::ModelInfo; +use std::collections::HashMap; use std::sync::LazyLock; +use crate::types::ModelInfo; + /// Built-in model catalog loaded from catalog.json (Section 2.9). /// The catalog is advisory, not restrictive -- unknown model strings pass through. static BUILT_IN_MODELS: LazyLock> = LazyLock::new(|| { @@ -52,6 +54,72 @@ pub fn list_models(provider: Option<&str>) -> Vec { ) } +/// Find the closest model on a target provider that matches the reference model's capabilities. +/// +/// Hard-filters on `supports_tools`, `supports_vision`, and `supports_reasoning`. +/// Among matches, picks the closest by `input_cost_per_million` (absolute diff). +/// Returns `None` if no model on the target provider matches all capabilities. +#[must_use] +pub fn closest_model(target_provider: &str, reference: &ModelInfo) -> Option { + BUILT_IN_MODELS + .iter() + .filter(|m| { + m.provider == target_provider + && m.supports_tools == reference.supports_tools + && m.supports_vision == reference.supports_vision + && m.supports_reasoning == reference.supports_reasoning + }) + .min_by(|a, b| { + let ref_cost = reference.input_cost_per_million.unwrap_or(0.0); + let cost_a = (a.input_cost_per_million.unwrap_or(0.0) - ref_cost).abs(); + let cost_b = (b.input_cost_per_million.unwrap_or(0.0) - ref_cost).abs(); + cost_a.partial_cmp(&cost_b).unwrap_or(std::cmp::Ordering::Equal) + }) + .cloned() +} + +/// A resolved fallback target: provider name + model ID. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FallbackTarget { + pub provider: String, + pub model: String, +} + +/// Build an ordered fallback chain for a primary provider/model. +/// +/// Looks up the primary model in the catalog, then for each fallback provider +/// in the configured order, finds the closest matching model. Providers where +/// no capability match exists are skipped. +/// +/// Returns an empty vec if the primary model is unknown or the provider is not +/// in the fallback map. +#[must_use] +pub fn build_fallback_chain( + primary_provider: &str, + primary_model: &str, + fallbacks: &HashMap>, +) -> Vec { + let reference = match get_model_info(primary_model) { + Some(info) => info, + None => return Vec::new(), + }; + + let fallback_providers = match fallbacks.get(primary_provider) { + Some(providers) => providers, + None => return Vec::new(), + }; + + fallback_providers + .iter() + .filter_map(|provider| { + closest_model(provider, &reference).map(|m| FallbackTarget { + provider: provider.clone(), + model: m.id, + }) + }) + .collect() +} + #[cfg(test)] mod tests { use super::*; @@ -243,6 +311,105 @@ mod tests { assert_eq!(get_model_info("mercury").unwrap().id, "mercury-2"); } + #[test] + fn closest_model_opus_to_gemini() { + let opus = get_model_info("claude-opus-4-6").unwrap(); + let result = closest_model("gemini", &opus).unwrap(); + // Opus ($15) → closest reasoning+vision+tools gemini model by cost + assert_eq!(result.id, "gemini-3.1-pro-preview"); + } + + #[test] + fn closest_model_sonnet_to_gemini() { + let sonnet = get_model_info("claude-sonnet-4-5").unwrap(); + let result = closest_model("gemini", &sonnet).unwrap(); + // Sonnet ($3) → gemini-3.1-pro ($2) is closer than gemini-3-flash ($0.50) + assert_eq!(result.id, "gemini-3.1-pro-preview"); + } + + #[test] + fn closest_model_haiku_to_openai_none() { + let haiku = get_model_info("claude-haiku-4-5").unwrap(); + // Haiku has reasoning=false; all openai models have reasoning=true + assert!(closest_model("openai", &haiku).is_none()); + } + + #[test] + fn closest_model_haiku_to_kimi() { + let haiku = get_model_info("claude-haiku-4-5").unwrap(); + let result = closest_model("kimi", &haiku).unwrap(); + // kimi-k2.5: no reasoning, vision, tools — matches haiku's caps + assert_eq!(result.id, "kimi-k2.5"); + } + + #[test] + fn closest_model_unknown_provider() { + let opus = get_model_info("claude-opus-4-6").unwrap(); + assert!(closest_model("nonexistent", &opus).is_none()); + } + + #[test] + fn closest_model_no_capability_match() { + // glm-4.7 has tools=true, vision=false, reasoning=false + // No gemini model matches vision=false (all gemini models have vision=true) + let glm = get_model_info("glm-4.7").unwrap(); + assert!(closest_model("gemini", &glm).is_none()); + } + + #[test] + fn build_fallback_chain_opus_anthropic() { + let fallbacks = HashMap::from([( + "anthropic".to_string(), + vec!["gemini".to_string(), "openai".to_string()], + )]); + let chain = build_fallback_chain("anthropic", "claude-opus-4-6", &fallbacks); + assert_eq!(chain.len(), 2); + assert_eq!(chain[0].provider, "gemini"); + assert_eq!(chain[0].model, "gemini-3.1-pro-preview"); + assert_eq!(chain[1].provider, "openai"); + assert_eq!(chain[1].model, "gpt-5.2"); + } + + #[test] + fn build_fallback_chain_provider_not_in_map() { + let fallbacks = HashMap::from([( + "openai".to_string(), + vec!["anthropic".to_string()], + )]); + let chain = build_fallback_chain("anthropic", "claude-opus-4-6", &fallbacks); + assert!(chain.is_empty()); + } + + #[test] + fn build_fallback_chain_skips_no_capability_match() { + // Haiku (no reasoning) → openai should be skipped (all have reasoning) + let fallbacks = HashMap::from([( + "anthropic".to_string(), + vec!["openai".to_string(), "kimi".to_string()], + )]); + let chain = build_fallback_chain("anthropic", "claude-haiku-4-5", &fallbacks); + assert_eq!(chain.len(), 1); + assert_eq!(chain[0].provider, "kimi"); + assert_eq!(chain[0].model, "kimi-k2.5"); + } + + #[test] + fn build_fallback_chain_empty_map() { + let fallbacks = HashMap::new(); + let chain = build_fallback_chain("anthropic", "claude-opus-4-6", &fallbacks); + assert!(chain.is_empty()); + } + + #[test] + fn build_fallback_chain_unknown_primary_model() { + let fallbacks = HashMap::from([( + "anthropic".to_string(), + vec!["gemini".to_string()], + )]); + let chain = build_fallback_chain("anthropic", "unknown-model-xyz", &fallbacks); + assert!(chain.is_empty()); + } + #[test] fn model_info_costs() { let claude = get_model_info("claude-opus-4-6").unwrap(); diff --git a/crates/arc-llm/src/error.rs b/crates/arc-llm/src/error.rs index 1a2716f23..f3240cd73 100644 --- a/crates/arc-llm/src/error.rs +++ b/crates/arc-llm/src/error.rs @@ -140,6 +140,26 @@ impl SdkError { } } + /// Whether this error is eligible for provider-level failover. + /// + /// Transient provider issues (rate limits, server errors, quota, timeouts, + /// network, stream) can be retried on a different provider. Deterministic + /// errors (auth, invalid request, context length, content filter) and + /// non-provider errors (abort, configuration) cannot. + #[must_use] + pub const fn failover_eligible(&self) -> bool { + match self { + Self::Provider { kind, .. } => matches!( + kind, + ProviderErrorKind::RateLimit + | ProviderErrorKind::Server + | ProviderErrorKind::QuotaExceeded + ), + Self::RequestTimeout { .. } | Self::Network { .. } | Self::Stream { .. } => true, + _ => false, + } + } + #[must_use] pub fn failure_signature_hint(&self) -> String { let provider = self.provider_name(); @@ -804,6 +824,104 @@ mod tests { assert_eq!(err.provider_name(), "unknown"); } + #[test] + fn failover_eligible_transient_provider_errors() { + let detail = || Box::new(ProviderErrorDetail::new("error", "openai")); + + assert!(SdkError::Provider { + kind: ProviderErrorKind::RateLimit, + detail: detail(), + } + .failover_eligible()); + + assert!(SdkError::Provider { + kind: ProviderErrorKind::Server, + detail: detail(), + } + .failover_eligible()); + + assert!(SdkError::Provider { + kind: ProviderErrorKind::QuotaExceeded, + detail: detail(), + } + .failover_eligible()); + } + + #[test] + fn failover_eligible_transient_non_provider_errors() { + assert!(SdkError::RequestTimeout { + message: "timed out".into() + } + .failover_eligible()); + + assert!(SdkError::Network { + message: "refused".into() + } + .failover_eligible()); + + assert!(SdkError::Stream { + message: "broken".into() + } + .failover_eligible()); + } + + #[test] + fn failover_not_eligible_deterministic_errors() { + let detail = || Box::new(ProviderErrorDetail::new("error", "openai")); + + assert!(!SdkError::Provider { + kind: ProviderErrorKind::Authentication, + detail: detail(), + } + .failover_eligible()); + + assert!(!SdkError::Provider { + kind: ProviderErrorKind::InvalidRequest, + detail: detail(), + } + .failover_eligible()); + + assert!(!SdkError::Provider { + kind: ProviderErrorKind::ContextLength, + detail: detail(), + } + .failover_eligible()); + + assert!(!SdkError::Provider { + kind: ProviderErrorKind::ContentFilter, + detail: detail(), + } + .failover_eligible()); + } + + #[test] + fn failover_not_eligible_non_provider_errors() { + assert!(!SdkError::Configuration { + message: "bad".into() + } + .failover_eligible()); + + assert!(!SdkError::Abort { + message: "cancelled".into() + } + .failover_eligible()); + + assert!(!SdkError::InvalidToolCall { + message: "bad".into() + } + .failover_eligible()); + + assert!(!SdkError::NoObjectGenerated { + message: "none".into() + } + .failover_eligible()); + + assert!(!SdkError::UnsupportedToolChoice { + message: "nope".into() + } + .failover_eligible()); + } + #[test] fn failure_signature_hint_provider_transient() { let err = SdkError::Provider { diff --git a/crates/arc-workflows/src/cli/backend.rs b/crates/arc-workflows/src/cli/backend.rs index dd5f88b2d..4de32dd42 100644 --- a/crates/arc-workflows/src/cli/backend.rs +++ b/crates/arc-workflows/src/cli/backend.rs @@ -8,11 +8,13 @@ use arc_agent::{ AgentEvent, AnthropicProfile, GeminiProfile, OpenAiProfile, ProviderProfile, Sandbox, Session, SessionConfig, Turn, }; +use arc_llm::catalog::FallbackTarget; use arc_llm::client::Client; use arc_llm::provider::Provider; use crate::context::Context; use crate::error::ArcError; +use crate::event::WorkflowRunEvent; use crate::graph::Node; use crate::handler::codergen::{CodergenBackend, CodergenResult}; use crate::outcome::StageUsage; @@ -24,15 +26,17 @@ use crate::outcome::StageUsage; pub struct AgentApiBackend { model: String, provider: Provider, + fallback_chain: Vec, sessions: Mutex>, } impl AgentApiBackend { #[must_use] - pub fn new(model: String, provider: Provider) -> Self { + pub fn new(model: String, provider: Provider, fallback_chain: Vec) -> Self { Self { model, provider, + fallback_chain, sessions: Mutex::new(HashMap::new()), } } @@ -150,7 +154,74 @@ impl CodergenBackend for AgentApiBackend { let _ = tokio::fs::write(stage_dir.join("api_request.json"), json).await; } - let response = client.complete(&request).await.map_err(ArcError::Llm)?; + // Build per-request fallback chain: if the node overrides the provider, + // compute a fresh chain for that provider; otherwise use the backend's. + let fallback_chain = if node.llm_provider().is_some() { + // Node-level override: build fallback chain if we have fallback config + // For node overrides without explicit fallbacks, no failover is available. + Vec::new() + } else { + self.fallback_chain.clone() + }; + + let result = client.complete(&request).await; + + let (response, actual_model, actual_provider) = match result { + Ok(resp) => ( + resp, + request.model.clone(), + request.provider.clone().unwrap_or_else(|| "anthropic".to_string()), + ), + Err(sdk_err) if sdk_err.failover_eligible() && !fallback_chain.is_empty() => { + let error_msg = sdk_err.to_string(); + let from_provider = request.provider.clone().unwrap_or_else(|| "anthropic".to_string()); + let from_model = request.model.clone(); + + let mut last_err = sdk_err; + let mut found = None; + + for target in &fallback_chain { + tracing::warn!( + stage = node.id.as_str(), + from_provider = from_provider.as_str(), + from_model = from_model.as_str(), + to_provider = target.provider.as_str(), + to_model = target.model.as_str(), + error = error_msg.as_str(), + "LLM provider failover (one_shot)" + ); + + let max_tokens = node.max_tokens().or_else(|| { + arc_llm::catalog::get_model_info(&target.model) + .and_then(|m| m.max_output) + }); + + let fallback_request = arc_llm::types::Request { + model: target.model.clone(), + provider: Some(target.provider.clone()), + max_tokens, + ..request.clone() + }; + + match client.complete(&fallback_request).await { + Ok(resp) => { + found = Some((resp, target.model.clone(), target.provider.clone())); + break; + } + Err(err) if err.failover_eligible() => { + last_err = err; + } + Err(err) => return Err(ArcError::Llm(err)), + } + } + + match found { + Some(triple) => triple, + None => return Err(ArcError::Llm(last_err)), + } + } + Err(sdk_err) => return Err(ArcError::Llm(sdk_err)), + }; if let Ok(json) = serde_json::to_string_pretty(&response) { let _ = tokio::fs::write(stage_dir.join("api_response.json"), json).await; @@ -158,15 +229,15 @@ impl CodergenBackend for AgentApiBackend { let provider_used = serde_json::json!({ "mode": "one_shot", - "provider": request.provider.as_deref().unwrap_or("anthropic"), - "model": &request.model, + "provider": &actual_provider, + "model": &actual_model, }); if let Ok(json) = serde_json::to_string_pretty(&provider_used) { let _ = tokio::fs::write(stage_dir.join("provider_used.json"), json).await; } let mut stage_usage = StageUsage { - model: model.to_string(), + model: actual_model, input_tokens: response.usage.input_tokens, output_tokens: response.usage.output_tokens, cache_read_tokens: response.usage.cache_read_tokens, @@ -291,14 +362,103 @@ impl CodergenBackend for AgentApiBackend { session.initialize().await; } - let result = session.process_input(prompt).await.map_err(|e| { - use arc_agent::AgentError; - match e { - AgentError::Llm(sdk_err) => ArcError::Llm(sdk_err), - AgentError::Aborted => ArcError::Cancelled, - other => ArcError::handler(format!("Agent session failed: {other}")), + let result = session.process_input(prompt).await; + + // On failover-eligible error, try fallback providers. + let result = match result { + Ok(()) => Ok(()), + Err(arc_agent::AgentError::Llm(ref sdk_err)) + if sdk_err.failover_eligible() && !self.fallback_chain.is_empty() => + { + let error_msg = sdk_err.to_string(); + let from_provider = self.provider.as_str().to_string(); + let from_model = self.model.clone(); + + let mut last_err = ArcError::Llm(sdk_err.clone()); + let mut succeeded = false; + + for target in &self.fallback_chain { + emitter.emit(&WorkflowRunEvent::Failover { + stage: node.id.clone(), + from_provider: from_provider.clone(), + from_model: from_model.clone(), + to_provider: target.provider.clone(), + to_model: target.model.clone(), + error: error_msg.clone(), + }); + + let target_provider: Provider = match target.provider.parse() { + Ok(p) => p, + Err(_) => continue, + }; + + // Create a temporary backend with fallback provider/model for session creation + let fallback_backend = AgentApiBackend::new( + target.model.clone(), + target_provider, + Vec::new(), + ); + let new_session = match fallback_backend.create_session(node, sandbox).await { + Ok(s) => s, + Err(e) => { + last_err = e; + continue; + } + }; + session = new_session; + + // Re-subscribe to forward events from the new session + let node_id2 = node.id.clone(); + let emitter2 = Arc::clone(emitter); + let mut rx2 = session.subscribe(); + tokio::spawn(async move { + while let Ok(event) = rx2.recv().await { + if !matches!( + &event.event, + AgentEvent::SessionStarted + | AgentEvent::SessionEnded + | AgentEvent::AssistantTextStart + | AgentEvent::TextDelta { .. } + | AgentEvent::ToolCallOutputDelta { .. } + | AgentEvent::SkillExpanded { .. } + ) { + emitter2.emit(&WorkflowRunEvent::Agent { + stage: node_id2.clone(), + event: event.event.clone(), + }); + } + } + }); + + session.initialize().await; + match session.process_input(prompt).await { + Ok(()) => { + succeeded = true; + break; + } + Err(arc_agent::AgentError::Llm(err)) if err.failover_eligible() => { + last_err = ArcError::Llm(err); + } + Err(arc_agent::AgentError::Llm(err)) => return Err(ArcError::Llm(err)), + Err(arc_agent::AgentError::Aborted) => return Err(ArcError::Cancelled), + Err(other) => { + return Err(ArcError::handler(format!( + "Agent session failed: {other}" + ))); + } + } + } + + if succeeded { + Ok(()) + } else { + Err(last_err) + } } - }); + Err(arc_agent::AgentError::Llm(sdk_err)) => Err(ArcError::Llm(sdk_err)), + Err(arc_agent::AgentError::Aborted) => Err(ArcError::Cancelled), + Err(other) => Err(ArcError::handler(format!("Agent session failed: {other}"))), + }; // On error, drop the session (don't cache failed state). result?; @@ -375,20 +535,20 @@ mod tests { #[test] fn agent_backend_stores_config() { - let backend = AgentApiBackend::new("claude-opus-4-6".to_string(), Provider::OpenAi); + let backend = AgentApiBackend::new("claude-opus-4-6".to_string(), Provider::OpenAi, Vec::new()); assert_eq!(backend.model, "claude-opus-4-6"); assert_eq!(backend.provider, Provider::OpenAi); } #[test] fn agent_backend_initializes_empty_sessions() { - let backend = AgentApiBackend::new("claude-opus-4-6".to_string(), Provider::Anthropic); + let backend = AgentApiBackend::new("claude-opus-4-6".to_string(), Provider::Anthropic, Vec::new()); assert!(backend.sessions.lock().unwrap().is_empty()); } #[test] fn build_profile_can_register_subagent_tools() { - let backend = AgentApiBackend::new("claude-opus-4-6".to_string(), Provider::Anthropic); + let backend = AgentApiBackend::new("claude-opus-4-6".to_string(), Provider::Anthropic, Vec::new()); let mut profile = backend.build_profile(); let manager = Arc::new(tokio::sync::Mutex::new(SubAgentManager::new(1))); let factory: SessionFactory = Arc::new(|| { diff --git a/crates/arc-workflows/src/cli/run.rs b/crates/arc-workflows/src/cli/run.rs index 3f3b8b199..3d22b0b3a 100644 --- a/crates/arc-workflows/src/cli/run.rs +++ b/crates/arc-workflows/src/cli/run.rs @@ -151,6 +151,29 @@ fn resolve_daytona_config( }) } +/// Resolve the fallback chain from config: task config wins, then run defaults. +fn resolve_fallback_chain( + provider: Provider, + model: &str, + run_cfg: Option<&WorkflowRunConfig>, + run_defaults: &RunDefaults, +) -> Vec { + let fallbacks = run_cfg + .and_then(|c| c.llm.as_ref()) + .and_then(|l| l.fallbacks.as_ref()) + .or_else(|| { + run_defaults + .llm + .as_ref() + .and_then(|l| l.fallbacks.as_ref()) + }); + + match fallbacks { + Some(map) => arc_llm::catalog::build_fallback_chain(provider.as_str(), model, map), + None => Vec::new(), + } +} + /// Accumulates token usage and cost across all workflow stages. #[derive(Default)] struct CostAccumulator { @@ -600,12 +623,20 @@ pub async fn run_command( .map_err(|e| anyhow::anyhow!("{e}"))? .unwrap_or(Provider::Anthropic); + // Resolve fallback chain from config + let fallback_chain = resolve_fallback_chain( + provider_enum, + &model, + run_cfg.as_ref(), + &run_defaults, + ); + // 7. Build engine let registry = default_registry(interviewer.clone(), || { if dry_run_mode { None } else { - let api = AgentApiBackend::new(model.clone(), provider_enum); + let api = AgentApiBackend::new(model.clone(), provider_enum, fallback_chain.clone()); let cli = AgentCliBackend::new(model.clone(), provider_enum); Some(Box::new(BackendRouter::new(Box::new(api), cli))) } @@ -988,11 +1019,14 @@ async fn run_from_branch( .map_err(|e| anyhow::anyhow!("{e}"))? .unwrap_or(arc_llm::provider::Provider::Anthropic); + // No fallback config available for branch resume; use empty chain. + let fallback_chain = Vec::new(); + let registry = crate::handler::default_registry(interviewer.clone(), || { if dry_run_mode { None } else { - let api = AgentApiBackend::new(model.clone(), provider_enum); + let api = AgentApiBackend::new(model.clone(), provider_enum, fallback_chain.clone()); let cli = AgentCliBackend::new(model.clone(), provider_enum); Some(Box::new(BackendRouter::new(Box::new(api), cli))) } @@ -1529,6 +1563,7 @@ mod tests { llm: Some(run_config::LlmConfig { model: Some("toml-model".to_string()), provider: Some("openai".to_string()), + fallbacks: None, }), setup: None, sandbox: None, @@ -1567,6 +1602,7 @@ mod tests { llm: Some(run_config::LlmConfig { model: Some("toml-model".to_string()), provider: Some("openai".to_string()), + fallbacks: None, }), setup: None, sandbox: None, @@ -1612,6 +1648,7 @@ mod tests { llm: Some(run_config::LlmConfig { model: Some("default-model".to_string()), provider: Some("openai".to_string()), + fallbacks: None, }), ..RunDefaults::default() }; @@ -1627,6 +1664,7 @@ mod tests { llm: Some(run_config::LlmConfig { model: Some("default-model".to_string()), provider: Some("anthropic".to_string()), + fallbacks: None, }), ..RunDefaults::default() }; @@ -1638,6 +1676,7 @@ mod tests { llm: Some(run_config::LlmConfig { model: Some("toml-model".to_string()), provider: Some("openai".to_string()), + fallbacks: None, }), setup: None, sandbox: None, diff --git a/crates/arc-workflows/src/cli/run_config.rs b/crates/arc-workflows/src/cli/run_config.rs index 12b76055d..28f60e21c 100644 --- a/crates/arc-workflows/src/cli/run_config.rs +++ b/crates/arc-workflows/src/cli/run_config.rs @@ -25,6 +25,8 @@ pub struct WorkflowRunConfig { pub struct LlmConfig { pub model: Option, pub provider: Option, + #[serde(default)] + pub fallbacks: Option>>, } #[derive(Clone, Debug, Deserialize, PartialEq)] @@ -70,6 +72,9 @@ impl WorkflowRunConfig { if task.provider.is_none() { task.provider = default.provider.clone(); } + if task.fallbacks.is_none() { + task.fallbacks = default.fallbacks.clone(); + } } (None, Some(_)) => self.llm = defaults.llm.clone(), _ => {} @@ -535,6 +540,7 @@ graph = "w.dot" llm: Some(LlmConfig { model: Some("default-model".into()), provider: Some("anthropic".into()), + fallbacks: None, }), ..RunDefaults::default() }; @@ -560,6 +566,7 @@ model = "task-model" llm: Some(LlmConfig { model: Some("default-model".into()), provider: None, + fallbacks: None, }), ..RunDefaults::default() }; @@ -633,6 +640,7 @@ model = "haiku" llm: Some(LlmConfig { model: None, provider: Some("anthropic".into()), + fallbacks: None, }), ..RunDefaults::default() }; @@ -928,6 +936,111 @@ auto_stop_interval = 60 assert_eq!(snapshot.memory, Some(8)); } + #[test] + fn parse_toml_with_fallbacks() { + let toml = r#" +version = 1 +goal = "Run tests" +graph = "workflow.dot" + +[llm] +model = "claude-opus-4-6" +provider = "anthropic" + +[llm.fallbacks] +anthropic = ["gemini", "openai"] +gemini = ["anthropic", "openai"] +"#; + let config = parse_run_config(toml).unwrap(); + let llm = config.llm.unwrap(); + let fallbacks = llm.fallbacks.unwrap(); + assert_eq!(fallbacks["anthropic"], vec!["gemini", "openai"]); + assert_eq!(fallbacks["gemini"], vec!["anthropic", "openai"]); + } + + #[test] + fn parse_toml_without_fallbacks() { + let toml = r#" +version = 1 +goal = "Run tests" +graph = "workflow.dot" + +[llm] +model = "claude-opus-4-6" +provider = "anthropic" +"#; + let config = parse_run_config(toml).unwrap(); + let llm = config.llm.unwrap(); + assert!(llm.fallbacks.is_none()); + } + + #[test] + fn apply_defaults_fallbacks_task_wins() { + let mut cfg = parse_run_config( + r#" +version = 1 +goal = "test" +graph = "w.dot" + +[llm] +model = "opus" + +[llm.fallbacks] +anthropic = ["gemini"] +"#, + ) + .unwrap(); + let defaults = RunDefaults { + llm: Some(LlmConfig { + model: None, + provider: Some("anthropic".into()), + fallbacks: Some(HashMap::from([( + "anthropic".into(), + vec!["openai".into()], + )])), + }), + ..RunDefaults::default() + }; + cfg.apply_defaults(&defaults); + let llm = cfg.llm.unwrap(); + assert_eq!( + llm.fallbacks.unwrap()["anthropic"], + vec!["gemini"] + ); + } + + #[test] + fn apply_defaults_fallbacks_inherited() { + let mut cfg = parse_run_config( + r#" +version = 1 +goal = "test" +graph = "w.dot" + +[llm] +model = "opus" +"#, + ) + .unwrap(); + let defaults = RunDefaults { + llm: Some(LlmConfig { + model: None, + provider: Some("anthropic".into()), + fallbacks: Some(HashMap::from([( + "anthropic".into(), + vec!["openai".into()], + )])), + }), + ..RunDefaults::default() + }; + cfg.apply_defaults(&defaults); + let llm = cfg.llm.unwrap(); + assert_eq!( + llm.fallbacks.unwrap()["anthropic"], + vec!["openai"] + ); + } + #[tokio::test] async fn run_setup_succeeds() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/arc-workflows/src/event.rs b/crates/arc-workflows/src/event.rs index 7375a7508..0966201f4 100644 --- a/crates/arc-workflows/src/event.rs +++ b/crates/arc-workflows/src/event.rs @@ -190,6 +190,14 @@ pub enum WorkflowRunEvent { SshAccessReady { ssh_command: String, }, + Failover { + stage: String, + from_provider: String, + from_model: String, + to_provider: String, + to_model: String, + error: String, + }, } impl WorkflowRunEvent { @@ -450,6 +458,24 @@ impl WorkflowRunEvent { Self::SshAccessReady { ssh_command } => { info!(ssh_command, "SSH access ready"); } + Self::Failover { + stage, + from_provider, + from_model, + to_provider, + to_model, + error, + } => { + warn!( + stage, + from_provider, + from_model, + to_provider, + to_model, + error, + "LLM provider failover" + ); + } } } } diff --git a/crates/arc-workflows/tests/integration.rs b/crates/arc-workflows/tests/integration.rs index 4b4304a14..e51488633 100644 --- a/crates/arc-workflows/tests/integration.rs +++ b/crates/arc-workflows/tests/integration.rs @@ -7203,6 +7203,7 @@ async fn arc_e2e_with_real_llm() { Some(Box::new(AgentApiBackend::new( model.clone(), Provider::Anthropic, + Vec::new(), )) as Box) });