Add provider-level model failover for LLM resilience

When an LLM provider returns a transient error (rate limit, server error,
quota exceeded, timeout, network, or stream failure), Arc now automatically
retries on fallback providers using the closest matching model from the
catalog based on capability filters and cost proximity.

Key changes:
- closest_model() and build_fallback_chain() in arc-llm catalog
- failover_eligible() on SdkError to classify transient vs deterministic errors
- fallbacks config field on LlmConfig with task-wins-over-defaults merging
- WorkflowRunEvent::Failover variant for observability
- Failover logic in both one_shot and agent session (run) code paths

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

crates/arc-api/src/serve.rs

@@ -113,6 +113,7 @@ pub async fn serve_command(args: ServeArgs, styles: &'static Styles) -> anyhow::
                 Some(Box::new(AgentApiBackend::new(
                     model.clone(),
                     provider_enum,
+                    Vec::new(),
                 )))
             }
         })

crates/arc-llm/src/catalog.rs

@@ -1,6 +1,8 @@
-use crate::types::ModelInfo;
+use std::collections::HashMap;
 use std::sync::LazyLock;
 
+use crate::types::ModelInfo;
+
 /// Built-in model catalog loaded from catalog.json (Section 2.9).
 /// The catalog is advisory, not restrictive -- unknown model strings pass through.
 static BUILT_IN_MODELS: LazyLock<Vec<ModelInfo>> = LazyLock::new(|| {
@@ -52,6 +54,72 @@ pub fn list_models(provider: Option<&str>) -> Vec<ModelInfo> {
     )
 }
 
+/// Find the closest model on a target provider that matches the reference model's capabilities.
+///
+/// Hard-filters on `supports_tools`, `supports_vision`, and `supports_reasoning`.
+/// Among matches, picks the closest by `input_cost_per_million` (absolute diff).
+/// Returns `None` if no model on the target provider matches all capabilities.
+#[must_use]
+pub fn closest_model(target_provider: &str, reference: &ModelInfo) -> Option<ModelInfo> {
+    BUILT_IN_MODELS
+        .iter()
+        .filter(|m| {
+            m.provider == target_provider
+                && m.supports_tools == reference.supports_tools
+                && m.supports_vision == reference.supports_vision
+                && m.supports_reasoning == reference.supports_reasoning
+        })
+        .min_by(|a, b| {
+            let ref_cost = reference.input_cost_per_million.unwrap_or(0.0);
+            let cost_a = (a.input_cost_per_million.unwrap_or(0.0) - ref_cost).abs();
+            let cost_b = (b.input_cost_per_million.unwrap_or(0.0) - ref_cost).abs();
+            cost_a.partial_cmp(&cost_b).unwrap_or(std::cmp::Ordering::Equal)
+        })
+        .cloned()
+}
+
+/// A resolved fallback target: provider name + model ID.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct FallbackTarget {
+    pub provider: String,
+    pub model: String,
+}
+
+/// Build an ordered fallback chain for a primary provider/model.
+///
+/// Looks up the primary model in the catalog, then for each fallback provider
+/// in the configured order, finds the closest matching model. Providers where
+/// no capability match exists are skipped.
+///
+/// Returns an empty vec if the primary model is unknown or the provider is not
+/// in the fallback map.
+#[must_use]
+pub fn build_fallback_chain(
+    primary_provider: &str,
+    primary_model: &str,
+    fallbacks: &HashMap<String, Vec<String>>,
+) -> Vec<FallbackTarget> {
+    let reference = match get_model_info(primary_model) {
+        Some(info) => info,
+        None => return Vec::new(),
+    };
+
+    let fallback_providers = match fallbacks.get(primary_provider) {
+        Some(providers) => providers,
+        None => return Vec::new(),
+    };
+
+    fallback_providers
+        .iter()
+        .filter_map(|provider| {
+            closest_model(provider, &reference).map(|m| FallbackTarget {
+                provider: provider.clone(),
+                model: m.id,
+            })
+        })
+        .collect()
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -243,6 +311,105 @@ mod tests {
         assert_eq!(get_model_info("mercury").unwrap().id, "mercury-2");
     }
 
+    #[test]
+    fn closest_model_opus_to_gemini() {
+        let opus = get_model_info("claude-opus-4-6").unwrap();
+        let result = closest_model("gemini", &opus).unwrap();
+        // Opus ($15) → closest reasoning+vision+tools gemini model by cost
+        assert_eq!(result.id, "gemini-3.1-pro-preview");
+    }
+
+    #[test]
+    fn closest_model_sonnet_to_gemini() {
+        let sonnet = get_model_info("claude-sonnet-4-5").unwrap();
+        let result = closest_model("gemini", &sonnet).unwrap();
+        // Sonnet ($3) → gemini-3.1-pro ($2) is closer than gemini-3-flash ($0.50)
+        assert_eq!(result.id, "gemini-3.1-pro-preview");
+    }
+
+    #[test]
+    fn closest_model_haiku_to_openai_none() {
+        let haiku = get_model_info("claude-haiku-4-5").unwrap();
+        // Haiku has reasoning=false; all openai models have reasoning=true
+        assert!(closest_model("openai", &haiku).is_none());
+    }
+
+    #[test]
+    fn closest_model_haiku_to_kimi() {
+        let haiku = get_model_info("claude-haiku-4-5").unwrap();
+        let result = closest_model("kimi", &haiku).unwrap();
+        // kimi-k2.5: no reasoning, vision, tools — matches haiku's caps
+        assert_eq!(result.id, "kimi-k2.5");
+    }
+
+    #[test]
+    fn closest_model_unknown_provider() {
+        let opus = get_model_info("claude-opus-4-6").unwrap();
+        assert!(closest_model("nonexistent", &opus).is_none());
+    }
+
+    #[test]
+    fn closest_model_no_capability_match() {
+        // glm-4.7 has tools=true, vision=false, reasoning=false
+        // No gemini model matches vision=false (all gemini models have vision=true)
+        let glm = get_model_info("glm-4.7").unwrap();
+        assert!(closest_model("gemini", &glm).is_none());
+    }
+
+    #[test]
+    fn build_fallback_chain_opus_anthropic() {
+        let fallbacks = HashMap::from([(
+            "anthropic".to_string(),
+            vec!["gemini".to_string(), "openai".to_string()],
+        )]);
+        let chain = build_fallback_chain("anthropic", "claude-opus-4-6", &fallbacks);
+        assert_eq!(chain.len(), 2);
+        assert_eq!(chain[0].provider, "gemini");
+        assert_eq!(chain[0].model, "gemini-3.1-pro-preview");
+        assert_eq!(chain[1].provider, "openai");
+        assert_eq!(chain[1].model, "gpt-5.2");
+    }
+
+    #[test]
+    fn build_fallback_chain_provider_not_in_map() {
+        let fallbacks = HashMap::from([(
+            "openai".to_string(),
+            vec!["anthropic".to_string()],
+        )]);
+        let chain = build_fallback_chain("anthropic", "claude-opus-4-6", &fallbacks);
+        assert!(chain.is_empty());
+    }
+
+    #[test]
+    fn build_fallback_chain_skips_no_capability_match() {
+        // Haiku (no reasoning) → openai should be skipped (all have reasoning)
+        let fallbacks = HashMap::from([(
+            "anthropic".to_string(),
+            vec!["openai".to_string(), "kimi".to_string()],
+        )]);
+        let chain = build_fallback_chain("anthropic", "claude-haiku-4-5", &fallbacks);
+        assert_eq!(chain.len(), 1);
+        assert_eq!(chain[0].provider, "kimi");
+        assert_eq!(chain[0].model, "kimi-k2.5");
+    }
+
+    #[test]
+    fn build_fallback_chain_empty_map() {
+        let fallbacks = HashMap::new();
+        let chain = build_fallback_chain("anthropic", "claude-opus-4-6", &fallbacks);
+        assert!(chain.is_empty());
+    }
+
+    #[test]
+    fn build_fallback_chain_unknown_primary_model() {
+        let fallbacks = HashMap::from([(
+            "anthropic".to_string(),
+            vec!["gemini".to_string()],
+        )]);
+        let chain = build_fallback_chain("anthropic", "unknown-model-xyz", &fallbacks);
+        assert!(chain.is_empty());
+    }
+
     #[test]
     fn model_info_costs() {
         let claude = get_model_info("claude-opus-4-6").unwrap();

crates/arc-llm/src/error.rs

@@ -140,6 +140,26 @@ impl SdkError {
         }
     }
 
+    /// Whether this error is eligible for provider-level failover.
+    ///
+    /// Transient provider issues (rate limits, server errors, quota, timeouts,
+    /// network, stream) can be retried on a different provider. Deterministic
+    /// errors (auth, invalid request, context length, content filter) and
+    /// non-provider errors (abort, configuration) cannot.
+    #[must_use]
+    pub const fn failover_eligible(&self) -> bool {
+        match self {
+            Self::Provider { kind, .. } => matches!(
+                kind,
+                ProviderErrorKind::RateLimit
+                    | ProviderErrorKind::Server
+                    | ProviderErrorKind::QuotaExceeded
+            ),
+            Self::RequestTimeout { .. } | Self::Network { .. } | Self::Stream { .. } => true,
+            _ => false,
+        }
+    }
+
     #[must_use]
     pub fn failure_signature_hint(&self) -> String {
         let provider = self.provider_name();
@@ -804,6 +824,104 @@ mod tests {
         assert_eq!(err.provider_name(), "unknown");
     }
 
+    #[test]
+    fn failover_eligible_transient_provider_errors() {
+        let detail = || Box::new(ProviderErrorDetail::new("error", "openai"));
+
+        assert!(SdkError::Provider {
+            kind: ProviderErrorKind::RateLimit,
+            detail: detail(),
+        }
+        .failover_eligible());
+
+        assert!(SdkError::Provider {
+            kind: ProviderErrorKind::Server,
+            detail: detail(),
+        }
+        .failover_eligible());
+
+        assert!(SdkError::Provider {
+            kind: ProviderErrorKind::QuotaExceeded,
+            detail: detail(),
+        }
+        .failover_eligible());
+    }
+
+    #[test]
+    fn failover_eligible_transient_non_provider_errors() {
+        assert!(SdkError::RequestTimeout {
+            message: "timed out".into()
+        }
+        .failover_eligible());
+
+        assert!(SdkError::Network {
+            message: "refused".into()
+        }
+        .failover_eligible());
+
+        assert!(SdkError::Stream {
+            message: "broken".into()
+        }
+        .failover_eligible());
+    }
+
+    #[test]
+    fn failover_not_eligible_deterministic_errors() {
+        let detail = || Box::new(ProviderErrorDetail::new("error", "openai"));
+
+        assert!(!SdkError::Provider {
+            kind: ProviderErrorKind::Authentication,
+            detail: detail(),
+        }
+        .failover_eligible());
+
+        assert!(!SdkError::Provider {
+            kind: ProviderErrorKind::InvalidRequest,
+            detail: detail(),
+        }
+        .failover_eligible());
+
+        assert!(!SdkError::Provider {
+            kind: ProviderErrorKind::ContextLength,
+            detail: detail(),
+        }
+        .failover_eligible());
+
+        assert!(!SdkError::Provider {
+            kind: ProviderErrorKind::ContentFilter,
+            detail: detail(),
+        }
+        .failover_eligible());
+    }
+
+    #[test]
+    fn failover_not_eligible_non_provider_errors() {
+        assert!(!SdkError::Configuration {
+            message: "bad".into()
+        }
+        .failover_eligible());
+
+        assert!(!SdkError::Abort {
+            message: "cancelled".into()
+        }
+        .failover_eligible());
+
+        assert!(!SdkError::InvalidToolCall {
+            message: "bad".into()
+        }
+        .failover_eligible());
+
+        assert!(!SdkError::NoObjectGenerated {
+            message: "none".into()
+        }
+        .failover_eligible());
+
+        assert!(!SdkError::UnsupportedToolChoice {
+            message: "nope".into()
+        }
+        .failover_eligible());
+    }
+
     #[test]
     fn failure_signature_hint_provider_transient() {
         let err = SdkError::Provider {

crates/arc-workflows/src/cli/backend.rs

@@ -8,11 +8,13 @@ use arc_agent::{
     AgentEvent, AnthropicProfile, GeminiProfile, OpenAiProfile, ProviderProfile, Sandbox, Session,
     SessionConfig, Turn,
 };
+use arc_llm::catalog::FallbackTarget;
 use arc_llm::client::Client;
 use arc_llm::provider::Provider;
 
 use crate::context::Context;
 use crate::error::ArcError;
+use crate::event::WorkflowRunEvent;
 use crate::graph::Node;
 use crate::handler::codergen::{CodergenBackend, CodergenResult};
 use crate::outcome::StageUsage;
@@ -24,15 +26,17 @@ use crate::outcome::StageUsage;
 pub struct AgentApiBackend {
     model: String,
     provider: Provider,
+    fallback_chain: Vec<FallbackTarget>,
     sessions: Mutex<HashMap<String, Session>>,
 }
 
 impl AgentApiBackend {
     #[must_use]
-    pub fn new(model: String, provider: Provider) -> Self {
+    pub fn new(model: String, provider: Provider, fallback_chain: Vec<FallbackTarget>) -> Self {
         Self {
             model,
             provider,
+            fallback_chain,
             sessions: Mutex::new(HashMap::new()),
         }
     }
@@ -150,7 +154,74 @@ impl CodergenBackend for AgentApiBackend {
             let _ = tokio::fs::write(stage_dir.join("api_request.json"), json).await;
         }
 
-        let response = client.complete(&request).await.map_err(ArcError::Llm)?;
+        // Build per-request fallback chain: if the node overrides the provider,
+        // compute a fresh chain for that provider; otherwise use the backend's.
+        let fallback_chain = if node.llm_provider().is_some() {
+            // Node-level override: build fallback chain if we have fallback config
+            // For node overrides without explicit fallbacks, no failover is available.
+            Vec::new()
+        } else {
+            self.fallback_chain.clone()
+        };
+
+        let result = client.complete(&request).await;
+
+        let (response, actual_model, actual_provider) = match result {
+            Ok(resp) => (
+                resp,
+                request.model.clone(),
+                request.provider.clone().unwrap_or_else(|| "anthropic".to_string()),
+            ),
+            Err(sdk_err) if sdk_err.failover_eligible() && !fallback_chain.is_empty() => {
+                let error_msg = sdk_err.to_string();
+                let from_provider = request.provider.clone().unwrap_or_else(|| "anthropic".to_string());
+                let from_model = request.model.clone();
+
+                let mut last_err = sdk_err;
+                let mut found = None;
+
+                for target in &fallback_chain {
+                    tracing::warn!(
+                        stage = node.id.as_str(),
+                        from_provider = from_provider.as_str(),
+                        from_model = from_model.as_str(),
+                        to_provider = target.provider.as_str(),
+                        to_model = target.model.as_str(),
+                        error = error_msg.as_str(),
+                        "LLM provider failover (one_shot)"
+                    );
+
+                    let max_tokens = node.max_tokens().or_else(|| {
+                        arc_llm::catalog::get_model_info(&target.model)
+                            .and_then(|m| m.max_output)
+                    });
+
+                    let fallback_request = arc_llm::types::Request {
+                        model: target.model.clone(),
+                        provider: Some(target.provider.clone()),
+                        max_tokens,
+                        ..request.clone()
+                    };
+
+                    match client.complete(&fallback_request).await {
+                        Ok(resp) => {
+                            found = Some((resp, target.model.clone(), target.provider.clone()));
+                            break;
+                        }
+                        Err(err) if err.failover_eligible() => {
+                            last_err = err;
+                        }
+                        Err(err) => return Err(ArcError::Llm(err)),
+                    }
+                }
+
+                match found {
+                    Some(triple) => triple,
+                    None => return Err(ArcError::Llm(last_err)),
+                }
+            }
+            Err(sdk_err) => return Err(ArcError::Llm(sdk_err)),
+        };
 
         if let Ok(json) = serde_json::to_string_pretty(&response) {
             let _ = tokio::fs::write(stage_dir.join("api_response.json"), json).await;
@@ -158,15 +229,15 @@ impl CodergenBackend for AgentApiBackend {
 
         let provider_used = serde_json::json!({
             "mode": "one_shot",
-            "provider": request.provider.as_deref().unwrap_or("anthropic"),
-            "model": &request.model,
+            "provider": &actual_provider,
+            "model": &actual_model,
         });
         if let Ok(json) = serde_json::to_string_pretty(&provider_used) {
             let _ = tokio::fs::write(stage_dir.join("provider_used.json"), json).await;
         }
 
         let mut stage_usage = StageUsage {
-            model: model.to_string(),
+            model: actual_model,
             input_tokens: response.usage.input_tokens,
             output_tokens: response.usage.output_tokens,
             cache_read_tokens: response.usage.cache_read_tokens,
@@ -291,14 +362,103 @@ impl CodergenBackend for AgentApiBackend {
             session.initialize().await;
         }
 
-        let result = session.process_input(prompt).await.map_err(|e| {
-            use arc_agent::AgentError;
-            match e {
-                AgentError::Llm(sdk_err) => ArcError::Llm(sdk_err),
-                AgentError::Aborted => ArcError::Cancelled,
-                other => ArcError::handler(format!("Agent session failed: {other}")),
+        let result = session.process_input(prompt).await;
+
+        // On failover-eligible error, try fallback providers.
+        let result = match result {
+            Ok(()) => Ok(()),
+            Err(arc_agent::AgentError::Llm(ref sdk_err))
+                if sdk_err.failover_eligible() && !self.fallback_chain.is_empty() =>
+            {
+                let error_msg = sdk_err.to_string();
+                let from_provider = self.provider.as_str().to_string();
+                let from_model = self.model.clone();
+
+                let mut last_err = ArcError::Llm(sdk_err.clone());
+                let mut succeeded = false;
+
+                for target in &self.fallback_chain {
+                    emitter.emit(&WorkflowRunEvent::Failover {
+                        stage: node.id.clone(),
+                        from_provider: from_provider.clone(),
+                        from_model: from_model.clone(),
+                        to_provider: target.provider.clone(),
+                        to_model: target.model.clone(),
+                        error: error_msg.clone(),
+                    });
+
+                    let target_provider: Provider = match target.provider.parse() {
+                        Ok(p) => p,
+                        Err(_) => continue,
+                    };
+
+                    // Create a temporary backend with fallback provider/model for session creation
+                    let fallback_backend = AgentApiBackend::new(
+                        target.model.clone(),
+                        target_provider,
+                        Vec::new(),
+                    );
+                    let new_session = match fallback_backend.create_session(node, sandbox).await {
+                        Ok(s) => s,
+                        Err(e) => {
+                            last_err = e;
+                            continue;
+                        }
+                    };
+                    session = new_session;
+
+                    // Re-subscribe to forward events from the new session
+                    let node_id2 = node.id.clone();
+                    let emitter2 = Arc::clone(emitter);
+                    let mut rx2 = session.subscribe();
+                    tokio::spawn(async move {
+                        while let Ok(event) = rx2.recv().await {
+                            if !matches!(
+                                &event.event,
+                                AgentEvent::SessionStarted
+                                    | AgentEvent::SessionEnded
+                                    | AgentEvent::AssistantTextStart
+                                    | AgentEvent::TextDelta { .. }
+                                    | AgentEvent::ToolCallOutputDelta { .. }
+                                    | AgentEvent::SkillExpanded { .. }
+                            ) {
+                                emitter2.emit(&WorkflowRunEvent::Agent {
+                                    stage: node_id2.clone(),
+                                    event: event.event.clone(),
+                                });
+                            }
+                        }
+                    });
+
+                    session.initialize().await;
+                    match session.process_input(prompt).await {
+                        Ok(()) => {
+                            succeeded = true;
+                            break;
+                        }
+                        Err(arc_agent::AgentError::Llm(err)) if err.failover_eligible() => {
+                            last_err = ArcError::Llm(err);
+                        }
+                        Err(arc_agent::AgentError::Llm(err)) => return Err(ArcError::Llm(err)),
+                        Err(arc_agent::AgentError::Aborted) => return Err(ArcError::Cancelled),
+                        Err(other) => {
+                            return Err(ArcError::handler(format!(
+                                "Agent session failed: {other}"
+                            )));
+                        }
+                    }
+                }
+
+                if succeeded {
+                    Ok(())
+                } else {
+                    Err(last_err)
+                }
             }
-        });
+            Err(arc_agent::AgentError::Llm(sdk_err)) => Err(ArcError::Llm(sdk_err)),
+            Err(arc_agent::AgentError::Aborted) => Err(ArcError::Cancelled),
+            Err(other) => Err(ArcError::handler(format!("Agent session failed: {other}"))),
+        };
 
         // On error, drop the session (don't cache failed state).
         result?;
@@ -375,20 +535,20 @@ mod tests {
 
     #[test]
     fn agent_backend_stores_config() {
-        let backend = AgentApiBackend::new("claude-opus-4-6".to_string(), Provider::OpenAi);
+        let backend = AgentApiBackend::new("claude-opus-4-6".to_string(), Provider::OpenAi, Vec::new());
         assert_eq!(backend.model, "claude-opus-4-6");
         assert_eq!(backend.provider, Provider::OpenAi);
     }
 
     #[test]
     fn agent_backend_initializes_empty_sessions() {
-        let backend = AgentApiBackend::new("claude-opus-4-6".to_string(), Provider::Anthropic);
+        let backend = AgentApiBackend::new("claude-opus-4-6".to_string(), Provider::Anthropic, Vec::new());
         assert!(backend.sessions.lock().unwrap().is_empty());
     }
 
     #[test]
     fn build_profile_can_register_subagent_tools() {
-        let backend = AgentApiBackend::new("claude-opus-4-6".to_string(), Provider::Anthropic);
+        let backend = AgentApiBackend::new("claude-opus-4-6".to_string(), Provider::Anthropic, Vec::new());
         let mut profile = backend.build_profile();
         let manager = Arc::new(tokio::sync::Mutex::new(SubAgentManager::new(1)));
         let factory: SessionFactory = Arc::new(|| {

crates/arc-workflows/src/cli/run.rs

@@ -151,6 +151,29 @@ fn resolve_daytona_config(
         })
 }
 
+/// Resolve the fallback chain from config: task config wins, then run defaults.
+fn resolve_fallback_chain(
+    provider: Provider,
+    model: &str,
+    run_cfg: Option<&WorkflowRunConfig>,
+    run_defaults: &RunDefaults,
+) -> Vec<arc_llm::catalog::FallbackTarget> {
+    let fallbacks = run_cfg
+        .and_then(|c| c.llm.as_ref())
+        .and_then(|l| l.fallbacks.as_ref())
+        .or_else(|| {
+            run_defaults
+                .llm
+                .as_ref()
+                .and_then(|l| l.fallbacks.as_ref())
+        });
+
+    match fallbacks {
+        Some(map) => arc_llm::catalog::build_fallback_chain(provider.as_str(), model, map),
+        None => Vec::new(),
+    }
+}
+
 /// Accumulates token usage and cost across all workflow stages.
 #[derive(Default)]
 struct CostAccumulator {
@@ -600,12 +623,20 @@ pub async fn run_command(
         .map_err(|e| anyhow::anyhow!("{e}"))?
         .unwrap_or(Provider::Anthropic);
 
+    // Resolve fallback chain from config
+    let fallback_chain = resolve_fallback_chain(
+        provider_enum,
+        &model,
+        run_cfg.as_ref(),
+        &run_defaults,
+    );
+
     // 7. Build engine
     let registry = default_registry(interviewer.clone(), || {
         if dry_run_mode {
             None
         } else {
-            let api = AgentApiBackend::new(model.clone(), provider_enum);
+            let api = AgentApiBackend::new(model.clone(), provider_enum, fallback_chain.clone());
             let cli = AgentCliBackend::new(model.clone(), provider_enum);
             Some(Box::new(BackendRouter::new(Box::new(api), cli)))
         }
@@ -988,11 +1019,14 @@ async fn run_from_branch(
         .map_err(|e| anyhow::anyhow!("{e}"))?
         .unwrap_or(arc_llm::provider::Provider::Anthropic);
 
+    // No fallback config available for branch resume; use empty chain.
+    let fallback_chain = Vec::new();
+
     let registry = crate::handler::default_registry(interviewer.clone(), || {
         if dry_run_mode {
             None
         } else {
-            let api = AgentApiBackend::new(model.clone(), provider_enum);
+            let api = AgentApiBackend::new(model.clone(), provider_enum, fallback_chain.clone());
             let cli = AgentCliBackend::new(model.clone(), provider_enum);
             Some(Box::new(BackendRouter::new(Box::new(api), cli)))
         }
@@ -1529,6 +1563,7 @@ mod tests {
             llm: Some(run_config::LlmConfig {
                 model: Some("toml-model".to_string()),
                 provider: Some("openai".to_string()),
+                fallbacks: None,
             }),
             setup: None,
             sandbox: None,
@@ -1567,6 +1602,7 @@ mod tests {
             llm: Some(run_config::LlmConfig {
                 model: Some("toml-model".to_string()),
                 provider: Some("openai".to_string()),
+                fallbacks: None,
             }),
             setup: None,
             sandbox: None,
@@ -1612,6 +1648,7 @@ mod tests {
             llm: Some(run_config::LlmConfig {
                 model: Some("default-model".to_string()),
                 provider: Some("openai".to_string()),
+                fallbacks: None,
             }),
             ..RunDefaults::default()
         };
@@ -1627,6 +1664,7 @@ mod tests {
             llm: Some(run_config::LlmConfig {
                 model: Some("default-model".to_string()),
                 provider: Some("anthropic".to_string()),
+                fallbacks: None,
             }),
             ..RunDefaults::default()
         };
@@ -1638,6 +1676,7 @@ mod tests {
             llm: Some(run_config::LlmConfig {
                 model: Some("toml-model".to_string()),
                 provider: Some("openai".to_string()),
+                fallbacks: None,
             }),
             setup: None,
             sandbox: None,

crates/arc-workflows/src/cli/run_config.rs

@@ -25,6 +25,8 @@ pub struct WorkflowRunConfig {
 pub struct LlmConfig {
     pub model: Option<String>,
     pub provider: Option<String>,
+    #[serde(default)]
+    pub fallbacks: Option<HashMap<String, Vec<String>>>,
 }
 
 #[derive(Clone, Debug, Deserialize, PartialEq)]
@@ -70,6 +72,9 @@ impl WorkflowRunConfig {
                 if task.provider.is_none() {
                     task.provider = default.provider.clone();
                 }
+                if task.fallbacks.is_none() {
+                    task.fallbacks = default.fallbacks.clone();
+                }
             }
             (None, Some(_)) => self.llm = defaults.llm.clone(),
             _ => {}
@@ -535,6 +540,7 @@ graph = "w.dot"
             llm: Some(LlmConfig {
                 model: Some("default-model".into()),
                 provider: Some("anthropic".into()),
+                fallbacks: None,
             }),
             ..RunDefaults::default()
         };
@@ -560,6 +566,7 @@ model = "task-model"
             llm: Some(LlmConfig {
                 model: Some("default-model".into()),
                 provider: None,
+                fallbacks: None,
             }),
             ..RunDefaults::default()
         };
@@ -633,6 +640,7 @@ model = "haiku"
             llm: Some(LlmConfig {
                 model: None,
                 provider: Some("anthropic".into()),
+                fallbacks: None,
             }),
             ..RunDefaults::default()
         };
@@ -928,6 +936,111 @@ auto_stop_interval = 60
         assert_eq!(snapshot.memory, Some(8));
     }
 
+    #[test]
+    fn parse_toml_with_fallbacks() {
+        let toml = r#"
+version = 1
+goal = "Run tests"
+graph = "workflow.dot"
+
+[llm]
+model = "claude-opus-4-6"
+provider = "anthropic"
+
+[llm.fallbacks]
+anthropic = ["gemini", "openai"]
+gemini = ["anthropic", "openai"]
+"#;
+        let config = parse_run_config(toml).unwrap();
+        let llm = config.llm.unwrap();
+        let fallbacks = llm.fallbacks.unwrap();
+        assert_eq!(fallbacks["anthropic"], vec!["gemini", "openai"]);
+        assert_eq!(fallbacks["gemini"], vec!["anthropic", "openai"]);
+    }
+
+    #[test]
+    fn parse_toml_without_fallbacks() {
+        let toml = r#"
+version = 1
+goal = "Run tests"
+graph = "workflow.dot"
+
+[llm]
+model = "claude-opus-4-6"
+provider = "anthropic"
+"#;
+        let config = parse_run_config(toml).unwrap();
+        let llm = config.llm.unwrap();
+        assert!(llm.fallbacks.is_none());
+    }
+
+    #[test]
+    fn apply_defaults_fallbacks_task_wins() {
+        let mut cfg = parse_run_config(
+            r#"
+version = 1
+goal = "test"
+graph = "w.dot"
+
+[llm]
+model = "opus"
+
+[llm.fallbacks]
+anthropic = ["gemini"]
+"#,
+        )
+        .unwrap();
+        let defaults = RunDefaults {
+            llm: Some(LlmConfig {
+                model: None,
+                provider: Some("anthropic".into()),
+                fallbacks: Some(HashMap::from([(
+                    "anthropic".into(),
+                    vec!["openai".into()],
+                )])),
+            }),
+            ..RunDefaults::default()
+        };
+        cfg.apply_defaults(&defaults);
+        let llm = cfg.llm.unwrap();
+        assert_eq!(
+            llm.fallbacks.unwrap()["anthropic"],
+            vec!["gemini"]
+        );
+    }
+
+    #[test]
+    fn apply_defaults_fallbacks_inherited() {
+        let mut cfg = parse_run_config(
+            r#"
+version = 1
+goal = "test"
+graph = "w.dot"
+
+[llm]
+model = "opus"
+"#,
+        )
+        .unwrap();
+        let defaults = RunDefaults {
+            llm: Some(LlmConfig {
+                model: None,
+                provider: Some("anthropic".into()),
+                fallbacks: Some(HashMap::from([(
+                    "anthropic".into(),
+                    vec!["openai".into()],
+                )])),
+            }),
+            ..RunDefaults::default()
+        };
+        cfg.apply_defaults(&defaults);
+        let llm = cfg.llm.unwrap();
+        assert_eq!(
+            llm.fallbacks.unwrap()["anthropic"],
+            vec!["openai"]
+        );
+    }
+
     #[tokio::test]
     async fn run_setup_succeeds() {
         let dir = tempfile::tempdir().unwrap();

crates/arc-workflows/src/event.rs

@@ -190,6 +190,14 @@ pub enum WorkflowRunEvent {
     SshAccessReady {
         ssh_command: String,
     },
+    Failover {
+        stage: String,
+        from_provider: String,
+        from_model: String,
+        to_provider: String,
+        to_model: String,
+        error: String,
+    },
 }
 
 impl WorkflowRunEvent {
@@ -450,6 +458,24 @@ impl WorkflowRunEvent {
             Self::SshAccessReady { ssh_command } => {
                 info!(ssh_command, "SSH access ready");
             }
+            Self::Failover {
+                stage,
+                from_provider,
+                from_model,
+                to_provider,
+                to_model,
+                error,
+            } => {
+                warn!(
+                    stage,
+                    from_provider,
+                    from_model,
+                    to_provider,
+                    to_model,
+                    error,
+                    "LLM provider failover"
+                );
+            }
         }
     }
 }

crates/arc-workflows/tests/integration.rs

@@ -7203,6 +7203,7 @@ async fn arc_e2e_with_real_llm() {
         Some(Box::new(AgentApiBackend::new(
             model.clone(),
             Provider::Anthropic,
+            Vec::new(),
         ))
             as Box<dyn arc_workflows::handler::codergen::CodergenBackend>)
     });