From ba8265665653efbec2a25df7929ce4fd7a078a2f Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Thu, 30 Jul 2026 12:02:41 -0400 Subject: [PATCH] feat: add model-keyed fallback policies --- .../administration/server-configuration.mdx | 4 +- docs/public/api-reference/fabro-api.yaml | 12 +- docs/public/core-concepts/models.mdx | 8 +- docs/public/examples/semantic-port.mdx | 30 +- docs/public/examples/solitaire.mdx | 4 +- docs/public/execution/failures.mdx | 15 +- docs/public/execution/run-configuration.mdx | 44 +- docs/public/reference/user-configuration.mdx | 6 +- lib/apps/fabro-cli/tests/it/cmd/attach.rs | 2 +- lib/apps/fabro-cli/tests/it/cmd/inspect.rs | 2 +- lib/apps/fabro-cli/tests/it/cmd/validate.rs | 17 + lib/apps/fabro-server/src/run_manifest.rs | 186 ++- .../tests/it/scenario/run_completion.rs | 2 +- lib/components/fabro-llm/src/error.rs | 38 +- lib/components/fabro-manifest/src/lib.rs | 2 +- .../fabro-workflow/src/event/convert.rs | 20 +- .../fabro-workflow/src/event/events.rs | 25 +- .../fabro-workflow/src/handler/llm/api.rs | 1242 ++++++++++++----- lib/components/fabro-workflow/src/lib.rs | 1 + .../fabro-workflow/src/model_fallback.rs | 514 +++++++ .../fabro-workflow/src/operations/start.rs | 821 +---------- .../src/pipeline/execute/tests.rs | 9 +- .../fabro-workflow/src/pipeline/initialize.rs | 18 +- .../fabro-workflow/src/pipeline/types.rs | 5 +- .../fabro-workflow/tests/it/integration.rs | 5 +- lib/foundation/fabro-config/src/builders.rs | 2 +- lib/foundation/fabro-config/src/layers/run.rs | 21 +- .../fabro-config/src/resolve/run.rs | 16 +- .../fabro-config/src/tests/combine.rs | 37 +- .../src/commands/docs_options_reference.rs | 4 +- lib/foundation/fabro-model/src/catalog.rs | 95 -- lib/foundation/fabro-model/src/reasoning.rs | 63 + .../fabro-types/src/run_event/misc.rs | 18 +- .../fabro-types/src/run_event/mod.rs | 27 + .../fabro-types/src/settings/model_ref.rs | 2 +- .../fabro-types/src/settings/run.rs | 9 +- .../src/models/run-model-settings.ts | 5 +- test/offline-fallbacks/workflow.fabro | 10 + test/offline-fallbacks/workflow.toml | 7 + 39 files changed, 2004 insertions(+), 1344 deletions(-) create mode 100644 lib/components/fabro-workflow/src/model_fallback.rs create mode 100644 test/offline-fallbacks/workflow.fabro create mode 100644 test/offline-fallbacks/workflow.toml diff --git a/docs/public/administration/server-configuration.mdx b/docs/public/administration/server-configuration.mdx index 9ec040d28..c806406f7 100644 --- a/docs/public/administration/server-configuration.mdx +++ b/docs/public/administration/server-configuration.mdx @@ -92,7 +92,9 @@ level = "info" [run.model] name = "claude-sonnet-4-5" provider = "anthropic" -fallbacks = ["gemini", "openai"] + +[run.model.fallbacks] +"claude-sonnet-4-5" = ["gemini", "openai"] [[run.prepare.steps]] script = "npm install" diff --git a/docs/public/api-reference/fabro-api.yaml b/docs/public/api-reference/fabro-api.yaml index bd6a3fdb4..8eac62712 100644 --- a/docs/public/api-reference/fabro-api.yaml +++ b/docs/public/api-reference/fabro-api.yaml @@ -14315,9 +14315,15 @@ components: name: type: ["string", "null"] fallbacks: - type: array - items: - $ref: "#/components/schemas/ModelRef" + type: object + description: > + Ordered fallback targets keyed by the originally requested model. + Each chain is independent; selecting a fallback target does not + activate that target model's own chain. + additionalProperties: + type: array + items: + $ref: "#/components/schemas/ModelRef" RunGitSettings: type: object diff --git a/docs/public/core-concepts/models.mdx b/docs/public/core-concepts/models.mdx index ecd5deb07..a4befa37f 100644 --- a/docs/public/core-concepts/models.mdx +++ b/docs/public/core-concepts/models.mdx @@ -285,7 +285,9 @@ goal = "Implement the feature" [run.model] name = "claude-sonnet-4-5" -fallbacks = ["gemini", "openai"] + +[run.model.fallbacks] +"claude-sonnet-4-5" = ["gemini", "openai"] ``` Then launch with: @@ -294,7 +296,9 @@ Then launch with: fabro run run.toml ``` -The `fallbacks` array is optional. Each entry may be a bare provider token (like `"gemini"`), a bare model ID or alias (like `"gpt-terra"`), or a qualified `"provider:selector"` reference. A qualified selector may be the provider's canonical model ID, alias, or API ID, including API IDs with slashes such as `"openrouter:moonshotai/kimi-k3"`. Fabro tries entries in order when the primary provider is unavailable, and qualified references remain provider pins. Legacy `provider/model` references remain accepted for compatibility. +The `[run.model.fallbacks]` table is optional. Each key names the originally requested model. Its value is the ordered list to try after that model's active provider fails. An entry may be a bare provider token (like `"gemini"`), a bare model ID or alias (like `"gpt-terra"`), or a qualified `"provider:selector"` reference. A qualified selector may be the provider's canonical model ID, alias, or API ID, including API IDs with slashes such as `"openrouter:moonshotai/kimi-k3"`. + +Fabro selects one chain from the original request. It does not switch to the chain configured for a fallback target. When the requested reasoning level is unavailable on a fallback target, Fabro uses the nearest supported level. Equal-distance choices round up. The precedence order is: node-level stylesheet > run config TOML > CLI flags > server defaults. More specific settings always win. diff --git a/docs/public/examples/semantic-port.mdx b/docs/public/examples/semantic-port.mdx index 9593facdf..10e559805 100644 --- a/docs/public/examples/semantic-port.mdx +++ b/docs/public/examples/semantic-port.mdx @@ -214,25 +214,29 @@ The stylesheet assigns three tiers of models: Pair the workflow with a run config TOML for repeatable execution: ```toml title="run.toml" -version = 1 -goal = "Port semantic changes from upstream openai-agents-python to our Go SDK" +_version = 1 + +[workflow] graph = "semport.fabro" -[llm] -model = "claude-sonnet-4-5" +[run] +goal = "Port semantic changes from upstream openai-agents-python to our Go SDK" + +[run.model] +name = "claude-sonnet-4-5" provider = "anthropic" -[llm.fallbacks] -anthropic = ["openai"] -gemini = ["anthropic"] +[run.model.fallbacks] +"claude-sonnet-4-5" = ["openai"] +"gemini-3.1-pro-preview" = ["anthropic"] -[setup] -commands = [ - "git clone https://github.com/openai/openai-agents-python upstream || (cd upstream && git pull)", - "pip install -r ledger/requirements.txt" -] +[[run.prepare.steps]] +script = "git clone https://github.com/openai/openai-agents-python upstream || (cd upstream && git pull)" -[vars] +[[run.prepare.steps]] +script = "pip install -r ledger/requirements.txt" + +[run.inputs] upstream_repo = "openai/openai-agents-python" downstream_lang = "go" ``` diff --git a/docs/public/examples/solitaire.mdx b/docs/public/examples/solitaire.mdx index 9b6abeb86..143be3f6e 100644 --- a/docs/public/examples/solitaire.mdx +++ b/docs/public/examples/solitaire.mdx @@ -266,7 +266,9 @@ goal = "Build a terminal-based solitaire (Klondike) game in Python" [run.model] name = "claude-sonnet-4-5" provider = "anthropic" -fallbacks = ["openai", "gemini"] + +[run.model.fallbacks] +"claude-sonnet-4-5" = ["openai", "gemini"] [[run.prepare.steps]] script = "python3 -m venv .venv && . .venv/bin/activate && pip install pytest curses" diff --git a/docs/public/execution/failures.mdx b/docs/public/execution/failures.mdx index bddd0dc7b..9cdad1d1e 100644 --- a/docs/public/execution/failures.mdx +++ b/docs/public/execution/failures.mdx @@ -112,13 +112,15 @@ When a handler reports a retryable failure, retries always proceed if attempts r ## Model fallbacks -When a model provider fails with a transient error or quota exhaustion, Fabro can automatically switch to a different provider. Configure fallback chains in your [run configuration](/execution/run-configuration): +When a model provider fails with a provider-local error, Fabro can automatically switch to another target. Configure one fixed chain for each requested model in your [run configuration](/execution/run-configuration): ```toml title="run.toml" [run.model] name = "claude-opus-4-6" provider = "anthropic" -fallbacks = ["gemini", "openai"] + +[run.model.fallbacks] +"claude-opus-4-6" = ["gemini", "openai"] ``` When Anthropic fails, Fabro tries Gemini first, then OpenAI. Fallback resolution is provider-aware: @@ -129,7 +131,9 @@ When Anthropic fails, Fabro tries Gemini first, then OpenAI. Fallback resolution Qualified fallback references always remain provider pins. For example, `"openai:gpt-5.6-sol"` pins the direct OpenAI offering. Legacy `provider/model` fallback references remain accepted for compatibility. -The primary provider and model were already resolved and persisted when the run was created; resuming does not re-run primary selection. Fallbacks are only considered after an eligible runtime failure. +Fabro selects the chain by the original requested model. A target in that chain never activates the target model's own chain. The same chain position is retained across structured-output repairs and cached agent sessions. + +The primary provider and model were already resolved and persisted when the run was created; resuming does not re-run primary selection. Fallbacks are only considered after an eligible runtime failure. If the fallback model does not support the requested reasoning level, Fabro uses the nearest supported level and rounds equal-distance choices up. ### What triggers failover @@ -141,7 +145,10 @@ Failover is a superset of LLM retry eligibility: | Server error (5xx) | Yes | Yes | | Timeout / network | Yes | Yes | | Quota exceeded | No | Yes | -| Authentication (401) | No | No | +| Authentication (401) | No | Yes | +| Access denied (403) | No | Yes | +| Model not found (404) | No | Yes | +| Model refusal | No | Yes | | Invalid request (400) | No | No | | Context length (413) | No | No | | Content filter | No | No | diff --git a/docs/public/execution/run-configuration.mdx b/docs/public/execution/run-configuration.mdx index eb81c82a9..dd8819052 100644 --- a/docs/public/execution/run-configuration.mdx +++ b/docs/public/execution/run-configuration.mdx @@ -45,11 +45,13 @@ working_dir = "/tmp/workdir" [run.model] name = "claude-sonnet-4-5" -fallbacks = ["openai", "gemini"] [run.model.controls] reasoning_effort = "high" +[run.model.fallbacks] +"claude-sonnet-4-5" = ["openai", "gemini"] + [[run.prepare.steps]] script = "git clone https://github.com/fabro-sh/fabro repo" @@ -140,25 +142,43 @@ name = "claude-sonnet-4-5" |---|---| | `name` | Canonical model slug or alias (e.g. `claude-sonnet-4-5`, `opus`, `gemini-pro`). See [Models](/core-concepts/models). | | `provider` | Optional provider pin. When omitted, Fabro selects among ready offerings by provider priority. When present, an unavailable provider is an error rather than permission to switch. | -| `fallbacks` | Ordered list of model references to try when the primary is unavailable. Entries can be bare provider tokens (`"openai"`), bare model IDs or aliases, or qualified `"provider:selector"` references. | +| `fallbacks` | Table of ordered fallback lists keyed by the originally requested model. | Provider values are catalog provider ID strings. Built-in IDs like `anthropic` and `openai` work, and settings-defined IDs like `proxy` work after they are added under `[llm.providers.]`. For a qualified fallback, the selector may be that provider's canonical model ID, alias, or API ID. Fabro splits on the first `:` when the part before it names a known provider, so provider API IDs may contain `/` or additional colons: ```toml title="run.toml" -[run.model] -fallbacks = [ +[run.model.fallbacks] +"kimi-k3" = [ + "kimi:kimi-k3", "openrouter:kimi-k3", - "gpt-terra", + "claude-opus", ] ``` -The first entry could equivalently be written as `"openrouter:moonshotai/kimi-k3"` using OpenRouter's API ID; both forms resolve to its canonical `kimi-k3` offering. The unqualified `gpt-terra` alias uses normal ready-provider priority selection. Legacy `provider/model` fallback references remain accepted but are normalized to `provider:model`. +This chain applies only when the original request resolves to `kimi-k3`. It tries direct Moonshot AI, then OpenRouter, then Claude Opus. The OpenRouter entry could equivalently be written as `"openrouter:moonshotai/kimi-k3"` using its API ID. Legacy `provider/model` fallback references remain accepted but are normalized to `provider:model`. A colon alone does not make a reference qualified. Many model IDs contain one — ollama `name:tag` values, Bedrock inference-profile IDs and ARNs — so Fabro treats the reference as qualified only when the text before the first `:` names a known provider. `"llama3:8b"` stays a single model ID, while `"ollama:llama3:8b"` pins the `ollama` provider and passes `llama3:8b` as the selector. -At run creation, Fabro resolves the primary selector and every node selector against the ready-provider snapshot. It persists the selected canonical model slug and provider, so resuming the run does not choose a different provider just because credentials or priorities changed. The configured fallback chain remains available for failures that occur while the materialized run is executing. +At run creation, Fabro resolves the primary selector, every node selector, and the fallback table against the server's ready-provider snapshot. It persists the selected canonical model slug and provider, so resuming the run does not choose a different provider just because credentials or priorities changed. `fabro validate` only checks the table's TOML shape because it is offline and has no server model catalog. Use `fabro preflight` for catalog and provider checks. + +Each original requested model selects one fixed chain. Fabro does not jump to the chain configured for a fallback target. When a target lacks the requested reasoning level, Fabro uses the nearest supported level. It rounds equal-distance choices up. + +For example, a server that selects Modal as the primary `kimi-k3` offering can define these independent chains: + +```toml title="settings.toml" +[run.model.fallbacks] +"kimi-k3" = ["kimi:kimi-k3", "openrouter:kimi-k3", "claude-opus"] +"glm-5.2" = ["gpt-sol"] +"gpt-sol" = ["claude-opus"] +"claude-opus" = ["gpt-sol"] +"gpt-terra" = ["claude-opus"] +"gpt-luna" = ["claude-sonnet"] +"claude-fable" = ["gpt-sol", "claude-opus"] +``` + +If `claude-fable` falls back to `gpt-sol`, Fabro continues with `claude-opus` from the Fable list. It does not restart from the separate `gpt-sol` list. Historical built-in provider API IDs are accepted for compatibility and normalize before this selection. For example, `name = "openai/gpt-5.6-sol"` is treated as the canonical `gpt-5.6-sol` selector; omit `provider` to use readiness and priority, or set `provider` separately to pin an offering. @@ -181,14 +201,14 @@ speed = "fast" | `reasoning_effort` | Native reasoning-effort value to request when the selected model allows it, such as `"low"`, `"medium"`, `"high"`, `"xhigh"`, or `"max"`. | | `speed` | Native speed value to request when the selected model declares it, such as `"fast"`. The standard speed is implicit and does not need to be set. | -#### Fallbacks with splice +#### Fallback lists with splice -Use the reserved `"..."` marker in `fallbacks` to splice in the inherited list from lower-precedence layers: +Use the reserved `"..."` marker in one model's list to splice in that model's inherited list from lower-precedence layers: ```toml title="run.toml" -[run.model] -# Prepend "anthropic" to whatever fallbacks the project config already defines. -fallbacks = ["anthropic", "..."] +[run.model.fallbacks] +# Prepend Anthropic to the inherited chain for gpt-5.6-sol. +"gpt-5.6-sol" = ["anthropic", "..."] ``` ### `[run.prepare]` diff --git a/docs/public/reference/user-configuration.mdx b/docs/public/reference/user-configuration.mdx index e60feb204..938468731 100644 --- a/docs/public/reference/user-configuration.mdx +++ b/docs/public/reference/user-configuration.mdx @@ -381,12 +381,14 @@ permissions = "read-write" [run.model] provider = "anthropic" name = "claude-sonnet-4-5" -fallbacks = ["openrouter:kimi-k3", "gpt-terra"] + +[run.model.fallbacks] +"claude-sonnet-4-5" = ["openrouter:kimi-k3", "gpt-terra"] ``` | Key | Type / values | Default | Description | |---|---|---|---| -| `fallbacks` | array | [] | Ordered fallback references: bare providers, bare model IDs or aliases,
or provider-qualified `provider:selector` values. A qualified selector
may be a model ID, alias, or provider API ID. Legacy `provider/model`
values remain accepted. Supports the `...` splice marker at layering
time — see [`super::splice_array`]. | +| `fallbacks` | table> | {} | Model-keyed fallback chains. Each value is an ordered list of bare
providers, bare model IDs or aliases, or provider-qualified
`provider:selector` values. A qualified selector may be a model ID,
alias, or provider API ID. Legacy `provider/model` values remain
accepted. Each list supports the `...` splice marker at layering time.
Fabro selects one chain from the original requested model. A fallback
target never activates another model's chain.

Model keys stay unresolved in this sparse layer because `fabro
validate` is offline and has no server model catalog. | | `name` | string | None | Model name for workflow runs. | | `provider` | string | None | Provider name for workflow model selection. | diff --git a/lib/apps/fabro-cli/tests/it/cmd/attach.rs b/lib/apps/fabro-cli/tests/it/cmd/attach.rs index c1ff07165..bbb802be5 100644 --- a/lib/apps/fabro-cli/tests/it/cmd/attach.rs +++ b/lib/apps/fabro-cli/tests/it/cmd/attach.rs @@ -997,7 +997,7 @@ fn attach_json_errors_without_prompting_for_human_input() { "reasoning_effort": null, "speed": null }, - "fallbacks": [], + "fallbacks": {}, "name": "[DEFAULT_MODEL]", "provider": "openai" }, diff --git a/lib/apps/fabro-cli/tests/it/cmd/inspect.rs b/lib/apps/fabro-cli/tests/it/cmd/inspect.rs index 086db6b32..2aed43ef4 100644 --- a/lib/apps/fabro-cli/tests/it/cmd/inspect.rs +++ b/lib/apps/fabro-cli/tests/it/cmd/inspect.rs @@ -129,7 +129,7 @@ fn inspect_resolves_selector_via_server_endpoint() { "model": { "provider": null, "name": null, - "fallbacks": [], + "fallbacks": {}, "controls": { "reasoning_effort": null, "speed": null diff --git a/lib/apps/fabro-cli/tests/it/cmd/validate.rs b/lib/apps/fabro-cli/tests/it/cmd/validate.rs index 65048ee61..f82f5dc9e 100644 --- a/lib/apps/fabro-cli/tests/it/cmd/validate.rs +++ b/lib/apps/fabro-cli/tests/it/cmd/validate.rs @@ -87,6 +87,23 @@ fn server_owned_provider_is_not_rejected_by_offline_validation() { "); } +#[test] +fn model_fallback_table_is_not_catalog_checked_by_offline_validation() { + let cli = LightweightCli::new(); + let mut cmd = cli.command(); + cmd.env("FABRO_SERVER", "http://127.0.0.1:9") + .arg("validate") + .arg(fixture("offline-fallbacks/workflow.fabro")); + + let output = cmd.output().expect("validate should execute"); + assert!( + output.status.success(), + "offline validation should parse model fallback tables without a server catalog\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); +} + #[test] fn branching() { let context = test_context!(); diff --git a/lib/apps/fabro-server/src/run_manifest.rs b/lib/apps/fabro-server/src/run_manifest.rs index 611fd7cd8..fd9fe790c 100644 --- a/lib/apps/fabro-server/src/run_manifest.rs +++ b/lib/apps/fabro-server/src/run_manifest.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::future::Future; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -25,15 +25,18 @@ use fabro_sandbox::from_environment::{ use fabro_sandbox::redact::redact_auth_url; use fabro_sandbox::{DockerSandboxOptions, Sandbox, SandboxSpec}; use fabro_static::EnvVars; +use fabro_types::settings::ModelRef; use fabro_types::settings::cli::OutputVerbosity; use fabro_types::settings::interp::InterpString; use fabro_types::settings::run::{EnvironmentProvider, McpServerSettings, RunGoal, RunNamespace}; use fabro_types::{ - ManifestPath, RunId, RunProvenance, SandboxProviderKind, ServerSettings, WorkflowSettings, + ManifestPath, RunId, RunNoticeLevel, RunProvenance, SandboxProviderKind, ServerSettings, + WorkflowSettings, }; use fabro_util::check_report::{CheckDetail, CheckReport, CheckResult, CheckSection, CheckStatus}; use fabro_validate::Severity; use fabro_workflow::Error as WorkflowError; +use fabro_workflow::model_fallback::resolve_model_fallbacks; use fabro_workflow::operations::{ CreateRunInput, ValidateInput, WorkflowInput, validate, validate_with_catalog, validate_with_ready_providers, @@ -544,6 +547,12 @@ async fn build_preflight_report( )); } run_environment_capability_check(&mut checks, &resolved_run); + let model_fallbacks_ok = run_model_fallback_check( + &mut checks, + catalog.as_ref(), + &ready_providers, + &resolved_run.model.fallbacks, + ); let needs_github_credentials = sandbox_provider.is_clone_based() || resolved_run.integrations.github.is_token_requested(); let github_app = if needs_github_credentials { @@ -592,7 +601,8 @@ async fn build_preflight_report( let github_token_ok = run_github_token_check(&mut checks, prepared, &resolved_run, github_app).await; - let checks_ok = sandbox_ok && repository_access_ok && llm_ok && github_token_ok; + let checks_ok = + model_fallbacks_ok && sandbox_ok && repository_access_ok && llm_ok && github_token_ok; Ok(( CheckReport { @@ -606,6 +616,76 @@ async fn build_preflight_report( )) } +fn run_model_fallback_check( + checks: &mut Vec, + catalog: &Catalog, + ready_providers: &[ProviderId], + configured: &BTreeMap>, +) -> bool { + if configured.is_empty() { + return true; + } + + let eligible = ready_providers.iter().cloned().collect::>(); + let resolved = match resolve_model_fallbacks(catalog, &eligible, configured) { + Ok(resolved) => resolved, + Err(error) => { + checks.push(CheckResult { + name: "Model Fallbacks".into(), + status: CheckStatus::Error, + summary: "invalid".into(), + details: configured + .keys() + .map(|model| CheckDetail::new(format!("Requested model: {model}"))) + .collect(), + remediation: Some(error.to_string()), + }); + return false; + } + }; + + let has_warning = resolved + .notices + .iter() + .any(|notice| notice.level() != RunNoticeLevel::Info); + let mut details = resolved + .policy + .iter() + .map(|(model, targets)| { + let chain = if targets.is_empty() { + "(none)".to_string() + } else { + targets + .iter() + .map(ToString::to_string) + .collect::>() + .join(" -> ") + }; + CheckDetail::new(format!("{model}: {chain}")) + }) + .collect::>(); + details.extend(resolved.notices.iter().map(|notice| CheckDetail { + text: notice.message(), + warn: notice.level() != RunNoticeLevel::Info, + })); + + checks.push(CheckResult { + name: "Model Fallbacks".into(), + status: if has_warning { + CheckStatus::Warning + } else { + CheckStatus::Pass + }, + summary: format!( + "{} requested model chain(s)", + resolved.policy.iter().count() + ), + details, + remediation: None, + }); + true +} + fn base_preflight_checks(prepared: &PreparedManifest, graph: &Graph) -> Vec { let setup_command_count = prepared.settings.run.prepare.steps.len(); let repo_summary = prepared.git.as_ref().map_or_else( @@ -1495,6 +1575,106 @@ mod tests { Arc::new(Catalog::from_builtin().unwrap()) } + fn openrouter_catalog() -> Catalog { + let overrides = toml::from_str( + r" +[providers.openrouter] +enabled = true +", + ) + .expect("catalog override should parse"); + Catalog::from_builtin_with_overrides(&overrides).expect("catalog should build") + } + + fn model_refs(values: &[&str]) -> Vec { + values + .iter() + .map(|value| value.parse().expect("fallback reference should parse")) + .collect() + } + + #[test] + fn model_fallback_preflight_resolves_each_requested_model_chain() { + let mut checks = Vec::new(); + let configured = std::collections::BTreeMap::from([ + ("gpt-sol".to_string(), model_refs(&["claude-opus"])), + ( + "claude-fable".to_string(), + model_refs(&["gpt-sol", "claude-opus"]), + ), + ]); + + assert!(run_model_fallback_check( + &mut checks, + &openrouter_catalog(), + &[ProviderId::new("openrouter")], + &configured, + )); + + let check = checks.last().expect("fallback check should be present"); + assert_eq!(check.status, CheckStatus::Pass); + assert!( + check + .details + .iter() + .any(|detail| detail.text == "gpt-5.6-sol: openrouter:claude-opus-5") + ); + assert!(check.details.iter().any(|detail| { + detail.text == "claude-fable-5: openrouter:gpt-5.6-sol -> openrouter:claude-opus-5" + })); + } + + #[test] + fn model_fallback_preflight_warns_when_a_provider_is_not_ready() { + let mut checks = Vec::new(); + let configured = std::collections::BTreeMap::from([( + "kimi-k3".to_string(), + model_refs(&["kimi:kimi-k3", "openrouter:kimi-k3"]), + )]); + + assert!(run_model_fallback_check( + &mut checks, + &openrouter_catalog(), + &[ProviderId::new("openrouter")], + &configured, + )); + + let check = checks.last().expect("fallback check should be present"); + assert_eq!(check.status, CheckStatus::Warning); + assert!( + check + .details + .iter() + .any(|detail| detail.warn + && detail.text.contains("provider `kimi` is not configured")) + ); + } + + #[test] + fn model_fallback_preflight_rejects_duplicate_canonical_keys() { + let mut checks = Vec::new(); + let configured = std::collections::BTreeMap::from([ + ("gpt-sol".to_string(), model_refs(&["claude-opus"])), + ("gpt-5.6-sol".to_string(), model_refs(&["claude-fable"])), + ]); + + assert!(!run_model_fallback_check( + &mut checks, + &openrouter_catalog(), + &[ProviderId::new("openrouter")], + &configured, + )); + + let check = checks.last().expect("fallback check should be present"); + assert_eq!(check.status, CheckStatus::Error); + assert!( + check + .remediation + .as_deref() + .is_some_and(|message| message.contains("both resolve to requested model")) + ); + } + fn openai_compatible_completion(model: &str) -> serde_json::Value { serde_json::json!({ "id": "chatcmpl_preflight", diff --git a/lib/apps/fabro-server/tests/it/scenario/run_completion.rs b/lib/apps/fabro-server/tests/it/scenario/run_completion.rs index 8c7e5aeb1..55ba6c938 100644 --- a/lib/apps/fabro-server/tests/it/scenario/run_completion.rs +++ b/lib/apps/fabro-server/tests/it/scenario/run_completion.rs @@ -63,7 +63,7 @@ fn test_app_with_openai_agent_backend(openai_base_url: String, api_key: String) fabro_workflow::handler::llm::AgentApiBackend::new_with_catalog( OPENAI_AGENT_MODEL.to_string(), ProviderId::openai(), - Vec::new(), + fabro_workflow::model_fallback::ModelFallbackPolicy::default(), Arc::clone(&llm_source), Arc::clone(&steering_hub), Arc::clone(&catalog), diff --git a/lib/components/fabro-llm/src/error.rs b/lib/components/fabro-llm/src/error.rs index 000f36943..753c12e6f 100644 --- a/lib/components/fabro-llm/src/error.rs +++ b/lib/components/fabro-llm/src/error.rs @@ -210,8 +210,9 @@ impl Error { /// Whether this error is eligible for provider-level failover. /// /// Includes everything that is `retryable()` (transient errors good for - /// same-provider retry) plus `QuotaExceeded` — a different provider won't - /// share the same quota. + /// same-provider retry), provider-local availability failures, and + /// `QuotaExceeded`. A different provider has independent credentials, + /// access policy, model inventory, and quota. #[must_use] pub fn failover_eligible(&self) -> bool { if self.retryable() { @@ -220,7 +221,10 @@ impl Error { matches!( self, Self::Provider { - kind: ProviderErrorKind::QuotaExceeded, + kind: ProviderErrorKind::Authentication + | ProviderErrorKind::AccessDenied + | ProviderErrorKind::NotFound + | ProviderErrorKind::QuotaExceeded, .. } | Self::RequestTimeout { .. } ) || self.refusal_content_filter() @@ -887,6 +891,26 @@ mod tests { ); } + #[test] + fn failover_eligible_provider_local_availability_errors() { + let detail = || Box::new(ProviderErrorDetail::new("error", "openai")); + + for kind in [ + ProviderErrorKind::Authentication, + ProviderErrorKind::AccessDenied, + ProviderErrorKind::NotFound, + ] { + assert!( + Error::Provider { + kind, + detail: detail(), + } + .failover_eligible(), + "{kind:?} should permit another provider" + ); + } + } + #[test] fn failover_eligible_transient_non_provider_errors() { assert!( @@ -918,14 +942,6 @@ mod tests { fn failover_not_eligible_deterministic_errors() { let detail = || Box::new(ProviderErrorDetail::new("error", "openai")); - assert!( - !Error::Provider { - kind: ProviderErrorKind::Authentication, - detail: detail(), - } - .failover_eligible() - ); - assert!( !Error::Provider { kind: ProviderErrorKind::InvalidRequest, diff --git a/lib/components/fabro-manifest/src/lib.rs b/lib/components/fabro-manifest/src/lib.rs index f2f1b066b..13bae0051 100644 --- a/lib/components/fabro-manifest/src/lib.rs +++ b/lib/components/fabro-manifest/src/lib.rs @@ -76,7 +76,7 @@ pub fn build_run_overrides(input: RunOverrideInput<'_>) -> RunLayer { let model = (input.model.is_some() || input.provider.is_some()).then(|| RunModelLayer { provider: input.provider.map(String::from), name: input.model.map(String::from), - fallbacks: Vec::new(), + fallbacks: MergeMap::default(), controls: None, }); let environment = (input.environment.is_some() diff --git a/lib/components/fabro-workflow/src/event/convert.rs b/lib/components/fabro-workflow/src/event/convert.rs index c7901d08c..f6a8b4371 100644 --- a/lib/components/fabro-workflow/src/event/convert.rs +++ b/lib/components/fabro-workflow/src/event/convert.rs @@ -1147,18 +1147,28 @@ fn event_body_from_event(event: &Event) -> EventBody { }) } Event::Failover { + original_provider, + original_model, + attempt, from_provider, from_model, to_provider, to_model, + requested_reasoning_effort, + effective_reasoning_effort, error, .. } => EventBody::Failover(fabro_types::FailoverProps { - from_provider: from_provider.clone(), - from_model: from_model.clone(), - to_provider: to_provider.clone(), - to_model: to_model.clone(), - error: error.clone(), + original_provider: Some(original_provider.clone()), + original_model: Some(original_model.clone()), + attempt: Some(*attempt), + from_provider: from_provider.clone(), + from_model: from_model.clone(), + to_provider: to_provider.clone(), + to_model: to_model.clone(), + requested_reasoning_effort: requested_reasoning_effort.clone(), + effective_reasoning_effort: effective_reasoning_effort.clone(), + error: error.clone(), }), Event::CommandStarted { script, diff --git a/lib/components/fabro-workflow/src/event/events.rs b/lib/components/fabro-workflow/src/event/events.rs index 77caf70cc..596a43ee4 100644 --- a/lib/components/fabro-workflow/src/event/events.rs +++ b/lib/components/fabro-workflow/src/event/events.rs @@ -583,12 +583,17 @@ pub enum Event { ssh_command: String, }, Failover { - stage: String, + stage: String, + original_provider: String, + original_model: String, + attempt: u32, from_provider: String, - from_model: String, - to_provider: String, - to_model: String, - error: String, + from_model: String, + to_provider: String, + to_model: String, + requested_reasoning_effort: Option, + effective_reasoning_effort: Option, + error: String, }, CommandStarted { node_id: String, @@ -1395,18 +1400,28 @@ impl Event { } Self::Failover { stage, + original_provider, + original_model, + attempt, from_provider, from_model, to_provider, to_model, + requested_reasoning_effort, + effective_reasoning_effort, error, } => { warn!( stage, + original_provider, + original_model, + attempt, from_provider, from_model, to_provider, to_model, + requested_reasoning_effort, + effective_reasoning_effort, error, "LLM provider failover" ); diff --git a/lib/components/fabro-workflow/src/handler/llm/api.rs b/lib/components/fabro-workflow/src/handler/llm/api.rs index 6523b8477..4f83d3fb4 100644 --- a/lib/components/fabro-workflow/src/handler/llm/api.rs +++ b/lib/components/fabro-workflow/src/handler/llm/api.rs @@ -41,7 +41,8 @@ use super::routing::ProviderContext; use crate::context::WorkflowContext; use crate::context::keys::Fidelity; use crate::error::Error; -use crate::event::{Emitter, Event, StageScope}; +use crate::event::{Emitter, Event, RunNoticeCode, RunNoticeLevel, StageScope}; +use crate::model_fallback::{ModelFallbackPolicy, canonical_model_id}; use crate::outcome::billed_model_usage_from_llm; use crate::services::FabroRunToolServices; use crate::steering_hub::{ActiveControlHandle, SteeringHub}; @@ -633,8 +634,8 @@ fn spawn_event_forwarder( pub struct AgentApiBackend { model: String, provider_id: ProviderId, - fallback_chain: Vec, - sessions: Mutex>, + fallbacks: ModelFallbackPolicy, + sessions: Mutex>, tool_env: Option>, mcp_servers: Vec, tool_secrets: ToolSecrets, @@ -645,6 +646,45 @@ pub struct AgentApiBackend { fabro_run_tools: Option, } +struct CachedAgentSession { + session: Session, + fallback_plan: FallbackPlan, +} + +#[derive(Clone, Debug)] +struct LlmRoute { + target: FallbackTarget, + controls: EffectiveRequestControls, +} + +#[derive(Clone, Debug)] +struct FallbackPlan { + original: FallbackTarget, + requested_controls: EffectiveRequestControls, + current: LlmRoute, + remaining: Vec, + next_index: usize, +} + +struct FallbackPlanNotice { + code: RunNoticeCode, + message: String, +} + +impl FallbackPlan { + #[must_use] + fn has_next(&self) -> bool { + self.next_index < self.remaining.len() + } + + fn advance(&mut self) -> Option<(LlmRoute, u32)> { + let route = self.remaining.get(self.next_index)?.clone(); + self.next_index = self.next_index.saturating_add(1); + self.current = route.clone(); + Some((route, u32::try_from(self.next_index).unwrap_or(u32::MAX))) + } +} + struct OneShotCompletion { response: Response, model: ModelRef, @@ -655,7 +695,7 @@ impl AgentApiBackend { pub fn new( model: String, provider_id: impl Into, - fallback_chain: Vec, + fallbacks: ModelFallbackPolicy, source: Arc, steering_hub: Arc, ) -> Self { @@ -663,7 +703,7 @@ impl AgentApiBackend { Self::new_with_catalog( model, provider_id.into(), - fallback_chain, + fallbacks, source, steering_hub, catalog, @@ -674,7 +714,7 @@ impl AgentApiBackend { pub fn new_with_catalog( model: String, provider_id: ProviderId, - fallback_chain: Vec, + fallbacks: ModelFallbackPolicy, source: Arc, steering_hub: Arc, catalog: Arc, @@ -682,7 +722,7 @@ impl AgentApiBackend { Self { model, provider_id, - fallback_chain, + fallbacks, sessions: Mutex::new(HashMap::new()), tool_env: None, mcp_servers: Vec::new(), @@ -751,12 +791,155 @@ impl AgentApiBackend { ) } + fn fallback_controls_for_target( + &self, + target: &FallbackTarget, + requested: EffectiveRequestControls, + ) -> Option { + let requested_effort = requested.reasoning_effort?; + let provider = ProviderId::new(&target.provider); + let offering = self.catalog.get_on_provider(&provider, &target.model)?; + let settings = self.catalog.settings_for(offering)?; + let effective_effort = + requested_effort.closest_supported(&settings.controls.reasoning_effort)?; + Some(EffectiveRequestControls { + reasoning_effort: Some(effective_effort), + speed: requested.speed, + }) + } + + fn fallback_plan( + &self, + model: &str, + provider: &ProviderId, + requested_controls: EffectiveRequestControls, + ) -> (FallbackPlan, Vec) { + let primary_model = canonical_model_id(self.catalog.as_ref(), provider, model); + let original = FallbackTarget::new(provider, primary_model); + let current = LlmRoute { + target: original.clone(), + controls: requested_controls, + }; + let Some(configured) = self + .fallbacks + .chain_for(self.catalog.as_ref(), provider, model) + else { + return ( + FallbackPlan { + original, + requested_controls, + current, + remaining: Vec::new(), + next_index: 0, + }, + Vec::new(), + ); + }; + + let mut remaining = Vec::new(); + let mut notices = Vec::new(); + for target in configured { + if target == &original + || remaining + .iter() + .any(|route: &LlmRoute| route.target == *target) + { + continue; + } + + let controls = match requested_controls.reasoning_effort { + None => requested_controls, + Some(requested_effort) => { + match self.fallback_controls_for_target(target, requested_controls) { + Some(controls) => controls, + None if self + .catalog + .get_on_provider(&ProviderId::new(&target.provider), &target.model) + .is_none() => + { + // A catalog-unknown passthrough target has no + // advertised controls. Preserve the request and let + // the provider validate it. + requested_controls + } + None => { + notices.push(FallbackPlanNotice { + code: RunNoticeCode::ModelFallbackSkipped, + message: format!( + "Model fallback `{target}` for requested model `{}` was skipped because it has no reasoning level near `{requested_effort}`.", + original.model + ), + }); + continue; + } + } + } + }; + remaining.push(LlmRoute { + target: target.clone(), + controls, + }); + } + + if !configured.is_empty() && remaining.is_empty() { + notices.push(FallbackPlanNotice { + code: RunNoticeCode::ModelFallbackChainEmpty, + message: format!( + "No usable model fallbacks remain for requested model `{}` and primary target `{original}`.", + original.model + ), + }); + } + + ( + FallbackPlan { + original, + requested_controls, + current, + remaining, + next_index: 0, + }, + notices, + ) + } + + fn emit_fallback_plan_notices( + notices: &[FallbackPlanNotice], + emitter: &Emitter, + stage_scope: &StageScope, + ) { + for notice in notices { + emitter.emit_scoped( + &Event::RunNotice { + level: RunNoticeLevel::Warn, + code: notice.code.to_string(), + message: notice.message.clone(), + exec_output_tail: None, + }, + stage_scope, + ); + } + } + + #[cfg(test)] async fn create_session( &self, node: &Node, sandbox: &Arc, tool_hooks: Option>, ) -> Result { + let (cached, _) = self + .create_session_with_plan(node, sandbox, tool_hooks) + .await?; + Ok(cached.session) + } + + async fn create_session_with_plan( + &self, + node: &Node, + sandbox: &Arc, + tool_hooks: Option>, + ) -> Result<(CachedAgentSession, Vec), Error> { let model = node.model().unwrap_or(&self.model); let provider = routing::resolve_node_provider_context( self.catalog.as_ref(), @@ -764,38 +947,49 @@ impl AgentApiBackend { &self.model, node, )?; - Self::create_session_for( - model, - provider, + let controls = self.resolve_effective_request_controls(node)?; + let (fallback_plan, notices) = self.fallback_plan(model, &provider.provider_id, controls); + let route = &fallback_plan.current; + let route_provider = + self.resolve_provider_context(&route.target.model, Some(&route.target.provider))?; + let session = Self::create_session_for( + &route.target.model, + route_provider, + controls, node, sandbox, self.source.as_ref(), Arc::clone(&self.catalog), - &self.run_model_controls, self.tool_env.as_ref(), tool_hooks, self.mcp_servers.clone(), self.tool_secrets.clone(), self.fabro_run_tools.clone(), ) - .await + .await?; + Ok(( + CachedAgentSession { + session, + fallback_plan, + }, + notices, + )) } async fn create_session_for( model: &str, provider: ProviderContext, + controls: EffectiveRequestControls, node: &Node, sandbox: &Arc, source: &dyn CredentialSource, catalog: Arc, - run_model_controls: &RunModelControls, tool_env: Option<&Arc>, tool_hooks: Option>, mcp_servers: Vec, tool_secrets: ToolSecrets, fabro_run_tools: Option, ) -> Result { - let controls = effective_request_controls(run_model_controls, node)?; let client = Client::from_source(source, Arc::clone(&catalog)) .await .map_err(|e| Error::handler_with_source("Failed to create LLM client", e))?; @@ -931,15 +1125,188 @@ impl AgentApiBackend { Ok(lease) } + #[expect( + clippy::too_many_arguments, + reason = "failover owns one agent invocation's live session, event bridge, accounting, and cancellation state" + )] + async fn failover_agent_session( + &self, + fallback_plan: &mut FallbackPlan, + initial_error: fabro_llm::Error, + node: &Node, + input: &str, + agent_tool_runtime: fabro_agent::AgentToolRuntime, + sandbox: &Arc, + tool_hooks: Option>, + cancel_token: &CancellationToken, + emitter: &Arc, + stage_scope: &StageScope, + stage_id: &StageId, + thread_id: Option<&str>, + file_tracking: &Arc>, + session: &mut Session, + bridge: &mut SessionCancelBridgeGuard, + lease: &mut Option>, + event_forwarder: &mut EventForwarder, + total_usage: &mut TokenCounts, + total_cost: &mut Option, + inference_duration: &mut Duration, + tool_duration: &mut Duration, + ) -> Result<(), Error> { + let mut error_message = initial_error.to_string(); + let mut last_error = Error::Llm(initial_error); + + bridge.abort(); + discard_session(session, lease, event_forwarder, emitter).await; + event_forwarder.abort(); + + while fallback_plan.has_next() { + let from = fallback_plan.current.clone(); + let Some((route, attempt)) = fallback_plan.advance() else { + break; + }; + Self::emit_failover( + node, + emitter, + stage_scope, + fallback_plan, + &from, + &route, + attempt, + &error_message, + ); + + let target_provider = match self + .resolve_provider_context(&route.target.model, Some(&route.target.provider)) + { + Ok(provider) => provider, + Err(error) => { + error_message = error.to_string(); + last_error = error; + continue; + } + }; + + if cancel_token.is_cancelled() { + return Err(Error::Cancelled); + } + let new_session = Self::create_session_for( + &route.target.model, + target_provider, + route.controls, + node, + sandbox, + self.source.as_ref(), + Arc::clone(&self.catalog), + self.tool_env.as_ref(), + tool_hooks.clone(), + self.mcp_servers.clone(), + self.tool_secrets.clone(), + self.fabro_run_tools.clone(), + ) + .await; + if cancel_token.is_cancelled() { + return Err(Error::Cancelled); + } + *session = match new_session { + Ok(session) => session, + Err(error) => { + error_message = error.to_string(); + last_error = error; + continue; + } + }; + bridge.replace(cancel_token.clone(), session); + *event_forwarder = spawn_event_forwarder( + session, + node.id.clone(), + stage_scope.clone(), + Arc::clone(emitter), + Arc::clone(file_tracking), + ); + + begin_session_lifecycle(session, emitter, None); + if let Err(error) = session.initialize().await { + match classify_agent_error(error, fallback_plan.has_next()) { + AgentApiErrorDisposition::Cancelled => { + bridge.abort(); + discard_session(session, lease, event_forwarder, emitter).await; + return Err(Error::Cancelled); + } + AgentApiErrorDisposition::Terminal(error) => { + bridge.abort(); + discard_session(session, lease, event_forwarder, emitter).await; + return Err(error); + } + AgentApiErrorDisposition::FailoverEligible(error) => { + error_message = error.to_string(); + last_error = Error::Llm(error); + bridge.abort(); + discard_session(session, lease, event_forwarder, emitter).await; + event_forwarder.abort(); + continue; + } + } + } + + match self.attach_session_to_hub(session, stage_id, thread_id, emitter) { + Ok(active_lease) => *lease = Some(active_lease), + Err(error) => { + bridge.abort(); + discard_session(session, lease, event_forwarder, emitter).await; + return Err(error); + } + } + emit_agent_tools_available(session, &node.id, stage_id, emitter); + + let process_result = session + .process_input_with_runtime(input, agent_tool_runtime.clone()) + .await; + let timing = session.last_input_timing(); + *inference_duration = inference_duration.saturating_add(timing.inference); + *tool_duration = tool_duration.saturating_add(timing.tool); + match process_result { + Ok(()) => { + event_forwarder.wait_for_processing_end().await; + *total_usage += session.last_input_usage(); + UsdMicros::accumulate(total_cost, session.last_input_cost()); + return Ok(()); + } + Err(error) => match classify_agent_error(error, fallback_plan.has_next()) { + AgentApiErrorDisposition::Cancelled => { + bridge.abort(); + discard_session(session, lease, event_forwarder, emitter).await; + return Err(Error::Cancelled); + } + AgentApiErrorDisposition::Terminal(error) => { + bridge.abort(); + discard_session(session, lease, event_forwarder, emitter).await; + return Err(error); + } + AgentApiErrorDisposition::FailoverEligible(error) => { + error_message = error.to_string(); + last_error = Error::Llm(error); + bridge.abort(); + discard_session(session, lease, event_forwarder, emitter).await; + event_forwarder.abort(); + } + }, + } + } + + Err(last_error) + } + async fn shutdown_cached_sessions(&self, emitter: &Arc) { - let sessions: Vec = self + let sessions: Vec = self .sessions .lock() .expect("sessions mutex is never poisoned: no code panics while holding this lock") .drain() .map(|(_, s)| s) .collect(); - for mut session in sessions { + for cached in sessions { + let mut session = cached.session; let session_id = session.id().to_string(); if session.shutdown(SessionShutdownReason::Completed).await { emitter.emit(&Event::AgentSessionEnded { @@ -950,6 +1317,40 @@ impl AgentApiBackend { } } + fn emit_failover( + node: &Node, + emitter: &Emitter, + stage_scope: &StageScope, + plan: &FallbackPlan, + from: &LlmRoute, + to: &LlmRoute, + attempt: u32, + error: &str, + ) { + emitter.emit_scoped( + &Event::Failover { + stage: node.id.clone(), + original_provider: plan.original.provider.clone(), + original_model: plan.original.model.clone(), + attempt, + from_provider: from.target.provider.clone(), + from_model: from.target.model.clone(), + to_provider: to.target.provider.clone(), + to_model: to.target.model.clone(), + requested_reasoning_effort: plan + .requested_controls + .reasoning_effort + .map(|effort| effort.to_string()), + effective_reasoning_effort: to + .controls + .reasoning_effort + .map(|effort| effort.to_string()), + error: error.to_string(), + }, + stage_scope, + ); + } + async fn complete_one_shot_request( &self, client: &Client, @@ -957,90 +1358,57 @@ impl AgentApiBackend { emitter: &Arc, stage_scope: &StageScope, request: &Request, - controls: EffectiveRequestControls, - fallback_chain: &[FallbackTarget], + plan: &mut FallbackPlan, ) -> Result { - let result = client.complete(request).await; - let default_provider = self.provider_id.to_string(); - - let (response, model) = match result { - Ok(resp) => (resp, ModelRef { - provider: ProviderId::from( - request - .provider - .clone() - .unwrap_or_else(|| default_provider.clone()), - ), - model_id: request.model.clone().into(), - speed: controls.speed, - }), - Err(sdk_err) if sdk_err.failover_eligible() && !fallback_chain.is_empty() => { - let error_msg = sdk_err.to_string(); - let from_provider = request - .provider - .clone() - .unwrap_or_else(|| default_provider.clone()); - let from_model = request.model.clone(); - - let mut last_err = sdk_err; - let mut found = None; - - for target in fallback_chain { - emitter.emit_scoped( - &Event::Failover { - stage: node.id.clone(), - from_provider: from_provider.clone(), - from_model: from_model.clone(), - to_provider: target.provider.clone(), - to_model: target.model.clone(), - error: error_msg.clone(), + let mut active_request = request.clone(); + loop { + match client.complete(&active_request).await { + Ok(response) => { + return Ok(OneShotCompletion { + response, + model: ModelRef { + provider: ProviderId::from(plan.current.target.provider.clone()), + model_id: plan.current.target.model.clone().into(), + speed: plan.current.controls.speed, }, + }); + } + Err(error) if error.failover_eligible() && plan.has_next() => { + let error_message = error.to_string(); + let from = plan.current.clone(); + let Some((to, attempt)) = plan.advance() else { + return Err(Error::Llm(error)); + }; + Self::emit_failover( + node, + emitter, stage_scope, + plan, + &from, + &to, + attempt, + &error_message, ); - let max_tokens = node.max_tokens().or_else(|| { self.catalog - .get_on_provider(&ProviderId::new(&target.provider), &target.model) + .get_on_provider( + &ProviderId::new(&to.target.provider), + &to.target.model, + ) .and_then(|model| model.limits.max_output) }); - - let fallback_request = Request { - model: target.model.clone(), - provider: Some(target.provider.clone()), + active_request = Request { + model: to.target.model, + provider: Some(to.target.provider), max_tokens, - reasoning_effort: controls.reasoning_effort, - speed: controls.speed, - ..request.clone() + reasoning_effort: to.controls.reasoning_effort, + speed: to.controls.speed, + ..active_request }; - - match client.complete(&fallback_request).await { - Ok(resp) => { - found = Some(OneShotCompletion { - response: resp, - model: ModelRef { - provider: ProviderId::from(target.provider.clone()), - model_id: target.model.clone().into(), - speed: controls.speed, - }, - }); - break; - } - Err(err) if err.failover_eligible() => { - last_err = err; - } - Err(err) => return Err(Error::Llm(err)), - } - } - - match found { - Some(completion) => return Ok(completion), - None => return Err(Error::Llm(last_err)), } + Err(error) => return Err(Error::Llm(error)), } - Err(sdk_err) => return Err(Error::Llm(sdk_err)), - }; - - Ok(OneShotCompletion { response, model }) + } } } @@ -1067,14 +1435,10 @@ impl CodergenBackend for AgentApiBackend { let model = node.model().unwrap_or(&self.model); let provider = self.resolve_provider_context(model, node.provider())?; - let provider_id = provider.provider_id.to_string(); let controls = self.resolve_effective_request_controls(node)?; - - let max_tokens = node.max_tokens().or_else(|| { - self.catalog - .get_on_provider(&provider.provider_id, model) - .and_then(|model| model.limits.max_output) - }); + let (mut fallback_plan, notices) = + self.fallback_plan(model, &provider.provider_id, controls); + Self::emit_fallback_plan_notices(¬ices, emitter, stage_scope); let mut messages = Vec::new(); if let Some(sys) = system_prompt { @@ -1082,14 +1446,6 @@ impl CodergenBackend for AgentApiBackend { } messages.push(Message::user(prompt)); - // Build per-request fallback chain: if the node overrides the provider, - // no failover is available; otherwise use the backend's. - let fallback_chain: &[FallbackTarget] = if node.provider().is_some() { - &[] - } else { - &self.fallback_chain - }; - let output_schema = structured_output::parse_node_output_schema(node)?; let response_format = output_schema .as_ref() @@ -1100,12 +1456,21 @@ impl CodergenBackend for AgentApiBackend { let mut inference_duration = Duration::ZERO; loop { + let route = &fallback_plan.current; + let max_tokens = node.max_tokens().or_else(|| { + self.catalog + .get_on_provider( + &ProviderId::new(&route.target.provider), + &route.target.model, + ) + .and_then(|model| model.limits.max_output) + }); let request = Request { - model: model.to_string(), + model: route.target.model.clone(), messages: messages.clone(), - provider: Some(provider_id.clone()), - reasoning_effort: controls.reasoning_effort, - speed: controls.speed, + provider: Some(route.target.provider.clone()), + reasoning_effort: route.controls.reasoning_effort, + speed: route.controls.speed, tools: None, tool_choice: None, response_format: response_format.clone(), @@ -1125,8 +1490,7 @@ impl CodergenBackend for AgentApiBackend { emitter, stage_scope, &request, - controls, - fallback_chain, + &mut fallback_plan, ) .await; inference_duration = inference_duration.saturating_add(inference_start.elapsed()); @@ -1206,28 +1570,38 @@ impl CodergenBackend for AgentApiBackend { if cancel_token.is_cancelled() { return Err(Error::Cancelled); } - let (mut session, is_reused) = if let Some(ref key) = reuse_key { + let (cached, is_reused, fallback_notices) = if let Some(ref key) = reuse_key { let existing = self .sessions .lock() .expect("sessions mutex is never poisoned: no code panics while holding this lock") .remove(key); - if let Some(s) = existing { - (s, true) + if let Some(cached) = existing { + (cached, true, Vec::new()) } else { - let created = self.create_session(node, sandbox, tool_hooks.clone()).await; + let created = self + .create_session_with_plan(node, sandbox, tool_hooks.clone()) + .await; if cancel_token.is_cancelled() { return Err(Error::Cancelled); } - (created?, false) + let (cached, notices) = created?; + (cached, false, notices) } } else { - let created = self.create_session(node, sandbox, tool_hooks.clone()).await; + let created = self + .create_session_with_plan(node, sandbox, tool_hooks.clone()) + .await; if cancel_token.is_cancelled() { return Err(Error::Cancelled); } - (created?, false) + let (cached, notices) = created?; + (cached, false, notices) }; + let CachedAgentSession { + mut session, + mut fallback_plan, + } = cached; if cancel_token.is_cancelled() { return Err(Error::Cancelled); } @@ -1247,6 +1621,7 @@ impl CodergenBackend for AgentApiBackend { last: None, })); let stage_scope = StageScope::for_handler(context, &node.id); + Self::emit_fallback_plan_notices(&fallback_notices, emitter, &stage_scope); // Subscribe to session events: forward to pipeline emitter + track files. let mut event_forwarder = spawn_event_forwarder( @@ -1269,7 +1644,7 @@ impl CodergenBackend for AgentApiBackend { let stage_id = stage_scope.stage_id(); let mut lease: Option> = None; - let allow_failover_primary = !self.fallback_chain.is_empty(); + let allow_failover_primary = fallback_plan.has_next(); let init_result = if is_reused { Ok(()) } else { @@ -1332,7 +1707,8 @@ impl CodergenBackend for AgentApiBackend { Err(err) => Err(err), }; - // On failover-eligible error, try fallback providers. + // On a provider-local failure, continue the fixed fallback plan that + // belongs to the originally requested model. let result: Result<(), Error> = match result { Ok(()) => Ok(()), Err(err) => match classify_agent_error(err, allow_failover_primary) { @@ -1347,189 +1723,30 @@ impl CodergenBackend for AgentApiBackend { return Err(err); } AgentApiErrorDisposition::FailoverEligible(sdk_err) => { - let error_msg = sdk_err.to_string(); - let from_provider = self.provider_id.to_string(); - let from_model = self.model.clone(); - - let mut last_err = Error::Llm(sdk_err); - let mut succeeded = false; - - bridge.abort(); - discard_session(&mut session, &mut lease, &mut event_forwarder, emitter).await; - event_forwarder.abort(); - - for (index, target) in self.fallback_chain.iter().enumerate() { - emitter.emit_scoped( - &Event::Failover { - stage: node.id.clone(), - from_provider: from_provider.clone(), - from_model: from_model.clone(), - to_provider: target.provider.clone(), - to_model: target.model.clone(), - error: error_msg.clone(), - }, - &stage_scope, - ); - - let Ok(target_provider) = - self.resolve_provider_context(&target.model, Some(&target.provider)) - else { - continue; - }; - - if cancel_token.is_cancelled() { - return Err(Error::Cancelled); - } - let new_session_result = Self::create_session_for( - &target.model, - target_provider, - node, - sandbox, - self.source.as_ref(), - Arc::clone(&self.catalog), - &self.run_model_controls, - self.tool_env.as_ref(), - tool_hooks.clone(), - self.mcp_servers.clone(), - self.tool_secrets.clone(), - self.fabro_run_tools.clone(), - ) - .await; - if cancel_token.is_cancelled() { - return Err(Error::Cancelled); - } - let new_session = match new_session_result { - Ok(s) => s, - Err(e) => { - last_err = e; - continue; - } - }; - session = new_session; - bridge.replace(cancel_token.clone(), &session); - - // Re-subscribe to forward events + track files from the new session - event_forwarder = spawn_event_forwarder( - &session, - node.id.clone(), - stage_scope.clone(), - Arc::clone(emitter), - Arc::clone(&file_tracking), - ); - - let allow_failover_next = index + 1 < self.fallback_chain.len(); - begin_session_lifecycle(&session, emitter, None); - if let Err(err) = session.initialize().await { - match classify_agent_error(err, allow_failover_next) { - AgentApiErrorDisposition::Cancelled => { - bridge.abort(); - discard_session( - &mut session, - &mut lease, - &mut event_forwarder, - emitter, - ) - .await; - return Err(Error::Cancelled); - } - AgentApiErrorDisposition::Terminal(err) => { - bridge.abort(); - discard_session( - &mut session, - &mut lease, - &mut event_forwarder, - emitter, - ) - .await; - return Err(err); - } - AgentApiErrorDisposition::FailoverEligible(sdk_err) => { - last_err = Error::Llm(sdk_err); - bridge.abort(); - discard_session( - &mut session, - &mut lease, - &mut event_forwarder, - emitter, - ) - .await; - continue; - } - } - } - match self.attach_session_to_hub( - &mut session, - &stage_id, - thread_id, - emitter, - ) { - Ok(active_lease) => lease = Some(active_lease), - Err(err) => { - bridge.abort(); - discard_session( - &mut session, - &mut lease, - &mut event_forwarder, - emitter, - ) - .await; - return Err(err); - } - } - emit_agent_tools_available(&session, &node.id, &stage_id, emitter); - let process_result = session - .process_input_with_runtime(prompt, agent_tool_runtime.clone()) - .await; - let timing = session.last_input_timing(); - inference_duration = inference_duration.saturating_add(timing.inference); - tool_duration = tool_duration.saturating_add(timing.tool); - match process_result { - Ok(()) => { - event_forwarder.wait_for_processing_end().await; - total_usage += session.last_input_usage(); - UsdMicros::accumulate(&mut total_cost, session.last_input_cost()); - succeeded = true; - break; - } - Err(err) => match classify_agent_error(err, allow_failover_next) { - AgentApiErrorDisposition::Cancelled => { - bridge.abort(); - discard_session( - &mut session, - &mut lease, - &mut event_forwarder, - emitter, - ) - .await; - return Err(Error::Cancelled); - } - AgentApiErrorDisposition::Terminal(err) => { - bridge.abort(); - discard_session( - &mut session, - &mut lease, - &mut event_forwarder, - emitter, - ) - .await; - return Err(err); - } - AgentApiErrorDisposition::FailoverEligible(sdk_err) => { - last_err = Error::Llm(sdk_err); - bridge.abort(); - discard_session( - &mut session, - &mut lease, - &mut event_forwarder, - emitter, - ) - .await; - } - }, - } - } - - if succeeded { Ok(()) } else { Err(last_err) } + self.failover_agent_session( + &mut fallback_plan, + sdk_err, + node, + prompt, + agent_tool_runtime.clone(), + sandbox, + tool_hooks.clone(), + &cancel_token, + emitter, + &stage_scope, + &stage_id, + thread_id, + &file_tracking, + &mut session, + &mut bridge, + &mut lease, + &mut event_forwarder, + &mut total_usage, + &mut total_cost, + &mut inference_duration, + &mut tool_duration, + ) + .await } }, }; @@ -1588,7 +1805,7 @@ impl CodergenBackend for AgentApiBackend { repair_attempts += 1; response = last_assistant_response(&session); } - Err(err) => match classify_agent_error(err, false) { + Err(err) => match classify_agent_error(err, fallback_plan.has_next()) { AgentApiErrorDisposition::Cancelled => { bridge.abort(); discard_session( @@ -1612,15 +1829,31 @@ impl CodergenBackend for AgentApiBackend { return Err(err); } AgentApiErrorDisposition::FailoverEligible(sdk_err) => { - bridge.abort(); - discard_session( + self.failover_agent_session( + &mut fallback_plan, + sdk_err, + node, + prompt, + agent_tool_runtime.clone(), + sandbox, + tool_hooks.clone(), + &cancel_token, + emitter, + &stage_scope, + &stage_id, + thread_id, + &file_tracking, &mut session, + &mut bridge, &mut lease, &mut event_forwarder, - emitter, + &mut total_usage, + &mut total_cost, + &mut inference_duration, + &mut tool_duration, ) - .await; - return Err(Error::Llm(sdk_err)); + .await?; + response = last_assistant_response(&session); } }, } @@ -1629,13 +1862,12 @@ impl CodergenBackend for AgentApiBackend { } } - let billing_controls = self.resolve_effective_request_controls(node)?; let stage_usage = billed_model_usage_from_llm( self.catalog.as_ref(), &ModelRef { provider: session.provider_id(), model_id: session.model().into(), - speed: billing_controls.speed, + speed: session.speed(), }, &total_usage, )? @@ -1653,7 +1885,10 @@ impl CodergenBackend for AgentApiBackend { self.sessions .lock() .expect("sessions mutex is never poisoned: no code panics while holding this lock") - .insert(key, session); + .insert(key, CachedAgentSession { + session, + fallback_plan, + }); } else { bridge.abort(); let session_id = session.id().to_string(); @@ -1894,6 +2129,20 @@ reasoning = false Arc::new(Catalog::from_builtin_with_overrides(&settings).unwrap()) } + fn enabled_fallback_catalog() -> Arc { + let settings: LlmCatalogSettings = toml::from_str( + r" +[providers.modal] +enabled = true + +[providers.openrouter] +enabled = true +", + ) + .expect("fallback catalog overrides should parse"); + Arc::new(Catalog::from_builtin_with_overrides(&settings).unwrap()) + } + fn mock_api_backend(server: &MockServer) -> AgentApiBackend { let source = auth_test_support::env_credential_source(|name| { if name == "MOCK_API_KEY" { @@ -1905,13 +2154,85 @@ reasoning = false AgentApiBackend::new_with_catalog( "mock-model".to_string(), ProviderId::from("mock"), - Vec::new(), + ModelFallbackPolicy::default(), source, SteeringHub::for_tests(), mock_llm_catalog(server), ) } + fn fallback_api_backend(server: &MockServer) -> AgentApiBackend { + let settings: LlmCatalogSettings = toml::from_str(&format!( + r#" +[providers.primary] +adapter = "openai_compatible" +agent_profile = "openai" +base_url = "{}/primary" + +[providers.primary.auth] +credentials = ["env:PRIMARY_API_KEY"] + +[providers.primary.models.test-model] +display_name = "Primary Test Model" +family = "test" +default = true + +[providers.primary.models.test-model.limits] +context_window = 8192 +max_output = 1024 + +[providers.primary.models.test-model.features] +tools = true +vision = false +reasoning = false + +[providers.fallback] +adapter = "openai_compatible" +agent_profile = "openai" +base_url = "{}/fallback" + +[providers.fallback.auth] +credentials = ["env:FALLBACK_API_KEY"] + +[providers.fallback.models.test-model] +display_name = "Fallback Test Model" +family = "test" +default = true + +[providers.fallback.models.test-model.limits] +context_window = 8192 +max_output = 1024 + +[providers.fallback.models.test-model.features] +tools = true +vision = false +reasoning = false +"#, + server.base_url(), + server.base_url(), + )) + .expect("fallback catalog should parse"); + let catalog = Arc::new( + Catalog::from_builtin_with_overrides(&settings).expect("catalog should build"), + ); + let source = auth_test_support::env_credential_source(|name| match name { + "PRIMARY_API_KEY" | "FALLBACK_API_KEY" => Some("sk-test".to_string()), + _ => None, + }); + let policy = ModelFallbackPolicy::new(std::collections::BTreeMap::from([( + "test-model".to_string(), + vec![FallbackTarget::new("fallback", "test-model")], + )])); + AgentApiBackend::new_with_catalog( + "test-model".to_string(), + ProviderId::new("primary"), + policy, + source, + SteeringHub::for_tests(), + catalog, + ) + } + fn chat_completion_response( text: &str, input_tokens: i64, @@ -2007,7 +2328,7 @@ reasoning = false let backend = AgentApiBackend::new( "claude-opus-4-6".to_string(), ProviderId::openai(), - Vec::new(), + ModelFallbackPolicy::default(), auth_test_support::vault_only_credential_source(), SteeringHub::for_tests(), ); @@ -2020,7 +2341,7 @@ reasoning = false let backend = AgentApiBackend::new( "claude-opus-4-6".to_string(), ProviderId::anthropic(), - Vec::new(), + ModelFallbackPolicy::default(), auth_test_support::vault_only_credential_source(), SteeringHub::for_tests(), ); @@ -2850,7 +3171,7 @@ enabled = true let backend = AgentApiBackend::new_with_catalog( "gpt-5.4".to_string(), ProviderId::from("openrouter"), - Vec::new(), + ModelFallbackPolicy::default(), auth_test_support::vault_only_credential_source(), SteeringHub::for_tests(), Arc::new(Catalog::from_builtin_with_overrides(&settings).unwrap()), @@ -2866,7 +3187,7 @@ enabled = true let backend = AgentApiBackend::new_with_catalog( "gpt-5.4".to_string(), ProviderId::from("openrouter"), - Vec::new(), + ModelFallbackPolicy::default(), auth_test_support::vault_only_credential_source(), SteeringHub::for_tests(), Arc::new(Catalog::from_builtin().unwrap()), @@ -2913,7 +3234,7 @@ reasoning = false let backend = AgentApiBackend::new_with_catalog( "acme-llama".to_string(), ProviderId::from("acme"), - Vec::new(), + ModelFallbackPolicy::default(), auth_test_support::vault_only_credential_source(), SteeringHub::for_tests(), catalog, @@ -2960,7 +3281,7 @@ reasoning = false let backend = AgentApiBackend::new_with_catalog( "acme-claude".to_string(), ProviderId::from("acme"), - Vec::new(), + ModelFallbackPolicy::default(), auth_test_support::vault_only_credential_source(), SteeringHub::for_tests(), catalog, @@ -2977,7 +3298,7 @@ reasoning = false let backend = AgentApiBackend::new_with_catalog( "claude-sonnet-5".to_string(), ProviderId::anthropic(), - Vec::new(), + ModelFallbackPolicy::default(), auth_test_support::vault_only_credential_source(), SteeringHub::for_tests(), Arc::new(Catalog::from_builtin().unwrap()), @@ -3004,7 +3325,7 @@ enabled = true let backend = AgentApiBackend::new_with_catalog( "openai/gpt-5.4".to_string(), ProviderId::from("openrouter"), - Vec::new(), + ModelFallbackPolicy::default(), auth_test_support::vault_only_credential_source(), SteeringHub::for_tests(), catalog, @@ -3023,7 +3344,7 @@ enabled = true let backend = AgentApiBackend::new( "gpt-5.4".to_string(), ProviderId::openai(), - Vec::new(), + ModelFallbackPolicy::default(), auth_test_support::vault_only_credential_source(), SteeringHub::for_tests(), ) @@ -3044,7 +3365,7 @@ enabled = true let backend = AgentApiBackend::new( "gpt-5.4".to_string(), ProviderId::openai(), - Vec::new(), + ModelFallbackPolicy::default(), auth_test_support::vault_only_credential_source(), SteeringHub::for_tests(), ) @@ -3073,7 +3394,7 @@ enabled = true let backend = AgentApiBackend::new( "gpt-5.4".to_string(), ProviderId::openai(), - Vec::new(), + ModelFallbackPolicy::default(), auth_test_support::vault_only_credential_source(), SteeringHub::for_tests(), ); @@ -3084,6 +3405,86 @@ enabled = true assert_eq!(controls.reasoning_effort, None); } + #[test] + fn fallback_plan_maps_reasoning_to_each_target_and_rounds_ties_up() { + let policy = ModelFallbackPolicy::new(std::collections::BTreeMap::from([( + "kimi-k3".to_string(), + vec![ + FallbackTarget::new("kimi", "kimi-k3"), + FallbackTarget::new("openrouter", "kimi-k3"), + FallbackTarget::new("anthropic", "claude-opus-5"), + ], + )])); + let backend = AgentApiBackend::new_with_catalog( + "kimi-k3".to_string(), + ProviderId::new("modal"), + policy, + auth_test_support::vault_only_credential_source(), + SteeringHub::for_tests(), + enabled_fallback_catalog(), + ); + + let (plan, notices) = backend.fallback_plan( + "kimi-k3", + &ProviderId::new("modal"), + EffectiveRequestControls { + reasoning_effort: Some(ReasoningEffort::Medium), + speed: None, + }, + ); + + assert!(notices.is_empty()); + assert_eq!( + plan.remaining + .iter() + .map(|route| route.controls.reasoning_effort) + .collect::>(), + vec![ + Some(ReasoningEffort::High), + Some(ReasoningEffort::High), + Some(ReasoningEffort::Medium), + ] + ); + } + + #[test] + fn advancing_a_fallback_plan_never_activates_the_target_models_chain() { + let policy = ModelFallbackPolicy::new(std::collections::BTreeMap::from([ + ("claude-fable-5".to_string(), vec![ + FallbackTarget::new("openai", "gpt-5.6-sol"), + FallbackTarget::new("anthropic", "claude-opus-5"), + ]), + ("gpt-5.6-sol".to_string(), vec![FallbackTarget::new( + "anthropic", + "claude-sonnet-5", + )]), + ])); + let backend = AgentApiBackend::new_with_catalog( + "claude-fable-5".to_string(), + ProviderId::anthropic(), + policy, + auth_test_support::vault_only_credential_source(), + SteeringHub::for_tests(), + enabled_fallback_catalog(), + ); + let (mut plan, notices) = backend.fallback_plan( + "claude-fable-5", + &ProviderId::anthropic(), + EffectiveRequestControls::default(), + ); + + assert!(notices.is_empty()); + let (sol, first_attempt) = plan.advance().expect("Sol should be first"); + let (opus, second_attempt) = plan.advance().expect("Opus should be second"); + assert_eq!(sol.target, FallbackTarget::new("openai", "gpt-5.6-sol")); + assert_eq!( + opus.target, + FallbackTarget::new("anthropic", "claude-opus-5") + ); + assert_eq!((first_attempt, second_attempt), (1, 2)); + assert!(!plan.has_next()); + } + #[tokio::test] async fn api_backend_uses_source_credentials() { let dir = tempfile::tempdir().unwrap(); @@ -3099,7 +3500,7 @@ enabled = true let backend = AgentApiBackend::new( "claude-opus-4-6".to_string(), ProviderId::anthropic(), - Vec::new(), + ModelFallbackPolicy::default(), Arc::new(VaultCredentialSource::with_env_lookup( Arc::new(AsyncRwLock::new(vault)), |_| None, @@ -3116,14 +3517,18 @@ enabled = true #[tokio::test] async fn one_shot_falls_back_after_refusal_error() { - let fallback_chain = vec![FallbackTarget { + let configured_targets = vec![FallbackTarget { provider: "openai".to_string(), model: "gpt-5.5".to_string(), }]; + let fallback_policy = ModelFallbackPolicy::new(std::collections::BTreeMap::from([( + "claude-fable-5".to_string(), + configured_targets, + )])); let backend = AgentApiBackend::new( "claude-fable-5".to_string(), ProviderId::anthropic(), - fallback_chain.clone(), + fallback_policy, auth_test_support::vault_only_credential_source(), SteeringHub::for_tests(), ); @@ -3144,6 +3549,13 @@ enabled = true let context = Context::new(); let stage_scope = StageScope::for_handler(&context, &node.id); let emitter = Arc::new(Emitter::new(fabro_types::RunId::new())); + let emitted_failover = Arc::new(Mutex::new(None)); + let emitted_failover_for_listener = Arc::clone(&emitted_failover); + emitter.on_event(move |event| { + if let fabro_types::EventBody::Failover(props) = &event.body { + *emitted_failover_for_listener.lock().unwrap() = Some(props.clone()); + } + }); let request = Request { model: "claude-fable-5".to_string(), messages: vec![Message::user("Hello")], @@ -3160,6 +3572,12 @@ enabled = true metadata: None, provider_options: None, }; + let (mut fallback_plan, notices) = backend.fallback_plan( + "claude-fable-5", + &ProviderId::anthropic(), + EffectiveRequestControls::default(), + ); + assert!(notices.is_empty()); let completion = backend .complete_one_shot_request( @@ -3168,8 +3586,7 @@ enabled = true &emitter, &stage_scope, &request, - EffectiveRequestControls::default(), - &fallback_chain, + &mut fallback_plan, ) .await .unwrap(); @@ -3177,6 +3594,91 @@ enabled = true assert_eq!(completion.response.text(), "fallback ok"); assert_eq!(completion.model.provider, ProviderId::openai()); assert_eq!(completion.model.model_id, "gpt-5.5"); + let failover = emitted_failover + .lock() + .unwrap() + .clone() + .expect("agent.failover should be emitted"); + assert_eq!(failover.original_provider.as_deref(), Some("anthropic")); + assert_eq!(failover.original_model.as_deref(), Some("claude-fable-5")); + assert_eq!(failover.attempt, Some(1)); + assert_eq!(failover.from_provider, "anthropic"); + assert_eq!(failover.from_model, "claude-fable-5"); + assert_eq!(failover.to_provider, "openai"); + assert_eq!(failover.to_model, "gpt-5.5"); + assert_eq!(failover.requested_reasoning_effort, None); + assert_eq!(failover.effective_reasoning_effort, None); + assert!(failover.error.contains("refused")); + } + + #[tokio::test] + async fn explicit_provider_one_shot_stays_on_fallback_during_output_repair() { + let server = MockServer::start(); + let primary_failure = server.mock(|when, then| { + when.method(POST).path("/primary/chat/completions"); + then.status(401) + .header("content-type", "application/json") + .json_body(serde_json::json!({ + "error": { + "message": "primary credential expired", + "type": "authentication_error" + } + })); + }); + let fallback_response = server.mock(|when, then| { + when.method(POST) + .path("/fallback/chat/completions") + .body_excludes(r#""role":"assistant""#); + then.status(200) + .header("content-type", "application/json") + .json_body(chat_completion_response("not json", 10, 1)); + }); + let fallback_repair = server.mock(|when, then| { + when.method(POST) + .path("/fallback/chat/completions") + .body_includes(r#""role":"assistant""#) + .body_includes("not json"); + then.status(200) + .header("content-type", "application/json") + .json_body(chat_completion_response(r#"{"passed":true}"#, 11, 2)); + }); + let backend = fallback_api_backend(&server); + let mut node = Node::new("audit"); + node.attrs.insert( + "provider".to_string(), + AttrValue::String("primary".to_string()), + ); + node.attrs + .insert("output_schema".to_string(), custom_output_schema_attr()); + node.attrs + .insert("output_retries".to_string(), AttrValue::Integer(1)); + let context = Context::new(); + let stage_scope = StageScope::for_handler(&context, &node.id); + let emitter = Arc::new(Emitter::new(fabro_types::RunId::new())); + let workspace = tempfile::tempdir().unwrap(); + let sandbox: Arc = + Arc::new(LocalSandbox::new(workspace.path().to_path_buf())); + + let result = backend + .one_shot(OneShotRequest { + node: &node, + prompt: "Audit the result", + system_prompt: None, + emitter: &emitter, + stage_scope: &stage_scope, + sandbox: &sandbox, + cancel_token: CancellationToken::new(), + }) + .await + .unwrap(); + + primary_failure.assert_calls(1); + fallback_response.assert_calls(1); + fallback_repair.assert_calls(1); + let CodergenResult::Text { text, .. } = result else { + panic!("one_shot should return text"); + }; + assert_eq!(text, r#"{"passed":true}"#); } #[tokio::test] @@ -3309,6 +3811,77 @@ enabled = true assert_eq!(usage.tokens().output_tokens, 7); } + #[tokio::test] + async fn agent_output_repair_continues_on_the_original_models_fallback_plan() { + let server = MockServer::start(); + let primary_response = server.mock(|when, then| { + when.method(POST) + .path("/primary/chat/completions") + .body_includes(r#""stream":true"#) + .body_excludes(r#""role":"assistant""#); + then.status(200) + .header("content-type", "text/event-stream") + .body(chat_completion_stream("not json", 20, 3)); + }); + let failed_repair = server.mock(|when, then| { + when.method(POST) + .path("/primary/chat/completions") + .body_includes(r#""role":"assistant""#) + .body_includes("not json"); + then.status(401) + .header("content-type", "application/json") + .json_body(serde_json::json!({ + "error": { + "message": "primary credential expired", + "type": "authentication_error" + } + })); + }); + let fallback_response = server.mock(|when, then| { + when.method(POST) + .path("/fallback/chat/completions") + .body_includes(r#""stream":true"#) + .body_excludes(r#""role":"assistant""#); + then.status(200) + .header("content-type", "text/event-stream") + .body(chat_completion_stream(r#"{"passed":true}"#, 22, 4)); + }); + let backend = fallback_api_backend(&server); + let mut node = Node::new("audit"); + node.attrs + .insert("output_schema".to_string(), custom_output_schema_attr()); + node.attrs + .insert("output_retries".to_string(), AttrValue::Integer(1)); + let context = Context::new(); + let emitter = Arc::new(Emitter::new(fabro_types::RunId::new())); + let workspace = tempfile::tempdir().unwrap(); + let sandbox: Arc = + Arc::new(LocalSandbox::new(workspace.path().to_path_buf())); + + let result = backend + .run(CodergenRunRequest { + node: &node, + prompt: "Audit the result", + context: &context, + thread_id: None, + emitter: &emitter, + sandbox: &sandbox, + tool_hooks: None, + cancel_token: CancellationToken::new(), + agent_tool_runtime: fabro_agent::AgentToolRuntime::default(), + }) + .await + .unwrap(); + + primary_response.assert_calls(1); + failed_repair.assert_calls(1); + fallback_response.assert_calls(1); + let CodergenResult::Text { text, .. } = result else { + panic!("run should return text"); + }; + assert_eq!(text, r#"{"passed":true}"#); + } + #[tokio::test] async fn agent_run_web_search_uses_configured_brave_search_key() { let server = MockServer::start(); @@ -3397,7 +3970,7 @@ enabled = true let backend = AgentApiBackend::new( "gpt-5.4".to_string(), ProviderId::openai(), - Vec::new(), + ModelFallbackPolicy::default(), auth_test_support::vault_only_credential_source(), SteeringHub::for_tests(), ); @@ -3426,12 +3999,21 @@ enabled = true SessionOptions::default(), None, ); + let (fallback_plan, notices) = backend.fallback_plan( + "gpt-5.4", + &ProviderId::openai(), + EffectiveRequestControls::default(), + ); + assert!(notices.is_empty()); begin_session_lifecycle(&session, &emitter, None); backend .sessions .lock() .unwrap() - .insert("thread-1".to_string(), session); + .insert("thread-1".to_string(), CachedAgentSession { + session, + fallback_plan, + }); backend.shutdown(&emitter).await; backend.shutdown(&emitter).await; @@ -3515,7 +4097,7 @@ enabled = true fn non_failover_llm_error() -> LlmError { LlmError::Provider { - kind: ProviderErrorKind::Authentication, + kind: ProviderErrorKind::InvalidRequest, detail: Box::new(ProviderErrorDetail { message: "bad key".into(), provider: "openai".into(), diff --git a/lib/components/fabro-workflow/src/lib.rs b/lib/components/fabro-workflow/src/lib.rs index ee3ec4566..bd7f3e665 100644 --- a/lib/components/fabro-workflow/src/lib.rs +++ b/lib/components/fabro-workflow/src/lib.rs @@ -303,6 +303,7 @@ mod interview_runtime; reason = "The lifecycle module remains crate-visible for tests and pending integrations." )] pub(crate) mod lifecycle; +pub mod model_fallback; pub(crate) mod node_handler; pub mod operations; pub mod outcome; diff --git a/lib/components/fabro-workflow/src/model_fallback.rs b/lib/components/fabro-workflow/src/model_fallback.rs new file mode 100644 index 000000000..7b13fe429 --- /dev/null +++ b/lib/components/fabro-workflow/src/model_fallback.rs @@ -0,0 +1,514 @@ +use std::collections::{BTreeMap, HashMap, HashSet}; + +use fabro_model::{Catalog, FallbackTarget, Model, ModelSelectionError, ProviderId}; +use fabro_types::settings::{ModelRef, ResolvedModelRef}; +use fabro_types::{RunNoticeCode, RunNoticeLevel}; + +use crate::Error; + +/// Catalog-resolved fallback chains keyed by canonical requested model ID. +/// +/// A chain is selected from the original request only. Targets never cause +/// another chain lookup. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ModelFallbackPolicy { + chains: BTreeMap>, +} + +impl ModelFallbackPolicy { + #[must_use] + pub fn new(chains: BTreeMap>) -> Self { + Self { chains } + } + + #[must_use] + pub fn chain_for<'a>( + &'a self, + catalog: &Catalog, + provider: &ProviderId, + model: &str, + ) -> Option<&'a [FallbackTarget]> { + let canonical = canonical_model_id(catalog, provider, model); + self.chains.get(&canonical).map(Vec::as_slice) + } + + pub fn iter(&self) -> impl Iterator { + self.chains + .iter() + .map(|(model, chain)| (model.as_str(), chain.as_slice())) + } + + #[must_use] + pub fn is_empty(&self) -> bool { + self.chains.is_empty() + } +} + +pub(crate) fn canonical_model_id(catalog: &Catalog, provider: &ProviderId, model: &str) -> String { + catalog.get_on_provider(provider, model).map_or_else( + || { + catalog + .select(model, None, &catalog.all_provider_ids()) + .map_or_else(|_| model.to_string(), |offering| offering.id.to_string()) + }, + |offering| offering.id.to_string(), + ) +} + +/// Server-side result of canonicalizing and filtering configured fallback +/// chains. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ResolvedModelFallbacks { + pub policy: ModelFallbackPolicy, + pub notices: Vec, +} + +/// Why a configured fallback candidate was removed from one model's chain. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ModelFallbackNotice { + ProviderUnconfigured { + requested_model: String, + reference: ModelRef, + provider: ProviderId, + }, + NoConfiguredOffering { + requested_model: String, + reference: ModelRef, + providers: Vec, + }, + PrimaryNotInCatalog { + requested_model: String, + reference: ModelRef, + primary: FallbackTarget, + }, + NoCompatibleModel { + requested_model: String, + reference: ModelRef, + provider: ProviderId, + }, + Duplicate { + requested_model: String, + reference: ModelRef, + target: FallbackTarget, + }, + ChainEmpty { + requested_model: String, + }, +} + +impl ModelFallbackNotice { + #[must_use] + pub fn code(&self) -> RunNoticeCode { + match self { + Self::ChainEmpty { .. } => RunNoticeCode::ModelFallbackChainEmpty, + Self::ProviderUnconfigured { .. } + | Self::NoConfiguredOffering { .. } + | Self::PrimaryNotInCatalog { .. } + | Self::NoCompatibleModel { .. } + | Self::Duplicate { .. } => RunNoticeCode::ModelFallbackSkipped, + } + } + + #[must_use] + pub fn level(&self) -> RunNoticeLevel { + match self { + Self::Duplicate { .. } => RunNoticeLevel::Info, + Self::ProviderUnconfigured { .. } + | Self::NoConfiguredOffering { .. } + | Self::PrimaryNotInCatalog { .. } + | Self::NoCompatibleModel { .. } + | Self::ChainEmpty { .. } => RunNoticeLevel::Warn, + } + } + + #[must_use] + pub fn message(&self) -> String { + match self { + Self::ProviderUnconfigured { + requested_model, + reference, + provider, + } => format!( + "Model fallback `{reference}` for requested model `{requested_model}` was skipped because provider `{provider}` is not configured." + ), + Self::NoConfiguredOffering { + requested_model, + reference, + providers, + } => { + let providers = providers + .iter() + .map(ProviderId::to_string) + .collect::>() + .join(", "); + format!( + "Model fallback `{reference}` for requested model `{requested_model}` was skipped because none of its providers are configured. It is offered by: {providers}." + ) + } + Self::PrimaryNotInCatalog { + requested_model, + reference, + primary, + } => format!( + "Model fallback `{reference}` for requested model `{requested_model}` was skipped because `{primary}` is not in the catalog, so there is no capability profile to match against." + ), + Self::NoCompatibleModel { + requested_model, + reference, + provider, + } => format!( + "Model fallback `{reference}` for requested model `{requested_model}` was skipped because provider `{provider}` has no compatible model." + ), + Self::Duplicate { + requested_model, + reference, + target, + } => format!( + "Model fallback `{reference}` for requested model `{requested_model}` was skipped because target `{target}` already appears in that chain." + ), + Self::ChainEmpty { requested_model } => format!( + "No usable model fallbacks remain for requested model `{requested_model}` after filtering its configured candidates." + ), + } + } +} + +/// Resolve every model-keyed fallback chain against the server's catalog and +/// configured-provider snapshot. +/// +/// This function must stay at server-side call sites. Offline validation only +/// parses the raw table and cannot canonicalize model aliases. +pub fn resolve_model_fallbacks( + catalog: &Catalog, + eligible: &HashSet, + configured: &BTreeMap>, +) -> Result { + let mut resolved = ResolvedModelFallbacks::default(); + let mut raw_key_by_canonical = HashMap::::new(); + + for (raw_key, references) in configured { + let selected = + catalog.resolve_selection_with_catalog_fallback(Some(raw_key), None, eligible)?; + let requested_model = selected.model; + + if let Some(previous) = + raw_key_by_canonical.insert(requested_model.clone(), raw_key.clone()) + { + return Err(Error::Precondition(format!( + "`run.model.fallbacks` keys `{previous}` and `{raw_key}` both resolve to requested model `{requested_model}`" + ))); + } + + let primary = FallbackTarget::new(&selected.provider, &requested_model); + let primary_model = catalog.get_on_provider(&selected.provider, &requested_model); + let mut targets = Vec::new(); + + for model_ref in references { + let target = match resolve_fallback_candidate( + catalog, + &requested_model, + &primary, + primary_model, + eligible, + model_ref, + )? { + FallbackCandidate::Skipped(notice) => { + resolved.notices.push(notice); + continue; + } + FallbackCandidate::Target(target) => target, + }; + + if targets.contains(&target) { + resolved.notices.push(ModelFallbackNotice::Duplicate { + requested_model: requested_model.clone(), + reference: model_ref.clone(), + target, + }); + } else { + targets.push(target); + } + } + + if targets.is_empty() { + resolved.notices.push(ModelFallbackNotice::ChainEmpty { + requested_model: requested_model.clone(), + }); + } + resolved.policy.chains.insert(requested_model, targets); + } + + Ok(resolved) +} + +enum FallbackCandidate { + Target(FallbackTarget), + Skipped(ModelFallbackNotice), +} + +fn resolve_fallback_candidate( + catalog: &Catalog, + requested_model: &str, + primary: &FallbackTarget, + primary_model: Option<&Model>, + eligible: &HashSet, + model_ref: &ModelRef, +) -> Result { + let reference = model_ref.clone(); + + Ok(match model_ref.resolve(catalog)? { + ResolvedModelRef::Provider(provider_name) => { + let provider = catalog.provider_id(&provider_name)?; + if !eligible.contains(&provider) { + return Ok(FallbackCandidate::Skipped( + ModelFallbackNotice::ProviderUnconfigured { + requested_model: requested_model.to_string(), + reference, + provider, + }, + )); + } + let Some(primary_model) = primary_model else { + return Ok(FallbackCandidate::Skipped( + ModelFallbackNotice::PrimaryNotInCatalog { + requested_model: requested_model.to_string(), + reference, + primary: primary.clone(), + }, + )); + }; + match catalog.closest(&provider, primary_model) { + Some(model) => FallbackCandidate::Target(FallbackTarget::new(provider, &model.id)), + None => FallbackCandidate::Skipped(ModelFallbackNotice::NoCompatibleModel { + requested_model: requested_model.to_string(), + reference, + provider, + }), + } + } + ResolvedModelRef::Model { + provider: Some(provider_name), + selector, + } => { + let provider = catalog.provider_id(&provider_name)?; + if !eligible.contains(&provider) { + return Ok(FallbackCandidate::Skipped( + ModelFallbackNotice::ProviderUnconfigured { + requested_model: requested_model.to_string(), + reference, + provider, + }, + )); + } + match catalog.resolve_on_provider(&provider, &selector) { + Ok(info) => { + FallbackCandidate::Target(FallbackTarget::new(&info.provider, &info.id)) + } + Err(ModelSelectionError::UnknownSelectorOnProvider { .. }) => { + FallbackCandidate::Target(FallbackTarget::new(provider, selector)) + } + Err(error) => return Err(error.into()), + } + } + ResolvedModelRef::Model { + provider: None, + selector, + } => match catalog.select(&selector, None, eligible) { + Ok(info) => FallbackCandidate::Target(FallbackTarget::new(&info.provider, &info.id)), + Err(ModelSelectionError::NoEligibleOffering { providers, .. }) => { + FallbackCandidate::Skipped(ModelFallbackNotice::NoConfiguredOffering { + requested_model: requested_model.to_string(), + reference, + providers, + }) + } + Err(ModelSelectionError::UnknownSelector { .. }) => { + FallbackCandidate::Target(FallbackTarget::new(&primary.provider, selector)) + } + Err(error) => return Err(error.into()), + }, + }) +} + +#[cfg(test)] +mod tests { + use std::collections::{BTreeMap, HashSet}; + + use fabro_model::{Catalog, FallbackTarget, ProviderId}; + + use super::{ModelFallbackNotice, resolve_model_fallbacks}; + + fn references(values: &[&str]) -> Vec { + values + .iter() + .map(|value| value.parse().expect("fixture reference should parse")) + .collect() + } + + fn openrouter_catalog() -> Catalog { + let overrides = toml::from_str( + r" +[providers.openrouter] +enabled = true +", + ) + .expect("catalog override should parse"); + Catalog::from_builtin_with_overrides(&overrides).expect("catalog should build") + } + + #[test] + fn canonicalizes_keys_and_keeps_each_chain_independent() { + let catalog = openrouter_catalog(); + let eligible = HashSet::from([ProviderId::new("openrouter")]); + let configured = BTreeMap::from([ + ("gpt-sol".to_string(), references(&["claude-opus"])), + ( + "claude-fable".to_string(), + references(&["gpt-sol", "claude-opus"]), + ), + ]); + + let resolved = resolve_model_fallbacks(&catalog, &eligible, &configured).unwrap(); + + assert_eq!( + resolved + .policy + .chain_for(&catalog, &ProviderId::new("openrouter"), "gpt-sol"), + Some([FallbackTarget::new("openrouter", "claude-opus-5")].as_slice()) + ); + assert_eq!( + resolved + .policy + .chain_for(&catalog, &ProviderId::new("openrouter"), "claude-fable"), + Some( + [ + FallbackTarget::new("openrouter", "gpt-5.6-sol"), + FallbackTarget::new("openrouter", "claude-opus-5"), + ] + .as_slice() + ) + ); + } + + #[test] + fn rejects_aliases_that_define_the_same_requested_model_twice() { + let catalog = openrouter_catalog(); + let eligible = HashSet::from([ProviderId::new("openrouter")]); + let configured = BTreeMap::from([ + ("gpt-sol".to_string(), references(&["claude-opus"])), + ("gpt-5.6-sol".to_string(), references(&["claude-fable"])), + ]); + + let error = resolve_model_fallbacks(&catalog, &eligible, &configured).unwrap_err(); + + assert!( + error + .to_string() + .contains("both resolve to requested model"), + "unexpected error: {error}" + ); + } + + #[test] + fn skips_unconfigured_candidates_per_requested_model() { + let catalog = openrouter_catalog(); + let eligible = HashSet::from([ProviderId::new("openrouter")]); + let configured = BTreeMap::from([( + "kimi-k3".to_string(), + references(&["kimi:kimi-k3", "openrouter:kimi-k3"]), + )]); + + let resolved = resolve_model_fallbacks(&catalog, &eligible, &configured).unwrap(); + + assert_eq!( + resolved + .policy + .chain_for(&catalog, &ProviderId::new("openrouter"), "kimi-k3"), + Some([FallbackTarget::new("openrouter", "kimi-k3")].as_slice()) + ); + assert!(matches!( + resolved.notices.as_slice(), + [ModelFallbackNotice::ProviderUnconfigured { + requested_model, + provider, + .. + }] if requested_model == "kimi-k3" && provider == &ProviderId::new("kimi") + )); + } + + #[test] + fn resolves_the_requested_production_policy_as_independent_chains() { + let catalog = { + let overrides = toml::from_str( + r" +[providers.modal] +enabled = true + +[providers.openrouter] +enabled = true +", + ) + .expect("catalog override should parse"); + Catalog::from_builtin_with_overrides(&overrides).expect("catalog should build") + }; + let eligible = HashSet::from([ + ProviderId::new("modal"), + ProviderId::new("kimi"), + ProviderId::new("openrouter"), + ]); + let configured = BTreeMap::from([ + ( + "kimi-k3".to_string(), + references(&["kimi:kimi-k3", "openrouter:kimi-k3", "claude-opus"]), + ), + ("glm-5.2".to_string(), references(&["gpt-sol"])), + ("gpt-sol".to_string(), references(&["claude-opus"])), + ("claude-opus".to_string(), references(&["gpt-sol"])), + ("gpt-terra".to_string(), references(&["claude-opus"])), + ("gpt-luna".to_string(), references(&["claude-sonnet"])), + ( + "claude-fable".to_string(), + references(&["gpt-sol", "claude-opus"]), + ), + ]); + + let resolved = resolve_model_fallbacks(&catalog, &eligible, &configured).unwrap(); + + assert!(resolved.notices.is_empty()); + let chain = |model: &str| { + resolved + .policy + .chain_for(&catalog, &ProviderId::new("openrouter"), model) + .expect("requested model should have a chain") + }; + assert_eq!(chain("kimi-k3"), [ + FallbackTarget::new("kimi", "kimi-k3"), + FallbackTarget::new("openrouter", "kimi-k3"), + FallbackTarget::new("openrouter", "claude-opus-5"), + ]); + assert_eq!(chain("glm-5.2"), [FallbackTarget::new( + "openrouter", + "gpt-5.6-sol" + )]); + assert_eq!(chain("gpt-sol"), [FallbackTarget::new( + "openrouter", + "claude-opus-5" + )]); + assert_eq!(chain("claude-opus"), [FallbackTarget::new( + "openrouter", + "gpt-5.6-sol" + )]); + assert_eq!(chain("gpt-terra"), [FallbackTarget::new( + "openrouter", + "claude-opus-5" + )]); + assert_eq!(chain("gpt-luna"), [FallbackTarget::new( + "openrouter", + "claude-sonnet-5" + )]); + assert_eq!(chain("claude-fable"), [ + FallbackTarget::new("openrouter", "gpt-5.6-sol"), + FallbackTarget::new("openrouter", "claude-opus-5"), + ]); + } +} diff --git a/lib/components/fabro-workflow/src/operations/start.rs b/lib/components/fabro-workflow/src/operations/start.rs index 54d3ae572..fc576892e 100644 --- a/lib/components/fabro-workflow/src/operations/start.rs +++ b/lib/components/fabro-workflow/src/operations/start.rs @@ -7,7 +7,7 @@ use fabro_auth::{CredentialSource, VaultCredentialSource}; use fabro_interview::{AutoApproveInterviewer, Interviewer}; use fabro_llm::client::Client as LlmClient; use fabro_mcp::config::McpServerSettings; -use fabro_model::{Catalog, FallbackTarget, Model, ModelSelectionError, ProviderId}; +use fabro_model::{Catalog, ProviderId}; use fabro_sandbox::daytona::DaytonaConfig; use fabro_sandbox::from_environment::{ daytona_config_from_environment, docker_config_from_environment_with_secrets, @@ -17,10 +17,9 @@ use fabro_sandbox::{DockerSandboxOptions, SandboxSpec}; use fabro_static::EnvVars; use fabro_types::settings::run::{ ApprovalMode, McpServerSettings as ResolvedMcpServerSettings, PullRequestSettings, - ResolvedMcpEntry, RunMode, RunModelSettings as ResolvedRunModelSettings, - RunNamespace as ResolvedRunSettings, RunPrepareSettings as ResolvedRunPrepareSettings, + ResolvedMcpEntry, RunMode, RunNamespace as ResolvedRunSettings, + RunPrepareSettings as ResolvedRunPrepareSettings, }; -use fabro_types::settings::{ModelRef, ResolvedModelRef}; use fabro_types::{ManifestPath, RunId, RunRunnableSource, SandboxProviderKind}; use fabro_vault::Vault; use tokio::runtime::Handle; @@ -32,10 +31,10 @@ use crate::artifact_upload::ArtifactSink; use crate::context::Context; use crate::error::{self, Error}; use crate::event::{ - Emitter, Event, EventBody, RunEventLogger, RunEventSink, RunNoticeCode, RunNoticeLevel, - append_event_to_sink, + Emitter, Event, EventBody, RunEventLogger, RunEventSink, RunNoticeLevel, append_event_to_sink, }; use crate::handler::HandlerRegistry; +use crate::model_fallback::{ModelFallbackNotice, ResolvedModelFallbacks, resolve_model_fallbacks}; use crate::outcome::{Outcome, StageOutcome}; use crate::pipeline::{ self, FinalizeOptions, Finalized, InitOptions, LlmSpec, Persisted, PublishOptions, ResumeState, @@ -91,132 +90,7 @@ struct RunSession { struct ResolvedStartLlm { model: String, provider_id: ProviderId, - fallbacks: ResolvedFallbackChain, -} - -#[derive(Debug, Default, PartialEq, Eq)] -struct ResolvedFallbackChain { - targets: Vec, - notices: Vec, -} - -/// Why one fallback candidate did not make it into the chain, or that the whole -/// chain came out empty. Resolution happens before the run's event sink is -/// wired up, so these are carried to [`RunSession::run`] and emitted there. -#[derive(Debug, PartialEq, Eq)] -enum ModelFallbackNotice { - ProviderUnconfigured { - reference: ModelRef, - provider: ProviderId, - }, - NoConfiguredOffering { - reference: ModelRef, - providers: Vec, - }, - /// The candidate named a provider but no model, and the primary model is - /// not in the catalog, so there is nothing to match its capabilities to. - PrimaryNotInCatalog { - reference: ModelRef, - primary: FallbackTarget, - }, - NoCompatibleModel { - reference: ModelRef, - provider: ProviderId, - }, - MatchesPrimary { - reference: ModelRef, - target: FallbackTarget, - }, - Duplicate { - reference: ModelRef, - target: FallbackTarget, - }, - ChainEmpty, -} - -impl ModelFallbackNotice { - fn code(&self) -> RunNoticeCode { - match self { - Self::ChainEmpty => RunNoticeCode::ModelFallbackChainEmpty, - Self::ProviderUnconfigured { .. } - | Self::NoConfiguredOffering { .. } - | Self::PrimaryNotInCatalog { .. } - | Self::NoCompatibleModel { .. } - | Self::MatchesPrimary { .. } - | Self::Duplicate { .. } => RunNoticeCode::ModelFallbackSkipped, - } - } - - fn level(&self) -> RunNoticeLevel { - match self { - Self::MatchesPrimary { .. } | Self::Duplicate { .. } => RunNoticeLevel::Info, - Self::ProviderUnconfigured { .. } - | Self::NoConfiguredOffering { .. } - | Self::PrimaryNotInCatalog { .. } - | Self::NoCompatibleModel { .. } - | Self::ChainEmpty => RunNoticeLevel::Warn, - } - } - - fn message(&self) -> String { - match self { - Self::ProviderUnconfigured { - reference, - provider, - } => { - format!( - "Model fallback `{reference}` was skipped because provider `{provider}` is not configured." - ) - } - Self::NoConfiguredOffering { - reference, - providers, - } => { - let providers = providers - .iter() - .map(ProviderId::to_string) - .collect::>() - .join(", "); - format!( - "Model fallback `{reference}` was skipped because none of its providers are configured. It is offered by: {providers}." - ) - } - Self::PrimaryNotInCatalog { reference, primary } => { - format!( - "Model fallback `{reference}` was skipped because the primary model `{primary}` is not in the catalog, so there is no capability profile to match against." - ) - } - Self::NoCompatibleModel { - reference, - provider, - } => { - format!( - "Model fallback `{reference}` was skipped because provider `{provider}` has no compatible model." - ) - } - Self::MatchesPrimary { reference, target } => { - format!( - "Model fallback `{reference}` was skipped because it resolves to the primary target `{target}`." - ) - } - Self::Duplicate { reference, target } => { - format!( - "Model fallback `{reference}` was skipped because target `{target}` already appears in the fallback chain." - ) - } - Self::ChainEmpty => { - "No usable model fallbacks remain after filtering the configured fallback candidates." - .to_string() - } - } - } - - /// Publish every notice on the run's event stream. - fn emit_all(notices: &[Self], emitter: &Emitter) { - for notice in notices { - emitter.notice(notice.level(), notice.code(), notice.message()); - } - } + fallbacks: ResolvedModelFallbacks, } pub struct StartServices { @@ -604,7 +478,7 @@ impl RunSession { llm: LlmSpec { model: llm.model.clone(), provider_id: llm.provider_id.clone(), - fallback_chain: llm.fallbacks.targets, + fallbacks: llm.fallbacks.policy, mcp_servers, model_controls: resolved.model.controls.clone(), dry_run: resolved.execution.mode == RunMode::DryRun, @@ -743,8 +617,7 @@ fn resolve_start_llm( settings.model.provider.as_deref(), false, )?; - let fallbacks = - resolve_fallback_chain(catalog, &provider_id, &model, &settings.model, &eligible)?; + let fallbacks = resolve_model_fallbacks(catalog, &eligible, &settings.model.fallbacks)?; Ok(ResolvedStartLlm { model, @@ -753,166 +626,6 @@ fn resolve_start_llm( }) } -/// Resolve fallback candidates against the configured provider snapshot. -/// -/// Candidates that cannot be used in this environment — an unconfigured -/// provider, no compatible model, a target equal to the primary, or a duplicate -/// — are dropped, and each drop records a [`ModelFallbackNotice`] that the run -/// emits at startup. Remaining candidates keep their configured order. -/// -/// A provider the catalog has never heard of is a different case: that is a -/// typo rather than an environment difference, so it fails the run instead of -/// being skipped. This is what keeps a chain portable without letting a -/// misspelled provider silently disappear. -fn resolve_fallback_chain( - catalog: &Catalog, - provider: &ProviderId, - model: &str, - settings: &ResolvedRunModelSettings, - eligible: &HashSet, -) -> Result { - if settings.fallbacks.is_empty() { - return Ok(ResolvedFallbackChain::default()); - } - - let primary_model = catalog.get_on_provider(provider, model); - let primary = FallbackTarget::new(provider, model); - let mut resolution = ResolvedFallbackChain::default(); - - for model_ref in &settings.fallbacks { - let target = match resolve_fallback_candidate( - catalog, - &primary, - primary_model, - eligible, - model_ref, - )? { - FallbackCandidate::Skipped(notice) => { - resolution.notices.push(notice); - continue; - } - FallbackCandidate::Target(target) => target, - }; - - let reference = model_ref.clone(); - if target == primary { - resolution - .notices - .push(ModelFallbackNotice::MatchesPrimary { reference, target }); - } else if resolution.targets.contains(&target) { - resolution - .notices - .push(ModelFallbackNotice::Duplicate { reference, target }); - } else { - resolution.targets.push(target); - } - } - - if resolution.targets.is_empty() { - resolution.notices.push(ModelFallbackNotice::ChainEmpty); - } - - Ok(resolution) -} - -/// The outcome of resolving one fallback candidate: either a dispatchable -/// target or the reason the candidate cannot be used. -enum FallbackCandidate { - Target(FallbackTarget), - Skipped(ModelFallbackNotice), -} - -/// Resolve one fallback reference against the configured provider snapshot. -/// -/// `primary_model` is the primary's catalog entry, used to pick the closest -/// capability match when a candidate names a provider but no model. It is -/// `None` when the primary is itself a passthrough selector. -/// -/// A selector the catalog does not know passes through verbatim so a model -/// newer than the catalog still dispatches. When the candidate named a -/// provider, it passes through on that provider; when it did not, it passes -/// through on the primary's provider, which means such a fallback gives no -/// cross-provider failover. -fn resolve_fallback_candidate( - catalog: &Catalog, - primary: &FallbackTarget, - primary_model: Option<&Model>, - eligible: &HashSet, - model_ref: &ModelRef, -) -> Result { - let reference = model_ref.clone(); - - Ok(match model_ref.resolve(catalog)? { - ResolvedModelRef::Provider(provider_name) => { - let provider = catalog.provider_id(&provider_name)?; - if !eligible.contains(&provider) { - return Ok(FallbackCandidate::Skipped( - ModelFallbackNotice::ProviderUnconfigured { - reference, - provider, - }, - )); - } - // Without a catalog entry for the primary there is no capability - // profile to match against, which is not the provider's fault. - let Some(primary_model) = primary_model else { - return Ok(FallbackCandidate::Skipped( - ModelFallbackNotice::PrimaryNotInCatalog { - reference, - primary: primary.clone(), - }, - )); - }; - match catalog.closest(&provider, primary_model) { - Some(model) => FallbackCandidate::Target(FallbackTarget::new(provider, &model.id)), - None => FallbackCandidate::Skipped(ModelFallbackNotice::NoCompatibleModel { - reference, - provider, - }), - } - } - ResolvedModelRef::Model { - provider: Some(provider_name), - selector, - } => { - let provider = catalog.provider_id(&provider_name)?; - if !eligible.contains(&provider) { - return Ok(FallbackCandidate::Skipped( - ModelFallbackNotice::ProviderUnconfigured { - reference, - provider, - }, - )); - } - match catalog.resolve_on_provider(&provider, &selector) { - Ok(info) => { - FallbackCandidate::Target(FallbackTarget::new(&info.provider, &info.id)) - } - Err(ModelSelectionError::UnknownSelectorOnProvider { .. }) => { - FallbackCandidate::Target(FallbackTarget::new(provider, selector)) - } - Err(error) => return Err(error.into()), - } - } - ResolvedModelRef::Model { - provider: None, - selector, - } => match catalog.select(&selector, None, eligible) { - Ok(info) => FallbackCandidate::Target(FallbackTarget::new(&info.provider, &info.id)), - Err(ModelSelectionError::NoEligibleOffering { providers, .. }) => { - FallbackCandidate::Skipped(ModelFallbackNotice::NoConfiguredOffering { - reference, - providers, - }) - } - Err(ModelSelectionError::UnknownSelector { .. }) => { - FallbackCandidate::Target(FallbackTarget::new(&primary.provider, selector)) - } - Err(error) => return Err(error.into()), - }, - }) -} - /// Build the launch-time MCP config from resolved settings. Secret tokens in /// the transport (`command`/`url`/`env`/`headers`) resolve from the vault at /// the run boundary. Unsupported tokens fail. @@ -1035,7 +748,10 @@ impl RunSession { store_progress_logger.register(self.emitter.as_ref()); // Emit after the logger is registered so the notices reach the run // store, and before `run.started` so they read as launch-time context. - ModelFallbackNotice::emit_all(&self.fallback_notices, self.emitter.as_ref()); + for notice in &self.fallback_notices { + self.emitter + .notice(notice.level(), notice.code(), notice.message()); + } let init_options = InitOptions { run_store: self.run_store.clone(), @@ -1334,11 +1050,11 @@ mod tests { RunEnvironmentLayer, RunExecutionLayer, RunLayer, StickyMap, WorkflowSettingsBuilder, }; use fabro_store::Database; + use fabro_types::settings::InterpString; use fabro_types::settings::run::{ McpTransport as ResolvedMcpTransport, PreparedStep, PreparedStepRun, RunMode, RunPrepareSettings, }; - use fabro_types::settings::{InterpString, ModelRef}; use fabro_types::{ BilledModelUsage, ManifestPath, StageTiming, WorkflowSettings, fixtures, test_support, }; @@ -1505,517 +1221,6 @@ reasoning = false Catalog::from_settings(&settings).unwrap() } - #[test] - fn resolve_start_llm_infers_primary_and_filters_global_fallbacks() { - let catalog = portable_model_catalog(); - let mut settings = ResolvedRunSettings::default(); - settings.model.name = Some("gpt-56-sol".to_string()); - settings.model.fallbacks = vec![ - "openai:gpt-56-sol".parse::().unwrap(), - "openrouter:gpt-56-sol".parse::().unwrap(), - "openrouter:openai/gpt-5.6-sol".parse::().unwrap(), - ]; - - let resolved = resolve_start_llm( - &catalog, - &[ProviderId::new("openrouter"), ProviderId::openai()], - &settings, - ) - .unwrap(); - - assert_eq!(resolved.provider_id, ProviderId::openai()); - assert_eq!(resolved.model, "gpt-5.6-sol"); - assert_eq!(resolved.fallbacks.targets, vec![FallbackTarget { - provider: "openrouter".to_string(), - model: "gpt-5.6-sol".to_string(), - }]); - assert_eq!(resolved.fallbacks.notices, vec![ - ModelFallbackNotice::MatchesPrimary { - reference: "openai:gpt-56-sol".parse().unwrap(), - target: FallbackTarget { - provider: "openai".to_string(), - model: "gpt-5.6-sol".to_string(), - }, - }, - ModelFallbackNotice::Duplicate { - reference: "openrouter:openai/gpt-5.6-sol".parse().unwrap(), - target: FallbackTarget { - provider: "openrouter".to_string(), - model: "gpt-5.6-sol".to_string(), - }, - }, - ]); - } - - #[test] - fn resolve_fallback_chain_skips_unconfigured_provider_and_preserves_order() { - let catalog = test_catalog(); - let settings = ResolvedRunModelSettings { - fallbacks: vec![ - "gemini".parse::().unwrap(), - "gemini:unused".parse::().unwrap(), - "openai:gpt-5.4-mini".parse::().unwrap(), - "anthropic:claude-fable-5".parse::().unwrap(), - ], - ..ResolvedRunModelSettings::default() - }; - - let resolution = resolve_fallback_chain( - catalog.as_ref(), - &ProviderId::anthropic(), - "claude-opus-4-6", - &settings, - &HashSet::from([ProviderId::anthropic(), ProviderId::openai()]), - ) - .unwrap(); - - assert_eq!(resolution.targets, vec![ - FallbackTarget { - provider: "openai".to_string(), - model: "gpt-5.4-mini".to_string(), - }, - FallbackTarget { - provider: "anthropic".to_string(), - model: "claude-fable-5".to_string(), - }, - ]); - assert_eq!(resolution.notices, vec![ - ModelFallbackNotice::ProviderUnconfigured { - reference: "gemini".parse().unwrap(), - provider: ProviderId::gemini(), - }, - ModelFallbackNotice::ProviderUnconfigured { - reference: "gemini:unused".parse().unwrap(), - provider: ProviderId::gemini(), - }, - ]); - assert_eq!(resolution.notices[0].level(), RunNoticeLevel::Warn); - assert_eq!( - resolution.notices[0].code(), - RunNoticeCode::ModelFallbackSkipped - ); - } - - /// The resolver builds notices before the run's event sink exists, so this - /// covers the hand-off: each notice must reach the event stream as a - /// `run.notice` carrying its own level, code, and rendered message. - #[test] - fn fallback_notices_reach_the_event_stream() { - let emitter = Arc::new(Emitter::new(fixtures::RUN_1)); - let captured = Arc::new(Mutex::new(Vec::new())); - let sink = Arc::clone(&captured); - emitter.on_event(move |event| sink.lock().unwrap().push(event.clone())); - - let notices = vec![ - ModelFallbackNotice::ProviderUnconfigured { - reference: "gemini".parse().unwrap(), - provider: ProviderId::gemini(), - }, - ModelFallbackNotice::MatchesPrimary { - reference: "openai:gpt-5.6-sol".parse().unwrap(), - target: FallbackTarget::new("openai", "gpt-5.6-sol"), - }, - ModelFallbackNotice::ChainEmpty, - ]; - - ModelFallbackNotice::emit_all(¬ices, emitter.as_ref()); - - let events = captured.lock().unwrap(); - let emitted = events - .iter() - .map(|event| match &event.body { - EventBody::RunNotice(props) => { - (props.level, props.code.clone(), props.message.clone()) - } - other => panic!("expected run.notice body, got {other:?}"), - }) - .collect::>(); - - assert_eq!(emitted, vec![ - ( - RunNoticeLevel::Warn, - RunNoticeCode::ModelFallbackSkipped.to_string(), - "Model fallback `gemini` was skipped because provider `gemini` is not configured." - .to_string(), - ), - ( - RunNoticeLevel::Info, - RunNoticeCode::ModelFallbackSkipped.to_string(), - "Model fallback `openai:gpt-5.6-sol` was skipped because it resolves to the primary target `openai:gpt-5.6-sol`." - .to_string(), - ), - ( - RunNoticeLevel::Warn, - RunNoticeCode::ModelFallbackChainEmpty.to_string(), - "No usable model fallbacks remain after filtering the configured fallback candidates." - .to_string(), - ), - ]); - } - - /// A provider-only fallback cannot be matched when the primary model is a - /// passthrough selector, because there is no capability profile to compare - /// against. The notice must name that cause rather than blaming the - /// provider, which may well have compatible models. - #[test] - fn resolve_fallback_chain_blames_missing_primary_not_the_fallback_provider() { - let catalog = portable_model_catalog(); - let settings = ResolvedRunModelSettings { - fallbacks: vec!["openrouter".parse::().unwrap()], - ..ResolvedRunModelSettings::default() - }; - - let resolution = resolve_fallback_chain( - &catalog, - &ProviderId::openai(), - "gpt-5.9-not-in-catalog", - &settings, - &HashSet::from([ProviderId::openai(), ProviderId::new("openrouter")]), - ) - .unwrap(); - - assert!(resolution.targets.is_empty()); - assert_eq!(resolution.notices, vec![ - ModelFallbackNotice::PrimaryNotInCatalog { - reference: "openrouter".parse().unwrap(), - primary: FallbackTarget::new("openai", "gpt-5.9-not-in-catalog"), - }, - ModelFallbackNotice::ChainEmpty, - ]); - let message = resolution.notices[0].message(); - assert!( - message.contains("primary model `openai:gpt-5.9-not-in-catalog` is not in the catalog"), - "notice should name the missing primary: {message}" - ); - } - - #[test] - fn resolve_fallback_chain_skips_model_without_configured_offering() { - let catalog = portable_model_catalog(); - let settings = ResolvedRunModelSettings { - fallbacks: vec!["mini".parse::().unwrap()], - ..ResolvedRunModelSettings::default() - }; - - let resolution = resolve_fallback_chain( - &catalog, - &ProviderId::new("openrouter"), - "gpt-5.6-sol", - &settings, - &HashSet::from([ProviderId::new("openrouter")]), - ) - .unwrap(); - - assert!(resolution.targets.is_empty()); - assert_eq!(resolution.notices, vec![ - ModelFallbackNotice::NoConfiguredOffering { - reference: "mini".parse().unwrap(), - providers: vec![ProviderId::openai()], - }, - ModelFallbackNotice::ChainEmpty, - ]); - assert_eq!( - resolution.notices[1].code(), - RunNoticeCode::ModelFallbackChainEmpty - ); - assert_eq!(resolution.notices[1].level(), RunNoticeLevel::Warn); - assert!( - resolution.notices[0] - .message() - .contains("offered by: openai"), - "notice should name the providers that offer the model: {}", - resolution.notices[0].message() - ); - } - - #[test] - fn resolve_fallback_chain_rejects_unknown_qualified_provider() { - let catalog = portable_model_catalog(); - let settings = ResolvedRunModelSettings { - fallbacks: vec!["missing/model".parse::().unwrap()], - ..ResolvedRunModelSettings::default() - }; - - let error = resolve_fallback_chain( - &catalog, - &ProviderId::openai(), - "gpt-5.6-sol", - &settings, - &catalog.all_provider_ids(), - ) - .unwrap_err(); - - assert!(matches!( - error, - Error::ModelSelection(ModelSelectionError::UnknownProvider { provider }) - if provider == ProviderId::new("missing") - )); - } - - #[test] - fn resolve_fallback_chain_resolves_provider_fallbacks() { - let catalog = test_catalog(); - let settings = ResolvedRunModelSettings { - fallbacks: vec!["openai".parse::().unwrap()], - ..ResolvedRunModelSettings::default() - }; - - let chain = resolve_fallback_chain( - catalog.as_ref(), - &ProviderId::anthropic(), - "claude-opus-4-6", - &settings, - &catalog.all_provider_ids(), - ) - .unwrap(); - - assert_eq!(chain.targets, vec![FallbackTarget { - provider: "openai".to_string(), - model: "gpt-5.5".to_string(), - }]); - } - - #[test] - fn resolve_fallback_chain_resolves_explicit_model_fallbacks() { - let catalog = test_catalog(); - let settings = ResolvedRunModelSettings { - fallbacks: vec!["openai:gpt-5.4-mini".parse::().unwrap()], - ..ResolvedRunModelSettings::default() - }; - - let chain = resolve_fallback_chain( - catalog.as_ref(), - &ProviderId::anthropic(), - "claude-opus-4-6", - &settings, - &catalog.all_provider_ids(), - ) - .unwrap(); - - assert_eq!(chain.targets, vec![FallbackTarget { - provider: "openai".to_string(), - model: "gpt-5.4-mini".to_string(), - }]); - } - - #[test] - fn resolve_fallback_chain_selects_shared_bare_alias_from_ready_providers() { - let catalog = portable_model_catalog(); - let settings = ResolvedRunModelSettings { - fallbacks: vec!["gpt-56-sol".parse::().unwrap()], - ..ResolvedRunModelSettings::default() - }; - - let chain = resolve_fallback_chain( - &catalog, - &ProviderId::openai(), - "gpt-5.6-sol", - &settings, - &HashSet::from([ProviderId::new("openrouter")]), - ) - .unwrap(); - - assert_eq!(chain.targets, vec![FallbackTarget { - provider: "openrouter".to_string(), - model: "gpt-5.6-sol".to_string(), - }]); - } - - #[test] - fn resolve_fallback_chain_resolves_provider_qualified_shared_alias() { - let catalog = portable_model_catalog(); - let settings = ResolvedRunModelSettings { - fallbacks: vec!["openrouter:gpt-56-sol".parse::().unwrap()], - ..ResolvedRunModelSettings::default() - }; - - let chain = resolve_fallback_chain( - &catalog, - &ProviderId::openai(), - "gpt-5.6-sol", - &settings, - &catalog.all_provider_ids(), - ) - .unwrap(); - - assert_eq!(chain.targets, vec![FallbackTarget { - provider: "openrouter".to_string(), - model: "gpt-5.6-sol".to_string(), - }]); - } - - /// A qualified fallback resolves to the same offering whether the selector - /// is the canonical model ID or the provider's API ID. The trailing bare - /// alias still goes through ready-provider priority selection. - #[test] - fn resolve_fallback_chain_resolves_qualified_model_id_and_api_id_alike() { - let overrides: fabro_model::catalog::LlmCatalogSettings = toml::from_str( - r" -[providers.openrouter] -enabled = true -", - ) - .unwrap(); - let catalog = Catalog::from_builtin_with_overrides(&overrides).unwrap(); - - for selector in ["openrouter:kimi-k3", "openrouter:moonshotai/kimi-k3"] { - let settings = ResolvedRunModelSettings { - fallbacks: vec![ - selector.parse::().unwrap(), - "gpt-terra".parse::().unwrap(), - ], - ..ResolvedRunModelSettings::default() - }; - - let chain = resolve_fallback_chain( - &catalog, - &ProviderId::new("kimi"), - "kimi-k3", - &settings, - &HashSet::from([ - ProviderId::new("kimi"), - ProviderId::new("openrouter"), - ProviderId::openai(), - ]), - ) - .unwrap(); - - assert_eq!( - chain.targets, - vec![ - FallbackTarget { - provider: "openrouter".to_string(), - model: "kimi-k3".to_string(), - }, - FallbackTarget { - provider: "openai".to_string(), - model: "gpt-5.6-terra".to_string(), - }, - ], - "{selector}" - ); - } - } - - /// A colon in a model ID does not make it provider-qualified, so an - /// unknown colon-bearing selector still passes through to a provider - /// instead of failing the run with an unknown-provider error. - #[test] - fn resolve_fallback_chain_passes_through_colon_bearing_model_ids() { - let catalog = portable_model_catalog(); - let settings = ResolvedRunModelSettings { - fallbacks: vec!["future-model:latest".parse::().unwrap()], - ..ResolvedRunModelSettings::default() - }; - - let chain = resolve_fallback_chain( - &catalog, - &ProviderId::openai(), - "gpt-5.6-sol", - &settings, - &HashSet::from([ProviderId::openai(), ProviderId::new("openrouter")]), - ) - .unwrap(); - - assert_eq!(chain.targets, vec![FallbackTarget { - provider: ProviderId::openai().to_string(), - model: "future-model:latest".to_string(), - }]); - } - - #[test] - fn resolve_fallback_chain_keeps_qualified_legacy_references_as_provider_pins() { - let catalog = test_catalog(); - let settings = ResolvedRunModelSettings { - fallbacks: vec![ - "openai/gpt-5.6-sol".parse::().unwrap(), - "anthropic/claude-fable-5".parse::().unwrap(), - ], - ..ResolvedRunModelSettings::default() - }; - - let chain = resolve_fallback_chain( - catalog.as_ref(), - &ProviderId::anthropic(), - "claude-opus-4-6", - &settings, - &catalog.all_provider_ids(), - ) - .unwrap(); - - assert_eq!(chain.targets, vec![ - FallbackTarget { - provider: "openai".to_string(), - model: "gpt-5.6-sol".to_string(), - }, - FallbackTarget { - provider: "anthropic".to_string(), - model: "claude-fable-5".to_string(), - }, - ]); - } - - #[test] - fn resolve_fallback_chain_propagates_provider_model_ambiguity() { - let settings: fabro_model::catalog::LlmCatalogSettings = toml::from_str( - r#" -[providers.shared] -display_name = "Shared Provider" -adapter = "openai" -agent_profile = "openai" - -[providers.shared.models.default] -display_name = "Default" -family = "test" -default = true - -[providers.shared.models.default.limits] -context_window = 1000 - -[providers.shared.models.default.features] -tools = false -vision = false -reasoning = false - -[providers.other] -display_name = "Other" -adapter = "openai" -agent_profile = "openai" - -[providers.other.models.model] -display_name = "Shared Alias" -family = "test" -aliases = ["shared"] -default = true - -[providers.other.models.model.limits] -context_window = 1000 - -[providers.other.models.model.features] -tools = false -vision = false -reasoning = false -"#, - ) - .unwrap(); - let catalog = Catalog::from_settings(&settings).unwrap(); - let run_model = ResolvedRunModelSettings { - fallbacks: vec!["shared".parse::().unwrap()], - ..ResolvedRunModelSettings::default() - }; - - let error = resolve_fallback_chain( - &catalog, - &ProviderId::new("other"), - "model", - &run_model, - &catalog.all_provider_ids(), - ) - .unwrap_err(); - - assert!(matches!(error, Error::ModelReference(_))); - } - #[test] fn materialized_provider_pin_is_not_reselected_when_readiness_changes() { let catalog = portable_model_catalog(); diff --git a/lib/components/fabro-workflow/src/pipeline/execute/tests.rs b/lib/components/fabro-workflow/src/pipeline/execute/tests.rs index b34810f64..6fd73a17c 100644 --- a/lib/components/fabro-workflow/src/pipeline/execute/tests.rs +++ b/lib/components/fabro-workflow/src/pipeline/execute/tests.rs @@ -31,6 +31,7 @@ use crate::error::Error; use crate::event::{Emitter, Event, StoreProgressLogger, append_event}; use crate::handler::start::StartHandler; use crate::handler::{Handler as HandlerTrait, HandlerRegistry}; +use crate::model_fallback::ModelFallbackPolicy; use crate::outcome::{Outcome, OutcomeExt, StageOutcome}; use crate::pipeline::initialize; use crate::pipeline::types::{InitOptions, LlmSpec, Persisted, ResumeState, SandboxEnvSpec}; @@ -267,7 +268,7 @@ async fn execute_test_run_with_options( llm: LlmSpec { model: "test-model".to_string(), provider_id: fabro_model::ProviderId::anthropic(), - fallback_chain: Vec::new(), + fallbacks: ModelFallbackPolicy::default(), mcp_servers: Vec::new(), model_controls: RunModelControls::default(), dry_run: true, @@ -327,7 +328,7 @@ async fn execute_runs_start_to_exit_and_returns_final_context() { llm: LlmSpec { model: "test-model".to_string(), provider_id: fabro_model::ProviderId::anthropic(), - fallback_chain: Vec::new(), + fallbacks: ModelFallbackPolicy::default(), mcp_servers: Vec::new(), model_controls: RunModelControls::default(), dry_run: true, @@ -468,7 +469,7 @@ async fn resumed_in_flight_node_starts_a_new_stage_execution() { llm: LlmSpec { model: "test-model".to_string(), provider_id: fabro_model::ProviderId::anthropic(), - fallback_chain: Vec::new(), + fallbacks: ModelFallbackPolicy::default(), mcp_servers: Vec::new(), model_controls: RunModelControls::default(), dry_run: true, @@ -582,7 +583,7 @@ async fn run_with_lifecycle( llm: LlmSpec { model: "test-model".to_string(), provider_id: fabro_model::ProviderId::anthropic(), - fallback_chain: Vec::new(), + fallbacks: ModelFallbackPolicy::default(), mcp_servers: Vec::new(), model_controls: RunModelControls::default(), dry_run: true, diff --git a/lib/components/fabro-workflow/src/pipeline/initialize.rs b/lib/components/fabro-workflow/src/pipeline/initialize.rs index 0f4bfb440..7e0fcb893 100644 --- a/lib/components/fabro-workflow/src/pipeline/initialize.rs +++ b/lib/components/fabro-workflow/src/pipeline/initialize.rs @@ -25,6 +25,8 @@ use crate::git::GitAuthor; use crate::github_token_source::{AppIatMinter, GitHubTokenSource}; use crate::handler::llm::{AgentAcpBackend, AgentApiBackend, BackendRouter, routing}; use crate::handler::{HandlerRegistry, default_registry}; +#[cfg(test)] +use crate::model_fallback::ModelFallbackPolicy; use crate::run_metadata::{RunMetadataRuntime, build_metadata_writer, metadata_branch_name}; use crate::run_options::{GitCheckpointOptions, RunOptions}; use crate::sandbox_git_runtime::SandboxGitRuntime; @@ -166,7 +168,7 @@ async fn build_registry( let build_llm_registry = || { let model = spec.model.clone(); let provider_id = spec.provider_id.clone(); - let fallback_chain = spec.fallback_chain.clone(); + let fallbacks = spec.fallbacks.clone(); let mcp_servers = spec.mcp_servers.clone(); let model_controls = spec.model_controls.clone(); let tool_secrets_for_api = tool_secrets.clone(); @@ -180,7 +182,7 @@ async fn build_registry( let mut api = AgentApiBackend::new_with_catalog( model.clone(), provider_id.clone(), - fallback_chain.clone(), + fallbacks.clone(), Arc::clone(&llm_source_for_api), Arc::clone(&steering_hub_for_api), Arc::clone(&catalog_for_api), @@ -830,7 +832,7 @@ mod tests { llm: LlmSpec { model: "test-model".to_string(), provider_id: fabro_model::ProviderId::anthropic(), - fallback_chain: Vec::new(), + fallbacks: ModelFallbackPolicy::default(), mcp_servers: Vec::new(), model_controls: RunModelControls::default(), dry_run: true, @@ -911,7 +913,7 @@ mod tests { llm: LlmSpec { model: "test-model".to_string(), provider_id: fabro_model::ProviderId::anthropic(), - fallback_chain: Vec::new(), + fallbacks: ModelFallbackPolicy::default(), mcp_servers: Vec::new(), model_controls: RunModelControls::default(), dry_run: true, @@ -1010,7 +1012,7 @@ mod tests { &LlmSpec { model: "claude-opus-4-6".to_string(), provider_id: fabro_model::ProviderId::anthropic(), - fallback_chain: Vec::new(), + fallbacks: ModelFallbackPolicy::default(), mcp_servers: Vec::new(), model_controls: RunModelControls::default(), dry_run: false, @@ -1132,7 +1134,7 @@ mod tests { llm: LlmSpec { model: "fake-acp".to_string(), provider_id: fabro_model::ProviderId::openai(), - fallback_chain: Vec::new(), + fallbacks: ModelFallbackPolicy::default(), mcp_servers: Vec::new(), model_controls: RunModelControls::default(), dry_run: false, @@ -1227,7 +1229,7 @@ mod tests { llm: LlmSpec { model: "test-model".to_string(), provider_id: fabro_model::ProviderId::anthropic(), - fallback_chain: Vec::new(), + fallbacks: ModelFallbackPolicy::default(), mcp_servers: Vec::new(), model_controls: RunModelControls::default(), dry_run: true, @@ -1369,7 +1371,7 @@ mod tests { llm: LlmSpec { model: "test-model".to_string(), provider_id: fabro_model::ProviderId::anthropic(), - fallback_chain: Vec::new(), + fallbacks: ModelFallbackPolicy::default(), mcp_servers: Vec::new(), model_controls: RunModelControls::default(), dry_run: true, diff --git a/lib/components/fabro-workflow/src/pipeline/types.rs b/lib/components/fabro-workflow/src/pipeline/types.rs index 8d257ea9a..ebbe3f4f8 100644 --- a/lib/components/fabro-workflow/src/pipeline/types.rs +++ b/lib/components/fabro-workflow/src/pipeline/types.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use fabro_graphviz::graph::Graph; use fabro_interview::Interviewer; use fabro_mcp::config::McpServerSettings; -use fabro_model::{Catalog, FallbackTarget, ProviderId}; +use fabro_model::{Catalog, ProviderId}; use fabro_sandbox::SandboxSpec; use fabro_template::TemplateContext; use fabro_types::settings::run::{PullRequestSettings, RunModelControls}; @@ -20,6 +20,7 @@ use crate::error::Error; use crate::event::Emitter; use crate::file_resolver::FileResolver; use crate::handler::HandlerRegistry; +use crate::model_fallback::ModelFallbackPolicy; use crate::outcome::Outcome; use crate::records::{Checkpoint, Conclusion, RunSpec}; use crate::run_control::RunControlState; @@ -236,7 +237,7 @@ impl Persisted { pub struct LlmSpec { pub model: String, pub provider_id: ProviderId, - pub fallback_chain: Vec, + pub fallbacks: ModelFallbackPolicy, pub mcp_servers: Vec, pub model_controls: RunModelControls, pub dry_run: bool, diff --git a/lib/components/fabro-workflow/tests/it/integration.rs b/lib/components/fabro-workflow/tests/it/integration.rs index 32b64e605..7ddb72647 100644 --- a/lib/components/fabro-workflow/tests/it/integration.rs +++ b/lib/components/fabro-workflow/tests/it/integration.rs @@ -50,6 +50,7 @@ use fabro_workflow::handler::manager_loop::SubWorkflowHandler; use fabro_workflow::handler::start::StartHandler; use fabro_workflow::handler::wait::WaitHandler; use fabro_workflow::handler::{Handler, HandlerRegistry}; +use fabro_workflow::model_fallback::ModelFallbackPolicy; use fabro_workflow::outcome::{Outcome, OutcomeExt, StageOutcome}; use fabro_workflow::records::{Checkpoint, CheckpointExt}; use fabro_workflow::run_options::{GitCheckpointOptions, RunOptions}; @@ -2332,7 +2333,7 @@ reasoning = false let backend = AgentApiBackend::new_with_catalog( "compact-model".to_string(), ProviderId::from("compact"), - Vec::new(), + ModelFallbackPolicy::default(), source, Arc::new(SteeringHub::new(Arc::new(Emitter::default()))), catalog, @@ -2487,7 +2488,7 @@ base_url = "{}" let backend = AgentApiBackend::new_with_catalog( "openai/gpt-5.4".to_string(), ProviderId::from("openrouter"), - Vec::new(), + ModelFallbackPolicy::default(), source, Arc::new(SteeringHub::new(Arc::new(Emitter::default()))), catalog, diff --git a/lib/foundation/fabro-config/src/builders.rs b/lib/foundation/fabro-config/src/builders.rs index 980faa3f3..08eb4c70b 100644 --- a/lib/foundation/fabro-config/src/builders.rs +++ b/lib/foundation/fabro-config/src/builders.rs @@ -746,7 +746,7 @@ command = ["demo-mcp"] model: Some(RunModelLayer { provider: Some("openai".to_string()), name: Some("gpt-5".to_string()), - fallbacks: Vec::new(), + fallbacks: crate::MergeMap::default(), controls: None, }), execution: Some(RunExecutionLayer { diff --git a/lib/foundation/fabro-config/src/layers/run.rs b/lib/foundation/fabro-config/src/layers/run.rs index 853ba7a0c..97d249c05 100644 --- a/lib/foundation/fabro-config/src/layers/run.rs +++ b/lib/foundation/fabro-config/src/layers/run.rs @@ -148,14 +148,19 @@ pub struct RunModelLayer { #[serde(default, skip_serializing_if = "Option::is_none")] #[option(value_type = "string")] pub name: Option, - /// Ordered fallback references: bare providers, bare model IDs or aliases, - /// or provider-qualified `provider:selector` values. A qualified selector - /// may be a model ID, alias, or provider API ID. Legacy `provider/model` - /// values remain accepted. Supports the `...` splice marker at layering - /// time — see [`super::splice_array`]. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - #[option(default = "[]", value_type = "array")] - pub fallbacks: Vec, + /// Model-keyed fallback chains. Each value is an ordered list of bare + /// providers, bare model IDs or aliases, or provider-qualified + /// `provider:selector` values. A qualified selector may be a model ID, + /// alias, or provider API ID. Legacy `provider/model` values remain + /// accepted. Each list supports the `...` splice marker at layering time. + /// Fabro selects one chain from the original requested model. A fallback + /// target never activates another model's chain. + /// + /// Model keys stay unresolved in this sparse layer because `fabro + /// validate` is offline and has no server model catalog. + #[serde(default, skip_serializing_if = "MergeMap::is_empty")] + #[option(default = "{}", value_type = "table>")] + pub fallbacks: MergeMap>, /// Run-level default values for typed model controls. Node attributes /// and style-applied attributes still win over these defaults. #[serde(default, skip_serializing_if = "Option::is_none")] diff --git a/lib/foundation/fabro-config/src/resolve/run.rs b/lib/foundation/fabro-config/src/resolve/run.rs index 1d20db7a6..4f67cc2d1 100644 --- a/lib/foundation/fabro-config/src/resolve/run.rs +++ b/lib/foundation/fabro-config/src/resolve/run.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use fabro_types::settings::InterpString; use fabro_types::settings::run::{ @@ -123,11 +123,17 @@ fn resolve_model(model: Option<&RunModelLayer>) -> RunModelSettings { fallbacks: model .fallbacks .iter() - .filter_map(|entry| match entry { - ModelRefOrSplice::ModelRef(model_ref) => Some(model_ref.clone()), - ModelRefOrSplice::Splice => None, + .map(|(requested_model, chain)| { + let chain = chain + .iter() + .filter_map(|entry| match entry { + ModelRefOrSplice::ModelRef(model_ref) => Some(model_ref.clone()), + ModelRefOrSplice::Splice => None, + }) + .collect(); + (requested_model.clone(), chain) }) - .collect(), + .collect::>(), controls: model .controls .as_ref() diff --git a/lib/foundation/fabro-config/src/tests/combine.rs b/lib/foundation/fabro-config/src/tests/combine.rs index 426656596..ef296087b 100644 --- a/lib/foundation/fabro-config/src/tests/combine.rs +++ b/lib/foundation/fabro-config/src/tests/combine.rs @@ -100,21 +100,46 @@ script = "higher-1" fn run_model_fallbacks_splice_inserts_inherited() { let lower = parse( r#" -[run.model] -fallbacks = ["openrouter:moonshotai/kimi-k3", "gpt-terra"] +[run.model.fallbacks] +"kimi-k3" = ["openrouter:moonshotai/kimi-k3", "gpt-terra"] +"gpt-sol" = ["claude-opus"] "#, ); let higher = parse( r#" -[run.model] -fallbacks = ["anthropic", "..."] +[run.model.fallbacks] +"kimi-k3" = ["anthropic", "..."] +"claude-fable" = ["gpt-sol", "claude-opus"] "#, ); let merged = higher.combine(lower); let fallbacks = merged.run.unwrap().model.unwrap().fallbacks; assert_eq!( - serde_json::to_value(&fallbacks).unwrap(), - serde_json::json!(["anthropic", "openrouter:moonshotai/kimi-k3", "gpt-terra",]) + serde_json::to_value(fallbacks.get("kimi-k3").unwrap()).unwrap(), + serde_json::json!(["anthropic", "openrouter:moonshotai/kimi-k3", "gpt-terra"]) + ); + assert_eq!( + serde_json::to_value(fallbacks.get("gpt-sol").unwrap()).unwrap(), + serde_json::json!(["claude-opus"]) + ); + assert_eq!( + serde_json::to_value(fallbacks.get("claude-fable").unwrap()).unwrap(), + serde_json::json!(["gpt-sol", "claude-opus"]) + ); +} + +#[test] +fn run_model_fallbacks_rejects_the_removed_array_shape() { + let error = r#" +[run.model] +fallbacks = ["anthropic"] +"# + .parse::() + .expect_err("the global fallback array must stay unsupported"); + + assert!( + error.to_string().contains("invalid type: sequence"), + "unexpected parse error: {error}" ); } diff --git a/lib/foundation/fabro-dev/src/commands/docs_options_reference.rs b/lib/foundation/fabro-dev/src/commands/docs_options_reference.rs index 2422b47bd..47cee4fb2 100644 --- a/lib/foundation/fabro-dev/src/commands/docs_options_reference.rs +++ b/lib/foundation/fabro-dev/src/commands/docs_options_reference.rs @@ -109,7 +109,9 @@ permissions = "read-write""#, r#"[run.model] provider = "anthropic" name = "claude-sonnet-4-5" -fallbacks = ["openrouter:kimi-k3", "gpt-terra"]"#, + +[run.model.fallbacks] +"claude-sonnet-4-5" = ["openrouter:kimi-k3", "gpt-terra"]"#, ), Section::of::( "[cli.logging]", diff --git a/lib/foundation/fabro-model/src/catalog.rs b/lib/foundation/fabro-model/src/catalog.rs index 2e16f378c..f92db6873 100644 --- a/lib/foundation/fabro-model/src/catalog.rs +++ b/lib/foundation/fabro-model/src/catalog.rs @@ -1482,36 +1482,6 @@ impl Catalog { .unwrap_or(std::cmp::Ordering::Equal) }) } - - /// Build an ordered fallback chain for a primary provider/model. - /// - /// For each fallback provider, finds the closest matching model. Providers - /// where no capability match exists (or the provider string doesn't - /// parse) are skipped. - #[must_use] - pub fn build_fallback_chain( - &self, - primary: &ProviderId, - model: &str, - fallbacks: &HashMap>, - ) -> Vec { - let Some(reference) = self.get_on_provider(primary, model) else { - return Vec::new(); - }; - - let Some(fallback_providers) = fallbacks.get(primary.as_str()) else { - return Vec::new(); - }; - - fallback_providers - .iter() - .filter_map(|provider_str| { - let provider = ProviderId::from(provider_str.clone()); - self.closest(&provider, reference) - .map(|m| FallbackTarget::new(provider_str, &m.id)) - }) - .collect() - } } type ModelIndexes = ( @@ -4200,71 +4170,6 @@ enabled = true ); } - #[test] - fn builtin_build_fallback_chain() { - let fallbacks = HashMap::from([("anthropic".to_string(), vec![ - "gemini".to_string(), - "openai".to_string(), - ])]); - let chain = Catalog::builtin().build_fallback_chain( - &ProviderId::anthropic(), - "claude-opus-4-6", - &fallbacks, - ); - assert_eq!(chain.len(), 2); - assert_eq!(chain[0].provider, "gemini"); - assert_eq!(chain[0].model, "gemini-3.1-pro-preview"); - assert_eq!(chain[1].provider, "openai"); - assert_eq!(chain[1].model, "gpt-5.5"); - } - - #[test] - fn builtin_build_fallback_chain_unknown_model() { - let fallbacks = HashMap::from([("anthropic".to_string(), vec!["gemini".to_string()])]); - let chain = Catalog::builtin().build_fallback_chain( - &ProviderId::anthropic(), - "unknown-xyz", - &fallbacks, - ); - assert!(chain.is_empty()); - } - - #[test] - fn builtin_build_fallback_chain_provider_not_in_map() { - let fallbacks = HashMap::from([("openai".to_string(), vec!["anthropic".to_string()])]); - let chain = Catalog::builtin().build_fallback_chain( - &ProviderId::anthropic(), - "claude-opus-4-6", - &fallbacks, - ); - assert!(chain.is_empty()); - } - - #[test] - fn builtin_build_fallback_chain_skips_no_capability_match() { - let fallbacks = HashMap::from([("anthropic".to_string(), vec![ - "openai".to_string(), - "kimi".to_string(), - ])]); - let chain = Catalog::builtin().build_fallback_chain( - &ProviderId::anthropic(), - "claude-haiku-4-5", - &fallbacks, - ); - assert!(chain.is_empty()); - } - - #[test] - fn builtin_build_fallback_chain_empty_map() { - let fallbacks = HashMap::new(); - let chain = Catalog::builtin().build_fallback_chain( - &ProviderId::anthropic(), - "claude-opus-4-6", - &fallbacks, - ); - assert!(chain.is_empty()); - } - #[test] fn builtin_catalog_is_loaded_from_provider_toml_settings() { let catalog = Catalog::builtin(); diff --git a/lib/foundation/fabro-model/src/reasoning.rs b/lib/foundation/fabro-model/src/reasoning.rs index 7420178d0..33f8c78cf 100644 --- a/lib/foundation/fabro-model/src/reasoning.rs +++ b/lib/foundation/fabro-model/src/reasoning.rs @@ -4,6 +4,8 @@ //! validation, OpenAPI replacement types, and the LLM client all share one //! enum so that adding a new effort value remains a Rust change. +use std::cmp::Reverse; + use serde::{Deserialize, Serialize}; #[derive( @@ -37,6 +39,24 @@ impl ReasoningEffort { pub fn variants() -> &'static [Self] { ::VARIANTS } + + /// Select the supported effort nearest to this value. + /// + /// The enum declaration defines the ordered progression. When two values + /// are equally distant, the higher effort wins. + #[must_use] + pub fn closest_supported(self, supported: &[Self]) -> Option { + let variants = Self::variants(); + let requested_rank = variants.iter().position(|effort| *effort == self)?; + + supported.iter().copied().min_by_key(|effort| { + let rank = variants + .iter() + .position(|candidate| candidate == effort) + .expect("supported reasoning effort must be an enum variant"); + (requested_rank.abs_diff(rank), Reverse(rank)) + }) + } } #[cfg(test)] @@ -91,6 +111,49 @@ mod tests { assert_eq!(v[v.len() - 1], ReasoningEffort::Max); } + #[test] + fn closest_supported_uses_exact_match() { + assert_eq!( + ReasoningEffort::High.closest_supported(&[ + ReasoningEffort::Low, + ReasoningEffort::High, + ReasoningEffort::Max, + ]), + Some(ReasoningEffort::High) + ); + } + + #[test] + fn closest_supported_rounds_equal_distance_up() { + let kimi = [ + ReasoningEffort::Low, + ReasoningEffort::High, + ReasoningEffort::Max, + ]; + assert_eq!( + ReasoningEffort::Medium.closest_supported(&kimi), + Some(ReasoningEffort::High) + ); + assert_eq!( + ReasoningEffort::XHigh.closest_supported(&kimi), + Some(ReasoningEffort::Max) + ); + } + + #[test] + fn closest_supported_uses_nearest_lower_value_when_needed() { + assert_eq!( + ReasoningEffort::Max + .closest_supported(&[ReasoningEffort::High, ReasoningEffort::XHigh]), + Some(ReasoningEffort::XHigh) + ); + } + + #[test] + fn closest_supported_returns_none_for_unsupported_control() { + assert_eq!(ReasoningEffort::High.closest_supported(&[]), None); + } + #[test] fn round_trip_through_json() { let json = serde_json::to_string(&ReasoningEffort::High).unwrap(); diff --git a/lib/foundation/fabro-types/src/run_event/misc.rs b/lib/foundation/fabro-types/src/run_event/misc.rs index 19a21b34e..4e2036ea4 100644 --- a/lib/foundation/fabro-types/src/run_event/misc.rs +++ b/lib/foundation/fabro-types/src/run_event/misc.rs @@ -187,11 +187,21 @@ pub struct SshAccessReadyProps { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct FailoverProps { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub original_provider: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub original_model: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub attempt: Option, pub from_provider: String, - pub from_model: String, - pub to_provider: String, - pub to_model: String, - pub error: String, + pub from_model: String, + pub to_provider: String, + pub to_model: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub requested_reasoning_effort: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub effective_reasoning_effort: Option, + pub error: String, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] diff --git a/lib/foundation/fabro-types/src/run_event/mod.rs b/lib/foundation/fabro-types/src/run_event/mod.rs index 6680fff2e..27c3b87e9 100644 --- a/lib/foundation/fabro-types/src/run_event/mod.rs +++ b/lib/foundation/fabro-types/src/run_event/mod.rs @@ -1013,6 +1013,33 @@ mod tests { assert!(matches!(parsed.body, EventBody::RunCreated(_))); } + #[test] + fn historical_failover_event_defaults_new_route_context() { + let line = json!({ + "id": "evt_failover", + "ts": "2026-04-04T12:00:00.000Z", + "run_id": fixtures::RUN_1, + "event": "agent.failover", + "properties": { + "from_provider": "anthropic", + "from_model": "claude-fable-5", + "to_provider": "openai", + "to_model": "gpt-5.6-sol", + "error": "provider unavailable" + } + }); + + let parsed = RunEvent::from_value(line).unwrap(); + let EventBody::Failover(props) = parsed.body else { + panic!("expected agent.failover"); + }; + assert_eq!(props.original_provider, None); + assert_eq!(props.original_model, None); + assert_eq!(props.attempt, None); + assert_eq!(props.requested_reasoning_effort, None); + assert_eq!(props.effective_reasoning_effort, None); + } + #[test] fn run_created_round_trip_preserves_manifest_blob() { let line = json!({ diff --git a/lib/foundation/fabro-types/src/settings/model_ref.rs b/lib/foundation/fabro-types/src/settings/model_ref.rs index 184bb8305..b551684b7 100644 --- a/lib/foundation/fabro-types/src/settings/model_ref.rs +++ b/lib/foundation/fabro-types/src/settings/model_ref.rs @@ -1,6 +1,6 @@ //! Model references for `run.model.fallbacks`. //! -//! Each entry is one of: +//! Each fallback-chain entry is one of: //! //! - a bare token such as `openai` or `gpt-5.4` — the parser cannot tell alone //! whether the token is a provider name or a model alias diff --git a/lib/foundation/fabro-types/src/settings/run.rs b/lib/foundation/fabro-types/src/settings/run.rs index c5f62b480..5ecebdd0c 100644 --- a/lib/foundation/fabro-types/src/settings/run.rs +++ b/lib/foundation/fabro-types/src/settings/run.rs @@ -6,7 +6,7 @@ //! notifications, interviews, agent knobs, hooks, SCM targeting, pull-request //! behavior, and artifact collection. -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use std::path::PathBuf; use std::time::Duration as StdDuration; @@ -684,7 +684,12 @@ pub enum RunGoal { pub struct RunModelSettings { pub provider: Option, pub name: Option, - pub fallbacks: Vec, + /// Ordered fallback references keyed by the originally requested model. + /// + /// Keys remain raw selectors during offline configuration resolution. + /// The server canonicalizes them against its model catalog before a run + /// starts. + pub fallbacks: BTreeMap>, /// Run-level default values for typed model controls /// (`reasoning_effort`, `speed`). Node and style attributes still win /// over these defaults. diff --git a/lib/packages/fabro-api-client/src/models/run-model-settings.ts b/lib/packages/fabro-api-client/src/models/run-model-settings.ts index d6bbf1994..a2e8c4d4e 100644 --- a/lib/packages/fabro-api-client/src/models/run-model-settings.ts +++ b/lib/packages/fabro-api-client/src/models/run-model-settings.ts @@ -17,5 +17,8 @@ export interface RunModelSettings { 'provider': string | null; 'name': string | null; - 'fallbacks': Array; + /** + * Ordered fallback targets keyed by the originally requested model. Each chain is independent; selecting a fallback target does not activate that target model\'s own chain. + */ + 'fallbacks': { [key: string]: Array; }; } diff --git a/test/offline-fallbacks/workflow.fabro b/test/offline-fallbacks/workflow.fabro new file mode 100644 index 000000000..c35213b73 --- /dev/null +++ b/test/offline-fallbacks/workflow.fabro @@ -0,0 +1,10 @@ +digraph OfflineFallbacks { + graph [goal="Validate a future model fallback offline"] + + start [shape=Mdiamond, label="Start"] + exit [shape=Msquare, label="Exit"] + + work [label="Work", prompt="Do work", model="future-primary"] + + start -> work -> exit +} diff --git a/test/offline-fallbacks/workflow.toml b/test/offline-fallbacks/workflow.toml new file mode 100644 index 000000000..565fc58a3 --- /dev/null +++ b/test/offline-fallbacks/workflow.toml @@ -0,0 +1,7 @@ +_version = 1 + +[workflow] +graph = "workflow.fabro" + +[run.model.fallbacks] +"future-primary" = ["future-provider:future-model"]