diff --git a/docs/public/administration/server-configuration.mdx b/docs/public/administration/server-configuration.mdx index 9ec040d28..c806406f7 100644 --- a/docs/public/administration/server-configuration.mdx +++ b/docs/public/administration/server-configuration.mdx @@ -92,7 +92,9 @@ level = "info" [run.model] name = "claude-sonnet-4-5" provider = "anthropic" -fallbacks = ["gemini", "openai"] + +[run.model.fallbacks] +"claude-sonnet-4-5" = ["gemini", "openai"] [[run.prepare.steps]] script = "npm install" diff --git a/docs/public/api-reference/fabro-api.yaml b/docs/public/api-reference/fabro-api.yaml index bd6a3fdb4..80681f93a 100644 --- a/docs/public/api-reference/fabro-api.yaml +++ b/docs/public/api-reference/fabro-api.yaml @@ -14315,9 +14315,29 @@ components: name: type: ["string", "null"] fallbacks: - type: array - items: - $ref: "#/components/schemas/ModelRef" + type: object + description: > + Ordered fallback targets keyed by the originally requested model. + Each chain is independent; selecting a fallback target does not + activate that target model's own chain. + additionalProperties: + type: array + items: + $ref: "#/components/schemas/ModelRef" + controls: + $ref: "#/components/schemas/RunModelControls" + + RunModelControls: + type: object + description: > + Run-level default values for typed model controls. Node and style + attributes still win over these defaults. + required: [reasoning_effort, speed] + properties: + reasoning_effort: + type: ["string", "null"] + speed: + type: ["string", "null"] RunGitSettings: type: object diff --git a/docs/public/core-concepts/models.mdx b/docs/public/core-concepts/models.mdx index ecd5deb07..a4befa37f 100644 --- a/docs/public/core-concepts/models.mdx +++ b/docs/public/core-concepts/models.mdx @@ -285,7 +285,9 @@ goal = "Implement the feature" [run.model] name = "claude-sonnet-4-5" -fallbacks = ["gemini", "openai"] + +[run.model.fallbacks] +"claude-sonnet-4-5" = ["gemini", "openai"] ``` Then launch with: @@ -294,7 +296,9 @@ Then launch with: fabro run run.toml ``` -The `fallbacks` array is optional. Each entry may be a bare provider token (like `"gemini"`), a bare model ID or alias (like `"gpt-terra"`), or a qualified `"provider:selector"` reference. A qualified selector may be the provider's canonical model ID, alias, or API ID, including API IDs with slashes such as `"openrouter:moonshotai/kimi-k3"`. Fabro tries entries in order when the primary provider is unavailable, and qualified references remain provider pins. Legacy `provider/model` references remain accepted for compatibility. +The `[run.model.fallbacks]` table is optional. Each key names the originally requested model. Its value is the ordered list to try after that model's active provider fails. An entry may be a bare provider token (like `"gemini"`), a bare model ID or alias (like `"gpt-terra"`), or a qualified `"provider:selector"` reference. A qualified selector may be the provider's canonical model ID, alias, or API ID, including API IDs with slashes such as `"openrouter:moonshotai/kimi-k3"`. + +Fabro selects one chain from the original request. It does not switch to the chain configured for a fallback target. When the requested reasoning level is unavailable on a fallback target, Fabro uses the nearest supported level. Equal-distance choices round up. The precedence order is: node-level stylesheet > run config TOML > CLI flags > server defaults. More specific settings always win. diff --git a/docs/public/examples/semantic-port.mdx b/docs/public/examples/semantic-port.mdx index 9593facdf..10e559805 100644 --- a/docs/public/examples/semantic-port.mdx +++ b/docs/public/examples/semantic-port.mdx @@ -214,25 +214,29 @@ The stylesheet assigns three tiers of models: Pair the workflow with a run config TOML for repeatable execution: ```toml title="run.toml" -version = 1 -goal = "Port semantic changes from upstream openai-agents-python to our Go SDK" +_version = 1 + +[workflow] graph = "semport.fabro" -[llm] -model = "claude-sonnet-4-5" +[run] +goal = "Port semantic changes from upstream openai-agents-python to our Go SDK" + +[run.model] +name = "claude-sonnet-4-5" provider = "anthropic" -[llm.fallbacks] -anthropic = ["openai"] -gemini = ["anthropic"] +[run.model.fallbacks] +"claude-sonnet-4-5" = ["openai"] +"gemini-3.1-pro-preview" = ["anthropic"] -[setup] -commands = [ - "git clone https://github.com/openai/openai-agents-python upstream || (cd upstream && git pull)", - "pip install -r ledger/requirements.txt" -] +[[run.prepare.steps]] +script = "git clone https://github.com/openai/openai-agents-python upstream || (cd upstream && git pull)" -[vars] +[[run.prepare.steps]] +script = "pip install -r ledger/requirements.txt" + +[run.inputs] upstream_repo = "openai/openai-agents-python" downstream_lang = "go" ``` diff --git a/docs/public/examples/solitaire.mdx b/docs/public/examples/solitaire.mdx index 9b6abeb86..143be3f6e 100644 --- a/docs/public/examples/solitaire.mdx +++ b/docs/public/examples/solitaire.mdx @@ -266,7 +266,9 @@ goal = "Build a terminal-based solitaire (Klondike) game in Python" [run.model] name = "claude-sonnet-4-5" provider = "anthropic" -fallbacks = ["openai", "gemini"] + +[run.model.fallbacks] +"claude-sonnet-4-5" = ["openai", "gemini"] [[run.prepare.steps]] script = "python3 -m venv .venv && . .venv/bin/activate && pip install pytest curses" diff --git a/docs/public/execution/failures.mdx b/docs/public/execution/failures.mdx index bddd0dc7b..9cdad1d1e 100644 --- a/docs/public/execution/failures.mdx +++ b/docs/public/execution/failures.mdx @@ -112,13 +112,15 @@ When a handler reports a retryable failure, retries always proceed if attempts r ## Model fallbacks -When a model provider fails with a transient error or quota exhaustion, Fabro can automatically switch to a different provider. Configure fallback chains in your [run configuration](/execution/run-configuration): +When a model provider fails with a provider-local error, Fabro can automatically switch to another target. Configure one fixed chain for each requested model in your [run configuration](/execution/run-configuration): ```toml title="run.toml" [run.model] name = "claude-opus-4-6" provider = "anthropic" -fallbacks = ["gemini", "openai"] + +[run.model.fallbacks] +"claude-opus-4-6" = ["gemini", "openai"] ``` When Anthropic fails, Fabro tries Gemini first, then OpenAI. Fallback resolution is provider-aware: @@ -129,7 +131,9 @@ When Anthropic fails, Fabro tries Gemini first, then OpenAI. Fallback resolution Qualified fallback references always remain provider pins. For example, `"openai:gpt-5.6-sol"` pins the direct OpenAI offering. Legacy `provider/model` fallback references remain accepted for compatibility. -The primary provider and model were already resolved and persisted when the run was created; resuming does not re-run primary selection. Fallbacks are only considered after an eligible runtime failure. +Fabro selects the chain by the original requested model. A target in that chain never activates the target model's own chain. The same chain position is retained across structured-output repairs and cached agent sessions. + +The primary provider and model were already resolved and persisted when the run was created; resuming does not re-run primary selection. Fallbacks are only considered after an eligible runtime failure. If the fallback model does not support the requested reasoning level, Fabro uses the nearest supported level and rounds equal-distance choices up. ### What triggers failover @@ -141,7 +145,10 @@ Failover is a superset of LLM retry eligibility: | Server error (5xx) | Yes | Yes | | Timeout / network | Yes | Yes | | Quota exceeded | No | Yes | -| Authentication (401) | No | No | +| Authentication (401) | No | Yes | +| Access denied (403) | No | Yes | +| Model not found (404) | No | Yes | +| Model refusal | No | Yes | | Invalid request (400) | No | No | | Context length (413) | No | No | | Content filter | No | No | diff --git a/docs/public/execution/run-configuration.mdx b/docs/public/execution/run-configuration.mdx index eb81c82a9..dd8819052 100644 --- a/docs/public/execution/run-configuration.mdx +++ b/docs/public/execution/run-configuration.mdx @@ -45,11 +45,13 @@ working_dir = "/tmp/workdir" [run.model] name = "claude-sonnet-4-5" -fallbacks = ["openai", "gemini"] [run.model.controls] reasoning_effort = "high" +[run.model.fallbacks] +"claude-sonnet-4-5" = ["openai", "gemini"] + [[run.prepare.steps]] script = "git clone https://github.com/fabro-sh/fabro repo" @@ -140,25 +142,43 @@ name = "claude-sonnet-4-5" |---|---| | `name` | Canonical model slug or alias (e.g. `claude-sonnet-4-5`, `opus`, `gemini-pro`). See [Models](/core-concepts/models). | | `provider` | Optional provider pin. When omitted, Fabro selects among ready offerings by provider priority. When present, an unavailable provider is an error rather than permission to switch. | -| `fallbacks` | Ordered list of model references to try when the primary is unavailable. Entries can be bare provider tokens (`"openai"`), bare model IDs or aliases, or qualified `"provider:selector"` references. | +| `fallbacks` | Table of ordered fallback lists keyed by the originally requested model. | Provider values are catalog provider ID strings. Built-in IDs like `anthropic` and `openai` work, and settings-defined IDs like `proxy` work after they are added under `[llm.providers.]`. For a qualified fallback, the selector may be that provider's canonical model ID, alias, or API ID. Fabro splits on the first `:` when the part before it names a known provider, so provider API IDs may contain `/` or additional colons: ```toml title="run.toml" -[run.model] -fallbacks = [ +[run.model.fallbacks] +"kimi-k3" = [ + "kimi:kimi-k3", "openrouter:kimi-k3", - "gpt-terra", + "claude-opus", ] ``` -The first entry could equivalently be written as `"openrouter:moonshotai/kimi-k3"` using OpenRouter's API ID; both forms resolve to its canonical `kimi-k3` offering. The unqualified `gpt-terra` alias uses normal ready-provider priority selection. Legacy `provider/model` fallback references remain accepted but are normalized to `provider:model`. +This chain applies only when the original request resolves to `kimi-k3`. It tries direct Moonshot AI, then OpenRouter, then Claude Opus. The OpenRouter entry could equivalently be written as `"openrouter:moonshotai/kimi-k3"` using its API ID. Legacy `provider/model` fallback references remain accepted but are normalized to `provider:model`. A colon alone does not make a reference qualified. Many model IDs contain one — ollama `name:tag` values, Bedrock inference-profile IDs and ARNs — so Fabro treats the reference as qualified only when the text before the first `:` names a known provider. `"llama3:8b"` stays a single model ID, while `"ollama:llama3:8b"` pins the `ollama` provider and passes `llama3:8b` as the selector. -At run creation, Fabro resolves the primary selector and every node selector against the ready-provider snapshot. It persists the selected canonical model slug and provider, so resuming the run does not choose a different provider just because credentials or priorities changed. The configured fallback chain remains available for failures that occur while the materialized run is executing. +At run creation, Fabro resolves the primary selector, every node selector, and the fallback table against the server's ready-provider snapshot. It persists the selected canonical model slug and provider, so resuming the run does not choose a different provider just because credentials or priorities changed. `fabro validate` only checks the table's TOML shape because it is offline and has no server model catalog. Use `fabro preflight` for catalog and provider checks. + +Each original requested model selects one fixed chain. Fabro does not jump to the chain configured for a fallback target. When a target lacks the requested reasoning level, Fabro uses the nearest supported level. It rounds equal-distance choices up. + +For example, a server that selects Modal as the primary `kimi-k3` offering can define these independent chains: + +```toml title="settings.toml" +[run.model.fallbacks] +"kimi-k3" = ["kimi:kimi-k3", "openrouter:kimi-k3", "claude-opus"] +"glm-5.2" = ["gpt-sol"] +"gpt-sol" = ["claude-opus"] +"claude-opus" = ["gpt-sol"] +"gpt-terra" = ["claude-opus"] +"gpt-luna" = ["claude-sonnet"] +"claude-fable" = ["gpt-sol", "claude-opus"] +``` + +If `claude-fable` falls back to `gpt-sol`, Fabro continues with `claude-opus` from the Fable list. It does not restart from the separate `gpt-sol` list. Historical built-in provider API IDs are accepted for compatibility and normalize before this selection. For example, `name = "openai/gpt-5.6-sol"` is treated as the canonical `gpt-5.6-sol` selector; omit `provider` to use readiness and priority, or set `provider` separately to pin an offering. @@ -181,14 +201,14 @@ speed = "fast" | `reasoning_effort` | Native reasoning-effort value to request when the selected model allows it, such as `"low"`, `"medium"`, `"high"`, `"xhigh"`, or `"max"`. | | `speed` | Native speed value to request when the selected model declares it, such as `"fast"`. The standard speed is implicit and does not need to be set. | -#### Fallbacks with splice +#### Fallback lists with splice -Use the reserved `"..."` marker in `fallbacks` to splice in the inherited list from lower-precedence layers: +Use the reserved `"..."` marker in one model's list to splice in that model's inherited list from lower-precedence layers: ```toml title="run.toml" -[run.model] -# Prepend "anthropic" to whatever fallbacks the project config already defines. -fallbacks = ["anthropic", "..."] +[run.model.fallbacks] +# Prepend Anthropic to the inherited chain for gpt-5.6-sol. +"gpt-5.6-sol" = ["anthropic", "..."] ``` ### `[run.prepare]` diff --git a/docs/public/reference/user-configuration.mdx b/docs/public/reference/user-configuration.mdx index e60feb204..938468731 100644 --- a/docs/public/reference/user-configuration.mdx +++ b/docs/public/reference/user-configuration.mdx @@ -381,12 +381,14 @@ permissions = "read-write" [run.model] provider = "anthropic" name = "claude-sonnet-4-5" -fallbacks = ["openrouter:kimi-k3", "gpt-terra"] + +[run.model.fallbacks] +"claude-sonnet-4-5" = ["openrouter:kimi-k3", "gpt-terra"] ``` | Key | Type / values | Default | Description | |---|---|---|---| -| `fallbacks` | array | [] | Ordered fallback references: bare providers, bare model IDs or aliases,
or provider-qualified `provider:selector` values. A qualified selector
may be a model ID, alias, or provider API ID. Legacy `provider/model`
values remain accepted. Supports the `...` splice marker at layering
time — see [`super::splice_array`]. | +| `fallbacks` | table> | {} | Model-keyed fallback chains. Each value is an ordered list of bare
providers, bare model IDs or aliases, or provider-qualified
`provider:selector` values. A qualified selector may be a model ID,
alias, or provider API ID. Legacy `provider/model` values remain
accepted. Each list supports the `...` splice marker at layering time.
Fabro selects one chain from the original requested model. A fallback
target never activates another model's chain.

Model keys stay unresolved in this sparse layer because `fabro
validate` is offline and has no server model catalog. | | `name` | string | None | Model name for workflow runs. | | `provider` | string | None | Provider name for workflow model selection. | diff --git a/lib/apps/fabro-cli/tests/it/cmd/attach.rs b/lib/apps/fabro-cli/tests/it/cmd/attach.rs index c1ff07165..bbb802be5 100644 --- a/lib/apps/fabro-cli/tests/it/cmd/attach.rs +++ b/lib/apps/fabro-cli/tests/it/cmd/attach.rs @@ -997,7 +997,7 @@ fn attach_json_errors_without_prompting_for_human_input() { "reasoning_effort": null, "speed": null }, - "fallbacks": [], + "fallbacks": {}, "name": "[DEFAULT_MODEL]", "provider": "openai" }, diff --git a/lib/apps/fabro-cli/tests/it/cmd/inspect.rs b/lib/apps/fabro-cli/tests/it/cmd/inspect.rs index 086db6b32..2aed43ef4 100644 --- a/lib/apps/fabro-cli/tests/it/cmd/inspect.rs +++ b/lib/apps/fabro-cli/tests/it/cmd/inspect.rs @@ -129,7 +129,7 @@ fn inspect_resolves_selector_via_server_endpoint() { "model": { "provider": null, "name": null, - "fallbacks": [], + "fallbacks": {}, "controls": { "reasoning_effort": null, "speed": null diff --git a/lib/apps/fabro-cli/tests/it/cmd/validate.rs b/lib/apps/fabro-cli/tests/it/cmd/validate.rs index 65048ee61..f82f5dc9e 100644 --- a/lib/apps/fabro-cli/tests/it/cmd/validate.rs +++ b/lib/apps/fabro-cli/tests/it/cmd/validate.rs @@ -87,6 +87,23 @@ fn server_owned_provider_is_not_rejected_by_offline_validation() { "); } +#[test] +fn model_fallback_table_is_not_catalog_checked_by_offline_validation() { + let cli = LightweightCli::new(); + let mut cmd = cli.command(); + cmd.env("FABRO_SERVER", "http://127.0.0.1:9") + .arg("validate") + .arg(fixture("offline-fallbacks/workflow.fabro")); + + let output = cmd.output().expect("validate should execute"); + assert!( + output.status.success(), + "offline validation should parse model fallback tables without a server catalog\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); +} + #[test] fn branching() { let context = test_context!(); diff --git a/lib/apps/fabro-server/src/run_manifest.rs b/lib/apps/fabro-server/src/run_manifest.rs index 611fd7cd8..fd9a1f50c 100644 --- a/lib/apps/fabro-server/src/run_manifest.rs +++ b/lib/apps/fabro-server/src/run_manifest.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use std::future::Future; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -25,15 +25,18 @@ use fabro_sandbox::from_environment::{ use fabro_sandbox::redact::redact_auth_url; use fabro_sandbox::{DockerSandboxOptions, Sandbox, SandboxSpec}; use fabro_static::EnvVars; +use fabro_types::settings::ModelRef; use fabro_types::settings::cli::OutputVerbosity; use fabro_types::settings::interp::InterpString; use fabro_types::settings::run::{EnvironmentProvider, McpServerSettings, RunGoal, RunNamespace}; use fabro_types::{ - ManifestPath, RunId, RunProvenance, SandboxProviderKind, ServerSettings, WorkflowSettings, + ManifestPath, RunId, RunNoticeLevel, RunProvenance, SandboxProviderKind, ServerSettings, + WorkflowSettings, }; use fabro_util::check_report::{CheckDetail, CheckReport, CheckResult, CheckSection, CheckStatus}; use fabro_validate::Severity; use fabro_workflow::Error as WorkflowError; +use fabro_workflow::model_fallback::resolve_model_fallbacks; use fabro_workflow::operations::{ CreateRunInput, ValidateInput, WorkflowInput, validate, validate_with_catalog, validate_with_ready_providers, @@ -544,6 +547,12 @@ async fn build_preflight_report( )); } run_environment_capability_check(&mut checks, &resolved_run); + let model_fallbacks_ok = run_model_fallback_check( + &mut checks, + catalog.as_ref(), + &ready_providers, + &resolved_run.model.fallbacks, + ); let needs_github_credentials = sandbox_provider.is_clone_based() || resolved_run.integrations.github.is_token_requested(); let github_app = if needs_github_credentials { @@ -592,7 +601,8 @@ async fn build_preflight_report( let github_token_ok = run_github_token_check(&mut checks, prepared, &resolved_run, github_app).await; - let checks_ok = sandbox_ok && repository_access_ok && llm_ok && github_token_ok; + let checks_ok = + model_fallbacks_ok && sandbox_ok && repository_access_ok && llm_ok && github_token_ok; Ok(( CheckReport { @@ -606,6 +616,72 @@ async fn build_preflight_report( )) } +fn run_model_fallback_check( + checks: &mut Vec, + catalog: &Catalog, + ready_providers: &[ProviderId], + configured: &BTreeMap>, +) -> bool { + if configured.is_empty() { + return true; + } + + let resolved = match resolve_model_fallbacks(catalog, ready_providers, configured) { + Ok(resolved) => resolved, + Err(error) => { + checks.push(CheckResult { + name: "Model Fallbacks".into(), + status: CheckStatus::Error, + summary: "invalid".into(), + details: configured + .keys() + .map(|model| CheckDetail::new(format!("Requested model: {model}"))) + .collect(), + remediation: Some(error.to_string()), + }); + return false; + } + }; + + let has_warning = resolved + .notices + .iter() + .any(|notice| notice.level() != RunNoticeLevel::Info); + let mut details = resolved + .policy + .iter() + .map(|(model, targets)| { + let chain = if targets.is_empty() { + "(none)".to_string() + } else { + targets + .iter() + .map(ToString::to_string) + .collect::>() + .join(" -> ") + }; + CheckDetail::new(format!("{model}: {chain}")) + }) + .collect::>(); + details.extend(resolved.notices.iter().map(|notice| CheckDetail { + text: notice.message(), + warn: notice.level() != RunNoticeLevel::Info, + })); + + checks.push(CheckResult { + name: "Model Fallbacks".into(), + status: if has_warning { + CheckStatus::Warning + } else { + CheckStatus::Pass + }, + summary: format!("{} requested model chain(s)", resolved.policy.len()), + details, + remediation: None, + }); + true +} + fn base_preflight_checks(prepared: &PreparedManifest, graph: &Graph) -> Vec { let setup_command_count = prepared.settings.run.prepare.steps.len(); let repo_summary = prepared.git.as_ref().map_or_else( @@ -1495,6 +1571,106 @@ mod tests { Arc::new(Catalog::from_builtin().unwrap()) } + fn openrouter_catalog() -> Catalog { + let overrides = toml::from_str( + r" +[providers.openrouter] +enabled = true +", + ) + .expect("catalog override should parse"); + Catalog::from_builtin_with_overrides(&overrides).expect("catalog should build") + } + + fn model_refs(values: &[&str]) -> Vec { + values + .iter() + .map(|value| value.parse().expect("fallback reference should parse")) + .collect() + } + + #[test] + fn model_fallback_preflight_resolves_each_requested_model_chain() { + let mut checks = Vec::new(); + let configured = std::collections::BTreeMap::from([ + ("gpt-sol".to_string(), model_refs(&["claude-opus"])), + ( + "claude-fable".to_string(), + model_refs(&["gpt-sol", "claude-opus"]), + ), + ]); + + assert!(run_model_fallback_check( + &mut checks, + &openrouter_catalog(), + &[ProviderId::new("openrouter")], + &configured, + )); + + let check = checks.last().expect("fallback check should be present"); + assert_eq!(check.status, CheckStatus::Pass); + assert!( + check + .details + .iter() + .any(|detail| detail.text == "gpt-5.6-sol: openrouter:claude-opus-5") + ); + assert!(check.details.iter().any(|detail| { + detail.text == "claude-fable-5: openrouter:gpt-5.6-sol -> openrouter:claude-opus-5" + })); + } + + #[test] + fn model_fallback_preflight_warns_when_a_provider_is_not_ready() { + let mut checks = Vec::new(); + let configured = std::collections::BTreeMap::from([( + "kimi-k3".to_string(), + model_refs(&["kimi:kimi-k3", "openrouter:kimi-k3"]), + )]); + + assert!(run_model_fallback_check( + &mut checks, + &openrouter_catalog(), + &[ProviderId::new("openrouter")], + &configured, + )); + + let check = checks.last().expect("fallback check should be present"); + assert_eq!(check.status, CheckStatus::Warning); + assert!( + check + .details + .iter() + .any(|detail| detail.warn + && detail.text.contains("provider `kimi` is not configured")) + ); + } + + #[test] + fn model_fallback_preflight_rejects_duplicate_canonical_keys() { + let mut checks = Vec::new(); + let configured = std::collections::BTreeMap::from([ + ("gpt-sol".to_string(), model_refs(&["claude-opus"])), + ("gpt-5.6-sol".to_string(), model_refs(&["claude-fable"])), + ]); + + assert!(!run_model_fallback_check( + &mut checks, + &openrouter_catalog(), + &[ProviderId::new("openrouter")], + &configured, + )); + + let check = checks.last().expect("fallback check should be present"); + assert_eq!(check.status, CheckStatus::Error); + assert!( + check + .remediation + .as_deref() + .is_some_and(|message| message.contains("both resolve to requested model")) + ); + } + fn openai_compatible_completion(model: &str) -> serde_json::Value { serde_json::json!({ "id": "chatcmpl_preflight", diff --git a/lib/apps/fabro-server/tests/it/scenario/run_completion.rs b/lib/apps/fabro-server/tests/it/scenario/run_completion.rs index 8c7e5aeb1..55ba6c938 100644 --- a/lib/apps/fabro-server/tests/it/scenario/run_completion.rs +++ b/lib/apps/fabro-server/tests/it/scenario/run_completion.rs @@ -63,7 +63,7 @@ fn test_app_with_openai_agent_backend(openai_base_url: String, api_key: String) fabro_workflow::handler::llm::AgentApiBackend::new_with_catalog( OPENAI_AGENT_MODEL.to_string(), ProviderId::openai(), - Vec::new(), + fabro_workflow::model_fallback::ModelFallbackPolicy::default(), Arc::clone(&llm_source), Arc::clone(&steering_hub), Arc::clone(&catalog), diff --git a/lib/components/fabro-llm/src/error.rs b/lib/components/fabro-llm/src/error.rs index 000f36943..753c12e6f 100644 --- a/lib/components/fabro-llm/src/error.rs +++ b/lib/components/fabro-llm/src/error.rs @@ -210,8 +210,9 @@ impl Error { /// Whether this error is eligible for provider-level failover. /// /// Includes everything that is `retryable()` (transient errors good for - /// same-provider retry) plus `QuotaExceeded` — a different provider won't - /// share the same quota. + /// same-provider retry), provider-local availability failures, and + /// `QuotaExceeded`. A different provider has independent credentials, + /// access policy, model inventory, and quota. #[must_use] pub fn failover_eligible(&self) -> bool { if self.retryable() { @@ -220,7 +221,10 @@ impl Error { matches!( self, Self::Provider { - kind: ProviderErrorKind::QuotaExceeded, + kind: ProviderErrorKind::Authentication + | ProviderErrorKind::AccessDenied + | ProviderErrorKind::NotFound + | ProviderErrorKind::QuotaExceeded, .. } | Self::RequestTimeout { .. } ) || self.refusal_content_filter() @@ -887,6 +891,26 @@ mod tests { ); } + #[test] + fn failover_eligible_provider_local_availability_errors() { + let detail = || Box::new(ProviderErrorDetail::new("error", "openai")); + + for kind in [ + ProviderErrorKind::Authentication, + ProviderErrorKind::AccessDenied, + ProviderErrorKind::NotFound, + ] { + assert!( + Error::Provider { + kind, + detail: detail(), + } + .failover_eligible(), + "{kind:?} should permit another provider" + ); + } + } + #[test] fn failover_eligible_transient_non_provider_errors() { assert!( @@ -918,14 +942,6 @@ mod tests { fn failover_not_eligible_deterministic_errors() { let detail = || Box::new(ProviderErrorDetail::new("error", "openai")); - assert!( - !Error::Provider { - kind: ProviderErrorKind::Authentication, - detail: detail(), - } - .failover_eligible() - ); - assert!( !Error::Provider { kind: ProviderErrorKind::InvalidRequest, diff --git a/lib/components/fabro-manifest/src/lib.rs b/lib/components/fabro-manifest/src/lib.rs index f2f1b066b..13bae0051 100644 --- a/lib/components/fabro-manifest/src/lib.rs +++ b/lib/components/fabro-manifest/src/lib.rs @@ -76,7 +76,7 @@ pub fn build_run_overrides(input: RunOverrideInput<'_>) -> RunLayer { let model = (input.model.is_some() || input.provider.is_some()).then(|| RunModelLayer { provider: input.provider.map(String::from), name: input.model.map(String::from), - fallbacks: Vec::new(), + fallbacks: MergeMap::default(), controls: None, }); let environment = (input.environment.is_some() diff --git a/lib/components/fabro-workflow/src/event/convert.rs b/lib/components/fabro-workflow/src/event/convert.rs index c7901d08c..a6275bfad 100644 --- a/lib/components/fabro-workflow/src/event/convert.rs +++ b/lib/components/fabro-workflow/src/event/convert.rs @@ -1146,20 +1146,7 @@ fn event_body_from_event(event: &Event) -> EventBody { ssh_command: ssh_command.clone(), }) } - Event::Failover { - from_provider, - from_model, - to_provider, - to_model, - error, - .. - } => EventBody::Failover(fabro_types::FailoverProps { - from_provider: from_provider.clone(), - from_model: from_model.clone(), - to_provider: to_provider.clone(), - to_model: to_model.clone(), - error: error.clone(), - }), + Event::Failover { props, .. } => EventBody::Failover(props.clone()), Event::CommandStarted { script, command, diff --git a/lib/components/fabro-workflow/src/event/emitter.rs b/lib/components/fabro-workflow/src/event/emitter.rs index 339919b18..89f4d8600 100644 --- a/lib/components/fabro-workflow/src/event/emitter.rs +++ b/lib/components/fabro-workflow/src/event/emitter.rs @@ -84,6 +84,24 @@ impl Emitter { }); } + pub fn notice_scoped( + &self, + level: RunNoticeLevel, + code: RunNoticeCode, + message: impl Into, + scope: &StageScope, + ) { + self.emit_scoped( + &Event::RunNotice { + level, + code: code.to_string(), + message: message.into(), + exec_output_tail: None, + }, + scope, + ); + } + pub fn notice_with_tail( &self, level: RunNoticeLevel, diff --git a/lib/components/fabro-workflow/src/event/events.rs b/lib/components/fabro-workflow/src/event/events.rs index 77caf70cc..126b23d90 100644 --- a/lib/components/fabro-workflow/src/event/events.rs +++ b/lib/components/fabro-workflow/src/event/events.rs @@ -583,12 +583,8 @@ pub enum Event { ssh_command: String, }, Failover { - stage: String, - from_provider: String, - from_model: String, - to_provider: String, - to_model: String, - error: String, + stage: String, + props: fabro_types::FailoverProps, }, CommandStarted { node_id: String, @@ -1393,21 +1389,19 @@ impl Event { Self::SshAccessReady { ssh_command } => { info!(ssh_command, "SSH access ready"); } - Self::Failover { - stage, - from_provider, - from_model, - to_provider, - to_model, - error, - } => { + Self::Failover { stage, props } => { warn!( stage, - from_provider, - from_model, - to_provider, - to_model, - error, + original_provider = ?props.original_provider, + original_model = ?props.original_model, + attempt = ?props.attempt, + from_provider = %props.from_provider, + from_model = %props.from_model, + to_provider = %props.to_provider, + to_model = %props.to_model, + requested_reasoning_effort = ?props.requested_reasoning_effort, + effective_reasoning_effort = ?props.effective_reasoning_effort, + error = %props.error, "LLM provider failover" ); } diff --git a/lib/components/fabro-workflow/src/handler/llm/api.rs b/lib/components/fabro-workflow/src/handler/llm/api.rs index 6523b8477..5dad09d34 100644 --- a/lib/components/fabro-workflow/src/handler/llm/api.rs +++ b/lib/components/fabro-workflow/src/handler/llm/api.rs @@ -14,7 +14,7 @@ use fabro_auth::CredentialSource; use fabro_graphviz::graph::{AttrValue, Node}; use fabro_llm::client::Client; use fabro_llm::types::{ - Message, ReasoningEffort, Request, Response, Speed, TokenCounts, + Message, ReasoningEffort, Request, Response, ResponseFormat, Speed, TokenCounts, ToolDefinition as LlmToolDefinition, }; use fabro_mcp::config::McpServerSettings; @@ -24,7 +24,7 @@ use fabro_model::{ AgentProfileKind, Catalog, FallbackTarget, ModelHandle, ModelRef, ProviderId, UsdMicros, }; use fabro_types::settings::run::RunModelControls; -use fabro_types::{PermissionLevel, RunId, SessionCapability, StageId, StageTiming}; +use fabro_types::{FailoverProps, PermissionLevel, RunId, SessionCapability, StageId, StageTiming}; use serde::de::DeserializeOwned; use tokio::sync::mpsc; use tokio::task::JoinHandle; @@ -42,6 +42,7 @@ use crate::context::WorkflowContext; use crate::context::keys::Fidelity; use crate::error::Error; use crate::event::{Emitter, Event, StageScope}; +use crate::model_fallback::{ModelFallbackNotice, ModelFallbackPolicy}; use crate::outcome::billed_model_usage_from_llm; use crate::services::FabroRunToolServices; use crate::steering_hub::{ActiveControlHandle, SteeringHub}; @@ -631,18 +632,86 @@ fn spawn_event_forwarder( /// For `full` fidelity nodes sharing a thread key, sessions are cached /// and reused so the LLM sees the full conversation history. pub struct AgentApiBackend { - model: String, - provider_id: ProviderId, - fallback_chain: Vec, - sessions: Mutex>, - tool_env: Option>, - mcp_servers: Vec, - tool_secrets: ToolSecrets, - run_model_controls: RunModelControls, - source: Arc, - steering_hub: Arc, - catalog: Arc, - fabro_run_tools: Option, + model: String, + provider_id: ProviderId, + fallbacks: ModelFallbackPolicy, + sessions: Mutex>, + /// Messages of fallback-plan notices already emitted for this run, so the + /// same configuration warning is not repeated on every LLM call. + emitted_plan_notices: Mutex>, + tool_env: Option>, + mcp_servers: Vec, + tool_secrets: ToolSecrets, + run_model_controls: RunModelControls, + source: Arc, + steering_hub: Arc, + catalog: Arc, + fabro_run_tools: Option, +} + +struct CachedAgentSession { + session: Session, + fallback_plan: FallbackPlan, +} + +#[derive(Clone, Debug)] +struct LlmRoute { + target: FallbackTarget, + controls: EffectiveRequestControls, +} + +#[derive(Clone, Debug)] +struct FallbackPlan { + original: LlmRoute, + remaining: Vec, + /// 0 addresses the original route; N addresses `remaining[N - 1]`. + position: usize, +} + +impl FallbackPlan { + fn current(&self) -> &LlmRoute { + self.route_at(self.position) + } + + /// The route that was active before the most recent [`Self::advance`]. + fn previous(&self) -> &LlmRoute { + self.route_at(self.position.saturating_sub(1)) + } + + fn route_at(&self, position: usize) -> &LlmRoute { + position + .checked_sub(1) + .map_or(&self.original, |index| &self.remaining[index]) + } + + fn attempt(&self) -> u32 { + u32::try_from(self.position).unwrap_or(u32::MAX) + } + + #[must_use] + fn has_next(&self) -> bool { + self.position < self.remaining.len() + } + + /// Move to the next fallback route. Returns false when the plan is + /// exhausted. + fn advance(&mut self) -> bool { + if self.has_next() { + self.position += 1; + true + } else { + false + } + } +} + +/// Request controls resolved for one fallback target. +enum FallbackControls { + /// The target can serve the request with these controls. + Usable(EffectiveRequestControls), + /// The target advertises reasoning levels, but none is near the requested + /// effort. + NoNearbyReasoningLevel(ReasoningEffort), } struct OneShotCompletion { @@ -650,12 +719,76 @@ struct OneShotCompletion { model: ModelRef, } +/// One agent invocation's live session, cancel bridge, activation lease, +/// event forwarding, and accounting state. +/// +/// Failover discards and replaces the session while the accumulated usage, +/// cost, and timing keep counting across routes. +struct LiveAgentInvocation { + session: Session, + bridge: SessionCancelBridgeGuard, + lease: Option>, + event_forwarder: EventForwarder, + file_tracking: Arc>, + total_usage: TokenCounts, + total_cost: Option, + inference_duration: Duration, + tool_duration: Duration, +} + +impl LiveAgentInvocation { + /// Tear down the current session: detach the cancel bridge, release the + /// lease, shut the session down, and stop event forwarding. + async fn abort_and_discard(&mut self, emitter: &Arc) { + self.bridge.abort(); + discard_session( + &mut self.session, + &mut self.lease, + &mut self.event_forwarder, + emitter, + ) + .await; + self.event_forwarder.abort(); + } + + /// Tear down the session for a failed agent call and classify the error. + /// Terminal and cancelled errors come back as `Err` for the caller to + /// propagate; a failover-eligible error comes back as `Ok` so the caller + /// can continue the fallback plan. + async fn discard_for_error( + &mut self, + error: fabro_agent::Error, + allow_failover: bool, + emitter: &Arc, + ) -> Result { + let disposition = classify_agent_error(error, allow_failover); + self.abort_and_discard(emitter).await; + match disposition { + AgentApiErrorDisposition::Cancelled => Err(Error::Cancelled), + AgentApiErrorDisposition::Terminal(error) => Err(error), + AgentApiErrorDisposition::FailoverEligible(error) => Ok(error), + } + } + + fn record_input_timing(&mut self) { + let timing = self.session.last_input_timing(); + self.inference_duration = self.inference_duration.saturating_add(timing.inference); + self.tool_duration = self.tool_duration.saturating_add(timing.tool); + } + + async fn record_input_usage(&mut self) { + self.event_forwarder.wait_for_processing_end().await; + self.total_usage += self.session.last_input_usage(); + UsdMicros::accumulate(&mut self.total_cost, self.session.last_input_cost()); + } +} + impl AgentApiBackend { #[must_use] pub fn new( model: String, provider_id: impl Into, - fallback_chain: Vec, + fallbacks: ModelFallbackPolicy, source: Arc, steering_hub: Arc, ) -> Self { @@ -663,7 +796,7 @@ impl AgentApiBackend { Self::new_with_catalog( model, provider_id.into(), - fallback_chain, + fallbacks, source, steering_hub, catalog, @@ -674,7 +807,7 @@ impl AgentApiBackend { pub fn new_with_catalog( model: String, provider_id: ProviderId, - fallback_chain: Vec, + fallbacks: ModelFallbackPolicy, source: Arc, steering_hub: Arc, catalog: Arc, @@ -682,8 +815,9 @@ impl AgentApiBackend { Self { model, provider_id, - fallback_chain, + fallbacks, sessions: Mutex::new(HashMap::new()), + emitted_plan_notices: Mutex::new(HashSet::new()), tool_env: None, mcp_servers: Vec::new(), tool_secrets: ToolSecrets::default(), @@ -751,12 +885,122 @@ impl AgentApiBackend { ) } - async fn create_session( + fn fallback_controls_for_target( + &self, + target: &FallbackTarget, + requested: EffectiveRequestControls, + ) -> FallbackControls { + let Some(requested_effort) = requested.reasoning_effort else { + return FallbackControls::Usable(requested); + }; + let Some(offering) = self + .catalog + .get_on_provider(&target.provider, target.model.as_str()) + else { + // A catalog-unknown passthrough target has no advertised controls. + // Preserve the request and let the provider validate it. + return FallbackControls::Usable(requested); + }; + let effective_effort = self.catalog.settings_for(offering).and_then(|settings| { + requested_effort.closest_supported(&settings.controls.reasoning_effort) + }); + match effective_effort { + Some(effort) => FallbackControls::Usable(EffectiveRequestControls { + reasoning_effort: Some(effort), + speed: requested.speed, + }), + None => FallbackControls::NoNearbyReasoningLevel(requested_effort), + } + } + + fn fallback_plan( + &self, + model: &str, + provider: &ProviderId, + requested_controls: EffectiveRequestControls, + ) -> (FallbackPlan, Vec) { + let primary_model = self.catalog.canonical_model_id(provider, model); + let original = LlmRoute { + target: FallbackTarget::new(provider, &primary_model), + controls: requested_controls, + }; + let Some(configured) = self.fallbacks.chain_for_canonical(&primary_model) else { + return ( + FallbackPlan { + original, + remaining: Vec::new(), + position: 0, + }, + Vec::new(), + ); + }; + + let mut remaining = Vec::new(); + let mut notices = Vec::new(); + for target in configured { + // The resolver already de-duplicated the chain; only the primary + // target, which the resolver cannot know, needs filtering here. + if *target == original.target { + continue; + } + + let controls = match self.fallback_controls_for_target(target, requested_controls) { + FallbackControls::Usable(controls) => controls, + FallbackControls::NoNearbyReasoningLevel(requested_effort) => { + notices.push(ModelFallbackNotice::NoNearbyReasoningLevel { + requested_model: original.target.model.to_string(), + target: target.clone(), + requested_effort, + }); + continue; + } + }; + remaining.push(LlmRoute { + target: target.clone(), + controls, + }); + } + + if !configured.is_empty() && remaining.is_empty() { + notices.push(ModelFallbackNotice::ChainEmpty { + requested_model: original.target.model.to_string(), + }); + } + + ( + FallbackPlan { + original, + remaining, + position: 0, + }, + notices, + ) + } + + fn emit_fallback_plan_notices( + &self, + notices: &[ModelFallbackNotice], + emitter: &Emitter, + stage_scope: &StageScope, + ) { + let mut emitted = self + .emitted_plan_notices + .lock() + .expect("notices mutex is never poisoned: no code panics while holding this lock"); + for notice in notices { + let message = notice.message(); + if emitted.insert(message.clone()) { + emitter.notice_scoped(notice.level(), notice.code(), message, stage_scope); + } + } + } + + async fn create_session_with_plan( &self, node: &Node, sandbox: &Arc, tool_hooks: Option>, - ) -> Result { + ) -> Result<(CachedAgentSession, Vec), Error> { let model = node.model().unwrap_or(&self.model); let provider = routing::resolve_node_provider_context( self.catalog.as_ref(), @@ -764,38 +1008,51 @@ impl AgentApiBackend { &self.model, node, )?; - Self::create_session_for( - model, - provider, + let controls = self.resolve_effective_request_controls(node)?; + let (fallback_plan, notices) = self.fallback_plan(model, &provider.provider_id, controls); + let route = fallback_plan.current(); + let route_provider = self.resolve_provider_context( + route.target.model.as_str(), + Some(route.target.provider.as_str()), + )?; + let session = Self::create_session_for( + route.target.model.as_str(), + route_provider, + route.controls, node, sandbox, self.source.as_ref(), Arc::clone(&self.catalog), - &self.run_model_controls, self.tool_env.as_ref(), tool_hooks, self.mcp_servers.clone(), self.tool_secrets.clone(), self.fabro_run_tools.clone(), ) - .await + .await?; + Ok(( + CachedAgentSession { + session, + fallback_plan, + }, + notices, + )) } async fn create_session_for( model: &str, provider: ProviderContext, + controls: EffectiveRequestControls, node: &Node, sandbox: &Arc, source: &dyn CredentialSource, catalog: Arc, - run_model_controls: &RunModelControls, tool_env: Option<&Arc>, tool_hooks: Option>, mcp_servers: Vec, tool_secrets: ToolSecrets, fabro_run_tools: Option, ) -> Result { - let controls = effective_request_controls(run_model_controls, node)?; let client = Client::from_source(source, Arc::clone(&catalog)) .await .map_err(|e| Error::handler_with_source("Failed to create LLM client", e))?; @@ -931,15 +1188,140 @@ impl AgentApiBackend { Ok(lease) } + /// Continue the fallback plan after a failover-eligible agent error. + /// + /// The caller must already have torn down the failed session (see + /// [`LiveAgentInvocation::discard_for_error`]); this method only builds + /// and drives replacement sessions. + async fn failover_agent_session( + &self, + fallback_plan: &mut FallbackPlan, + initial_error: fabro_llm::Error, + request: &CodergenRunRequest<'_>, + input: &str, + stage_scope: &StageScope, + stage_id: &StageId, + live: &mut LiveAgentInvocation, + ) -> Result<(), Error> { + let emitter = request.emitter; + let mut last_error = Error::Llm(initial_error); + + while fallback_plan.advance() { + Self::emit_failover( + request.node, + emitter, + stage_scope, + fallback_plan, + &last_error.to_string(), + ); + let route = fallback_plan.current().clone(); + + let target_provider = match self.resolve_provider_context( + route.target.model.as_str(), + Some(route.target.provider.as_str()), + ) { + Ok(provider) => provider, + Err(error) => { + last_error = error; + continue; + } + }; + + if request.cancel_token.is_cancelled() { + return Err(Error::Cancelled); + } + let new_session = Self::create_session_for( + route.target.model.as_str(), + target_provider, + route.controls, + request.node, + request.sandbox, + self.source.as_ref(), + Arc::clone(&self.catalog), + self.tool_env.as_ref(), + request.tool_hooks.clone(), + self.mcp_servers.clone(), + self.tool_secrets.clone(), + self.fabro_run_tools.clone(), + ) + .await; + if request.cancel_token.is_cancelled() { + return Err(Error::Cancelled); + } + live.session = match new_session { + Ok(session) => session, + Err(error) => { + last_error = error; + continue; + } + }; + live.bridge + .replace(request.cancel_token.clone(), &live.session); + live.event_forwarder = spawn_event_forwarder( + &live.session, + request.node.id.clone(), + stage_scope.clone(), + Arc::clone(emitter), + Arc::clone(&live.file_tracking), + ); + + begin_session_lifecycle(&live.session, emitter, None); + if let Err(error) = live.session.initialize().await { + let allow_failover = fallback_plan.has_next(); + last_error = Error::Llm( + live.discard_for_error(error, allow_failover, emitter) + .await?, + ); + continue; + } + + match self.attach_session_to_hub( + &mut live.session, + stage_id, + request.thread_id, + emitter, + ) { + Ok(active_lease) => live.lease = Some(active_lease), + Err(error) => { + live.abort_and_discard(emitter).await; + return Err(error); + } + } + emit_agent_tools_available(&live.session, &request.node.id, stage_id, emitter); + + let process_result = live + .session + .process_input_with_runtime(input, request.agent_tool_runtime.clone()) + .await; + live.record_input_timing(); + match process_result { + Ok(()) => { + live.record_input_usage().await; + return Ok(()); + } + Err(error) => { + let allow_failover = fallback_plan.has_next(); + last_error = Error::Llm( + live.discard_for_error(error, allow_failover, emitter) + .await?, + ); + } + } + } + + Err(last_error) + } + async fn shutdown_cached_sessions(&self, emitter: &Arc) { - let sessions: Vec = self + let sessions: Vec = self .sessions .lock() .expect("sessions mutex is never poisoned: no code panics while holding this lock") .drain() .map(|(_, s)| s) .collect(); - for mut session in sessions { + for cached in sessions { + let mut session = cached.session; let session_id = session.id().to_string(); if session.shutdown(SessionShutdownReason::Completed).await { emitter.emit(&Event::AgentSessionEnded { @@ -950,97 +1332,112 @@ impl AgentApiBackend { } } + /// Emit `agent.failover` for the plan's most recent + /// [`FallbackPlan::advance`]. + /// + /// `from` is the previously attempted candidate, which may have failed + /// during activation without ever serving traffic; `error` says why it + /// was abandoned. Consecutive events therefore chain — one event's `to` + /// is the next event's `from` — recording every candidate the plan tried. + fn emit_failover( + node: &Node, + emitter: &Emitter, + stage_scope: &StageScope, + plan: &FallbackPlan, + error: &str, + ) { + let from = plan.previous(); + let to = plan.current(); + emitter.emit_scoped( + &Event::Failover { + stage: node.id.clone(), + props: FailoverProps { + original_provider: Some(plan.original.target.provider.to_string()), + original_model: Some(plan.original.target.model.to_string()), + attempt: Some(plan.attempt()), + from_provider: from.target.provider.to_string(), + from_model: from.target.model.to_string(), + to_provider: to.target.provider.to_string(), + to_model: to.target.model.to_string(), + requested_reasoning_effort: plan.original.controls.reasoning_effort, + effective_reasoning_effort: to.controls.reasoning_effort, + error: error.to_string(), + }, + }, + stage_scope, + ); + } + + fn route_max_tokens(&self, node: &Node, route: &LlmRoute) -> Option { + node.max_tokens().or_else(|| { + self.catalog + .get_on_provider(&route.target.provider, route.target.model.as_str()) + .and_then(|model| model.limits.max_output) + }) + } + + /// Build a one-shot completion request addressed to `route`. + fn route_request( + &self, + node: &Node, + route: &LlmRoute, + messages: Vec, + response_format: Option, + ) -> Request { + Request { + model: route.target.model.to_string(), + messages, + provider: Some(route.target.provider.to_string()), + tools: None, + tool_choice: None, + response_format, + temperature: None, + top_p: None, + max_tokens: self.route_max_tokens(node, route), + stop_sequences: None, + reasoning_effort: route.controls.reasoning_effort, + speed: route.controls.speed, + metadata: None, + provider_options: None, + } + } + async fn complete_one_shot_request( &self, client: &Client, node: &Node, emitter: &Arc, stage_scope: &StageScope, - request: &Request, - controls: EffectiveRequestControls, - fallback_chain: &[FallbackTarget], + mut request: Request, + plan: &mut FallbackPlan, ) -> Result { - let result = client.complete(request).await; - let default_provider = self.provider_id.to_string(); - - let (response, model) = match result { - Ok(resp) => (resp, ModelRef { - provider: ProviderId::from( - request - .provider - .clone() - .unwrap_or_else(|| default_provider.clone()), - ), - model_id: request.model.clone().into(), - speed: controls.speed, - }), - Err(sdk_err) if sdk_err.failover_eligible() && !fallback_chain.is_empty() => { - let error_msg = sdk_err.to_string(); - let from_provider = request - .provider - .clone() - .unwrap_or_else(|| default_provider.clone()); - let from_model = request.model.clone(); - - let mut last_err = sdk_err; - let mut found = None; - - for target in fallback_chain { - emitter.emit_scoped( - &Event::Failover { - stage: node.id.clone(), - from_provider: from_provider.clone(), - from_model: from_model.clone(), - to_provider: target.provider.clone(), - to_model: target.model.clone(), - error: error_msg.clone(), + loop { + match client.complete(&request).await { + Ok(response) => { + let route = plan.current(); + return Ok(OneShotCompletion { + response, + model: ModelRef { + provider: route.target.provider.clone(), + model_id: route.target.model.clone(), + speed: route.controls.speed, }, - stage_scope, - ); - - let max_tokens = node.max_tokens().or_else(|| { - self.catalog - .get_on_provider(&ProviderId::new(&target.provider), &target.model) - .and_then(|model| model.limits.max_output) }); - - let fallback_request = Request { - model: target.model.clone(), - provider: Some(target.provider.clone()), - max_tokens, - reasoning_effort: controls.reasoning_effort, - speed: controls.speed, - ..request.clone() - }; - - match client.complete(&fallback_request).await { - Ok(resp) => { - found = Some(OneShotCompletion { - response: resp, - model: ModelRef { - provider: ProviderId::from(target.provider.clone()), - model_id: target.model.clone().into(), - speed: controls.speed, - }, - }); - break; - } - Err(err) if err.failover_eligible() => { - last_err = err; - } - Err(err) => return Err(Error::Llm(err)), - } } - - match found { - Some(completion) => return Ok(completion), - None => return Err(Error::Llm(last_err)), + Err(error) if error.failover_eligible() && plan.has_next() => { + let error_message = error.to_string(); + plan.advance(); + Self::emit_failover(node, emitter, stage_scope, plan, &error_message); + request = self.route_request( + node, + plan.current(), + request.messages, + request.response_format, + ); } + Err(error) => return Err(Error::Llm(error)), } - Err(sdk_err) => return Err(Error::Llm(sdk_err)), - }; - - Ok(OneShotCompletion { response, model }) + } } } @@ -1067,14 +1464,10 @@ impl CodergenBackend for AgentApiBackend { let model = node.model().unwrap_or(&self.model); let provider = self.resolve_provider_context(model, node.provider())?; - let provider_id = provider.provider_id.to_string(); let controls = self.resolve_effective_request_controls(node)?; - - let max_tokens = node.max_tokens().or_else(|| { - self.catalog - .get_on_provider(&provider.provider_id, model) - .and_then(|model| model.limits.max_output) - }); + let (mut fallback_plan, notices) = + self.fallback_plan(model, &provider.provider_id, controls); + self.emit_fallback_plan_notices(¬ices, emitter, stage_scope); let mut messages = Vec::new(); if let Some(sys) = system_prompt { @@ -1082,14 +1475,6 @@ impl CodergenBackend for AgentApiBackend { } messages.push(Message::user(prompt)); - // Build per-request fallback chain: if the node overrides the provider, - // no failover is available; otherwise use the backend's. - let fallback_chain: &[FallbackTarget] = if node.provider().is_some() { - &[] - } else { - &self.fallback_chain - }; - let output_schema = structured_output::parse_node_output_schema(node)?; let response_format = output_schema .as_ref() @@ -1100,22 +1485,12 @@ impl CodergenBackend for AgentApiBackend { let mut inference_duration = Duration::ZERO; loop { - let request = Request { - model: model.to_string(), - messages: messages.clone(), - provider: Some(provider_id.clone()), - reasoning_effort: controls.reasoning_effort, - speed: controls.speed, - tools: None, - tool_choice: None, - response_format: response_format.clone(), - temperature: None, - top_p: None, - max_tokens, - stop_sequences: None, - metadata: None, - provider_options: None, - }; + let request = self.route_request( + node, + fallback_plan.current(), + messages.clone(), + response_format.clone(), + ); let inference_start = Instant::now(); let completion_result = self @@ -1124,9 +1499,8 @@ impl CodergenBackend for AgentApiBackend { node, emitter, stage_scope, - &request, - controls, - fallback_chain, + request, + &mut fallback_plan, ) .await; inference_duration = inference_duration.saturating_add(inference_start.elapsed()); @@ -1181,57 +1555,49 @@ impl CodergenBackend for AgentApiBackend { async fn run(&self, request: CodergenRunRequest<'_>) -> Result { let node = request.node; - let prompt = request.prompt; - let context = request.context; - let thread_id = request.thread_id; let emitter = request.emitter; - let sandbox = request.sandbox; - let tool_hooks = request.tool_hooks; - let cancel_token = request.cancel_token; - let agent_tool_runtime = request.agent_tool_runtime; let output_schema = structured_output::parse_node_output_schema(node)?; - let fidelity = context.fidelity(); + let fidelity = request.context.fidelity(); let reuse_key = if fidelity == Fidelity::Full { - thread_id.map(String::from) + request.thread_id.map(String::from) } else { None }; - let mut bridge = SessionCancelBridgeGuard::new(); - // Take a cached session if reusing, otherwise create a new one. Cancel // checks bracket `Client::from_source(...)` so cancellation arriving // during credential refresh is not lost. - if cancel_token.is_cancelled() { + if request.cancel_token.is_cancelled() { return Err(Error::Cancelled); } - let (mut session, is_reused) = if let Some(ref key) = reuse_key { - let existing = self - .sessions + let cached_session = reuse_key.as_ref().and_then(|key| { + self.sessions .lock() .expect("sessions mutex is never poisoned: no code panics while holding this lock") - .remove(key); - if let Some(s) = existing { - (s, true) - } else { - let created = self.create_session(node, sandbox, tool_hooks.clone()).await; - if cancel_token.is_cancelled() { - return Err(Error::Cancelled); - } - (created?, false) - } + .remove(key) + }); + let is_reused = cached_session.is_some(); + let (cached, fallback_notices) = if let Some(cached) = cached_session { + (cached, Vec::new()) } else { - let created = self.create_session(node, sandbox, tool_hooks.clone()).await; - if cancel_token.is_cancelled() { + let created = self + .create_session_with_plan(node, request.sandbox, request.tool_hooks.clone()) + .await; + if request.cancel_token.is_cancelled() { return Err(Error::Cancelled); } - (created?, false) + created? }; - if cancel_token.is_cancelled() { + let CachedAgentSession { + session, + mut fallback_plan, + } = cached; + if request.cancel_token.is_cancelled() { return Err(Error::Cancelled); } - bridge.replace(cancel_token.clone(), &session); + let mut bridge = SessionCancelBridgeGuard::new(); + bridge.replace(request.cancel_token.clone(), &session); tracing::info!( node = %node.id, @@ -1246,10 +1612,11 @@ impl CodergenBackend for AgentApiBackend { touched: HashSet::new(), last: None, })); - let stage_scope = StageScope::for_handler(context, &node.id); + let stage_scope = StageScope::for_handler(request.context, &node.id); + self.emit_fallback_plan_notices(&fallback_notices, emitter, &stage_scope); // Subscribe to session events: forward to pipeline emitter + track files. - let mut event_forwarder = spawn_event_forwarder( + let event_forwarder = spawn_event_forwarder( &session, node.id.clone(), stage_scope.clone(), @@ -1257,55 +1624,44 @@ impl CodergenBackend for AgentApiBackend { Arc::clone(&file_tracking), ); - let mut total_usage = TokenCounts::default(); - let mut total_cost = None; - let mut inference_duration = Duration::ZERO; - let mut tool_duration = Duration::ZERO; - // Activate with the steering hub after initialization so HTTP // `POST /runs/{id}/steer` calls reach this session. The activation // lease is shared with the natural-completion coordinator and is // released on every exit path. let stage_id = stage_scope.stage_id(); - let mut lease: Option> = None; + let mut live = LiveAgentInvocation { + session, + bridge, + lease: None, + event_forwarder, + file_tracking, + total_usage: TokenCounts::default(), + total_cost: None, + inference_duration: Duration::ZERO, + tool_duration: Duration::ZERO, + }; - let allow_failover_primary = !self.fallback_chain.is_empty(); + let allow_failover_primary = fallback_plan.has_next(); let init_result = if is_reused { Ok(()) } else { - begin_session_lifecycle(&session, emitter, None); - match session.initialize().await { - Ok(()) => Ok(()), - Err(err) => match classify_agent_error(err, allow_failover_primary) { - AgentApiErrorDisposition::Cancelled => { - bridge.abort(); - discard_session(&mut session, &mut lease, &mut event_forwarder, emitter) - .await; - return Err(Error::Cancelled); - } - AgentApiErrorDisposition::Terminal(err) => { - bridge.abort(); - discard_session(&mut session, &mut lease, &mut event_forwarder, emitter) - .await; - return Err(err); - } - AgentApiErrorDisposition::FailoverEligible(sdk_err) => { - Err(fabro_agent::Error::Llm(sdk_err)) - } - }, - } + begin_session_lifecycle(&live.session, emitter, None); + live.session.initialize().await }; // If initialize failed with a failover-eligible error, treat as a // process_input failover trigger; otherwise run process_input. let result = match init_result { Ok(()) => { - match self.attach_session_to_hub(&mut session, &stage_id, thread_id, emitter) { - Ok(active_lease) => lease = Some(active_lease), + match self.attach_session_to_hub( + &mut live.session, + &stage_id, + request.thread_id, + emitter, + ) { + Ok(active_lease) => live.lease = Some(active_lease), Err(err) => { - bridge.abort(); - discard_session(&mut session, &mut lease, &mut event_forwarder, emitter) - .await; + live.abort_and_discard(emitter).await; return Err(err); } } @@ -1314,243 +1670,58 @@ impl CodergenBackend for AgentApiBackend { // and exposure mode are immutable for the session's lifetime, // so re-emitting on every subsequent prompt is wasted work. if !is_reused { - emit_agent_tools_available(&session, &node.id, &stage_id, emitter); + emit_agent_tools_available(&live.session, &node.id, &stage_id, emitter); } - let process_result = session - .process_input_with_runtime(prompt, agent_tool_runtime.clone()) + let process_result = live + .session + .process_input_with_runtime(request.prompt, request.agent_tool_runtime.clone()) .await; - let timing = session.last_input_timing(); - inference_duration = inference_duration.saturating_add(timing.inference); - tool_duration = tool_duration.saturating_add(timing.tool); + live.record_input_timing(); if process_result.is_ok() { - event_forwarder.wait_for_processing_end().await; - total_usage += session.last_input_usage(); - UsdMicros::accumulate(&mut total_cost, session.last_input_cost()); + live.record_input_usage().await; } process_result } Err(err) => Err(err), }; - // On failover-eligible error, try fallback providers. + // On a provider-local failure, continue the fixed fallback plan that + // belongs to the originally requested model. let result: Result<(), Error> = match result { Ok(()) => Ok(()), - Err(err) => match classify_agent_error(err, allow_failover_primary) { - AgentApiErrorDisposition::Cancelled => { - bridge.abort(); - discard_session(&mut session, &mut lease, &mut event_forwarder, emitter).await; - return Err(Error::Cancelled); - } - AgentApiErrorDisposition::Terminal(err) => { - bridge.abort(); - discard_session(&mut session, &mut lease, &mut event_forwarder, emitter).await; - return Err(err); - } - AgentApiErrorDisposition::FailoverEligible(sdk_err) => { - let error_msg = sdk_err.to_string(); - let from_provider = self.provider_id.to_string(); - let from_model = self.model.clone(); - - let mut last_err = Error::Llm(sdk_err); - let mut succeeded = false; - - bridge.abort(); - discard_session(&mut session, &mut lease, &mut event_forwarder, emitter).await; - event_forwarder.abort(); - - for (index, target) in self.fallback_chain.iter().enumerate() { - emitter.emit_scoped( - &Event::Failover { - stage: node.id.clone(), - from_provider: from_provider.clone(), - from_model: from_model.clone(), - to_provider: target.provider.clone(), - to_model: target.model.clone(), - error: error_msg.clone(), - }, - &stage_scope, - ); - - let Ok(target_provider) = - self.resolve_provider_context(&target.model, Some(&target.provider)) - else { - continue; - }; - - if cancel_token.is_cancelled() { - return Err(Error::Cancelled); - } - let new_session_result = Self::create_session_for( - &target.model, - target_provider, - node, - sandbox, - self.source.as_ref(), - Arc::clone(&self.catalog), - &self.run_model_controls, - self.tool_env.as_ref(), - tool_hooks.clone(), - self.mcp_servers.clone(), - self.tool_secrets.clone(), - self.fabro_run_tools.clone(), - ) - .await; - if cancel_token.is_cancelled() { - return Err(Error::Cancelled); - } - let new_session = match new_session_result { - Ok(s) => s, - Err(e) => { - last_err = e; - continue; - } - }; - session = new_session; - bridge.replace(cancel_token.clone(), &session); - - // Re-subscribe to forward events + track files from the new session - event_forwarder = spawn_event_forwarder( - &session, - node.id.clone(), - stage_scope.clone(), - Arc::clone(emitter), - Arc::clone(&file_tracking), - ); - - let allow_failover_next = index + 1 < self.fallback_chain.len(); - begin_session_lifecycle(&session, emitter, None); - if let Err(err) = session.initialize().await { - match classify_agent_error(err, allow_failover_next) { - AgentApiErrorDisposition::Cancelled => { - bridge.abort(); - discard_session( - &mut session, - &mut lease, - &mut event_forwarder, - emitter, - ) - .await; - return Err(Error::Cancelled); - } - AgentApiErrorDisposition::Terminal(err) => { - bridge.abort(); - discard_session( - &mut session, - &mut lease, - &mut event_forwarder, - emitter, - ) - .await; - return Err(err); - } - AgentApiErrorDisposition::FailoverEligible(sdk_err) => { - last_err = Error::Llm(sdk_err); - bridge.abort(); - discard_session( - &mut session, - &mut lease, - &mut event_forwarder, - emitter, - ) - .await; - continue; - } - } - } - match self.attach_session_to_hub( - &mut session, - &stage_id, - thread_id, - emitter, - ) { - Ok(active_lease) => lease = Some(active_lease), - Err(err) => { - bridge.abort(); - discard_session( - &mut session, - &mut lease, - &mut event_forwarder, - emitter, - ) - .await; - return Err(err); - } - } - emit_agent_tools_available(&session, &node.id, &stage_id, emitter); - let process_result = session - .process_input_with_runtime(prompt, agent_tool_runtime.clone()) - .await; - let timing = session.last_input_timing(); - inference_duration = inference_duration.saturating_add(timing.inference); - tool_duration = tool_duration.saturating_add(timing.tool); - match process_result { - Ok(()) => { - event_forwarder.wait_for_processing_end().await; - total_usage += session.last_input_usage(); - UsdMicros::accumulate(&mut total_cost, session.last_input_cost()); - succeeded = true; - break; - } - Err(err) => match classify_agent_error(err, allow_failover_next) { - AgentApiErrorDisposition::Cancelled => { - bridge.abort(); - discard_session( - &mut session, - &mut lease, - &mut event_forwarder, - emitter, - ) - .await; - return Err(Error::Cancelled); - } - AgentApiErrorDisposition::Terminal(err) => { - bridge.abort(); - discard_session( - &mut session, - &mut lease, - &mut event_forwarder, - emitter, - ) - .await; - return Err(err); - } - AgentApiErrorDisposition::FailoverEligible(sdk_err) => { - last_err = Error::Llm(sdk_err); - bridge.abort(); - discard_session( - &mut session, - &mut lease, - &mut event_forwarder, - emitter, - ) - .await; - } - }, - } - } - - if succeeded { Ok(()) } else { Err(last_err) } - } - }, + Err(err) => { + let sdk_err = live + .discard_for_error(err, allow_failover_primary, emitter) + .await?; + self.failover_agent_session( + &mut fallback_plan, + sdk_err, + &request, + request.prompt, + &stage_scope, + &stage_id, + &mut live, + ) + .await + } }; // On error, discard the session (don't cache failed state). The // bridge's `Drop` will abort the spawned task on early return. if let Err(err) = result { - bridge.abort(); - discard_session(&mut session, &mut lease, &mut event_forwarder, emitter).await; + live.abort_and_discard(emitter).await; return Err(err); } - let mut response = last_assistant_response(&session); + let mut response = last_assistant_response(&live.session); if let Some(schema) = &output_schema { let mut repair_attempts = 0_i64; loop { - let last_file_touched = last_touched_file(&file_tracking); + let last_file_touched = last_touched_file(&live.file_tracking); match validate_agent_output_sources( schema, &response, - sandbox, + request.sandbox, last_file_touched.as_deref(), ) .await @@ -1558,104 +1729,86 @@ impl CodergenBackend for AgentApiBackend { Ok(_) => break, Err(error) => { if repair_attempts >= node.output_retries() { - bridge.abort(); - discard_session( - &mut session, - &mut lease, - &mut event_forwarder, - emitter, - ) - .await; + live.abort_and_discard(emitter).await; return Err(Error::OutputSchemaValidation( structured_output::exhausted_failure_reason(node.output_retries()), )); } let repair_message = error.repair_message(schema); - let repair_result = session + let repair_result = live + .session .process_input_with_runtime( &repair_message, fabro_agent::AgentToolRuntime::default(), ) .await; - let timing = session.last_input_timing(); - inference_duration = inference_duration.saturating_add(timing.inference); - tool_duration = tool_duration.saturating_add(timing.tool); + live.record_input_timing(); match repair_result { Ok(()) => { - event_forwarder.wait_for_processing_end().await; - total_usage += session.last_input_usage(); - UsdMicros::accumulate(&mut total_cost, session.last_input_cost()); + live.record_input_usage().await; repair_attempts += 1; - response = last_assistant_response(&session); + response = last_assistant_response(&live.session); + } + Err(err) => { + let allow_failover = fallback_plan.has_next(); + let sdk_err = + live.discard_for_error(err, allow_failover, emitter).await?; + self.failover_agent_session( + &mut fallback_plan, + sdk_err, + &request, + request.prompt, + &stage_scope, + &stage_id, + &mut live, + ) + .await?; + response = last_assistant_response(&live.session); } - Err(err) => match classify_agent_error(err, false) { - AgentApiErrorDisposition::Cancelled => { - bridge.abort(); - discard_session( - &mut session, - &mut lease, - &mut event_forwarder, - emitter, - ) - .await; - return Err(Error::Cancelled); - } - AgentApiErrorDisposition::Terminal(err) => { - bridge.abort(); - discard_session( - &mut session, - &mut lease, - &mut event_forwarder, - emitter, - ) - .await; - return Err(err); - } - AgentApiErrorDisposition::FailoverEligible(sdk_err) => { - bridge.abort(); - discard_session( - &mut session, - &mut lease, - &mut event_forwarder, - emitter, - ) - .await; - return Err(Error::Llm(sdk_err)); - } - }, } } } } } - let billing_controls = self.resolve_effective_request_controls(node)?; let stage_usage = billed_model_usage_from_llm( self.catalog.as_ref(), &ModelRef { - provider: session.provider_id(), - model_id: session.model().into(), - speed: billing_controls.speed, + provider: live.session.provider_id(), + model_id: live.session.model().into(), + speed: live.session.speed(), }, - &total_usage, + &live.total_usage, )? - .with_reported_cost(total_cost); + .with_reported_cost(live.total_cost); - if let Some(lease) = lease.take() { + if let Some(lease) = live.lease.take() { lease.release(); } // Cache session back for reuse on success. Detach the bridge first so // the cached session is not left wired to this run's cancel token. + live.bridge.abort(); + let LiveAgentInvocation { + session, + event_forwarder, + file_tracking, + inference_duration, + tool_duration, + .. + } = live; if let Some(key) = reuse_key { - bridge.abort(); drop(event_forwarder); self.sessions .lock() .expect("sessions mutex is never poisoned: no code panics while holding this lock") - .insert(key, session); + .insert(key, CachedAgentSession { + session, + fallback_plan, + }); } else { - bridge.abort(); + let mut session = session; + let mut event_forwarder = event_forwarder; let session_id = session.id().to_string(); session.shutdown(SessionShutdownReason::Completed).await; event_forwarder.wait_for_session_end().await; @@ -1894,6 +2047,20 @@ reasoning = false Arc::new(Catalog::from_builtin_with_overrides(&settings).unwrap()) } + fn enabled_fallback_catalog() -> Arc { + let settings: LlmCatalogSettings = toml::from_str( + r" +[providers.modal] +enabled = true + +[providers.openrouter] +enabled = true +", + ) + .expect("fallback catalog overrides should parse"); + Arc::new(Catalog::from_builtin_with_overrides(&settings).unwrap()) + } + fn mock_api_backend(server: &MockServer) -> AgentApiBackend { let source = auth_test_support::env_credential_source(|name| { if name == "MOCK_API_KEY" { @@ -1905,13 +2072,85 @@ reasoning = false AgentApiBackend::new_with_catalog( "mock-model".to_string(), ProviderId::from("mock"), - Vec::new(), + ModelFallbackPolicy::default(), source, SteeringHub::for_tests(), mock_llm_catalog(server), ) } + fn fallback_api_backend(server: &MockServer) -> AgentApiBackend { + let settings: LlmCatalogSettings = toml::from_str(&format!( + r#" +[providers.primary] +adapter = "openai_compatible" +agent_profile = "openai" +base_url = "{}/primary" + +[providers.primary.auth] +credentials = ["env:PRIMARY_API_KEY"] + +[providers.primary.models.test-model] +display_name = "Primary Test Model" +family = "test" +default = true + +[providers.primary.models.test-model.limits] +context_window = 8192 +max_output = 1024 + +[providers.primary.models.test-model.features] +tools = true +vision = false +reasoning = false + +[providers.fallback] +adapter = "openai_compatible" +agent_profile = "openai" +base_url = "{}/fallback" + +[providers.fallback.auth] +credentials = ["env:FALLBACK_API_KEY"] + +[providers.fallback.models.test-model] +display_name = "Fallback Test Model" +family = "test" +default = true + +[providers.fallback.models.test-model.limits] +context_window = 8192 +max_output = 1024 + +[providers.fallback.models.test-model.features] +tools = true +vision = false +reasoning = false +"#, + server.base_url(), + server.base_url(), + )) + .expect("fallback catalog should parse"); + let catalog = Arc::new( + Catalog::from_builtin_with_overrides(&settings).expect("catalog should build"), + ); + let source = auth_test_support::env_credential_source(|name| match name { + "PRIMARY_API_KEY" | "FALLBACK_API_KEY" => Some("sk-test".to_string()), + _ => None, + }); + let policy = ModelFallbackPolicy::new(std::collections::BTreeMap::from([( + "test-model".to_string(), + vec![FallbackTarget::new("fallback", "test-model")], + )])); + AgentApiBackend::new_with_catalog( + "test-model".to_string(), + ProviderId::new("primary"), + policy, + source, + SteeringHub::for_tests(), + catalog, + ) + } + fn chat_completion_response( text: &str, input_tokens: i64, @@ -2007,7 +2246,7 @@ reasoning = false let backend = AgentApiBackend::new( "claude-opus-4-6".to_string(), ProviderId::openai(), - Vec::new(), + ModelFallbackPolicy::default(), auth_test_support::vault_only_credential_source(), SteeringHub::for_tests(), ); @@ -2020,7 +2259,7 @@ reasoning = false let backend = AgentApiBackend::new( "claude-opus-4-6".to_string(), ProviderId::anthropic(), - Vec::new(), + ModelFallbackPolicy::default(), auth_test_support::vault_only_credential_source(), SteeringHub::for_tests(), ); @@ -2803,9 +3042,11 @@ reasoning = false Arc::new(LocalSandbox::new(workspace.path().to_path_buf())); let mut session = backend - .create_session(&node, &sandbox, Some(hooks)) + .create_session_with_plan(&node, &sandbox, Some(hooks)) .await - .unwrap(); + .unwrap() + .0 + .session; session .process_input("PARENT_PROMPT_MARKER: spawn a helper subagent") .await @@ -2850,7 +3091,7 @@ enabled = true let backend = AgentApiBackend::new_with_catalog( "gpt-5.4".to_string(), ProviderId::from("openrouter"), - Vec::new(), + ModelFallbackPolicy::default(), auth_test_support::vault_only_credential_source(), SteeringHub::for_tests(), Arc::new(Catalog::from_builtin_with_overrides(&settings).unwrap()), @@ -2866,7 +3107,7 @@ enabled = true let backend = AgentApiBackend::new_with_catalog( "gpt-5.4".to_string(), ProviderId::from("openrouter"), - Vec::new(), + ModelFallbackPolicy::default(), auth_test_support::vault_only_credential_source(), SteeringHub::for_tests(), Arc::new(Catalog::from_builtin().unwrap()), @@ -2913,7 +3154,7 @@ reasoning = false let backend = AgentApiBackend::new_with_catalog( "acme-llama".to_string(), ProviderId::from("acme"), - Vec::new(), + ModelFallbackPolicy::default(), auth_test_support::vault_only_credential_source(), SteeringHub::for_tests(), catalog, @@ -2960,7 +3201,7 @@ reasoning = false let backend = AgentApiBackend::new_with_catalog( "acme-claude".to_string(), ProviderId::from("acme"), - Vec::new(), + ModelFallbackPolicy::default(), auth_test_support::vault_only_credential_source(), SteeringHub::for_tests(), catalog, @@ -2977,7 +3218,7 @@ reasoning = false let backend = AgentApiBackend::new_with_catalog( "claude-sonnet-5".to_string(), ProviderId::anthropic(), - Vec::new(), + ModelFallbackPolicy::default(), auth_test_support::vault_only_credential_source(), SteeringHub::for_tests(), Arc::new(Catalog::from_builtin().unwrap()), @@ -3004,7 +3245,7 @@ enabled = true let backend = AgentApiBackend::new_with_catalog( "openai/gpt-5.4".to_string(), ProviderId::from("openrouter"), - Vec::new(), + ModelFallbackPolicy::default(), auth_test_support::vault_only_credential_source(), SteeringHub::for_tests(), catalog, @@ -3023,7 +3264,7 @@ enabled = true let backend = AgentApiBackend::new( "gpt-5.4".to_string(), ProviderId::openai(), - Vec::new(), + ModelFallbackPolicy::default(), auth_test_support::vault_only_credential_source(), SteeringHub::for_tests(), ) @@ -3044,7 +3285,7 @@ enabled = true let backend = AgentApiBackend::new( "gpt-5.4".to_string(), ProviderId::openai(), - Vec::new(), + ModelFallbackPolicy::default(), auth_test_support::vault_only_credential_source(), SteeringHub::for_tests(), ) @@ -3073,7 +3314,7 @@ enabled = true let backend = AgentApiBackend::new( "gpt-5.4".to_string(), ProviderId::openai(), - Vec::new(), + ModelFallbackPolicy::default(), auth_test_support::vault_only_credential_source(), SteeringHub::for_tests(), ); @@ -3084,6 +3325,91 @@ enabled = true assert_eq!(controls.reasoning_effort, None); } + #[test] + fn fallback_plan_maps_reasoning_to_each_target_and_rounds_ties_up() { + let policy = ModelFallbackPolicy::new(std::collections::BTreeMap::from([( + "kimi-k3".to_string(), + vec![ + FallbackTarget::new("kimi", "kimi-k3"), + FallbackTarget::new("openrouter", "kimi-k3"), + FallbackTarget::new("anthropic", "claude-opus-5"), + ], + )])); + let backend = AgentApiBackend::new_with_catalog( + "kimi-k3".to_string(), + ProviderId::new("modal"), + policy, + auth_test_support::vault_only_credential_source(), + SteeringHub::for_tests(), + enabled_fallback_catalog(), + ); + + let (plan, notices) = backend.fallback_plan( + "kimi-k3", + &ProviderId::new("modal"), + EffectiveRequestControls { + reasoning_effort: Some(ReasoningEffort::Medium), + speed: None, + }, + ); + + assert!(notices.is_empty()); + assert_eq!( + plan.remaining + .iter() + .map(|route| route.controls.reasoning_effort) + .collect::>(), + vec![ + Some(ReasoningEffort::High), + Some(ReasoningEffort::High), + Some(ReasoningEffort::Medium), + ] + ); + } + + #[test] + fn advancing_a_fallback_plan_never_activates_the_target_models_chain() { + let policy = ModelFallbackPolicy::new(std::collections::BTreeMap::from([ + ("claude-fable-5".to_string(), vec![ + FallbackTarget::new("openai", "gpt-5.6-sol"), + FallbackTarget::new("anthropic", "claude-opus-5"), + ]), + ("gpt-5.6-sol".to_string(), vec![FallbackTarget::new( + "anthropic", + "claude-sonnet-5", + )]), + ])); + let backend = AgentApiBackend::new_with_catalog( + "claude-fable-5".to_string(), + ProviderId::anthropic(), + policy, + auth_test_support::vault_only_credential_source(), + SteeringHub::for_tests(), + enabled_fallback_catalog(), + ); + let (mut plan, notices) = backend.fallback_plan( + "claude-fable-5", + &ProviderId::anthropic(), + EffectiveRequestControls::default(), + ); + + assert!(notices.is_empty()); + assert!(plan.advance(), "Sol should be first"); + assert_eq!( + plan.current().target, + FallbackTarget::new("openai", "gpt-5.6-sol") + ); + assert_eq!(plan.attempt(), 1); + assert!(plan.advance(), "Opus should be second"); + assert_eq!( + plan.current().target, + FallbackTarget::new("anthropic", "claude-opus-5") + ); + assert_eq!(plan.attempt(), 2); + assert!(!plan.has_next()); + assert!(!plan.advance()); + } + #[tokio::test] async fn api_backend_uses_source_credentials() { let dir = tempfile::tempdir().unwrap(); @@ -3099,7 +3425,7 @@ enabled = true let backend = AgentApiBackend::new( "claude-opus-4-6".to_string(), ProviderId::anthropic(), - Vec::new(), + ModelFallbackPolicy::default(), Arc::new(VaultCredentialSource::with_env_lookup( Arc::new(AsyncRwLock::new(vault)), |_| None, @@ -3116,14 +3442,15 @@ enabled = true #[tokio::test] async fn one_shot_falls_back_after_refusal_error() { - let fallback_chain = vec![FallbackTarget { - provider: "openai".to_string(), - model: "gpt-5.5".to_string(), - }]; + let configured_targets = vec![FallbackTarget::new("openai", "gpt-5.5")]; + let fallback_policy = ModelFallbackPolicy::new(std::collections::BTreeMap::from([( + "claude-fable-5".to_string(), + configured_targets, + )])); let backend = AgentApiBackend::new( "claude-fable-5".to_string(), ProviderId::anthropic(), - fallback_chain.clone(), + fallback_policy, auth_test_support::vault_only_credential_source(), SteeringHub::for_tests(), ); @@ -3144,6 +3471,13 @@ enabled = true let context = Context::new(); let stage_scope = StageScope::for_handler(&context, &node.id); let emitter = Arc::new(Emitter::new(fabro_types::RunId::new())); + let emitted_failover = Arc::new(Mutex::new(None)); + let emitted_failover_for_listener = Arc::clone(&emitted_failover); + emitter.on_event(move |event| { + if let fabro_types::EventBody::Failover(props) = &event.body { + *emitted_failover_for_listener.lock().unwrap() = Some(props.clone()); + } + }); let request = Request { model: "claude-fable-5".to_string(), messages: vec![Message::user("Hello")], @@ -3160,6 +3494,12 @@ enabled = true metadata: None, provider_options: None, }; + let (mut fallback_plan, notices) = backend.fallback_plan( + "claude-fable-5", + &ProviderId::anthropic(), + EffectiveRequestControls::default(), + ); + assert!(notices.is_empty()); let completion = backend .complete_one_shot_request( @@ -3167,9 +3507,8 @@ enabled = true &node, &emitter, &stage_scope, - &request, - EffectiveRequestControls::default(), - &fallback_chain, + request, + &mut fallback_plan, ) .await .unwrap(); @@ -3177,6 +3516,91 @@ enabled = true assert_eq!(completion.response.text(), "fallback ok"); assert_eq!(completion.model.provider, ProviderId::openai()); assert_eq!(completion.model.model_id, "gpt-5.5"); + let failover = emitted_failover + .lock() + .unwrap() + .clone() + .expect("agent.failover should be emitted"); + assert_eq!(failover.original_provider.as_deref(), Some("anthropic")); + assert_eq!(failover.original_model.as_deref(), Some("claude-fable-5")); + assert_eq!(failover.attempt, Some(1)); + assert_eq!(failover.from_provider, "anthropic"); + assert_eq!(failover.from_model, "claude-fable-5"); + assert_eq!(failover.to_provider, "openai"); + assert_eq!(failover.to_model, "gpt-5.5"); + assert_eq!(failover.requested_reasoning_effort, None); + assert_eq!(failover.effective_reasoning_effort, None); + assert!(failover.error.contains("refused")); + } + + #[tokio::test] + async fn explicit_provider_one_shot_stays_on_fallback_during_output_repair() { + let server = MockServer::start(); + let primary_failure = server.mock(|when, then| { + when.method(POST).path("/primary/chat/completions"); + then.status(401) + .header("content-type", "application/json") + .json_body(serde_json::json!({ + "error": { + "message": "primary credential expired", + "type": "authentication_error" + } + })); + }); + let fallback_response = server.mock(|when, then| { + when.method(POST) + .path("/fallback/chat/completions") + .body_excludes(r#""role":"assistant""#); + then.status(200) + .header("content-type", "application/json") + .json_body(chat_completion_response("not json", 10, 1)); + }); + let fallback_repair = server.mock(|when, then| { + when.method(POST) + .path("/fallback/chat/completions") + .body_includes(r#""role":"assistant""#) + .body_includes("not json"); + then.status(200) + .header("content-type", "application/json") + .json_body(chat_completion_response(r#"{"passed":true}"#, 11, 2)); + }); + let backend = fallback_api_backend(&server); + let mut node = Node::new("audit"); + node.attrs.insert( + "provider".to_string(), + AttrValue::String("primary".to_string()), + ); + node.attrs + .insert("output_schema".to_string(), custom_output_schema_attr()); + node.attrs + .insert("output_retries".to_string(), AttrValue::Integer(1)); + let context = Context::new(); + let stage_scope = StageScope::for_handler(&context, &node.id); + let emitter = Arc::new(Emitter::new(fabro_types::RunId::new())); + let workspace = tempfile::tempdir().unwrap(); + let sandbox: Arc = + Arc::new(LocalSandbox::new(workspace.path().to_path_buf())); + + let result = backend + .one_shot(OneShotRequest { + node: &node, + prompt: "Audit the result", + system_prompt: None, + emitter: &emitter, + stage_scope: &stage_scope, + sandbox: &sandbox, + cancel_token: CancellationToken::new(), + }) + .await + .unwrap(); + + primary_failure.assert_calls(1); + fallback_response.assert_calls(1); + fallback_repair.assert_calls(1); + let CodergenResult::Text { text, .. } = result else { + panic!("one_shot should return text"); + }; + assert_eq!(text, r#"{"passed":true}"#); } #[tokio::test] @@ -3309,6 +3733,77 @@ enabled = true assert_eq!(usage.tokens().output_tokens, 7); } + #[tokio::test] + async fn agent_output_repair_continues_on_the_original_models_fallback_plan() { + let server = MockServer::start(); + let primary_response = server.mock(|when, then| { + when.method(POST) + .path("/primary/chat/completions") + .body_includes(r#""stream":true"#) + .body_excludes(r#""role":"assistant""#); + then.status(200) + .header("content-type", "text/event-stream") + .body(chat_completion_stream("not json", 20, 3)); + }); + let failed_repair = server.mock(|when, then| { + when.method(POST) + .path("/primary/chat/completions") + .body_includes(r#""role":"assistant""#) + .body_includes("not json"); + then.status(401) + .header("content-type", "application/json") + .json_body(serde_json::json!({ + "error": { + "message": "primary credential expired", + "type": "authentication_error" + } + })); + }); + let fallback_response = server.mock(|when, then| { + when.method(POST) + .path("/fallback/chat/completions") + .body_includes(r#""stream":true"#) + .body_excludes(r#""role":"assistant""#); + then.status(200) + .header("content-type", "text/event-stream") + .body(chat_completion_stream(r#"{"passed":true}"#, 22, 4)); + }); + let backend = fallback_api_backend(&server); + let mut node = Node::new("audit"); + node.attrs + .insert("output_schema".to_string(), custom_output_schema_attr()); + node.attrs + .insert("output_retries".to_string(), AttrValue::Integer(1)); + let context = Context::new(); + let emitter = Arc::new(Emitter::new(fabro_types::RunId::new())); + let workspace = tempfile::tempdir().unwrap(); + let sandbox: Arc = + Arc::new(LocalSandbox::new(workspace.path().to_path_buf())); + + let result = backend + .run(CodergenRunRequest { + node: &node, + prompt: "Audit the result", + context: &context, + thread_id: None, + emitter: &emitter, + sandbox: &sandbox, + tool_hooks: None, + cancel_token: CancellationToken::new(), + agent_tool_runtime: fabro_agent::AgentToolRuntime::default(), + }) + .await + .unwrap(); + + primary_response.assert_calls(1); + failed_repair.assert_calls(1); + fallback_response.assert_calls(1); + let CodergenResult::Text { text, .. } = result else { + panic!("run should return text"); + }; + assert_eq!(text, r#"{"passed":true}"#); + } + #[tokio::test] async fn agent_run_web_search_uses_configured_brave_search_key() { let server = MockServer::start(); @@ -3397,7 +3892,7 @@ enabled = true let backend = AgentApiBackend::new( "gpt-5.4".to_string(), ProviderId::openai(), - Vec::new(), + ModelFallbackPolicy::default(), auth_test_support::vault_only_credential_source(), SteeringHub::for_tests(), ); @@ -3426,12 +3921,21 @@ enabled = true SessionOptions::default(), None, ); + let (fallback_plan, notices) = backend.fallback_plan( + "gpt-5.4", + &ProviderId::openai(), + EffectiveRequestControls::default(), + ); + assert!(notices.is_empty()); begin_session_lifecycle(&session, &emitter, None); backend .sessions .lock() .unwrap() - .insert("thread-1".to_string(), session); + .insert("thread-1".to_string(), CachedAgentSession { + session, + fallback_plan, + }); backend.shutdown(&emitter).await; backend.shutdown(&emitter).await; @@ -3515,7 +4019,7 @@ enabled = true fn non_failover_llm_error() -> LlmError { LlmError::Provider { - kind: ProviderErrorKind::Authentication, + kind: ProviderErrorKind::InvalidRequest, detail: Box::new(ProviderErrorDetail { message: "bad key".into(), provider: "openai".into(), diff --git a/lib/components/fabro-workflow/src/lib.rs b/lib/components/fabro-workflow/src/lib.rs index ee3ec4566..bd7f3e665 100644 --- a/lib/components/fabro-workflow/src/lib.rs +++ b/lib/components/fabro-workflow/src/lib.rs @@ -303,6 +303,7 @@ mod interview_runtime; reason = "The lifecycle module remains crate-visible for tests and pending integrations." )] pub(crate) mod lifecycle; +pub mod model_fallback; pub(crate) mod node_handler; pub mod operations; pub mod outcome; diff --git a/lib/components/fabro-workflow/src/model_fallback.rs b/lib/components/fabro-workflow/src/model_fallback.rs new file mode 100644 index 000000000..b795cb6d5 --- /dev/null +++ b/lib/components/fabro-workflow/src/model_fallback.rs @@ -0,0 +1,573 @@ +use std::collections::{BTreeMap, HashMap, HashSet}; + +use fabro_model::{ + Catalog, FallbackTarget, Model, ModelSelectionError, ProviderId, ReasoningEffort, +}; +use fabro_types::settings::{ModelRef, ResolvedModelRef}; +use fabro_types::{RunNoticeCode, RunNoticeLevel}; + +use crate::Error; + +/// Catalog-resolved fallback chains keyed by canonical requested model ID. +/// +/// A chain is selected from the original request only. Targets never cause +/// another chain lookup. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ModelFallbackPolicy { + chains: BTreeMap>, +} + +impl ModelFallbackPolicy { + #[cfg(test)] + #[must_use] + pub fn new(chains: BTreeMap>) -> Self { + Self { chains } + } + + #[must_use] + pub fn chain_for<'a>( + &'a self, + catalog: &Catalog, + provider: &ProviderId, + model: &str, + ) -> Option<&'a [FallbackTarget]> { + self.chain_for_canonical(&catalog.canonical_model_id(provider, model)) + } + + /// Look up a chain by an already-canonicalized requested model ID. + #[must_use] + pub fn chain_for_canonical(&self, canonical_model: &str) -> Option<&[FallbackTarget]> { + self.chains.get(canonical_model).map(Vec::as_slice) + } + + pub fn iter(&self) -> impl Iterator { + self.chains + .iter() + .map(|(model, chain)| (model.as_str(), chain.as_slice())) + } + + #[must_use] + pub fn len(&self) -> usize { + self.chains.len() + } + + #[must_use] + pub fn is_empty(&self) -> bool { + self.chains.is_empty() + } +} + +/// Server-side result of canonicalizing and filtering configured fallback +/// chains. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ResolvedModelFallbacks { + pub policy: ModelFallbackPolicy, + pub notices: Vec, +} + +/// Why a configured fallback candidate was removed from one model's chain. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ModelFallbackNotice { + ProviderUnconfigured { + requested_model: String, + reference: ModelRef, + provider: ProviderId, + }, + NoConfiguredOffering { + requested_model: String, + reference: ModelRef, + providers: Vec, + }, + PrimaryNotInCatalog { + requested_model: String, + reference: ModelRef, + primary: FallbackTarget, + }, + NoCompatibleModel { + requested_model: String, + reference: ModelRef, + provider: ProviderId, + }, + Duplicate { + requested_model: String, + reference: ModelRef, + target: FallbackTarget, + }, + NoNearbyReasoningLevel { + requested_model: String, + target: FallbackTarget, + requested_effort: ReasoningEffort, + }, + ChainEmpty { + requested_model: String, + }, +} + +impl ModelFallbackNotice { + #[must_use] + pub fn code(&self) -> RunNoticeCode { + match self { + Self::ChainEmpty { .. } => RunNoticeCode::ModelFallbackChainEmpty, + Self::ProviderUnconfigured { .. } + | Self::NoConfiguredOffering { .. } + | Self::PrimaryNotInCatalog { .. } + | Self::NoCompatibleModel { .. } + | Self::Duplicate { .. } + | Self::NoNearbyReasoningLevel { .. } => RunNoticeCode::ModelFallbackSkipped, + } + } + + #[must_use] + pub fn level(&self) -> RunNoticeLevel { + match self { + Self::Duplicate { .. } => RunNoticeLevel::Info, + Self::ProviderUnconfigured { .. } + | Self::NoConfiguredOffering { .. } + | Self::PrimaryNotInCatalog { .. } + | Self::NoCompatibleModel { .. } + | Self::NoNearbyReasoningLevel { .. } + | Self::ChainEmpty { .. } => RunNoticeLevel::Warn, + } + } + + #[must_use] + pub fn message(&self) -> String { + match self { + Self::ProviderUnconfigured { + requested_model, + reference, + provider, + } => format!( + "Model fallback `{reference}` for requested model `{requested_model}` was skipped because provider `{provider}` is not configured." + ), + Self::NoConfiguredOffering { + requested_model, + reference, + providers, + } => { + let providers = providers + .iter() + .map(ProviderId::to_string) + .collect::>() + .join(", "); + format!( + "Model fallback `{reference}` for requested model `{requested_model}` was skipped because none of its providers are configured. It is offered by: {providers}." + ) + } + Self::PrimaryNotInCatalog { + requested_model, + reference, + primary, + } => format!( + "Model fallback `{reference}` for requested model `{requested_model}` was skipped because `{primary}` is not in the catalog, so there is no capability profile to match against." + ), + Self::NoCompatibleModel { + requested_model, + reference, + provider, + } => format!( + "Model fallback `{reference}` for requested model `{requested_model}` was skipped because provider `{provider}` has no compatible model." + ), + Self::Duplicate { + requested_model, + reference, + target, + } => format!( + "Model fallback `{reference}` for requested model `{requested_model}` was skipped because target `{target}` already appears in that chain." + ), + Self::NoNearbyReasoningLevel { + requested_model, + target, + requested_effort, + } => format!( + "Model fallback `{target}` for requested model `{requested_model}` was skipped because it has no reasoning level near `{requested_effort}`." + ), + Self::ChainEmpty { requested_model } => format!( + "No usable model fallbacks remain for requested model `{requested_model}` after filtering its configured candidates." + ), + } + } +} + +/// Resolve every model-keyed fallback chain against the server's catalog and +/// configured-provider snapshot. +/// +/// This function must stay at server-side call sites. Offline validation only +/// parses the raw table and cannot canonicalize model aliases. +pub fn resolve_model_fallbacks( + catalog: &Catalog, + configured_providers: &[ProviderId], + configured: &BTreeMap>, +) -> Result { + let eligible = configured_providers.iter().cloned().collect::>(); + let mut resolved = ResolvedModelFallbacks::default(); + let mut raw_key_by_canonical = HashMap::::new(); + + for (raw_key, references) in configured { + require_bare_model_key(catalog, raw_key)?; + let selected = + catalog.resolve_selection_with_catalog_fallback(Some(raw_key), None, &eligible)?; + let requested_model = selected.model; + + if let Some(previous) = + raw_key_by_canonical.insert(requested_model.clone(), raw_key.clone()) + { + return Err(Error::Precondition(format!( + "`run.model.fallbacks` keys `{previous}` and `{raw_key}` both resolve to requested model `{requested_model}`" + ))); + } + + let primary = FallbackTarget::new(&selected.provider, &requested_model); + let primary_model = catalog.get_on_provider(&selected.provider, &requested_model); + let mut targets = Vec::new(); + + for model_ref in references { + let target = match resolve_fallback_candidate( + catalog, + &requested_model, + &primary, + primary_model, + &eligible, + model_ref, + )? { + FallbackCandidate::Skipped(notice) => { + resolved.notices.push(notice); + continue; + } + FallbackCandidate::Target(target) => target, + }; + + if targets.contains(&target) { + resolved.notices.push(ModelFallbackNotice::Duplicate { + requested_model: requested_model.clone(), + reference: model_ref.clone(), + target, + }); + } else { + targets.push(target); + } + } + + if targets.is_empty() { + resolved.notices.push(ModelFallbackNotice::ChainEmpty { + requested_model: requested_model.clone(), + }); + } + resolved.policy.chains.insert(requested_model, targets); + } + + Ok(resolved) +} + +/// Reject chain keys that name a provider. Keys are requested-model selectors; +/// a provider-qualified key can never match a dispatch-time canonical model +/// ID, so it would be silently dead configuration. +fn require_bare_model_key(catalog: &Catalog, raw_key: &str) -> Result<(), Error> { + let reference: ModelRef = raw_key + .parse() + .map_err(|error| Error::Precondition(format!("`run.model.fallbacks` key: {error}")))?; + match reference.resolve(catalog) { + Ok(ResolvedModelRef::Model { provider: None, .. }) => Ok(()), + Ok(ResolvedModelRef::Model { + provider: Some(_), + selector, + }) => Err(Error::Precondition(format!( + "`run.model.fallbacks` keys name a requested model; use `{selector}` instead of `{raw_key}`" + ))), + Ok(ResolvedModelRef::Provider(provider)) => Err(Error::Precondition(format!( + "`run.model.fallbacks` key `{raw_key}` names provider `{provider}`; keys must name a requested model" + ))), + Err(ambiguous) => Err(Error::Precondition(format!( + "`run.model.fallbacks` key: {ambiguous}" + ))), + } +} + +enum FallbackCandidate { + Target(FallbackTarget), + Skipped(ModelFallbackNotice), +} + +fn resolve_fallback_candidate( + catalog: &Catalog, + requested_model: &str, + primary: &FallbackTarget, + primary_model: Option<&Model>, + eligible: &HashSet, + model_ref: &ModelRef, +) -> Result { + let reference = model_ref.clone(); + + Ok(match model_ref.resolve(catalog)? { + ResolvedModelRef::Provider(provider_name) => { + let provider = catalog.provider_id(&provider_name)?; + if !eligible.contains(&provider) { + return Ok(FallbackCandidate::Skipped( + ModelFallbackNotice::ProviderUnconfigured { + requested_model: requested_model.to_string(), + reference, + provider, + }, + )); + } + let Some(primary_model) = primary_model else { + return Ok(FallbackCandidate::Skipped( + ModelFallbackNotice::PrimaryNotInCatalog { + requested_model: requested_model.to_string(), + reference, + primary: primary.clone(), + }, + )); + }; + match catalog.closest(&provider, primary_model) { + Some(model) => FallbackCandidate::Target(FallbackTarget::new(provider, &model.id)), + None => FallbackCandidate::Skipped(ModelFallbackNotice::NoCompatibleModel { + requested_model: requested_model.to_string(), + reference, + provider, + }), + } + } + ResolvedModelRef::Model { + provider: Some(provider_name), + selector, + } => { + let provider = catalog.provider_id(&provider_name)?; + if !eligible.contains(&provider) { + return Ok(FallbackCandidate::Skipped( + ModelFallbackNotice::ProviderUnconfigured { + requested_model: requested_model.to_string(), + reference, + provider, + }, + )); + } + match catalog.resolve_on_provider(&provider, &selector) { + Ok(info) => { + FallbackCandidate::Target(FallbackTarget::new(&info.provider, &info.id)) + } + Err(ModelSelectionError::UnknownSelectorOnProvider { .. }) => { + FallbackCandidate::Target(FallbackTarget::new(provider, selector)) + } + Err(error) => return Err(error.into()), + } + } + ResolvedModelRef::Model { + provider: None, + selector, + } => match catalog.select(&selector, None, eligible) { + Ok(info) => FallbackCandidate::Target(FallbackTarget::new(&info.provider, &info.id)), + Err(ModelSelectionError::NoEligibleOffering { providers, .. }) => { + FallbackCandidate::Skipped(ModelFallbackNotice::NoConfiguredOffering { + requested_model: requested_model.to_string(), + reference, + providers, + }) + } + Err(ModelSelectionError::UnknownSelector { .. }) => { + FallbackCandidate::Target(FallbackTarget::new(&primary.provider, selector)) + } + Err(error) => return Err(error.into()), + }, + }) +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use fabro_model::{Catalog, FallbackTarget, ProviderId}; + + use super::{ModelFallbackNotice, resolve_model_fallbacks}; + + fn references(values: &[&str]) -> Vec { + values + .iter() + .map(|value| value.parse().expect("fixture reference should parse")) + .collect() + } + + fn openrouter_catalog() -> Catalog { + let overrides = toml::from_str( + r" +[providers.openrouter] +enabled = true +", + ) + .expect("catalog override should parse"); + Catalog::from_builtin_with_overrides(&overrides).expect("catalog should build") + } + + #[test] + fn canonicalizes_keys_and_keeps_each_chain_independent() { + let catalog = openrouter_catalog(); + let eligible = [ProviderId::new("openrouter")]; + let configured = BTreeMap::from([ + ("gpt-sol".to_string(), references(&["claude-opus"])), + ( + "claude-fable".to_string(), + references(&["gpt-sol", "claude-opus"]), + ), + ]); + + let resolved = resolve_model_fallbacks(&catalog, &eligible, &configured).unwrap(); + + assert_eq!( + resolved + .policy + .chain_for(&catalog, &ProviderId::new("openrouter"), "gpt-sol"), + Some([FallbackTarget::new("openrouter", "claude-opus-5")].as_slice()) + ); + assert_eq!( + resolved + .policy + .chain_for(&catalog, &ProviderId::new("openrouter"), "claude-fable"), + Some( + [ + FallbackTarget::new("openrouter", "gpt-5.6-sol"), + FallbackTarget::new("openrouter", "claude-opus-5"), + ] + .as_slice() + ) + ); + } + + #[test] + fn rejects_aliases_that_define_the_same_requested_model_twice() { + let catalog = openrouter_catalog(); + let eligible = [ProviderId::new("openrouter")]; + let configured = BTreeMap::from([ + ("gpt-sol".to_string(), references(&["claude-opus"])), + ("gpt-5.6-sol".to_string(), references(&["claude-fable"])), + ]); + + let error = resolve_model_fallbacks(&catalog, &eligible, &configured).unwrap_err(); + + assert!( + error + .to_string() + .contains("both resolve to requested model"), + "unexpected error: {error}" + ); + } + + #[test] + fn rejects_provider_qualified_keys() { + let catalog = openrouter_catalog(); + let eligible = [ProviderId::new("openrouter")]; + let configured = BTreeMap::from([( + "openrouter:gpt-sol".to_string(), + references(&["claude-opus"]), + )]); + + let error = resolve_model_fallbacks(&catalog, &eligible, &configured).unwrap_err(); + + assert!( + error.to_string().contains("keys name a requested model"), + "unexpected error: {error}" + ); + } + + #[test] + fn skips_unconfigured_candidates_per_requested_model() { + let catalog = openrouter_catalog(); + let eligible = [ProviderId::new("openrouter")]; + let configured = BTreeMap::from([( + "kimi-k3".to_string(), + references(&["kimi:kimi-k3", "openrouter:kimi-k3"]), + )]); + + let resolved = resolve_model_fallbacks(&catalog, &eligible, &configured).unwrap(); + + assert_eq!( + resolved + .policy + .chain_for(&catalog, &ProviderId::new("openrouter"), "kimi-k3"), + Some([FallbackTarget::new("openrouter", "kimi-k3")].as_slice()) + ); + assert!(matches!( + resolved.notices.as_slice(), + [ModelFallbackNotice::ProviderUnconfigured { + requested_model, + provider, + .. + }] if requested_model == "kimi-k3" && provider == &ProviderId::new("kimi") + )); + } + + #[test] + fn resolves_the_requested_production_policy_as_independent_chains() { + let catalog = { + let overrides = toml::from_str( + r" +[providers.modal] +enabled = true + +[providers.openrouter] +enabled = true +", + ) + .expect("catalog override should parse"); + Catalog::from_builtin_with_overrides(&overrides).expect("catalog should build") + }; + let eligible = [ + ProviderId::new("modal"), + ProviderId::new("kimi"), + ProviderId::new("openrouter"), + ]; + let configured = BTreeMap::from([ + ( + "kimi-k3".to_string(), + references(&["kimi:kimi-k3", "openrouter:kimi-k3", "claude-opus"]), + ), + ("glm-5.2".to_string(), references(&["gpt-sol"])), + ("gpt-sol".to_string(), references(&["claude-opus"])), + ("claude-opus".to_string(), references(&["gpt-sol"])), + ("gpt-terra".to_string(), references(&["claude-opus"])), + ("gpt-luna".to_string(), references(&["claude-sonnet"])), + ( + "claude-fable".to_string(), + references(&["gpt-sol", "claude-opus"]), + ), + ]); + + let resolved = resolve_model_fallbacks(&catalog, &eligible, &configured).unwrap(); + + assert!(resolved.notices.is_empty()); + let chain = |model: &str| { + resolved + .policy + .chain_for(&catalog, &ProviderId::new("openrouter"), model) + .expect("requested model should have a chain") + }; + assert_eq!(chain("kimi-k3"), [ + FallbackTarget::new("kimi", "kimi-k3"), + FallbackTarget::new("openrouter", "kimi-k3"), + FallbackTarget::new("openrouter", "claude-opus-5"), + ]); + assert_eq!(chain("glm-5.2"), [FallbackTarget::new( + "openrouter", + "gpt-5.6-sol" + )]); + assert_eq!(chain("gpt-sol"), [FallbackTarget::new( + "openrouter", + "claude-opus-5" + )]); + assert_eq!(chain("claude-opus"), [FallbackTarget::new( + "openrouter", + "gpt-5.6-sol" + )]); + assert_eq!(chain("gpt-terra"), [FallbackTarget::new( + "openrouter", + "claude-opus-5" + )]); + assert_eq!(chain("gpt-luna"), [FallbackTarget::new( + "openrouter", + "claude-sonnet-5" + )]); + assert_eq!(chain("claude-fable"), [ + FallbackTarget::new("openrouter", "gpt-5.6-sol"), + FallbackTarget::new("openrouter", "claude-opus-5"), + ]); + } +} diff --git a/lib/components/fabro-workflow/src/operations/start.rs b/lib/components/fabro-workflow/src/operations/start.rs index 54d3ae572..12b7455a9 100644 --- a/lib/components/fabro-workflow/src/operations/start.rs +++ b/lib/components/fabro-workflow/src/operations/start.rs @@ -7,7 +7,7 @@ use fabro_auth::{CredentialSource, VaultCredentialSource}; use fabro_interview::{AutoApproveInterviewer, Interviewer}; use fabro_llm::client::Client as LlmClient; use fabro_mcp::config::McpServerSettings; -use fabro_model::{Catalog, FallbackTarget, Model, ModelSelectionError, ProviderId}; +use fabro_model::{Catalog, ProviderId}; use fabro_sandbox::daytona::DaytonaConfig; use fabro_sandbox::from_environment::{ daytona_config_from_environment, docker_config_from_environment_with_secrets, @@ -17,10 +17,9 @@ use fabro_sandbox::{DockerSandboxOptions, SandboxSpec}; use fabro_static::EnvVars; use fabro_types::settings::run::{ ApprovalMode, McpServerSettings as ResolvedMcpServerSettings, PullRequestSettings, - ResolvedMcpEntry, RunMode, RunModelSettings as ResolvedRunModelSettings, - RunNamespace as ResolvedRunSettings, RunPrepareSettings as ResolvedRunPrepareSettings, + ResolvedMcpEntry, RunMode, RunNamespace as ResolvedRunSettings, + RunPrepareSettings as ResolvedRunPrepareSettings, }; -use fabro_types::settings::{ModelRef, ResolvedModelRef}; use fabro_types::{ManifestPath, RunId, RunRunnableSource, SandboxProviderKind}; use fabro_vault::Vault; use tokio::runtime::Handle; @@ -32,10 +31,10 @@ use crate::artifact_upload::ArtifactSink; use crate::context::Context; use crate::error::{self, Error}; use crate::event::{ - Emitter, Event, EventBody, RunEventLogger, RunEventSink, RunNoticeCode, RunNoticeLevel, - append_event_to_sink, + Emitter, Event, EventBody, RunEventLogger, RunEventSink, RunNoticeLevel, append_event_to_sink, }; use crate::handler::HandlerRegistry; +use crate::model_fallback::{ModelFallbackNotice, ResolvedModelFallbacks, resolve_model_fallbacks}; use crate::outcome::{Outcome, StageOutcome}; use crate::pipeline::{ self, FinalizeOptions, Finalized, InitOptions, LlmSpec, Persisted, PublishOptions, ResumeState, @@ -91,132 +90,7 @@ struct RunSession { struct ResolvedStartLlm { model: String, provider_id: ProviderId, - fallbacks: ResolvedFallbackChain, -} - -#[derive(Debug, Default, PartialEq, Eq)] -struct ResolvedFallbackChain { - targets: Vec, - notices: Vec, -} - -/// Why one fallback candidate did not make it into the chain, or that the whole -/// chain came out empty. Resolution happens before the run's event sink is -/// wired up, so these are carried to [`RunSession::run`] and emitted there. -#[derive(Debug, PartialEq, Eq)] -enum ModelFallbackNotice { - ProviderUnconfigured { - reference: ModelRef, - provider: ProviderId, - }, - NoConfiguredOffering { - reference: ModelRef, - providers: Vec, - }, - /// The candidate named a provider but no model, and the primary model is - /// not in the catalog, so there is nothing to match its capabilities to. - PrimaryNotInCatalog { - reference: ModelRef, - primary: FallbackTarget, - }, - NoCompatibleModel { - reference: ModelRef, - provider: ProviderId, - }, - MatchesPrimary { - reference: ModelRef, - target: FallbackTarget, - }, - Duplicate { - reference: ModelRef, - target: FallbackTarget, - }, - ChainEmpty, -} - -impl ModelFallbackNotice { - fn code(&self) -> RunNoticeCode { - match self { - Self::ChainEmpty => RunNoticeCode::ModelFallbackChainEmpty, - Self::ProviderUnconfigured { .. } - | Self::NoConfiguredOffering { .. } - | Self::PrimaryNotInCatalog { .. } - | Self::NoCompatibleModel { .. } - | Self::MatchesPrimary { .. } - | Self::Duplicate { .. } => RunNoticeCode::ModelFallbackSkipped, - } - } - - fn level(&self) -> RunNoticeLevel { - match self { - Self::MatchesPrimary { .. } | Self::Duplicate { .. } => RunNoticeLevel::Info, - Self::ProviderUnconfigured { .. } - | Self::NoConfiguredOffering { .. } - | Self::PrimaryNotInCatalog { .. } - | Self::NoCompatibleModel { .. } - | Self::ChainEmpty => RunNoticeLevel::Warn, - } - } - - fn message(&self) -> String { - match self { - Self::ProviderUnconfigured { - reference, - provider, - } => { - format!( - "Model fallback `{reference}` was skipped because provider `{provider}` is not configured." - ) - } - Self::NoConfiguredOffering { - reference, - providers, - } => { - let providers = providers - .iter() - .map(ProviderId::to_string) - .collect::>() - .join(", "); - format!( - "Model fallback `{reference}` was skipped because none of its providers are configured. It is offered by: {providers}." - ) - } - Self::PrimaryNotInCatalog { reference, primary } => { - format!( - "Model fallback `{reference}` was skipped because the primary model `{primary}` is not in the catalog, so there is no capability profile to match against." - ) - } - Self::NoCompatibleModel { - reference, - provider, - } => { - format!( - "Model fallback `{reference}` was skipped because provider `{provider}` has no compatible model." - ) - } - Self::MatchesPrimary { reference, target } => { - format!( - "Model fallback `{reference}` was skipped because it resolves to the primary target `{target}`." - ) - } - Self::Duplicate { reference, target } => { - format!( - "Model fallback `{reference}` was skipped because target `{target}` already appears in the fallback chain." - ) - } - Self::ChainEmpty => { - "No usable model fallbacks remain after filtering the configured fallback candidates." - .to_string() - } - } - } - - /// Publish every notice on the run's event stream. - fn emit_all(notices: &[Self], emitter: &Emitter) { - for notice in notices { - emitter.notice(notice.level(), notice.code(), notice.message()); - } - } + fallbacks: ResolvedModelFallbacks, } pub struct StartServices { @@ -604,7 +478,7 @@ impl RunSession { llm: LlmSpec { model: llm.model.clone(), provider_id: llm.provider_id.clone(), - fallback_chain: llm.fallbacks.targets, + fallbacks: llm.fallbacks.policy, mcp_servers, model_controls: resolved.model.controls.clone(), dry_run: resolved.execution.mode == RunMode::DryRun, @@ -743,8 +617,7 @@ fn resolve_start_llm( settings.model.provider.as_deref(), false, )?; - let fallbacks = - resolve_fallback_chain(catalog, &provider_id, &model, &settings.model, &eligible)?; + let fallbacks = resolve_model_fallbacks(catalog, configured, &settings.model.fallbacks)?; Ok(ResolvedStartLlm { model, @@ -753,166 +626,6 @@ fn resolve_start_llm( }) } -/// Resolve fallback candidates against the configured provider snapshot. -/// -/// Candidates that cannot be used in this environment — an unconfigured -/// provider, no compatible model, a target equal to the primary, or a duplicate -/// — are dropped, and each drop records a [`ModelFallbackNotice`] that the run -/// emits at startup. Remaining candidates keep their configured order. -/// -/// A provider the catalog has never heard of is a different case: that is a -/// typo rather than an environment difference, so it fails the run instead of -/// being skipped. This is what keeps a chain portable without letting a -/// misspelled provider silently disappear. -fn resolve_fallback_chain( - catalog: &Catalog, - provider: &ProviderId, - model: &str, - settings: &ResolvedRunModelSettings, - eligible: &HashSet, -) -> Result { - if settings.fallbacks.is_empty() { - return Ok(ResolvedFallbackChain::default()); - } - - let primary_model = catalog.get_on_provider(provider, model); - let primary = FallbackTarget::new(provider, model); - let mut resolution = ResolvedFallbackChain::default(); - - for model_ref in &settings.fallbacks { - let target = match resolve_fallback_candidate( - catalog, - &primary, - primary_model, - eligible, - model_ref, - )? { - FallbackCandidate::Skipped(notice) => { - resolution.notices.push(notice); - continue; - } - FallbackCandidate::Target(target) => target, - }; - - let reference = model_ref.clone(); - if target == primary { - resolution - .notices - .push(ModelFallbackNotice::MatchesPrimary { reference, target }); - } else if resolution.targets.contains(&target) { - resolution - .notices - .push(ModelFallbackNotice::Duplicate { reference, target }); - } else { - resolution.targets.push(target); - } - } - - if resolution.targets.is_empty() { - resolution.notices.push(ModelFallbackNotice::ChainEmpty); - } - - Ok(resolution) -} - -/// The outcome of resolving one fallback candidate: either a dispatchable -/// target or the reason the candidate cannot be used. -enum FallbackCandidate { - Target(FallbackTarget), - Skipped(ModelFallbackNotice), -} - -/// Resolve one fallback reference against the configured provider snapshot. -/// -/// `primary_model` is the primary's catalog entry, used to pick the closest -/// capability match when a candidate names a provider but no model. It is -/// `None` when the primary is itself a passthrough selector. -/// -/// A selector the catalog does not know passes through verbatim so a model -/// newer than the catalog still dispatches. When the candidate named a -/// provider, it passes through on that provider; when it did not, it passes -/// through on the primary's provider, which means such a fallback gives no -/// cross-provider failover. -fn resolve_fallback_candidate( - catalog: &Catalog, - primary: &FallbackTarget, - primary_model: Option<&Model>, - eligible: &HashSet, - model_ref: &ModelRef, -) -> Result { - let reference = model_ref.clone(); - - Ok(match model_ref.resolve(catalog)? { - ResolvedModelRef::Provider(provider_name) => { - let provider = catalog.provider_id(&provider_name)?; - if !eligible.contains(&provider) { - return Ok(FallbackCandidate::Skipped( - ModelFallbackNotice::ProviderUnconfigured { - reference, - provider, - }, - )); - } - // Without a catalog entry for the primary there is no capability - // profile to match against, which is not the provider's fault. - let Some(primary_model) = primary_model else { - return Ok(FallbackCandidate::Skipped( - ModelFallbackNotice::PrimaryNotInCatalog { - reference, - primary: primary.clone(), - }, - )); - }; - match catalog.closest(&provider, primary_model) { - Some(model) => FallbackCandidate::Target(FallbackTarget::new(provider, &model.id)), - None => FallbackCandidate::Skipped(ModelFallbackNotice::NoCompatibleModel { - reference, - provider, - }), - } - } - ResolvedModelRef::Model { - provider: Some(provider_name), - selector, - } => { - let provider = catalog.provider_id(&provider_name)?; - if !eligible.contains(&provider) { - return Ok(FallbackCandidate::Skipped( - ModelFallbackNotice::ProviderUnconfigured { - reference, - provider, - }, - )); - } - match catalog.resolve_on_provider(&provider, &selector) { - Ok(info) => { - FallbackCandidate::Target(FallbackTarget::new(&info.provider, &info.id)) - } - Err(ModelSelectionError::UnknownSelectorOnProvider { .. }) => { - FallbackCandidate::Target(FallbackTarget::new(provider, selector)) - } - Err(error) => return Err(error.into()), - } - } - ResolvedModelRef::Model { - provider: None, - selector, - } => match catalog.select(&selector, None, eligible) { - Ok(info) => FallbackCandidate::Target(FallbackTarget::new(&info.provider, &info.id)), - Err(ModelSelectionError::NoEligibleOffering { providers, .. }) => { - FallbackCandidate::Skipped(ModelFallbackNotice::NoConfiguredOffering { - reference, - providers, - }) - } - Err(ModelSelectionError::UnknownSelector { .. }) => { - FallbackCandidate::Target(FallbackTarget::new(&primary.provider, selector)) - } - Err(error) => return Err(error.into()), - }, - }) -} - /// Build the launch-time MCP config from resolved settings. Secret tokens in /// the transport (`command`/`url`/`env`/`headers`) resolve from the vault at /// the run boundary. Unsupported tokens fail. @@ -1035,7 +748,10 @@ impl RunSession { store_progress_logger.register(self.emitter.as_ref()); // Emit after the logger is registered so the notices reach the run // store, and before `run.started` so they read as launch-time context. - ModelFallbackNotice::emit_all(&self.fallback_notices, self.emitter.as_ref()); + for notice in &self.fallback_notices { + self.emitter + .notice(notice.level(), notice.code(), notice.message()); + } let init_options = InitOptions { run_store: self.run_store.clone(), @@ -1334,11 +1050,11 @@ mod tests { RunEnvironmentLayer, RunExecutionLayer, RunLayer, StickyMap, WorkflowSettingsBuilder, }; use fabro_store::Database; + use fabro_types::settings::InterpString; use fabro_types::settings::run::{ McpTransport as ResolvedMcpTransport, PreparedStep, PreparedStepRun, RunMode, RunPrepareSettings, }; - use fabro_types::settings::{InterpString, ModelRef}; use fabro_types::{ BilledModelUsage, ManifestPath, StageTiming, WorkflowSettings, fixtures, test_support, }; @@ -1505,517 +1221,6 @@ reasoning = false Catalog::from_settings(&settings).unwrap() } - #[test] - fn resolve_start_llm_infers_primary_and_filters_global_fallbacks() { - let catalog = portable_model_catalog(); - let mut settings = ResolvedRunSettings::default(); - settings.model.name = Some("gpt-56-sol".to_string()); - settings.model.fallbacks = vec![ - "openai:gpt-56-sol".parse::().unwrap(), - "openrouter:gpt-56-sol".parse::().unwrap(), - "openrouter:openai/gpt-5.6-sol".parse::().unwrap(), - ]; - - let resolved = resolve_start_llm( - &catalog, - &[ProviderId::new("openrouter"), ProviderId::openai()], - &settings, - ) - .unwrap(); - - assert_eq!(resolved.provider_id, ProviderId::openai()); - assert_eq!(resolved.model, "gpt-5.6-sol"); - assert_eq!(resolved.fallbacks.targets, vec![FallbackTarget { - provider: "openrouter".to_string(), - model: "gpt-5.6-sol".to_string(), - }]); - assert_eq!(resolved.fallbacks.notices, vec![ - ModelFallbackNotice::MatchesPrimary { - reference: "openai:gpt-56-sol".parse().unwrap(), - target: FallbackTarget { - provider: "openai".to_string(), - model: "gpt-5.6-sol".to_string(), - }, - }, - ModelFallbackNotice::Duplicate { - reference: "openrouter:openai/gpt-5.6-sol".parse().unwrap(), - target: FallbackTarget { - provider: "openrouter".to_string(), - model: "gpt-5.6-sol".to_string(), - }, - }, - ]); - } - - #[test] - fn resolve_fallback_chain_skips_unconfigured_provider_and_preserves_order() { - let catalog = test_catalog(); - let settings = ResolvedRunModelSettings { - fallbacks: vec![ - "gemini".parse::().unwrap(), - "gemini:unused".parse::().unwrap(), - "openai:gpt-5.4-mini".parse::().unwrap(), - "anthropic:claude-fable-5".parse::().unwrap(), - ], - ..ResolvedRunModelSettings::default() - }; - - let resolution = resolve_fallback_chain( - catalog.as_ref(), - &ProviderId::anthropic(), - "claude-opus-4-6", - &settings, - &HashSet::from([ProviderId::anthropic(), ProviderId::openai()]), - ) - .unwrap(); - - assert_eq!(resolution.targets, vec![ - FallbackTarget { - provider: "openai".to_string(), - model: "gpt-5.4-mini".to_string(), - }, - FallbackTarget { - provider: "anthropic".to_string(), - model: "claude-fable-5".to_string(), - }, - ]); - assert_eq!(resolution.notices, vec![ - ModelFallbackNotice::ProviderUnconfigured { - reference: "gemini".parse().unwrap(), - provider: ProviderId::gemini(), - }, - ModelFallbackNotice::ProviderUnconfigured { - reference: "gemini:unused".parse().unwrap(), - provider: ProviderId::gemini(), - }, - ]); - assert_eq!(resolution.notices[0].level(), RunNoticeLevel::Warn); - assert_eq!( - resolution.notices[0].code(), - RunNoticeCode::ModelFallbackSkipped - ); - } - - /// The resolver builds notices before the run's event sink exists, so this - /// covers the hand-off: each notice must reach the event stream as a - /// `run.notice` carrying its own level, code, and rendered message. - #[test] - fn fallback_notices_reach_the_event_stream() { - let emitter = Arc::new(Emitter::new(fixtures::RUN_1)); - let captured = Arc::new(Mutex::new(Vec::new())); - let sink = Arc::clone(&captured); - emitter.on_event(move |event| sink.lock().unwrap().push(event.clone())); - - let notices = vec![ - ModelFallbackNotice::ProviderUnconfigured { - reference: "gemini".parse().unwrap(), - provider: ProviderId::gemini(), - }, - ModelFallbackNotice::MatchesPrimary { - reference: "openai:gpt-5.6-sol".parse().unwrap(), - target: FallbackTarget::new("openai", "gpt-5.6-sol"), - }, - ModelFallbackNotice::ChainEmpty, - ]; - - ModelFallbackNotice::emit_all(¬ices, emitter.as_ref()); - - let events = captured.lock().unwrap(); - let emitted = events - .iter() - .map(|event| match &event.body { - EventBody::RunNotice(props) => { - (props.level, props.code.clone(), props.message.clone()) - } - other => panic!("expected run.notice body, got {other:?}"), - }) - .collect::>(); - - assert_eq!(emitted, vec![ - ( - RunNoticeLevel::Warn, - RunNoticeCode::ModelFallbackSkipped.to_string(), - "Model fallback `gemini` was skipped because provider `gemini` is not configured." - .to_string(), - ), - ( - RunNoticeLevel::Info, - RunNoticeCode::ModelFallbackSkipped.to_string(), - "Model fallback `openai:gpt-5.6-sol` was skipped because it resolves to the primary target `openai:gpt-5.6-sol`." - .to_string(), - ), - ( - RunNoticeLevel::Warn, - RunNoticeCode::ModelFallbackChainEmpty.to_string(), - "No usable model fallbacks remain after filtering the configured fallback candidates." - .to_string(), - ), - ]); - } - - /// A provider-only fallback cannot be matched when the primary model is a - /// passthrough selector, because there is no capability profile to compare - /// against. The notice must name that cause rather than blaming the - /// provider, which may well have compatible models. - #[test] - fn resolve_fallback_chain_blames_missing_primary_not_the_fallback_provider() { - let catalog = portable_model_catalog(); - let settings = ResolvedRunModelSettings { - fallbacks: vec!["openrouter".parse::().unwrap()], - ..ResolvedRunModelSettings::default() - }; - - let resolution = resolve_fallback_chain( - &catalog, - &ProviderId::openai(), - "gpt-5.9-not-in-catalog", - &settings, - &HashSet::from([ProviderId::openai(), ProviderId::new("openrouter")]), - ) - .unwrap(); - - assert!(resolution.targets.is_empty()); - assert_eq!(resolution.notices, vec![ - ModelFallbackNotice::PrimaryNotInCatalog { - reference: "openrouter".parse().unwrap(), - primary: FallbackTarget::new("openai", "gpt-5.9-not-in-catalog"), - }, - ModelFallbackNotice::ChainEmpty, - ]); - let message = resolution.notices[0].message(); - assert!( - message.contains("primary model `openai:gpt-5.9-not-in-catalog` is not in the catalog"), - "notice should name the missing primary: {message}" - ); - } - - #[test] - fn resolve_fallback_chain_skips_model_without_configured_offering() { - let catalog = portable_model_catalog(); - let settings = ResolvedRunModelSettings { - fallbacks: vec!["mini".parse::().unwrap()], - ..ResolvedRunModelSettings::default() - }; - - let resolution = resolve_fallback_chain( - &catalog, - &ProviderId::new("openrouter"), - "gpt-5.6-sol", - &settings, - &HashSet::from([ProviderId::new("openrouter")]), - ) - .unwrap(); - - assert!(resolution.targets.is_empty()); - assert_eq!(resolution.notices, vec![ - ModelFallbackNotice::NoConfiguredOffering { - reference: "mini".parse().unwrap(), - providers: vec![ProviderId::openai()], - }, - ModelFallbackNotice::ChainEmpty, - ]); - assert_eq!( - resolution.notices[1].code(), - RunNoticeCode::ModelFallbackChainEmpty - ); - assert_eq!(resolution.notices[1].level(), RunNoticeLevel::Warn); - assert!( - resolution.notices[0] - .message() - .contains("offered by: openai"), - "notice should name the providers that offer the model: {}", - resolution.notices[0].message() - ); - } - - #[test] - fn resolve_fallback_chain_rejects_unknown_qualified_provider() { - let catalog = portable_model_catalog(); - let settings = ResolvedRunModelSettings { - fallbacks: vec!["missing/model".parse::().unwrap()], - ..ResolvedRunModelSettings::default() - }; - - let error = resolve_fallback_chain( - &catalog, - &ProviderId::openai(), - "gpt-5.6-sol", - &settings, - &catalog.all_provider_ids(), - ) - .unwrap_err(); - - assert!(matches!( - error, - Error::ModelSelection(ModelSelectionError::UnknownProvider { provider }) - if provider == ProviderId::new("missing") - )); - } - - #[test] - fn resolve_fallback_chain_resolves_provider_fallbacks() { - let catalog = test_catalog(); - let settings = ResolvedRunModelSettings { - fallbacks: vec!["openai".parse::().unwrap()], - ..ResolvedRunModelSettings::default() - }; - - let chain = resolve_fallback_chain( - catalog.as_ref(), - &ProviderId::anthropic(), - "claude-opus-4-6", - &settings, - &catalog.all_provider_ids(), - ) - .unwrap(); - - assert_eq!(chain.targets, vec![FallbackTarget { - provider: "openai".to_string(), - model: "gpt-5.5".to_string(), - }]); - } - - #[test] - fn resolve_fallback_chain_resolves_explicit_model_fallbacks() { - let catalog = test_catalog(); - let settings = ResolvedRunModelSettings { - fallbacks: vec!["openai:gpt-5.4-mini".parse::().unwrap()], - ..ResolvedRunModelSettings::default() - }; - - let chain = resolve_fallback_chain( - catalog.as_ref(), - &ProviderId::anthropic(), - "claude-opus-4-6", - &settings, - &catalog.all_provider_ids(), - ) - .unwrap(); - - assert_eq!(chain.targets, vec![FallbackTarget { - provider: "openai".to_string(), - model: "gpt-5.4-mini".to_string(), - }]); - } - - #[test] - fn resolve_fallback_chain_selects_shared_bare_alias_from_ready_providers() { - let catalog = portable_model_catalog(); - let settings = ResolvedRunModelSettings { - fallbacks: vec!["gpt-56-sol".parse::().unwrap()], - ..ResolvedRunModelSettings::default() - }; - - let chain = resolve_fallback_chain( - &catalog, - &ProviderId::openai(), - "gpt-5.6-sol", - &settings, - &HashSet::from([ProviderId::new("openrouter")]), - ) - .unwrap(); - - assert_eq!(chain.targets, vec![FallbackTarget { - provider: "openrouter".to_string(), - model: "gpt-5.6-sol".to_string(), - }]); - } - - #[test] - fn resolve_fallback_chain_resolves_provider_qualified_shared_alias() { - let catalog = portable_model_catalog(); - let settings = ResolvedRunModelSettings { - fallbacks: vec!["openrouter:gpt-56-sol".parse::().unwrap()], - ..ResolvedRunModelSettings::default() - }; - - let chain = resolve_fallback_chain( - &catalog, - &ProviderId::openai(), - "gpt-5.6-sol", - &settings, - &catalog.all_provider_ids(), - ) - .unwrap(); - - assert_eq!(chain.targets, vec![FallbackTarget { - provider: "openrouter".to_string(), - model: "gpt-5.6-sol".to_string(), - }]); - } - - /// A qualified fallback resolves to the same offering whether the selector - /// is the canonical model ID or the provider's API ID. The trailing bare - /// alias still goes through ready-provider priority selection. - #[test] - fn resolve_fallback_chain_resolves_qualified_model_id_and_api_id_alike() { - let overrides: fabro_model::catalog::LlmCatalogSettings = toml::from_str( - r" -[providers.openrouter] -enabled = true -", - ) - .unwrap(); - let catalog = Catalog::from_builtin_with_overrides(&overrides).unwrap(); - - for selector in ["openrouter:kimi-k3", "openrouter:moonshotai/kimi-k3"] { - let settings = ResolvedRunModelSettings { - fallbacks: vec![ - selector.parse::().unwrap(), - "gpt-terra".parse::().unwrap(), - ], - ..ResolvedRunModelSettings::default() - }; - - let chain = resolve_fallback_chain( - &catalog, - &ProviderId::new("kimi"), - "kimi-k3", - &settings, - &HashSet::from([ - ProviderId::new("kimi"), - ProviderId::new("openrouter"), - ProviderId::openai(), - ]), - ) - .unwrap(); - - assert_eq!( - chain.targets, - vec![ - FallbackTarget { - provider: "openrouter".to_string(), - model: "kimi-k3".to_string(), - }, - FallbackTarget { - provider: "openai".to_string(), - model: "gpt-5.6-terra".to_string(), - }, - ], - "{selector}" - ); - } - } - - /// A colon in a model ID does not make it provider-qualified, so an - /// unknown colon-bearing selector still passes through to a provider - /// instead of failing the run with an unknown-provider error. - #[test] - fn resolve_fallback_chain_passes_through_colon_bearing_model_ids() { - let catalog = portable_model_catalog(); - let settings = ResolvedRunModelSettings { - fallbacks: vec!["future-model:latest".parse::().unwrap()], - ..ResolvedRunModelSettings::default() - }; - - let chain = resolve_fallback_chain( - &catalog, - &ProviderId::openai(), - "gpt-5.6-sol", - &settings, - &HashSet::from([ProviderId::openai(), ProviderId::new("openrouter")]), - ) - .unwrap(); - - assert_eq!(chain.targets, vec![FallbackTarget { - provider: ProviderId::openai().to_string(), - model: "future-model:latest".to_string(), - }]); - } - - #[test] - fn resolve_fallback_chain_keeps_qualified_legacy_references_as_provider_pins() { - let catalog = test_catalog(); - let settings = ResolvedRunModelSettings { - fallbacks: vec![ - "openai/gpt-5.6-sol".parse::().unwrap(), - "anthropic/claude-fable-5".parse::().unwrap(), - ], - ..ResolvedRunModelSettings::default() - }; - - let chain = resolve_fallback_chain( - catalog.as_ref(), - &ProviderId::anthropic(), - "claude-opus-4-6", - &settings, - &catalog.all_provider_ids(), - ) - .unwrap(); - - assert_eq!(chain.targets, vec![ - FallbackTarget { - provider: "openai".to_string(), - model: "gpt-5.6-sol".to_string(), - }, - FallbackTarget { - provider: "anthropic".to_string(), - model: "claude-fable-5".to_string(), - }, - ]); - } - - #[test] - fn resolve_fallback_chain_propagates_provider_model_ambiguity() { - let settings: fabro_model::catalog::LlmCatalogSettings = toml::from_str( - r#" -[providers.shared] -display_name = "Shared Provider" -adapter = "openai" -agent_profile = "openai" - -[providers.shared.models.default] -display_name = "Default" -family = "test" -default = true - -[providers.shared.models.default.limits] -context_window = 1000 - -[providers.shared.models.default.features] -tools = false -vision = false -reasoning = false - -[providers.other] -display_name = "Other" -adapter = "openai" -agent_profile = "openai" - -[providers.other.models.model] -display_name = "Shared Alias" -family = "test" -aliases = ["shared"] -default = true - -[providers.other.models.model.limits] -context_window = 1000 - -[providers.other.models.model.features] -tools = false -vision = false -reasoning = false -"#, - ) - .unwrap(); - let catalog = Catalog::from_settings(&settings).unwrap(); - let run_model = ResolvedRunModelSettings { - fallbacks: vec!["shared".parse::().unwrap()], - ..ResolvedRunModelSettings::default() - }; - - let error = resolve_fallback_chain( - &catalog, - &ProviderId::new("other"), - "model", - &run_model, - &catalog.all_provider_ids(), - ) - .unwrap_err(); - - assert!(matches!(error, Error::ModelReference(_))); - } - #[test] fn materialized_provider_pin_is_not_reselected_when_readiness_changes() { let catalog = portable_model_catalog(); diff --git a/lib/components/fabro-workflow/src/pipeline/execute/tests.rs b/lib/components/fabro-workflow/src/pipeline/execute/tests.rs index b34810f64..6fd73a17c 100644 --- a/lib/components/fabro-workflow/src/pipeline/execute/tests.rs +++ b/lib/components/fabro-workflow/src/pipeline/execute/tests.rs @@ -31,6 +31,7 @@ use crate::error::Error; use crate::event::{Emitter, Event, StoreProgressLogger, append_event}; use crate::handler::start::StartHandler; use crate::handler::{Handler as HandlerTrait, HandlerRegistry}; +use crate::model_fallback::ModelFallbackPolicy; use crate::outcome::{Outcome, OutcomeExt, StageOutcome}; use crate::pipeline::initialize; use crate::pipeline::types::{InitOptions, LlmSpec, Persisted, ResumeState, SandboxEnvSpec}; @@ -267,7 +268,7 @@ async fn execute_test_run_with_options( llm: LlmSpec { model: "test-model".to_string(), provider_id: fabro_model::ProviderId::anthropic(), - fallback_chain: Vec::new(), + fallbacks: ModelFallbackPolicy::default(), mcp_servers: Vec::new(), model_controls: RunModelControls::default(), dry_run: true, @@ -327,7 +328,7 @@ async fn execute_runs_start_to_exit_and_returns_final_context() { llm: LlmSpec { model: "test-model".to_string(), provider_id: fabro_model::ProviderId::anthropic(), - fallback_chain: Vec::new(), + fallbacks: ModelFallbackPolicy::default(), mcp_servers: Vec::new(), model_controls: RunModelControls::default(), dry_run: true, @@ -468,7 +469,7 @@ async fn resumed_in_flight_node_starts_a_new_stage_execution() { llm: LlmSpec { model: "test-model".to_string(), provider_id: fabro_model::ProviderId::anthropic(), - fallback_chain: Vec::new(), + fallbacks: ModelFallbackPolicy::default(), mcp_servers: Vec::new(), model_controls: RunModelControls::default(), dry_run: true, @@ -582,7 +583,7 @@ async fn run_with_lifecycle( llm: LlmSpec { model: "test-model".to_string(), provider_id: fabro_model::ProviderId::anthropic(), - fallback_chain: Vec::new(), + fallbacks: ModelFallbackPolicy::default(), mcp_servers: Vec::new(), model_controls: RunModelControls::default(), dry_run: true, diff --git a/lib/components/fabro-workflow/src/pipeline/initialize.rs b/lib/components/fabro-workflow/src/pipeline/initialize.rs index 0f4bfb440..7e0fcb893 100644 --- a/lib/components/fabro-workflow/src/pipeline/initialize.rs +++ b/lib/components/fabro-workflow/src/pipeline/initialize.rs @@ -25,6 +25,8 @@ use crate::git::GitAuthor; use crate::github_token_source::{AppIatMinter, GitHubTokenSource}; use crate::handler::llm::{AgentAcpBackend, AgentApiBackend, BackendRouter, routing}; use crate::handler::{HandlerRegistry, default_registry}; +#[cfg(test)] +use crate::model_fallback::ModelFallbackPolicy; use crate::run_metadata::{RunMetadataRuntime, build_metadata_writer, metadata_branch_name}; use crate::run_options::{GitCheckpointOptions, RunOptions}; use crate::sandbox_git_runtime::SandboxGitRuntime; @@ -166,7 +168,7 @@ async fn build_registry( let build_llm_registry = || { let model = spec.model.clone(); let provider_id = spec.provider_id.clone(); - let fallback_chain = spec.fallback_chain.clone(); + let fallbacks = spec.fallbacks.clone(); let mcp_servers = spec.mcp_servers.clone(); let model_controls = spec.model_controls.clone(); let tool_secrets_for_api = tool_secrets.clone(); @@ -180,7 +182,7 @@ async fn build_registry( let mut api = AgentApiBackend::new_with_catalog( model.clone(), provider_id.clone(), - fallback_chain.clone(), + fallbacks.clone(), Arc::clone(&llm_source_for_api), Arc::clone(&steering_hub_for_api), Arc::clone(&catalog_for_api), @@ -830,7 +832,7 @@ mod tests { llm: LlmSpec { model: "test-model".to_string(), provider_id: fabro_model::ProviderId::anthropic(), - fallback_chain: Vec::new(), + fallbacks: ModelFallbackPolicy::default(), mcp_servers: Vec::new(), model_controls: RunModelControls::default(), dry_run: true, @@ -911,7 +913,7 @@ mod tests { llm: LlmSpec { model: "test-model".to_string(), provider_id: fabro_model::ProviderId::anthropic(), - fallback_chain: Vec::new(), + fallbacks: ModelFallbackPolicy::default(), mcp_servers: Vec::new(), model_controls: RunModelControls::default(), dry_run: true, @@ -1010,7 +1012,7 @@ mod tests { &LlmSpec { model: "claude-opus-4-6".to_string(), provider_id: fabro_model::ProviderId::anthropic(), - fallback_chain: Vec::new(), + fallbacks: ModelFallbackPolicy::default(), mcp_servers: Vec::new(), model_controls: RunModelControls::default(), dry_run: false, @@ -1132,7 +1134,7 @@ mod tests { llm: LlmSpec { model: "fake-acp".to_string(), provider_id: fabro_model::ProviderId::openai(), - fallback_chain: Vec::new(), + fallbacks: ModelFallbackPolicy::default(), mcp_servers: Vec::new(), model_controls: RunModelControls::default(), dry_run: false, @@ -1227,7 +1229,7 @@ mod tests { llm: LlmSpec { model: "test-model".to_string(), provider_id: fabro_model::ProviderId::anthropic(), - fallback_chain: Vec::new(), + fallbacks: ModelFallbackPolicy::default(), mcp_servers: Vec::new(), model_controls: RunModelControls::default(), dry_run: true, @@ -1369,7 +1371,7 @@ mod tests { llm: LlmSpec { model: "test-model".to_string(), provider_id: fabro_model::ProviderId::anthropic(), - fallback_chain: Vec::new(), + fallbacks: ModelFallbackPolicy::default(), mcp_servers: Vec::new(), model_controls: RunModelControls::default(), dry_run: true, diff --git a/lib/components/fabro-workflow/src/pipeline/types.rs b/lib/components/fabro-workflow/src/pipeline/types.rs index 8d257ea9a..ebbe3f4f8 100644 --- a/lib/components/fabro-workflow/src/pipeline/types.rs +++ b/lib/components/fabro-workflow/src/pipeline/types.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use fabro_graphviz::graph::Graph; use fabro_interview::Interviewer; use fabro_mcp::config::McpServerSettings; -use fabro_model::{Catalog, FallbackTarget, ProviderId}; +use fabro_model::{Catalog, ProviderId}; use fabro_sandbox::SandboxSpec; use fabro_template::TemplateContext; use fabro_types::settings::run::{PullRequestSettings, RunModelControls}; @@ -20,6 +20,7 @@ use crate::error::Error; use crate::event::Emitter; use crate::file_resolver::FileResolver; use crate::handler::HandlerRegistry; +use crate::model_fallback::ModelFallbackPolicy; use crate::outcome::Outcome; use crate::records::{Checkpoint, Conclusion, RunSpec}; use crate::run_control::RunControlState; @@ -236,7 +237,7 @@ impl Persisted { pub struct LlmSpec { pub model: String, pub provider_id: ProviderId, - pub fallback_chain: Vec, + pub fallbacks: ModelFallbackPolicy, pub mcp_servers: Vec, pub model_controls: RunModelControls, pub dry_run: bool, diff --git a/lib/components/fabro-workflow/tests/it/integration.rs b/lib/components/fabro-workflow/tests/it/integration.rs index 32b64e605..7ddb72647 100644 --- a/lib/components/fabro-workflow/tests/it/integration.rs +++ b/lib/components/fabro-workflow/tests/it/integration.rs @@ -50,6 +50,7 @@ use fabro_workflow::handler::manager_loop::SubWorkflowHandler; use fabro_workflow::handler::start::StartHandler; use fabro_workflow::handler::wait::WaitHandler; use fabro_workflow::handler::{Handler, HandlerRegistry}; +use fabro_workflow::model_fallback::ModelFallbackPolicy; use fabro_workflow::outcome::{Outcome, OutcomeExt, StageOutcome}; use fabro_workflow::records::{Checkpoint, CheckpointExt}; use fabro_workflow::run_options::{GitCheckpointOptions, RunOptions}; @@ -2332,7 +2333,7 @@ reasoning = false let backend = AgentApiBackend::new_with_catalog( "compact-model".to_string(), ProviderId::from("compact"), - Vec::new(), + ModelFallbackPolicy::default(), source, Arc::new(SteeringHub::new(Arc::new(Emitter::default()))), catalog, @@ -2487,7 +2488,7 @@ base_url = "{}" let backend = AgentApiBackend::new_with_catalog( "openai/gpt-5.4".to_string(), ProviderId::from("openrouter"), - Vec::new(), + ModelFallbackPolicy::default(), source, Arc::new(SteeringHub::new(Arc::new(Emitter::default()))), catalog, diff --git a/lib/foundation/fabro-api/build.rs b/lib/foundation/fabro-api/build.rs index c7e057851..8d3a68284 100644 --- a/lib/foundation/fabro-api/build.rs +++ b/lib/foundation/fabro-api/build.rs @@ -244,6 +244,16 @@ fn main() { "fabro_types::settings::server::ServerAuthGithubSettings", &[], ), + ( + "RunModelSettings", + "fabro_types::settings::run::RunModelSettings", + &[], + ), + ( + "RunModelControls", + "fabro_types::settings::run::RunModelControls", + &[], + ), ( "ServerSandboxSettings", "fabro_types::settings::server::ServerSandboxSettings", diff --git a/lib/foundation/fabro-api/src/lib.rs b/lib/foundation/fabro-api/src/lib.rs index 05f4e0afe..73a9e39be 100644 --- a/lib/foundation/fabro-api/src/lib.rs +++ b/lib/foundation/fabro-api/src/lib.rs @@ -25,7 +25,7 @@ pub mod types { ReasoningEffortFeature, Speed as BillingSpeed, TokenCounts as CompletionUsage, }; pub use fabro_types::run_event::AgentSessionActivatedProps; - pub use fabro_types::settings::run::McpHttpProtocol; + pub use fabro_types::settings::run::{McpHttpProtocol, RunModelControls, RunModelSettings}; pub use fabro_types::settings::server::{ GithubIntegrationSettings, GithubIntegrationStrategy, IntegrationWebhooksSettings, LogDestination, ObjectStoreSettings, ServerApiSettings, ServerArtifactsSettings, diff --git a/lib/foundation/fabro-api/tests/run_model_settings_round_trip.rs b/lib/foundation/fabro-api/tests/run_model_settings_round_trip.rs new file mode 100644 index 000000000..a2b49ef39 --- /dev/null +++ b/lib/foundation/fabro-api/tests/run_model_settings_round_trip.rs @@ -0,0 +1,67 @@ +use std::any::{TypeId, type_name}; +use std::collections::BTreeMap; + +use fabro_api::types::{ + RunModelControls as ApiRunModelControls, RunModelSettings as ApiRunModelSettings, +}; +use fabro_types::settings::run::{RunModelControls, RunModelSettings}; + +#[test] +fn run_model_settings_reuses_domain_types() { + assert_same_type::(); + assert_same_type::(); +} + +#[test] +fn run_model_settings_json_matches_openapi_shape() { + let settings = RunModelSettings { + provider: Some("openrouter".to_string()), + name: Some("claude-fable".to_string()), + fallbacks: BTreeMap::from([("claude-fable".to_string(), vec![ + "gpt-sol".parse().expect("fixture reference should parse"), + "openrouter:claude-opus" + .parse() + .expect("fixture reference should parse"), + ])]), + controls: RunModelControls { + reasoning_effort: Some("high".to_string()), + speed: None, + }, + }; + + let json = serde_json::to_value(&settings).expect("run model settings should serialize"); + assert_eq!(json["provider"], "openrouter"); + assert_eq!(json["name"], "claude-fable"); + assert_eq!(json["fallbacks"]["claude-fable"][0], "gpt-sol"); + assert_eq!( + json["fallbacks"]["claude-fable"][1], + "openrouter:claude-opus" + ); + assert_eq!(json["controls"]["reasoning_effort"], "high"); + assert_eq!(json["controls"]["speed"], serde_json::Value::Null); + + let round_trip: ApiRunModelSettings = + serde_json::from_value(json).expect("run model settings should deserialize"); + assert_eq!(round_trip, settings); +} + +#[test] +fn run_model_settings_tolerates_absent_controls() { + let parsed: RunModelSettings = serde_json::from_value(serde_json::json!({ + "provider": null, + "name": null, + "fallbacks": {} + })) + .expect("settings without controls should deserialize"); + assert_eq!(parsed.controls, RunModelControls::default()); +} + +fn assert_same_type() { + assert_eq!( + TypeId::of::(), + TypeId::of::(), + "{} should be the same type as {}", + type_name::(), + type_name::() + ); +} diff --git a/lib/foundation/fabro-config/src/builders.rs b/lib/foundation/fabro-config/src/builders.rs index 980faa3f3..08eb4c70b 100644 --- a/lib/foundation/fabro-config/src/builders.rs +++ b/lib/foundation/fabro-config/src/builders.rs @@ -746,7 +746,7 @@ command = ["demo-mcp"] model: Some(RunModelLayer { provider: Some("openai".to_string()), name: Some("gpt-5".to_string()), - fallbacks: Vec::new(), + fallbacks: crate::MergeMap::default(), controls: None, }), execution: Some(RunExecutionLayer { diff --git a/lib/foundation/fabro-config/src/layers/run.rs b/lib/foundation/fabro-config/src/layers/run.rs index 853ba7a0c..97d249c05 100644 --- a/lib/foundation/fabro-config/src/layers/run.rs +++ b/lib/foundation/fabro-config/src/layers/run.rs @@ -148,14 +148,19 @@ pub struct RunModelLayer { #[serde(default, skip_serializing_if = "Option::is_none")] #[option(value_type = "string")] pub name: Option, - /// Ordered fallback references: bare providers, bare model IDs or aliases, - /// or provider-qualified `provider:selector` values. A qualified selector - /// may be a model ID, alias, or provider API ID. Legacy `provider/model` - /// values remain accepted. Supports the `...` splice marker at layering - /// time — see [`super::splice_array`]. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - #[option(default = "[]", value_type = "array")] - pub fallbacks: Vec, + /// Model-keyed fallback chains. Each value is an ordered list of bare + /// providers, bare model IDs or aliases, or provider-qualified + /// `provider:selector` values. A qualified selector may be a model ID, + /// alias, or provider API ID. Legacy `provider/model` values remain + /// accepted. Each list supports the `...` splice marker at layering time. + /// Fabro selects one chain from the original requested model. A fallback + /// target never activates another model's chain. + /// + /// Model keys stay unresolved in this sparse layer because `fabro + /// validate` is offline and has no server model catalog. + #[serde(default, skip_serializing_if = "MergeMap::is_empty")] + #[option(default = "{}", value_type = "table>")] + pub fallbacks: MergeMap>, /// Run-level default values for typed model controls. Node attributes /// and style-applied attributes still win over these defaults. #[serde(default, skip_serializing_if = "Option::is_none")] diff --git a/lib/foundation/fabro-config/src/resolve/run.rs b/lib/foundation/fabro-config/src/resolve/run.rs index 1d20db7a6..4f67cc2d1 100644 --- a/lib/foundation/fabro-config/src/resolve/run.rs +++ b/lib/foundation/fabro-config/src/resolve/run.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use fabro_types::settings::InterpString; use fabro_types::settings::run::{ @@ -123,11 +123,17 @@ fn resolve_model(model: Option<&RunModelLayer>) -> RunModelSettings { fallbacks: model .fallbacks .iter() - .filter_map(|entry| match entry { - ModelRefOrSplice::ModelRef(model_ref) => Some(model_ref.clone()), - ModelRefOrSplice::Splice => None, + .map(|(requested_model, chain)| { + let chain = chain + .iter() + .filter_map(|entry| match entry { + ModelRefOrSplice::ModelRef(model_ref) => Some(model_ref.clone()), + ModelRefOrSplice::Splice => None, + }) + .collect(); + (requested_model.clone(), chain) }) - .collect(), + .collect::>(), controls: model .controls .as_ref() diff --git a/lib/foundation/fabro-config/src/tests/combine.rs b/lib/foundation/fabro-config/src/tests/combine.rs index 426656596..ef296087b 100644 --- a/lib/foundation/fabro-config/src/tests/combine.rs +++ b/lib/foundation/fabro-config/src/tests/combine.rs @@ -100,21 +100,46 @@ script = "higher-1" fn run_model_fallbacks_splice_inserts_inherited() { let lower = parse( r#" -[run.model] -fallbacks = ["openrouter:moonshotai/kimi-k3", "gpt-terra"] +[run.model.fallbacks] +"kimi-k3" = ["openrouter:moonshotai/kimi-k3", "gpt-terra"] +"gpt-sol" = ["claude-opus"] "#, ); let higher = parse( r#" -[run.model] -fallbacks = ["anthropic", "..."] +[run.model.fallbacks] +"kimi-k3" = ["anthropic", "..."] +"claude-fable" = ["gpt-sol", "claude-opus"] "#, ); let merged = higher.combine(lower); let fallbacks = merged.run.unwrap().model.unwrap().fallbacks; assert_eq!( - serde_json::to_value(&fallbacks).unwrap(), - serde_json::json!(["anthropic", "openrouter:moonshotai/kimi-k3", "gpt-terra",]) + serde_json::to_value(fallbacks.get("kimi-k3").unwrap()).unwrap(), + serde_json::json!(["anthropic", "openrouter:moonshotai/kimi-k3", "gpt-terra"]) + ); + assert_eq!( + serde_json::to_value(fallbacks.get("gpt-sol").unwrap()).unwrap(), + serde_json::json!(["claude-opus"]) + ); + assert_eq!( + serde_json::to_value(fallbacks.get("claude-fable").unwrap()).unwrap(), + serde_json::json!(["gpt-sol", "claude-opus"]) + ); +} + +#[test] +fn run_model_fallbacks_rejects_the_removed_array_shape() { + let error = r#" +[run.model] +fallbacks = ["anthropic"] +"# + .parse::() + .expect_err("the global fallback array must stay unsupported"); + + assert!( + error.to_string().contains("invalid type: sequence"), + "unexpected parse error: {error}" ); } diff --git a/lib/foundation/fabro-dev/src/commands/docs_options_reference.rs b/lib/foundation/fabro-dev/src/commands/docs_options_reference.rs index 2422b47bd..47cee4fb2 100644 --- a/lib/foundation/fabro-dev/src/commands/docs_options_reference.rs +++ b/lib/foundation/fabro-dev/src/commands/docs_options_reference.rs @@ -109,7 +109,9 @@ permissions = "read-write""#, r#"[run.model] provider = "anthropic" name = "claude-sonnet-4-5" -fallbacks = ["openrouter:kimi-k3", "gpt-terra"]"#, + +[run.model.fallbacks] +"claude-sonnet-4-5" = ["openrouter:kimi-k3", "gpt-terra"]"#, ), Section::of::( "[cli.logging]", diff --git a/lib/foundation/fabro-model/src/catalog.rs b/lib/foundation/fabro-model/src/catalog.rs index 2e16f378c..21aa04888 100644 --- a/lib/foundation/fabro-model/src/catalog.rs +++ b/lib/foundation/fabro-model/src/catalog.rs @@ -401,8 +401,8 @@ static GLOBAL_CATALOG: LazyLock = LazyLock::new(|| { /// A resolved fallback target: provider name + model ID. #[derive(Debug, Clone, PartialEq, Eq)] pub struct FallbackTarget { - pub provider: String, - pub model: String, + pub provider: ProviderId, + pub model: ModelId, } impl FallbackTarget { @@ -411,8 +411,8 @@ impl FallbackTarget { /// selectors all use one constructor. pub fn new(provider: impl std::fmt::Display, model: impl std::fmt::Display) -> Self { Self { - provider: provider.to_string(), - model: model.to_string(), + provider: ProviderId::new(provider.to_string()), + model: ModelId::new(model.to_string()), } } } @@ -1051,6 +1051,23 @@ impl Catalog { .collect() } + /// Canonicalize a model selector to its catalog model ID, preferring the + /// given provider's offering. Unknown selectors pass through verbatim. + /// + /// Model-keyed fallback chains are written and read through this one + /// function so a configured chain key and a dispatch-time lookup cannot + /// silently disagree. + #[must_use] + pub fn canonical_model_id(&self, provider: &ProviderId, selector: &str) -> String { + self.get_on_provider(provider, selector).map_or_else( + || { + self.select(selector, None, &self.all_provider_ids()) + .map_or_else(|_| selector.to_string(), |offering| offering.id.to_string()) + }, + |offering| offering.id.to_string(), + ) + } + /// Select the highest-priority default model on an eligible provider. pub fn select_default( &self, @@ -1482,36 +1499,6 @@ impl Catalog { .unwrap_or(std::cmp::Ordering::Equal) }) } - - /// Build an ordered fallback chain for a primary provider/model. - /// - /// For each fallback provider, finds the closest matching model. Providers - /// where no capability match exists (or the provider string doesn't - /// parse) are skipped. - #[must_use] - pub fn build_fallback_chain( - &self, - primary: &ProviderId, - model: &str, - fallbacks: &HashMap>, - ) -> Vec { - let Some(reference) = self.get_on_provider(primary, model) else { - return Vec::new(); - }; - - let Some(fallback_providers) = fallbacks.get(primary.as_str()) else { - return Vec::new(); - }; - - fallback_providers - .iter() - .filter_map(|provider_str| { - let provider = ProviderId::from(provider_str.clone()); - self.closest(&provider, reference) - .map(|m| FallbackTarget::new(provider_str, &m.id)) - }) - .collect() - } } type ModelIndexes = ( @@ -4200,71 +4187,6 @@ enabled = true ); } - #[test] - fn builtin_build_fallback_chain() { - let fallbacks = HashMap::from([("anthropic".to_string(), vec![ - "gemini".to_string(), - "openai".to_string(), - ])]); - let chain = Catalog::builtin().build_fallback_chain( - &ProviderId::anthropic(), - "claude-opus-4-6", - &fallbacks, - ); - assert_eq!(chain.len(), 2); - assert_eq!(chain[0].provider, "gemini"); - assert_eq!(chain[0].model, "gemini-3.1-pro-preview"); - assert_eq!(chain[1].provider, "openai"); - assert_eq!(chain[1].model, "gpt-5.5"); - } - - #[test] - fn builtin_build_fallback_chain_unknown_model() { - let fallbacks = HashMap::from([("anthropic".to_string(), vec!["gemini".to_string()])]); - let chain = Catalog::builtin().build_fallback_chain( - &ProviderId::anthropic(), - "unknown-xyz", - &fallbacks, - ); - assert!(chain.is_empty()); - } - - #[test] - fn builtin_build_fallback_chain_provider_not_in_map() { - let fallbacks = HashMap::from([("openai".to_string(), vec!["anthropic".to_string()])]); - let chain = Catalog::builtin().build_fallback_chain( - &ProviderId::anthropic(), - "claude-opus-4-6", - &fallbacks, - ); - assert!(chain.is_empty()); - } - - #[test] - fn builtin_build_fallback_chain_skips_no_capability_match() { - let fallbacks = HashMap::from([("anthropic".to_string(), vec![ - "openai".to_string(), - "kimi".to_string(), - ])]); - let chain = Catalog::builtin().build_fallback_chain( - &ProviderId::anthropic(), - "claude-haiku-4-5", - &fallbacks, - ); - assert!(chain.is_empty()); - } - - #[test] - fn builtin_build_fallback_chain_empty_map() { - let fallbacks = HashMap::new(); - let chain = Catalog::builtin().build_fallback_chain( - &ProviderId::anthropic(), - "claude-opus-4-6", - &fallbacks, - ); - assert!(chain.is_empty()); - } - #[test] fn builtin_catalog_is_loaded_from_provider_toml_settings() { let catalog = Catalog::builtin(); diff --git a/lib/foundation/fabro-model/src/reasoning.rs b/lib/foundation/fabro-model/src/reasoning.rs index 7420178d0..44587df4a 100644 --- a/lib/foundation/fabro-model/src/reasoning.rs +++ b/lib/foundation/fabro-model/src/reasoning.rs @@ -4,6 +4,8 @@ //! validation, OpenAPI replacement types, and the LLM client all share one //! enum so that adding a new effort value remains a Rust change. +use std::cmp::Reverse; + use serde::{Deserialize, Serialize}; #[derive( @@ -37,6 +39,18 @@ impl ReasoningEffort { pub fn variants() -> &'static [Self] { ::VARIANTS } + + /// Select the supported effort nearest to this value. + /// + /// The enum declaration defines the ordered progression. When two values + /// are equally distant, the higher effort wins. + #[must_use] + pub fn closest_supported(self, supported: &[Self]) -> Option { + supported + .iter() + .copied() + .min_by_key(|effort| ((self as u8).abs_diff(*effort as u8), Reverse(*effort))) + } } #[cfg(test)] @@ -91,6 +105,49 @@ mod tests { assert_eq!(v[v.len() - 1], ReasoningEffort::Max); } + #[test] + fn closest_supported_uses_exact_match() { + assert_eq!( + ReasoningEffort::High.closest_supported(&[ + ReasoningEffort::Low, + ReasoningEffort::High, + ReasoningEffort::Max, + ]), + Some(ReasoningEffort::High) + ); + } + + #[test] + fn closest_supported_rounds_equal_distance_up() { + let kimi = [ + ReasoningEffort::Low, + ReasoningEffort::High, + ReasoningEffort::Max, + ]; + assert_eq!( + ReasoningEffort::Medium.closest_supported(&kimi), + Some(ReasoningEffort::High) + ); + assert_eq!( + ReasoningEffort::XHigh.closest_supported(&kimi), + Some(ReasoningEffort::Max) + ); + } + + #[test] + fn closest_supported_uses_nearest_lower_value_when_needed() { + assert_eq!( + ReasoningEffort::Max + .closest_supported(&[ReasoningEffort::High, ReasoningEffort::XHigh]), + Some(ReasoningEffort::XHigh) + ); + } + + #[test] + fn closest_supported_returns_none_for_unsupported_control() { + assert_eq!(ReasoningEffort::High.closest_supported(&[]), None); + } + #[test] fn round_trip_through_json() { let json = serde_json::to_string(&ReasoningEffort::High).unwrap(); diff --git a/lib/foundation/fabro-types/src/lib.rs b/lib/foundation/fabro-types/src/lib.rs index ad9142e51..7bbde04ab 100644 --- a/lib/foundation/fabro-types/src/lib.rs +++ b/lib/foundation/fabro-types/src/lib.rs @@ -114,10 +114,10 @@ pub use run_blob_id::RunBlobId; pub use run_event::{ AgentMcpToolSummary, AgentMemoryFileProps, AgentSkillActivationSource, AgentSkillSummary, AgentToolCategory, AgentToolSource, AgentToolSummary, AgentToolsAvailableProps, EventBody, - ExecOutputTail, InterviewOption, LlmOutputKind, LlmRetryPhase, MetadataSnapshotFailureKind, - MetadataSnapshotPhase, RunEvent, RunNoticeCode, RunNoticeLevel, RunPairEndedReason, - RunPairFailedReason, RunRunnableSource, SessionCapability, TodoCreatedProps, TodoDeletedProps, - TodoUpdatedProps, + ExecOutputTail, FailoverProps, InterviewOption, LlmOutputKind, LlmRetryPhase, + MetadataSnapshotFailureKind, MetadataSnapshotPhase, RunEvent, RunNoticeCode, RunNoticeLevel, + RunPairEndedReason, RunPairFailedReason, RunRunnableSource, SessionCapability, + TodoCreatedProps, TodoDeletedProps, TodoUpdatedProps, }; pub use run_failure::RunFailure; pub use run_id::{RunId, fixtures}; diff --git a/lib/foundation/fabro-types/src/run_event/misc.rs b/lib/foundation/fabro-types/src/run_event/misc.rs index 19a21b34e..e1505c6d3 100644 --- a/lib/foundation/fabro-types/src/run_event/misc.rs +++ b/lib/foundation/fabro-types/src/run_event/misc.rs @@ -1,3 +1,4 @@ +use fabro_model::ReasoningEffort; use serde::{Deserialize, Serialize}; use super::ExecOutputTail; @@ -187,11 +188,24 @@ pub struct SshAccessReadyProps { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct FailoverProps { + /// `original_*` and `attempt` are `Option` only because failover events + /// recorded before model-keyed fallbacks lack them. New events always set + /// them; stored events are immutable, so absence stays a supported input. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub original_provider: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub original_model: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub attempt: Option, pub from_provider: String, - pub from_model: String, - pub to_provider: String, - pub to_model: String, - pub error: String, + pub from_model: String, + pub to_provider: String, + pub to_model: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub requested_reasoning_effort: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub effective_reasoning_effort: Option, + pub error: String, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] diff --git a/lib/foundation/fabro-types/src/run_event/mod.rs b/lib/foundation/fabro-types/src/run_event/mod.rs index 6680fff2e..27c3b87e9 100644 --- a/lib/foundation/fabro-types/src/run_event/mod.rs +++ b/lib/foundation/fabro-types/src/run_event/mod.rs @@ -1013,6 +1013,33 @@ mod tests { assert!(matches!(parsed.body, EventBody::RunCreated(_))); } + #[test] + fn historical_failover_event_defaults_new_route_context() { + let line = json!({ + "id": "evt_failover", + "ts": "2026-04-04T12:00:00.000Z", + "run_id": fixtures::RUN_1, + "event": "agent.failover", + "properties": { + "from_provider": "anthropic", + "from_model": "claude-fable-5", + "to_provider": "openai", + "to_model": "gpt-5.6-sol", + "error": "provider unavailable" + } + }); + + let parsed = RunEvent::from_value(line).unwrap(); + let EventBody::Failover(props) = parsed.body else { + panic!("expected agent.failover"); + }; + assert_eq!(props.original_provider, None); + assert_eq!(props.original_model, None); + assert_eq!(props.attempt, None); + assert_eq!(props.requested_reasoning_effort, None); + assert_eq!(props.effective_reasoning_effort, None); + } + #[test] fn run_created_round_trip_preserves_manifest_blob() { let line = json!({ diff --git a/lib/foundation/fabro-types/src/settings/model_ref.rs b/lib/foundation/fabro-types/src/settings/model_ref.rs index 184bb8305..b551684b7 100644 --- a/lib/foundation/fabro-types/src/settings/model_ref.rs +++ b/lib/foundation/fabro-types/src/settings/model_ref.rs @@ -1,6 +1,6 @@ //! Model references for `run.model.fallbacks`. //! -//! Each entry is one of: +//! Each fallback-chain entry is one of: //! //! - a bare token such as `openai` or `gpt-5.4` — the parser cannot tell alone //! whether the token is a provider name or a model alias diff --git a/lib/foundation/fabro-types/src/settings/run.rs b/lib/foundation/fabro-types/src/settings/run.rs index c5f62b480..ace7cb3ab 100644 --- a/lib/foundation/fabro-types/src/settings/run.rs +++ b/lib/foundation/fabro-types/src/settings/run.rs @@ -6,7 +6,7 @@ //! notifications, interviews, agent knobs, hooks, SCM targeting, pull-request //! behavior, and artifact collection. -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use std::path::PathBuf; use std::time::Duration as StdDuration; @@ -680,11 +680,16 @@ pub enum RunGoal { File(InterpString), } -#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, Default, PartialEq, Serialize)] pub struct RunModelSettings { pub provider: Option, pub name: Option, - pub fallbacks: Vec, + /// Ordered fallback references keyed by the originally requested model. + /// + /// Keys remain raw selectors during offline configuration resolution. + /// The server canonicalizes them against its model catalog before a run + /// starts. + pub fallbacks: BTreeMap>, /// Run-level default values for typed model controls /// (`reasoning_effort`, `speed`). Node and style attributes still win /// over these defaults. @@ -692,6 +697,94 @@ pub struct RunModelSettings { pub controls: RunModelControls, } +/// Temporary compatibility deserializer: releases before model-keyed +/// fallbacks serialized `fallbacks` as a flat array that applied to the run's +/// requested model, and that shape persists inside stored `run.created` +/// events. A legacy array is keyed under `name` when one is set; a legacy +/// chain for the implicit default model cannot be keyed and is dropped. +/// +/// Remove once run logs recorded by pre-model-keyed releases (< 0.311) are +/// out of the support window; then restore `derive(Deserialize)`. +impl<'de> serde::Deserialize<'de> for RunModelSettings { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + #[derive(Deserialize)] + struct Shadow { + provider: Option, + name: Option, + fallbacks: FallbacksCompat, + #[serde(default)] + controls: RunModelControls, + } + + #[derive(Deserialize)] + #[serde(untagged)] + enum FallbacksCompat { + Keyed(BTreeMap>), + Legacy(Vec), + } + + let shadow = Shadow::deserialize(deserializer)?; + let fallbacks = match shadow.fallbacks { + FallbacksCompat::Keyed(chains) => chains, + FallbacksCompat::Legacy(chain) => match (&shadow.name, chain) { + (Some(name), chain) if !chain.is_empty() => BTreeMap::from([(name.clone(), chain)]), + _ => BTreeMap::new(), + }, + }; + Ok(Self { + provider: shadow.provider, + name: shadow.name, + fallbacks, + controls: shadow.controls, + }) + } +} + +#[cfg(test)] +mod run_model_settings_compat_tests { + use super::RunModelSettings; + + #[test] + fn keyed_fallbacks_round_trip() { + let settings: RunModelSettings = serde_json::from_value(serde_json::json!({ + "provider": "openrouter", + "name": "claude-fable", + "fallbacks": {"claude-fable": ["gpt-sol"]} + })) + .unwrap(); + let chain = &settings.fallbacks["claude-fable"]; + assert_eq!(chain.len(), 1); + assert_eq!(chain[0].to_string(), "gpt-sol"); + } + + #[test] + fn legacy_array_fallbacks_key_under_the_requested_model() { + let settings: RunModelSettings = serde_json::from_value(serde_json::json!({ + "provider": null, + "name": "claude-fable", + "fallbacks": ["gpt-sol", "openrouter:claude-opus"] + })) + .unwrap(); + let chain = &settings.fallbacks["claude-fable"]; + assert_eq!(chain.len(), 2); + assert_eq!(chain[1].to_string(), "openrouter:claude-opus"); + } + + #[test] + fn legacy_array_without_a_requested_model_is_dropped() { + let settings: RunModelSettings = serde_json::from_value(serde_json::json!({ + "provider": null, + "name": null, + "fallbacks": ["gpt-sol"] + })) + .unwrap(); + assert!(settings.fallbacks.is_empty()); + } +} + #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] pub struct RunModelControls { pub reasoning_effort: Option, diff --git a/lib/packages/fabro-api-client/src/.openapi-generator/FILES b/lib/packages/fabro-api-client/src/.openapi-generator/FILES index d0ea6bd28..55cf966cd 100644 --- a/lib/packages/fabro-api-client/src/.openapi-generator/FILES +++ b/lib/packages/fabro-api-client/src/.openapi-generator/FILES @@ -382,6 +382,7 @@ models/run-links.ts models/run-manifest.ts models/run-meta-branch-settings.ts models/run-mode.ts +models/run-model-controls.ts models/run-model-settings.ts models/run-model.ts models/run-namespace.ts diff --git a/lib/packages/fabro-api-client/src/models/index.ts b/lib/packages/fabro-api-client/src/models/index.ts index ac6fc24a4..862749945 100644 --- a/lib/packages/fabro-api-client/src/models/index.ts +++ b/lib/packages/fabro-api-client/src/models/index.ts @@ -354,6 +354,7 @@ export * from './run-manifest'; export * from './run-meta-branch-settings'; export * from './run-mode'; export * from './run-model'; +export * from './run-model-controls'; export * from './run-model-settings'; export * from './run-namespace'; export * from './run-origin'; diff --git a/lib/packages/fabro-api-client/src/models/run-model-controls.ts b/lib/packages/fabro-api-client/src/models/run-model-controls.ts new file mode 100644 index 000000000..5d60f0fa4 --- /dev/null +++ b/lib/packages/fabro-api-client/src/models/run-model-controls.ts @@ -0,0 +1,23 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * Fabro Run API + * HTTP API for managing Fabro workflow run executions. + * + * The version of the OpenAPI document: 0.1.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + + +/** + * Run-level default values for typed model controls. Node and style attributes still win over these defaults. + */ +export interface RunModelControls { + 'reasoning_effort': string | null; + 'speed': string | null; +} diff --git a/lib/packages/fabro-api-client/src/models/run-model-settings.ts b/lib/packages/fabro-api-client/src/models/run-model-settings.ts index d6bbf1994..599b6a26a 100644 --- a/lib/packages/fabro-api-client/src/models/run-model-settings.ts +++ b/lib/packages/fabro-api-client/src/models/run-model-settings.ts @@ -13,9 +13,16 @@ */ +// May contain unused imports in some cases +// @ts-ignore +import type { RunModelControls } from './run-model-controls'; export interface RunModelSettings { 'provider': string | null; 'name': string | null; - 'fallbacks': Array; + /** + * Ordered fallback targets keyed by the originally requested model. Each chain is independent; selecting a fallback target does not activate that target model\'s own chain. + */ + 'fallbacks': { [key: string]: Array; }; + 'controls'?: RunModelControls; } diff --git a/test/offline-fallbacks/workflow.fabro b/test/offline-fallbacks/workflow.fabro new file mode 100644 index 000000000..c35213b73 --- /dev/null +++ b/test/offline-fallbacks/workflow.fabro @@ -0,0 +1,10 @@ +digraph OfflineFallbacks { + graph [goal="Validate a future model fallback offline"] + + start [shape=Mdiamond, label="Start"] + exit [shape=Msquare, label="Exit"] + + work [label="Work", prompt="Do work", model="future-primary"] + + start -> work -> exit +} diff --git a/test/offline-fallbacks/workflow.toml b/test/offline-fallbacks/workflow.toml new file mode 100644 index 000000000..565fc58a3 --- /dev/null +++ b/test/offline-fallbacks/workflow.toml @@ -0,0 +1,7 @@ +_version = 1 + +[workflow] +graph = "workflow.fabro" + +[run.model.fallbacks] +"future-primary" = ["future-provider:future-model"]