mirror of
https://github.com/fabro-sh/fabro.git
synced 2026-08-28 05:27:41 +00:00
feat: add model-keyed fallback policies
This commit is contained in:
Bryan Helmkamp
parent
4ab090cae3
commit
ba82656656
39 changed files with 2004 additions and 1344 deletions
|
|
@ -92,7 +92,9 @@ level = "info"
|
|||
[run.model]
|
||||
name = "claude-sonnet-4-5"
|
||||
provider = "anthropic"
|
||||
fallbacks = ["gemini", "openai"]
|
||||
|
||||
[run.model.fallbacks]
|
||||
"claude-sonnet-4-5" = ["gemini", "openai"]
|
||||
|
||||
[[run.prepare.steps]]
|
||||
script = "npm install"
|
||||
|
|
|
|||
|
|
@ -14315,9 +14315,15 @@ components:
|
|||
name:
|
||||
type: ["string", "null"]
|
||||
fallbacks:
|
||||
type: array
|
||||
items:
|
||||
$ref: "#/components/schemas/ModelRef"
|
||||
type: object
|
||||
description: >
|
||||
Ordered fallback targets keyed by the originally requested model.
|
||||
Each chain is independent; selecting a fallback target does not
|
||||
activate that target model's own chain.
|
||||
additionalProperties:
|
||||
type: array
|
||||
items:
|
||||
$ref: "#/components/schemas/ModelRef"
|
||||
|
||||
RunGitSettings:
|
||||
type: object
|
||||
|
|
|
|||
|
|
@ -285,7 +285,9 @@ goal = "Implement the feature"
|
|||
|
||||
[run.model]
|
||||
name = "claude-sonnet-4-5"
|
||||
fallbacks = ["gemini", "openai"]
|
||||
|
||||
[run.model.fallbacks]
|
||||
"claude-sonnet-4-5" = ["gemini", "openai"]
|
||||
```
|
||||
|
||||
Then launch with:
|
||||
|
|
@ -294,7 +296,9 @@ Then launch with:
|
|||
fabro run run.toml
|
||||
```
|
||||
|
||||
The `fallbacks` array is optional. Each entry may be a bare provider token (like `"gemini"`), a bare model ID or alias (like `"gpt-terra"`), or a qualified `"provider:selector"` reference. A qualified selector may be the provider's canonical model ID, alias, or API ID, including API IDs with slashes such as `"openrouter:moonshotai/kimi-k3"`. Fabro tries entries in order when the primary provider is unavailable, and qualified references remain provider pins. Legacy `provider/model` references remain accepted for compatibility.
|
||||
The `[run.model.fallbacks]` table is optional. Each key names the originally requested model. Its value is the ordered list to try after that model's active provider fails. An entry may be a bare provider token (like `"gemini"`), a bare model ID or alias (like `"gpt-terra"`), or a qualified `"provider:selector"` reference. A qualified selector may be the provider's canonical model ID, alias, or API ID, including API IDs with slashes such as `"openrouter:moonshotai/kimi-k3"`.
|
||||
|
||||
Fabro selects one chain from the original request. It does not switch to the chain configured for a fallback target. When the requested reasoning level is unavailable on a fallback target, Fabro uses the nearest supported level. Equal-distance choices round up.
|
||||
|
||||
<Note>
|
||||
The precedence order is: node-level stylesheet > run config TOML > CLI flags > server defaults. More specific settings always win.
|
||||
|
|
|
|||
|
|
@ -214,25 +214,29 @@ The stylesheet assigns three tiers of models:
|
|||
Pair the workflow with a run config TOML for repeatable execution:
|
||||
|
||||
```toml title="run.toml"
|
||||
version = 1
|
||||
goal = "Port semantic changes from upstream openai-agents-python to our Go SDK"
|
||||
_version = 1
|
||||
|
||||
[workflow]
|
||||
graph = "semport.fabro"
|
||||
|
||||
[llm]
|
||||
model = "claude-sonnet-4-5"
|
||||
[run]
|
||||
goal = "Port semantic changes from upstream openai-agents-python to our Go SDK"
|
||||
|
||||
[run.model]
|
||||
name = "claude-sonnet-4-5"
|
||||
provider = "anthropic"
|
||||
|
||||
[llm.fallbacks]
|
||||
anthropic = ["openai"]
|
||||
gemini = ["anthropic"]
|
||||
[run.model.fallbacks]
|
||||
"claude-sonnet-4-5" = ["openai"]
|
||||
"gemini-3.1-pro-preview" = ["anthropic"]
|
||||
|
||||
[setup]
|
||||
commands = [
|
||||
"git clone https://github.com/openai/openai-agents-python upstream || (cd upstream && git pull)",
|
||||
"pip install -r ledger/requirements.txt"
|
||||
]
|
||||
[[run.prepare.steps]]
|
||||
script = "git clone https://github.com/openai/openai-agents-python upstream || (cd upstream && git pull)"
|
||||
|
||||
[vars]
|
||||
[[run.prepare.steps]]
|
||||
script = "pip install -r ledger/requirements.txt"
|
||||
|
||||
[run.inputs]
|
||||
upstream_repo = "openai/openai-agents-python"
|
||||
downstream_lang = "go"
|
||||
```
|
||||
|
|
|
|||
|
|
@ -266,7 +266,9 @@ goal = "Build a terminal-based solitaire (Klondike) game in Python"
|
|||
[run.model]
|
||||
name = "claude-sonnet-4-5"
|
||||
provider = "anthropic"
|
||||
fallbacks = ["openai", "gemini"]
|
||||
|
||||
[run.model.fallbacks]
|
||||
"claude-sonnet-4-5" = ["openai", "gemini"]
|
||||
|
||||
[[run.prepare.steps]]
|
||||
script = "python3 -m venv .venv && . .venv/bin/activate && pip install pytest curses"
|
||||
|
|
|
|||
|
|
@ -112,13 +112,15 @@ When a handler reports a retryable failure, retries always proceed if attempts r
|
|||
|
||||
## Model fallbacks
|
||||
|
||||
When a model provider fails with a transient error or quota exhaustion, Fabro can automatically switch to a different provider. Configure fallback chains in your [run configuration](/execution/run-configuration):
|
||||
When a model provider fails with a provider-local error, Fabro can automatically switch to another target. Configure one fixed chain for each requested model in your [run configuration](/execution/run-configuration):
|
||||
|
||||
```toml title="run.toml"
|
||||
[run.model]
|
||||
name = "claude-opus-4-6"
|
||||
provider = "anthropic"
|
||||
fallbacks = ["gemini", "openai"]
|
||||
|
||||
[run.model.fallbacks]
|
||||
"claude-opus-4-6" = ["gemini", "openai"]
|
||||
```
|
||||
|
||||
When Anthropic fails, Fabro tries Gemini first, then OpenAI. Fallback resolution is provider-aware:
|
||||
|
|
@ -129,7 +131,9 @@ When Anthropic fails, Fabro tries Gemini first, then OpenAI. Fallback resolution
|
|||
|
||||
Qualified fallback references always remain provider pins. For example, `"openai:gpt-5.6-sol"` pins the direct OpenAI offering. Legacy `provider/model` fallback references remain accepted for compatibility.
|
||||
|
||||
The primary provider and model were already resolved and persisted when the run was created; resuming does not re-run primary selection. Fallbacks are only considered after an eligible runtime failure.
|
||||
Fabro selects the chain by the original requested model. A target in that chain never activates the target model's own chain. The same chain position is retained across structured-output repairs and cached agent sessions.
|
||||
|
||||
The primary provider and model were already resolved and persisted when the run was created; resuming does not re-run primary selection. Fallbacks are only considered after an eligible runtime failure. If the fallback model does not support the requested reasoning level, Fabro uses the nearest supported level and rounds equal-distance choices up.
|
||||
|
||||
### What triggers failover
|
||||
|
||||
|
|
@ -141,7 +145,10 @@ Failover is a superset of LLM retry eligibility:
|
|||
| Server error (5xx) | Yes | Yes |
|
||||
| Timeout / network | Yes | Yes |
|
||||
| Quota exceeded | No | Yes |
|
||||
| Authentication (401) | No | No |
|
||||
| Authentication (401) | No | Yes |
|
||||
| Access denied (403) | No | Yes |
|
||||
| Model not found (404) | No | Yes |
|
||||
| Model refusal | No | Yes |
|
||||
| Invalid request (400) | No | No |
|
||||
| Context length (413) | No | No |
|
||||
| Content filter | No | No |
|
||||
|
|
|
|||
|
|
@ -45,11 +45,13 @@ working_dir = "/tmp/workdir"
|
|||
|
||||
[run.model]
|
||||
name = "claude-sonnet-4-5"
|
||||
fallbacks = ["openai", "gemini"]
|
||||
|
||||
[run.model.controls]
|
||||
reasoning_effort = "high"
|
||||
|
||||
[run.model.fallbacks]
|
||||
"claude-sonnet-4-5" = ["openai", "gemini"]
|
||||
|
||||
[[run.prepare.steps]]
|
||||
script = "git clone https://github.com/fabro-sh/fabro repo"
|
||||
|
||||
|
|
@ -140,25 +142,43 @@ name = "claude-sonnet-4-5"
|
|||
|---|---|
|
||||
| `name` | Canonical model slug or alias (e.g. `claude-sonnet-4-5`, `opus`, `gemini-pro`). See [Models](/core-concepts/models). |
|
||||
| `provider` | Optional provider pin. When omitted, Fabro selects among ready offerings by provider priority. When present, an unavailable provider is an error rather than permission to switch. |
|
||||
| `fallbacks` | Ordered list of model references to try when the primary is unavailable. Entries can be bare provider tokens (`"openai"`), bare model IDs or aliases, or qualified `"provider:selector"` references. |
|
||||
| `fallbacks` | Table of ordered fallback lists keyed by the originally requested model. |
|
||||
|
||||
Provider values are catalog provider ID strings. Built-in IDs like `anthropic` and `openai` work, and settings-defined IDs like `proxy` work after they are added under `[llm.providers.<id>]`.
|
||||
|
||||
For a qualified fallback, the selector may be that provider's canonical model ID, alias, or API ID. Fabro splits on the first `:` when the part before it names a known provider, so provider API IDs may contain `/` or additional colons:
|
||||
|
||||
```toml title="run.toml"
|
||||
[run.model]
|
||||
fallbacks = [
|
||||
[run.model.fallbacks]
|
||||
"kimi-k3" = [
|
||||
"kimi:kimi-k3",
|
||||
"openrouter:kimi-k3",
|
||||
"gpt-terra",
|
||||
"claude-opus",
|
||||
]
|
||||
```
|
||||
|
||||
The first entry could equivalently be written as `"openrouter:moonshotai/kimi-k3"` using OpenRouter's API ID; both forms resolve to its canonical `kimi-k3` offering. The unqualified `gpt-terra` alias uses normal ready-provider priority selection. Legacy `provider/model` fallback references remain accepted but are normalized to `provider:model`.
|
||||
This chain applies only when the original request resolves to `kimi-k3`. It tries direct Moonshot AI, then OpenRouter, then Claude Opus. The OpenRouter entry could equivalently be written as `"openrouter:moonshotai/kimi-k3"` using its API ID. Legacy `provider/model` fallback references remain accepted but are normalized to `provider:model`.
|
||||
|
||||
A colon alone does not make a reference qualified. Many model IDs contain one — ollama `name:tag` values, Bedrock inference-profile IDs and ARNs — so Fabro treats the reference as qualified only when the text before the first `:` names a known provider. `"llama3:8b"` stays a single model ID, while `"ollama:llama3:8b"` pins the `ollama` provider and passes `llama3:8b` as the selector.
|
||||
|
||||
At run creation, Fabro resolves the primary selector and every node selector against the ready-provider snapshot. It persists the selected canonical model slug and provider, so resuming the run does not choose a different provider just because credentials or priorities changed. The configured fallback chain remains available for failures that occur while the materialized run is executing.
|
||||
At run creation, Fabro resolves the primary selector, every node selector, and the fallback table against the server's ready-provider snapshot. It persists the selected canonical model slug and provider, so resuming the run does not choose a different provider just because credentials or priorities changed. `fabro validate` only checks the table's TOML shape because it is offline and has no server model catalog. Use `fabro preflight` for catalog and provider checks.
|
||||
|
||||
Each original requested model selects one fixed chain. Fabro does not jump to the chain configured for a fallback target. When a target lacks the requested reasoning level, Fabro uses the nearest supported level. It rounds equal-distance choices up.
|
||||
|
||||
For example, a server that selects Modal as the primary `kimi-k3` offering can define these independent chains:
|
||||
|
||||
```toml title="settings.toml"
|
||||
[run.model.fallbacks]
|
||||
"kimi-k3" = ["kimi:kimi-k3", "openrouter:kimi-k3", "claude-opus"]
|
||||
"glm-5.2" = ["gpt-sol"]
|
||||
"gpt-sol" = ["claude-opus"]
|
||||
"claude-opus" = ["gpt-sol"]
|
||||
"gpt-terra" = ["claude-opus"]
|
||||
"gpt-luna" = ["claude-sonnet"]
|
||||
"claude-fable" = ["gpt-sol", "claude-opus"]
|
||||
```
|
||||
|
||||
If `claude-fable` falls back to `gpt-sol`, Fabro continues with `claude-opus` from the Fable list. It does not restart from the separate `gpt-sol` list.
|
||||
|
||||
Historical built-in provider API IDs are accepted for compatibility and normalize before this selection. For example, `name = "openai/gpt-5.6-sol"` is treated as the canonical `gpt-5.6-sol` selector; omit `provider` to use readiness and priority, or set `provider` separately to pin an offering.
|
||||
|
||||
|
|
@ -181,14 +201,14 @@ speed = "fast"
|
|||
| `reasoning_effort` | Native reasoning-effort value to request when the selected model allows it, such as `"low"`, `"medium"`, `"high"`, `"xhigh"`, or `"max"`. |
|
||||
| `speed` | Native speed value to request when the selected model declares it, such as `"fast"`. The standard speed is implicit and does not need to be set. |
|
||||
|
||||
#### Fallbacks with splice
|
||||
#### Fallback lists with splice
|
||||
|
||||
Use the reserved `"..."` marker in `fallbacks` to splice in the inherited list from lower-precedence layers:
|
||||
Use the reserved `"..."` marker in one model's list to splice in that model's inherited list from lower-precedence layers:
|
||||
|
||||
```toml title="run.toml"
|
||||
[run.model]
|
||||
# Prepend "anthropic" to whatever fallbacks the project config already defines.
|
||||
fallbacks = ["anthropic", "..."]
|
||||
[run.model.fallbacks]
|
||||
# Prepend Anthropic to the inherited chain for gpt-5.6-sol.
|
||||
"gpt-5.6-sol" = ["anthropic", "..."]
|
||||
```
|
||||
|
||||
### `[run.prepare]`
|
||||
|
|
|
|||
|
|
@ -381,12 +381,14 @@ permissions = "read-write"
|
|||
[run.model]
|
||||
provider = "anthropic"
|
||||
name = "claude-sonnet-4-5"
|
||||
fallbacks = ["openrouter:kimi-k3", "gpt-terra"]
|
||||
|
||||
[run.model.fallbacks]
|
||||
"claude-sonnet-4-5" = ["openrouter:kimi-k3", "gpt-terra"]
|
||||
```
|
||||
|
||||
| Key | Type / values | Default | Description |
|
||||
|---|---|---|---|
|
||||
| `fallbacks` | array<string> | [] | Ordered fallback references: bare providers, bare model IDs or aliases,<br />or provider-qualified `provider:selector` values. A qualified selector<br />may be a model ID, alias, or provider API ID. Legacy `provider/model`<br />values remain accepted. Supports the `...` splice marker at layering<br />time — see [`super::splice_array`]. |
|
||||
| `fallbacks` | table<string, array<string>> | {} | Model-keyed fallback chains. Each value is an ordered list of bare<br />providers, bare model IDs or aliases, or provider-qualified<br />`provider:selector` values. A qualified selector may be a model ID,<br />alias, or provider API ID. Legacy `provider/model` values remain<br />accepted. Each list supports the `...` splice marker at layering time.<br />Fabro selects one chain from the original requested model. A fallback<br />target never activates another model's chain.<br /><br />Model keys stay unresolved in this sparse layer because `fabro<br />validate` is offline and has no server model catalog. |
|
||||
| `name` | string | None | Model name for workflow runs. |
|
||||
| `provider` | string | None | Provider name for workflow model selection. |
|
||||
|
||||
|
|
|
|||
|
|
@ -997,7 +997,7 @@ fn attach_json_errors_without_prompting_for_human_input() {
|
|||
"reasoning_effort": null,
|
||||
"speed": null
|
||||
},
|
||||
"fallbacks": [],
|
||||
"fallbacks": {},
|
||||
"name": "[DEFAULT_MODEL]",
|
||||
"provider": "openai"
|
||||
},
|
||||
|
|
|
|||
|
|
@ -129,7 +129,7 @@ fn inspect_resolves_selector_via_server_endpoint() {
|
|||
"model": {
|
||||
"provider": null,
|
||||
"name": null,
|
||||
"fallbacks": [],
|
||||
"fallbacks": {},
|
||||
"controls": {
|
||||
"reasoning_effort": null,
|
||||
"speed": null
|
||||
|
|
|
|||
|
|
@ -87,6 +87,23 @@ fn server_owned_provider_is_not_rejected_by_offline_validation() {
|
|||
");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn model_fallback_table_is_not_catalog_checked_by_offline_validation() {
|
||||
let cli = LightweightCli::new();
|
||||
let mut cmd = cli.command();
|
||||
cmd.env("FABRO_SERVER", "http://127.0.0.1:9")
|
||||
.arg("validate")
|
||||
.arg(fixture("offline-fallbacks/workflow.fabro"));
|
||||
|
||||
let output = cmd.output().expect("validate should execute");
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"offline validation should parse model fallback tables without a server catalog\nstdout:\n{}\nstderr:\n{}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn branching() {
|
||||
let context = test_context!();
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
use std::collections::HashMap;
|
||||
use std::collections::{BTreeMap, HashMap, HashSet};
|
||||
use std::future::Future;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
|
@ -25,15 +25,18 @@ use fabro_sandbox::from_environment::{
|
|||
use fabro_sandbox::redact::redact_auth_url;
|
||||
use fabro_sandbox::{DockerSandboxOptions, Sandbox, SandboxSpec};
|
||||
use fabro_static::EnvVars;
|
||||
use fabro_types::settings::ModelRef;
|
||||
use fabro_types::settings::cli::OutputVerbosity;
|
||||
use fabro_types::settings::interp::InterpString;
|
||||
use fabro_types::settings::run::{EnvironmentProvider, McpServerSettings, RunGoal, RunNamespace};
|
||||
use fabro_types::{
|
||||
ManifestPath, RunId, RunProvenance, SandboxProviderKind, ServerSettings, WorkflowSettings,
|
||||
ManifestPath, RunId, RunNoticeLevel, RunProvenance, SandboxProviderKind, ServerSettings,
|
||||
WorkflowSettings,
|
||||
};
|
||||
use fabro_util::check_report::{CheckDetail, CheckReport, CheckResult, CheckSection, CheckStatus};
|
||||
use fabro_validate::Severity;
|
||||
use fabro_workflow::Error as WorkflowError;
|
||||
use fabro_workflow::model_fallback::resolve_model_fallbacks;
|
||||
use fabro_workflow::operations::{
|
||||
CreateRunInput, ValidateInput, WorkflowInput, validate, validate_with_catalog,
|
||||
validate_with_ready_providers,
|
||||
|
|
@ -544,6 +547,12 @@ async fn build_preflight_report(
|
|||
));
|
||||
}
|
||||
run_environment_capability_check(&mut checks, &resolved_run);
|
||||
let model_fallbacks_ok = run_model_fallback_check(
|
||||
&mut checks,
|
||||
catalog.as_ref(),
|
||||
&ready_providers,
|
||||
&resolved_run.model.fallbacks,
|
||||
);
|
||||
let needs_github_credentials =
|
||||
sandbox_provider.is_clone_based() || resolved_run.integrations.github.is_token_requested();
|
||||
let github_app = if needs_github_credentials {
|
||||
|
|
@ -592,7 +601,8 @@ async fn build_preflight_report(
|
|||
let github_token_ok =
|
||||
run_github_token_check(&mut checks, prepared, &resolved_run, github_app).await;
|
||||
|
||||
let checks_ok = sandbox_ok && repository_access_ok && llm_ok && github_token_ok;
|
||||
let checks_ok =
|
||||
model_fallbacks_ok && sandbox_ok && repository_access_ok && llm_ok && github_token_ok;
|
||||
|
||||
Ok((
|
||||
CheckReport {
|
||||
|
|
@ -606,6 +616,76 @@ async fn build_preflight_report(
|
|||
))
|
||||
}
|
||||
|
||||
fn run_model_fallback_check(
|
||||
checks: &mut Vec<CheckResult>,
|
||||
catalog: &Catalog,
|
||||
ready_providers: &[ProviderId],
|
||||
configured: &BTreeMap<String, Vec<ModelRef>>,
|
||||
) -> bool {
|
||||
if configured.is_empty() {
|
||||
return true;
|
||||
}
|
||||
|
||||
let eligible = ready_providers.iter().cloned().collect::<HashSet<_>>();
|
||||
let resolved = match resolve_model_fallbacks(catalog, &eligible, configured) {
|
||||
Ok(resolved) => resolved,
|
||||
Err(error) => {
|
||||
checks.push(CheckResult {
|
||||
name: "Model Fallbacks".into(),
|
||||
status: CheckStatus::Error,
|
||||
summary: "invalid".into(),
|
||||
details: configured
|
||||
.keys()
|
||||
.map(|model| CheckDetail::new(format!("Requested model: {model}")))
|
||||
.collect(),
|
||||
remediation: Some(error.to_string()),
|
||||
});
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
let has_warning = resolved
|
||||
.notices
|
||||
.iter()
|
||||
.any(|notice| notice.level() != RunNoticeLevel::Info);
|
||||
let mut details = resolved
|
||||
.policy
|
||||
.iter()
|
||||
.map(|(model, targets)| {
|
||||
let chain = if targets.is_empty() {
|
||||
"(none)".to_string()
|
||||
} else {
|
||||
targets
|
||||
.iter()
|
||||
.map(ToString::to_string)
|
||||
.collect::<Vec<_>>()
|
||||
.join(" -> ")
|
||||
};
|
||||
CheckDetail::new(format!("{model}: {chain}"))
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
details.extend(resolved.notices.iter().map(|notice| CheckDetail {
|
||||
text: notice.message(),
|
||||
warn: notice.level() != RunNoticeLevel::Info,
|
||||
}));
|
||||
|
||||
checks.push(CheckResult {
|
||||
name: "Model Fallbacks".into(),
|
||||
status: if has_warning {
|
||||
CheckStatus::Warning
|
||||
} else {
|
||||
CheckStatus::Pass
|
||||
},
|
||||
summary: format!(
|
||||
"{} requested model chain(s)",
|
||||
resolved.policy.iter().count()
|
||||
),
|
||||
details,
|
||||
remediation: None,
|
||||
});
|
||||
true
|
||||
}
|
||||
|
||||
fn base_preflight_checks(prepared: &PreparedManifest, graph: &Graph) -> Vec<CheckResult> {
|
||||
let setup_command_count = prepared.settings.run.prepare.steps.len();
|
||||
let repo_summary = prepared.git.as_ref().map_or_else(
|
||||
|
|
@ -1495,6 +1575,106 @@ mod tests {
|
|||
Arc::new(Catalog::from_builtin().unwrap())
|
||||
}
|
||||
|
||||
fn openrouter_catalog() -> Catalog {
|
||||
let overrides = toml::from_str(
|
||||
r"
|
||||
[providers.openrouter]
|
||||
enabled = true
|
||||
",
|
||||
)
|
||||
.expect("catalog override should parse");
|
||||
Catalog::from_builtin_with_overrides(&overrides).expect("catalog should build")
|
||||
}
|
||||
|
||||
fn model_refs(values: &[&str]) -> Vec<fabro_types::settings::ModelRef> {
|
||||
values
|
||||
.iter()
|
||||
.map(|value| value.parse().expect("fallback reference should parse"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn model_fallback_preflight_resolves_each_requested_model_chain() {
|
||||
let mut checks = Vec::new();
|
||||
let configured = std::collections::BTreeMap::from([
|
||||
("gpt-sol".to_string(), model_refs(&["claude-opus"])),
|
||||
(
|
||||
"claude-fable".to_string(),
|
||||
model_refs(&["gpt-sol", "claude-opus"]),
|
||||
),
|
||||
]);
|
||||
|
||||
assert!(run_model_fallback_check(
|
||||
&mut checks,
|
||||
&openrouter_catalog(),
|
||||
&[ProviderId::new("openrouter")],
|
||||
&configured,
|
||||
));
|
||||
|
||||
let check = checks.last().expect("fallback check should be present");
|
||||
assert_eq!(check.status, CheckStatus::Pass);
|
||||
assert!(
|
||||
check
|
||||
.details
|
||||
.iter()
|
||||
.any(|detail| detail.text == "gpt-5.6-sol: openrouter:claude-opus-5")
|
||||
);
|
||||
assert!(check.details.iter().any(|detail| {
|
||||
detail.text == "claude-fable-5: openrouter:gpt-5.6-sol -> openrouter:claude-opus-5"
|
||||
}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn model_fallback_preflight_warns_when_a_provider_is_not_ready() {
|
||||
let mut checks = Vec::new();
|
||||
let configured = std::collections::BTreeMap::from([(
|
||||
"kimi-k3".to_string(),
|
||||
model_refs(&["kimi:kimi-k3", "openrouter:kimi-k3"]),
|
||||
)]);
|
||||
|
||||
assert!(run_model_fallback_check(
|
||||
&mut checks,
|
||||
&openrouter_catalog(),
|
||||
&[ProviderId::new("openrouter")],
|
||||
&configured,
|
||||
));
|
||||
|
||||
let check = checks.last().expect("fallback check should be present");
|
||||
assert_eq!(check.status, CheckStatus::Warning);
|
||||
assert!(
|
||||
check
|
||||
.details
|
||||
.iter()
|
||||
.any(|detail| detail.warn
|
||||
&& detail.text.contains("provider `kimi` is not configured"))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn model_fallback_preflight_rejects_duplicate_canonical_keys() {
|
||||
let mut checks = Vec::new();
|
||||
let configured = std::collections::BTreeMap::from([
|
||||
("gpt-sol".to_string(), model_refs(&["claude-opus"])),
|
||||
("gpt-5.6-sol".to_string(), model_refs(&["claude-fable"])),
|
||||
]);
|
||||
|
||||
assert!(!run_model_fallback_check(
|
||||
&mut checks,
|
||||
&openrouter_catalog(),
|
||||
&[ProviderId::new("openrouter")],
|
||||
&configured,
|
||||
));
|
||||
|
||||
let check = checks.last().expect("fallback check should be present");
|
||||
assert_eq!(check.status, CheckStatus::Error);
|
||||
assert!(
|
||||
check
|
||||
.remediation
|
||||
.as_deref()
|
||||
.is_some_and(|message| message.contains("both resolve to requested model"))
|
||||
);
|
||||
}
|
||||
|
||||
fn openai_compatible_completion(model: &str) -> serde_json::Value {
|
||||
serde_json::json!({
|
||||
"id": "chatcmpl_preflight",
|
||||
|
|
|
|||
|
|
@ -63,7 +63,7 @@ fn test_app_with_openai_agent_backend(openai_base_url: String, api_key: String)
|
|||
fabro_workflow::handler::llm::AgentApiBackend::new_with_catalog(
|
||||
OPENAI_AGENT_MODEL.to_string(),
|
||||
ProviderId::openai(),
|
||||
Vec::new(),
|
||||
fabro_workflow::model_fallback::ModelFallbackPolicy::default(),
|
||||
Arc::clone(&llm_source),
|
||||
Arc::clone(&steering_hub),
|
||||
Arc::clone(&catalog),
|
||||
|
|
|
|||
|
|
@ -210,8 +210,9 @@ impl Error {
|
|||
/// Whether this error is eligible for provider-level failover.
|
||||
///
|
||||
/// Includes everything that is `retryable()` (transient errors good for
|
||||
/// same-provider retry) plus `QuotaExceeded` — a different provider won't
|
||||
/// share the same quota.
|
||||
/// same-provider retry), provider-local availability failures, and
|
||||
/// `QuotaExceeded`. A different provider has independent credentials,
|
||||
/// access policy, model inventory, and quota.
|
||||
#[must_use]
|
||||
pub fn failover_eligible(&self) -> bool {
|
||||
if self.retryable() {
|
||||
|
|
@ -220,7 +221,10 @@ impl Error {
|
|||
matches!(
|
||||
self,
|
||||
Self::Provider {
|
||||
kind: ProviderErrorKind::QuotaExceeded,
|
||||
kind: ProviderErrorKind::Authentication
|
||||
| ProviderErrorKind::AccessDenied
|
||||
| ProviderErrorKind::NotFound
|
||||
| ProviderErrorKind::QuotaExceeded,
|
||||
..
|
||||
} | Self::RequestTimeout { .. }
|
||||
) || self.refusal_content_filter()
|
||||
|
|
@ -887,6 +891,26 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failover_eligible_provider_local_availability_errors() {
|
||||
let detail = || Box::new(ProviderErrorDetail::new("error", "openai"));
|
||||
|
||||
for kind in [
|
||||
ProviderErrorKind::Authentication,
|
||||
ProviderErrorKind::AccessDenied,
|
||||
ProviderErrorKind::NotFound,
|
||||
] {
|
||||
assert!(
|
||||
Error::Provider {
|
||||
kind,
|
||||
detail: detail(),
|
||||
}
|
||||
.failover_eligible(),
|
||||
"{kind:?} should permit another provider"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failover_eligible_transient_non_provider_errors() {
|
||||
assert!(
|
||||
|
|
@ -918,14 +942,6 @@ mod tests {
|
|||
fn failover_not_eligible_deterministic_errors() {
|
||||
let detail = || Box::new(ProviderErrorDetail::new("error", "openai"));
|
||||
|
||||
assert!(
|
||||
!Error::Provider {
|
||||
kind: ProviderErrorKind::Authentication,
|
||||
detail: detail(),
|
||||
}
|
||||
.failover_eligible()
|
||||
);
|
||||
|
||||
assert!(
|
||||
!Error::Provider {
|
||||
kind: ProviderErrorKind::InvalidRequest,
|
||||
|
|
|
|||
|
|
@ -76,7 +76,7 @@ pub fn build_run_overrides(input: RunOverrideInput<'_>) -> RunLayer {
|
|||
let model = (input.model.is_some() || input.provider.is_some()).then(|| RunModelLayer {
|
||||
provider: input.provider.map(String::from),
|
||||
name: input.model.map(String::from),
|
||||
fallbacks: Vec::new(),
|
||||
fallbacks: MergeMap::default(),
|
||||
controls: None,
|
||||
});
|
||||
let environment = (input.environment.is_some()
|
||||
|
|
|
|||
|
|
@ -1147,18 +1147,28 @@ fn event_body_from_event(event: &Event) -> EventBody {
|
|||
})
|
||||
}
|
||||
Event::Failover {
|
||||
original_provider,
|
||||
original_model,
|
||||
attempt,
|
||||
from_provider,
|
||||
from_model,
|
||||
to_provider,
|
||||
to_model,
|
||||
requested_reasoning_effort,
|
||||
effective_reasoning_effort,
|
||||
error,
|
||||
..
|
||||
} => EventBody::Failover(fabro_types::FailoverProps {
|
||||
from_provider: from_provider.clone(),
|
||||
from_model: from_model.clone(),
|
||||
to_provider: to_provider.clone(),
|
||||
to_model: to_model.clone(),
|
||||
error: error.clone(),
|
||||
original_provider: Some(original_provider.clone()),
|
||||
original_model: Some(original_model.clone()),
|
||||
attempt: Some(*attempt),
|
||||
from_provider: from_provider.clone(),
|
||||
from_model: from_model.clone(),
|
||||
to_provider: to_provider.clone(),
|
||||
to_model: to_model.clone(),
|
||||
requested_reasoning_effort: requested_reasoning_effort.clone(),
|
||||
effective_reasoning_effort: effective_reasoning_effort.clone(),
|
||||
error: error.clone(),
|
||||
}),
|
||||
Event::CommandStarted {
|
||||
script,
|
||||
|
|
|
|||
|
|
@ -583,12 +583,17 @@ pub enum Event {
|
|||
ssh_command: String,
|
||||
},
|
||||
Failover {
|
||||
stage: String,
|
||||
stage: String,
|
||||
original_provider: String,
|
||||
original_model: String,
|
||||
attempt: u32,
|
||||
from_provider: String,
|
||||
from_model: String,
|
||||
to_provider: String,
|
||||
to_model: String,
|
||||
error: String,
|
||||
from_model: String,
|
||||
to_provider: String,
|
||||
to_model: String,
|
||||
requested_reasoning_effort: Option<String>,
|
||||
effective_reasoning_effort: Option<String>,
|
||||
error: String,
|
||||
},
|
||||
CommandStarted {
|
||||
node_id: String,
|
||||
|
|
@ -1395,18 +1400,28 @@ impl Event {
|
|||
}
|
||||
Self::Failover {
|
||||
stage,
|
||||
original_provider,
|
||||
original_model,
|
||||
attempt,
|
||||
from_provider,
|
||||
from_model,
|
||||
to_provider,
|
||||
to_model,
|
||||
requested_reasoning_effort,
|
||||
effective_reasoning_effort,
|
||||
error,
|
||||
} => {
|
||||
warn!(
|
||||
stage,
|
||||
original_provider,
|
||||
original_model,
|
||||
attempt,
|
||||
from_provider,
|
||||
from_model,
|
||||
to_provider,
|
||||
to_model,
|
||||
requested_reasoning_effort,
|
||||
effective_reasoning_effort,
|
||||
error,
|
||||
"LLM provider failover"
|
||||
);
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -303,6 +303,7 @@ mod interview_runtime;
|
|||
reason = "The lifecycle module remains crate-visible for tests and pending integrations."
|
||||
)]
|
||||
pub(crate) mod lifecycle;
|
||||
pub mod model_fallback;
|
||||
pub(crate) mod node_handler;
|
||||
pub mod operations;
|
||||
pub mod outcome;
|
||||
|
|
|
|||
514
lib/components/fabro-workflow/src/model_fallback.rs
Normal file
514
lib/components/fabro-workflow/src/model_fallback.rs
Normal file
|
|
@ -0,0 +1,514 @@
|
|||
use std::collections::{BTreeMap, HashMap, HashSet};
|
||||
|
||||
use fabro_model::{Catalog, FallbackTarget, Model, ModelSelectionError, ProviderId};
|
||||
use fabro_types::settings::{ModelRef, ResolvedModelRef};
|
||||
use fabro_types::{RunNoticeCode, RunNoticeLevel};
|
||||
|
||||
use crate::Error;
|
||||
|
||||
/// Catalog-resolved fallback chains keyed by canonical requested model ID.
|
||||
///
|
||||
/// A chain is selected from the original request only. Targets never cause
|
||||
/// another chain lookup.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct ModelFallbackPolicy {
|
||||
chains: BTreeMap<String, Vec<FallbackTarget>>,
|
||||
}
|
||||
|
||||
impl ModelFallbackPolicy {
|
||||
#[must_use]
|
||||
pub fn new(chains: BTreeMap<String, Vec<FallbackTarget>>) -> Self {
|
||||
Self { chains }
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn chain_for<'a>(
|
||||
&'a self,
|
||||
catalog: &Catalog,
|
||||
provider: &ProviderId,
|
||||
model: &str,
|
||||
) -> Option<&'a [FallbackTarget]> {
|
||||
let canonical = canonical_model_id(catalog, provider, model);
|
||||
self.chains.get(&canonical).map(Vec::as_slice)
|
||||
}
|
||||
|
||||
pub fn iter(&self) -> impl Iterator<Item = (&str, &[FallbackTarget])> {
|
||||
self.chains
|
||||
.iter()
|
||||
.map(|(model, chain)| (model.as_str(), chain.as_slice()))
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.chains.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn canonical_model_id(catalog: &Catalog, provider: &ProviderId, model: &str) -> String {
|
||||
catalog.get_on_provider(provider, model).map_or_else(
|
||||
|| {
|
||||
catalog
|
||||
.select(model, None, &catalog.all_provider_ids())
|
||||
.map_or_else(|_| model.to_string(), |offering| offering.id.to_string())
|
||||
},
|
||||
|offering| offering.id.to_string(),
|
||||
)
|
||||
}
|
||||
|
||||
/// Server-side result of canonicalizing and filtering configured fallback
|
||||
/// chains.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct ResolvedModelFallbacks {
|
||||
pub policy: ModelFallbackPolicy,
|
||||
pub notices: Vec<ModelFallbackNotice>,
|
||||
}
|
||||
|
||||
/// Why a configured fallback candidate was removed from one model's chain.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum ModelFallbackNotice {
|
||||
ProviderUnconfigured {
|
||||
requested_model: String,
|
||||
reference: ModelRef,
|
||||
provider: ProviderId,
|
||||
},
|
||||
NoConfiguredOffering {
|
||||
requested_model: String,
|
||||
reference: ModelRef,
|
||||
providers: Vec<ProviderId>,
|
||||
},
|
||||
PrimaryNotInCatalog {
|
||||
requested_model: String,
|
||||
reference: ModelRef,
|
||||
primary: FallbackTarget,
|
||||
},
|
||||
NoCompatibleModel {
|
||||
requested_model: String,
|
||||
reference: ModelRef,
|
||||
provider: ProviderId,
|
||||
},
|
||||
Duplicate {
|
||||
requested_model: String,
|
||||
reference: ModelRef,
|
||||
target: FallbackTarget,
|
||||
},
|
||||
ChainEmpty {
|
||||
requested_model: String,
|
||||
},
|
||||
}
|
||||
|
||||
impl ModelFallbackNotice {
|
||||
#[must_use]
|
||||
pub fn code(&self) -> RunNoticeCode {
|
||||
match self {
|
||||
Self::ChainEmpty { .. } => RunNoticeCode::ModelFallbackChainEmpty,
|
||||
Self::ProviderUnconfigured { .. }
|
||||
| Self::NoConfiguredOffering { .. }
|
||||
| Self::PrimaryNotInCatalog { .. }
|
||||
| Self::NoCompatibleModel { .. }
|
||||
| Self::Duplicate { .. } => RunNoticeCode::ModelFallbackSkipped,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn level(&self) -> RunNoticeLevel {
|
||||
match self {
|
||||
Self::Duplicate { .. } => RunNoticeLevel::Info,
|
||||
Self::ProviderUnconfigured { .. }
|
||||
| Self::NoConfiguredOffering { .. }
|
||||
| Self::PrimaryNotInCatalog { .. }
|
||||
| Self::NoCompatibleModel { .. }
|
||||
| Self::ChainEmpty { .. } => RunNoticeLevel::Warn,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn message(&self) -> String {
|
||||
match self {
|
||||
Self::ProviderUnconfigured {
|
||||
requested_model,
|
||||
reference,
|
||||
provider,
|
||||
} => format!(
|
||||
"Model fallback `{reference}` for requested model `{requested_model}` was skipped because provider `{provider}` is not configured."
|
||||
),
|
||||
Self::NoConfiguredOffering {
|
||||
requested_model,
|
||||
reference,
|
||||
providers,
|
||||
} => {
|
||||
let providers = providers
|
||||
.iter()
|
||||
.map(ProviderId::to_string)
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ");
|
||||
format!(
|
||||
"Model fallback `{reference}` for requested model `{requested_model}` was skipped because none of its providers are configured. It is offered by: {providers}."
|
||||
)
|
||||
}
|
||||
Self::PrimaryNotInCatalog {
|
||||
requested_model,
|
||||
reference,
|
||||
primary,
|
||||
} => format!(
|
||||
"Model fallback `{reference}` for requested model `{requested_model}` was skipped because `{primary}` is not in the catalog, so there is no capability profile to match against."
|
||||
),
|
||||
Self::NoCompatibleModel {
|
||||
requested_model,
|
||||
reference,
|
||||
provider,
|
||||
} => format!(
|
||||
"Model fallback `{reference}` for requested model `{requested_model}` was skipped because provider `{provider}` has no compatible model."
|
||||
),
|
||||
Self::Duplicate {
|
||||
requested_model,
|
||||
reference,
|
||||
target,
|
||||
} => format!(
|
||||
"Model fallback `{reference}` for requested model `{requested_model}` was skipped because target `{target}` already appears in that chain."
|
||||
),
|
||||
Self::ChainEmpty { requested_model } => format!(
|
||||
"No usable model fallbacks remain for requested model `{requested_model}` after filtering its configured candidates."
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve every model-keyed fallback chain against the server's catalog and
|
||||
/// configured-provider snapshot.
|
||||
///
|
||||
/// This function must stay at server-side call sites. Offline validation only
|
||||
/// parses the raw table and cannot canonicalize model aliases.
|
||||
pub fn resolve_model_fallbacks(
|
||||
catalog: &Catalog,
|
||||
eligible: &HashSet<ProviderId>,
|
||||
configured: &BTreeMap<String, Vec<ModelRef>>,
|
||||
) -> Result<ResolvedModelFallbacks, Error> {
|
||||
let mut resolved = ResolvedModelFallbacks::default();
|
||||
let mut raw_key_by_canonical = HashMap::<String, String>::new();
|
||||
|
||||
for (raw_key, references) in configured {
|
||||
let selected =
|
||||
catalog.resolve_selection_with_catalog_fallback(Some(raw_key), None, eligible)?;
|
||||
let requested_model = selected.model;
|
||||
|
||||
if let Some(previous) =
|
||||
raw_key_by_canonical.insert(requested_model.clone(), raw_key.clone())
|
||||
{
|
||||
return Err(Error::Precondition(format!(
|
||||
"`run.model.fallbacks` keys `{previous}` and `{raw_key}` both resolve to requested model `{requested_model}`"
|
||||
)));
|
||||
}
|
||||
|
||||
let primary = FallbackTarget::new(&selected.provider, &requested_model);
|
||||
let primary_model = catalog.get_on_provider(&selected.provider, &requested_model);
|
||||
let mut targets = Vec::new();
|
||||
|
||||
for model_ref in references {
|
||||
let target = match resolve_fallback_candidate(
|
||||
catalog,
|
||||
&requested_model,
|
||||
&primary,
|
||||
primary_model,
|
||||
eligible,
|
||||
model_ref,
|
||||
)? {
|
||||
FallbackCandidate::Skipped(notice) => {
|
||||
resolved.notices.push(notice);
|
||||
continue;
|
||||
}
|
||||
FallbackCandidate::Target(target) => target,
|
||||
};
|
||||
|
||||
if targets.contains(&target) {
|
||||
resolved.notices.push(ModelFallbackNotice::Duplicate {
|
||||
requested_model: requested_model.clone(),
|
||||
reference: model_ref.clone(),
|
||||
target,
|
||||
});
|
||||
} else {
|
||||
targets.push(target);
|
||||
}
|
||||
}
|
||||
|
||||
if targets.is_empty() {
|
||||
resolved.notices.push(ModelFallbackNotice::ChainEmpty {
|
||||
requested_model: requested_model.clone(),
|
||||
});
|
||||
}
|
||||
resolved.policy.chains.insert(requested_model, targets);
|
||||
}
|
||||
|
||||
Ok(resolved)
|
||||
}
|
||||
|
||||
enum FallbackCandidate {
|
||||
Target(FallbackTarget),
|
||||
Skipped(ModelFallbackNotice),
|
||||
}
|
||||
|
||||
fn resolve_fallback_candidate(
|
||||
catalog: &Catalog,
|
||||
requested_model: &str,
|
||||
primary: &FallbackTarget,
|
||||
primary_model: Option<&Model>,
|
||||
eligible: &HashSet<ProviderId>,
|
||||
model_ref: &ModelRef,
|
||||
) -> Result<FallbackCandidate, Error> {
|
||||
let reference = model_ref.clone();
|
||||
|
||||
Ok(match model_ref.resolve(catalog)? {
|
||||
ResolvedModelRef::Provider(provider_name) => {
|
||||
let provider = catalog.provider_id(&provider_name)?;
|
||||
if !eligible.contains(&provider) {
|
||||
return Ok(FallbackCandidate::Skipped(
|
||||
ModelFallbackNotice::ProviderUnconfigured {
|
||||
requested_model: requested_model.to_string(),
|
||||
reference,
|
||||
provider,
|
||||
},
|
||||
));
|
||||
}
|
||||
let Some(primary_model) = primary_model else {
|
||||
return Ok(FallbackCandidate::Skipped(
|
||||
ModelFallbackNotice::PrimaryNotInCatalog {
|
||||
requested_model: requested_model.to_string(),
|
||||
reference,
|
||||
primary: primary.clone(),
|
||||
},
|
||||
));
|
||||
};
|
||||
match catalog.closest(&provider, primary_model) {
|
||||
Some(model) => FallbackCandidate::Target(FallbackTarget::new(provider, &model.id)),
|
||||
None => FallbackCandidate::Skipped(ModelFallbackNotice::NoCompatibleModel {
|
||||
requested_model: requested_model.to_string(),
|
||||
reference,
|
||||
provider,
|
||||
}),
|
||||
}
|
||||
}
|
||||
ResolvedModelRef::Model {
|
||||
provider: Some(provider_name),
|
||||
selector,
|
||||
} => {
|
||||
let provider = catalog.provider_id(&provider_name)?;
|
||||
if !eligible.contains(&provider) {
|
||||
return Ok(FallbackCandidate::Skipped(
|
||||
ModelFallbackNotice::ProviderUnconfigured {
|
||||
requested_model: requested_model.to_string(),
|
||||
reference,
|
||||
provider,
|
||||
},
|
||||
));
|
||||
}
|
||||
match catalog.resolve_on_provider(&provider, &selector) {
|
||||
Ok(info) => {
|
||||
FallbackCandidate::Target(FallbackTarget::new(&info.provider, &info.id))
|
||||
}
|
||||
Err(ModelSelectionError::UnknownSelectorOnProvider { .. }) => {
|
||||
FallbackCandidate::Target(FallbackTarget::new(provider, selector))
|
||||
}
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
}
|
||||
ResolvedModelRef::Model {
|
||||
provider: None,
|
||||
selector,
|
||||
} => match catalog.select(&selector, None, eligible) {
|
||||
Ok(info) => FallbackCandidate::Target(FallbackTarget::new(&info.provider, &info.id)),
|
||||
Err(ModelSelectionError::NoEligibleOffering { providers, .. }) => {
|
||||
FallbackCandidate::Skipped(ModelFallbackNotice::NoConfiguredOffering {
|
||||
requested_model: requested_model.to_string(),
|
||||
reference,
|
||||
providers,
|
||||
})
|
||||
}
|
||||
Err(ModelSelectionError::UnknownSelector { .. }) => {
|
||||
FallbackCandidate::Target(FallbackTarget::new(&primary.provider, selector))
|
||||
}
|
||||
Err(error) => return Err(error.into()),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::{BTreeMap, HashSet};
|
||||
|
||||
use fabro_model::{Catalog, FallbackTarget, ProviderId};
|
||||
|
||||
use super::{ModelFallbackNotice, resolve_model_fallbacks};
|
||||
|
||||
fn references(values: &[&str]) -> Vec<fabro_types::settings::ModelRef> {
|
||||
values
|
||||
.iter()
|
||||
.map(|value| value.parse().expect("fixture reference should parse"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn openrouter_catalog() -> Catalog {
|
||||
let overrides = toml::from_str(
|
||||
r"
|
||||
[providers.openrouter]
|
||||
enabled = true
|
||||
",
|
||||
)
|
||||
.expect("catalog override should parse");
|
||||
Catalog::from_builtin_with_overrides(&overrides).expect("catalog should build")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonicalizes_keys_and_keeps_each_chain_independent() {
|
||||
let catalog = openrouter_catalog();
|
||||
let eligible = HashSet::from([ProviderId::new("openrouter")]);
|
||||
let configured = BTreeMap::from([
|
||||
("gpt-sol".to_string(), references(&["claude-opus"])),
|
||||
(
|
||||
"claude-fable".to_string(),
|
||||
references(&["gpt-sol", "claude-opus"]),
|
||||
),
|
||||
]);
|
||||
|
||||
let resolved = resolve_model_fallbacks(&catalog, &eligible, &configured).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
resolved
|
||||
.policy
|
||||
.chain_for(&catalog, &ProviderId::new("openrouter"), "gpt-sol"),
|
||||
Some([FallbackTarget::new("openrouter", "claude-opus-5")].as_slice())
|
||||
);
|
||||
assert_eq!(
|
||||
resolved
|
||||
.policy
|
||||
.chain_for(&catalog, &ProviderId::new("openrouter"), "claude-fable"),
|
||||
Some(
|
||||
[
|
||||
FallbackTarget::new("openrouter", "gpt-5.6-sol"),
|
||||
FallbackTarget::new("openrouter", "claude-opus-5"),
|
||||
]
|
||||
.as_slice()
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_aliases_that_define_the_same_requested_model_twice() {
|
||||
let catalog = openrouter_catalog();
|
||||
let eligible = HashSet::from([ProviderId::new("openrouter")]);
|
||||
let configured = BTreeMap::from([
|
||||
("gpt-sol".to_string(), references(&["claude-opus"])),
|
||||
("gpt-5.6-sol".to_string(), references(&["claude-fable"])),
|
||||
]);
|
||||
|
||||
let error = resolve_model_fallbacks(&catalog, &eligible, &configured).unwrap_err();
|
||||
|
||||
assert!(
|
||||
error
|
||||
.to_string()
|
||||
.contains("both resolve to requested model"),
|
||||
"unexpected error: {error}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn skips_unconfigured_candidates_per_requested_model() {
|
||||
let catalog = openrouter_catalog();
|
||||
let eligible = HashSet::from([ProviderId::new("openrouter")]);
|
||||
let configured = BTreeMap::from([(
|
||||
"kimi-k3".to_string(),
|
||||
references(&["kimi:kimi-k3", "openrouter:kimi-k3"]),
|
||||
)]);
|
||||
|
||||
let resolved = resolve_model_fallbacks(&catalog, &eligible, &configured).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
resolved
|
||||
.policy
|
||||
.chain_for(&catalog, &ProviderId::new("openrouter"), "kimi-k3"),
|
||||
Some([FallbackTarget::new("openrouter", "kimi-k3")].as_slice())
|
||||
);
|
||||
assert!(matches!(
|
||||
resolved.notices.as_slice(),
|
||||
[ModelFallbackNotice::ProviderUnconfigured {
|
||||
requested_model,
|
||||
provider,
|
||||
..
|
||||
}] if requested_model == "kimi-k3" && provider == &ProviderId::new("kimi")
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolves_the_requested_production_policy_as_independent_chains() {
|
||||
let catalog = {
|
||||
let overrides = toml::from_str(
|
||||
r"
|
||||
[providers.modal]
|
||||
enabled = true
|
||||
|
||||
[providers.openrouter]
|
||||
enabled = true
|
||||
",
|
||||
)
|
||||
.expect("catalog override should parse");
|
||||
Catalog::from_builtin_with_overrides(&overrides).expect("catalog should build")
|
||||
};
|
||||
let eligible = HashSet::from([
|
||||
ProviderId::new("modal"),
|
||||
ProviderId::new("kimi"),
|
||||
ProviderId::new("openrouter"),
|
||||
]);
|
||||
let configured = BTreeMap::from([
|
||||
(
|
||||
"kimi-k3".to_string(),
|
||||
references(&["kimi:kimi-k3", "openrouter:kimi-k3", "claude-opus"]),
|
||||
),
|
||||
("glm-5.2".to_string(), references(&["gpt-sol"])),
|
||||
("gpt-sol".to_string(), references(&["claude-opus"])),
|
||||
("claude-opus".to_string(), references(&["gpt-sol"])),
|
||||
("gpt-terra".to_string(), references(&["claude-opus"])),
|
||||
("gpt-luna".to_string(), references(&["claude-sonnet"])),
|
||||
(
|
||||
"claude-fable".to_string(),
|
||||
references(&["gpt-sol", "claude-opus"]),
|
||||
),
|
||||
]);
|
||||
|
||||
let resolved = resolve_model_fallbacks(&catalog, &eligible, &configured).unwrap();
|
||||
|
||||
assert!(resolved.notices.is_empty());
|
||||
let chain = |model: &str| {
|
||||
resolved
|
||||
.policy
|
||||
.chain_for(&catalog, &ProviderId::new("openrouter"), model)
|
||||
.expect("requested model should have a chain")
|
||||
};
|
||||
assert_eq!(chain("kimi-k3"), [
|
||||
FallbackTarget::new("kimi", "kimi-k3"),
|
||||
FallbackTarget::new("openrouter", "kimi-k3"),
|
||||
FallbackTarget::new("openrouter", "claude-opus-5"),
|
||||
]);
|
||||
assert_eq!(chain("glm-5.2"), [FallbackTarget::new(
|
||||
"openrouter",
|
||||
"gpt-5.6-sol"
|
||||
)]);
|
||||
assert_eq!(chain("gpt-sol"), [FallbackTarget::new(
|
||||
"openrouter",
|
||||
"claude-opus-5"
|
||||
)]);
|
||||
assert_eq!(chain("claude-opus"), [FallbackTarget::new(
|
||||
"openrouter",
|
||||
"gpt-5.6-sol"
|
||||
)]);
|
||||
assert_eq!(chain("gpt-terra"), [FallbackTarget::new(
|
||||
"openrouter",
|
||||
"claude-opus-5"
|
||||
)]);
|
||||
assert_eq!(chain("gpt-luna"), [FallbackTarget::new(
|
||||
"openrouter",
|
||||
"claude-sonnet-5"
|
||||
)]);
|
||||
assert_eq!(chain("claude-fable"), [
|
||||
FallbackTarget::new("openrouter", "gpt-5.6-sol"),
|
||||
FallbackTarget::new("openrouter", "claude-opus-5"),
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
|
@ -7,7 +7,7 @@ use fabro_auth::{CredentialSource, VaultCredentialSource};
|
|||
use fabro_interview::{AutoApproveInterviewer, Interviewer};
|
||||
use fabro_llm::client::Client as LlmClient;
|
||||
use fabro_mcp::config::McpServerSettings;
|
||||
use fabro_model::{Catalog, FallbackTarget, Model, ModelSelectionError, ProviderId};
|
||||
use fabro_model::{Catalog, ProviderId};
|
||||
use fabro_sandbox::daytona::DaytonaConfig;
|
||||
use fabro_sandbox::from_environment::{
|
||||
daytona_config_from_environment, docker_config_from_environment_with_secrets,
|
||||
|
|
@ -17,10 +17,9 @@ use fabro_sandbox::{DockerSandboxOptions, SandboxSpec};
|
|||
use fabro_static::EnvVars;
|
||||
use fabro_types::settings::run::{
|
||||
ApprovalMode, McpServerSettings as ResolvedMcpServerSettings, PullRequestSettings,
|
||||
ResolvedMcpEntry, RunMode, RunModelSettings as ResolvedRunModelSettings,
|
||||
RunNamespace as ResolvedRunSettings, RunPrepareSettings as ResolvedRunPrepareSettings,
|
||||
ResolvedMcpEntry, RunMode, RunNamespace as ResolvedRunSettings,
|
||||
RunPrepareSettings as ResolvedRunPrepareSettings,
|
||||
};
|
||||
use fabro_types::settings::{ModelRef, ResolvedModelRef};
|
||||
use fabro_types::{ManifestPath, RunId, RunRunnableSource, SandboxProviderKind};
|
||||
use fabro_vault::Vault;
|
||||
use tokio::runtime::Handle;
|
||||
|
|
@ -32,10 +31,10 @@ use crate::artifact_upload::ArtifactSink;
|
|||
use crate::context::Context;
|
||||
use crate::error::{self, Error};
|
||||
use crate::event::{
|
||||
Emitter, Event, EventBody, RunEventLogger, RunEventSink, RunNoticeCode, RunNoticeLevel,
|
||||
append_event_to_sink,
|
||||
Emitter, Event, EventBody, RunEventLogger, RunEventSink, RunNoticeLevel, append_event_to_sink,
|
||||
};
|
||||
use crate::handler::HandlerRegistry;
|
||||
use crate::model_fallback::{ModelFallbackNotice, ResolvedModelFallbacks, resolve_model_fallbacks};
|
||||
use crate::outcome::{Outcome, StageOutcome};
|
||||
use crate::pipeline::{
|
||||
self, FinalizeOptions, Finalized, InitOptions, LlmSpec, Persisted, PublishOptions, ResumeState,
|
||||
|
|
@ -91,132 +90,7 @@ struct RunSession {
|
|||
struct ResolvedStartLlm {
|
||||
model: String,
|
||||
provider_id: ProviderId,
|
||||
fallbacks: ResolvedFallbackChain,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, PartialEq, Eq)]
|
||||
struct ResolvedFallbackChain {
|
||||
targets: Vec<FallbackTarget>,
|
||||
notices: Vec<ModelFallbackNotice>,
|
||||
}
|
||||
|
||||
/// Why one fallback candidate did not make it into the chain, or that the whole
|
||||
/// chain came out empty. Resolution happens before the run's event sink is
|
||||
/// wired up, so these are carried to [`RunSession::run`] and emitted there.
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
enum ModelFallbackNotice {
|
||||
ProviderUnconfigured {
|
||||
reference: ModelRef,
|
||||
provider: ProviderId,
|
||||
},
|
||||
NoConfiguredOffering {
|
||||
reference: ModelRef,
|
||||
providers: Vec<ProviderId>,
|
||||
},
|
||||
/// The candidate named a provider but no model, and the primary model is
|
||||
/// not in the catalog, so there is nothing to match its capabilities to.
|
||||
PrimaryNotInCatalog {
|
||||
reference: ModelRef,
|
||||
primary: FallbackTarget,
|
||||
},
|
||||
NoCompatibleModel {
|
||||
reference: ModelRef,
|
||||
provider: ProviderId,
|
||||
},
|
||||
MatchesPrimary {
|
||||
reference: ModelRef,
|
||||
target: FallbackTarget,
|
||||
},
|
||||
Duplicate {
|
||||
reference: ModelRef,
|
||||
target: FallbackTarget,
|
||||
},
|
||||
ChainEmpty,
|
||||
}
|
||||
|
||||
impl ModelFallbackNotice {
|
||||
fn code(&self) -> RunNoticeCode {
|
||||
match self {
|
||||
Self::ChainEmpty => RunNoticeCode::ModelFallbackChainEmpty,
|
||||
Self::ProviderUnconfigured { .. }
|
||||
| Self::NoConfiguredOffering { .. }
|
||||
| Self::PrimaryNotInCatalog { .. }
|
||||
| Self::NoCompatibleModel { .. }
|
||||
| Self::MatchesPrimary { .. }
|
||||
| Self::Duplicate { .. } => RunNoticeCode::ModelFallbackSkipped,
|
||||
}
|
||||
}
|
||||
|
||||
fn level(&self) -> RunNoticeLevel {
|
||||
match self {
|
||||
Self::MatchesPrimary { .. } | Self::Duplicate { .. } => RunNoticeLevel::Info,
|
||||
Self::ProviderUnconfigured { .. }
|
||||
| Self::NoConfiguredOffering { .. }
|
||||
| Self::PrimaryNotInCatalog { .. }
|
||||
| Self::NoCompatibleModel { .. }
|
||||
| Self::ChainEmpty => RunNoticeLevel::Warn,
|
||||
}
|
||||
}
|
||||
|
||||
fn message(&self) -> String {
|
||||
match self {
|
||||
Self::ProviderUnconfigured {
|
||||
reference,
|
||||
provider,
|
||||
} => {
|
||||
format!(
|
||||
"Model fallback `{reference}` was skipped because provider `{provider}` is not configured."
|
||||
)
|
||||
}
|
||||
Self::NoConfiguredOffering {
|
||||
reference,
|
||||
providers,
|
||||
} => {
|
||||
let providers = providers
|
||||
.iter()
|
||||
.map(ProviderId::to_string)
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ");
|
||||
format!(
|
||||
"Model fallback `{reference}` was skipped because none of its providers are configured. It is offered by: {providers}."
|
||||
)
|
||||
}
|
||||
Self::PrimaryNotInCatalog { reference, primary } => {
|
||||
format!(
|
||||
"Model fallback `{reference}` was skipped because the primary model `{primary}` is not in the catalog, so there is no capability profile to match against."
|
||||
)
|
||||
}
|
||||
Self::NoCompatibleModel {
|
||||
reference,
|
||||
provider,
|
||||
} => {
|
||||
format!(
|
||||
"Model fallback `{reference}` was skipped because provider `{provider}` has no compatible model."
|
||||
)
|
||||
}
|
||||
Self::MatchesPrimary { reference, target } => {
|
||||
format!(
|
||||
"Model fallback `{reference}` was skipped because it resolves to the primary target `{target}`."
|
||||
)
|
||||
}
|
||||
Self::Duplicate { reference, target } => {
|
||||
format!(
|
||||
"Model fallback `{reference}` was skipped because target `{target}` already appears in the fallback chain."
|
||||
)
|
||||
}
|
||||
Self::ChainEmpty => {
|
||||
"No usable model fallbacks remain after filtering the configured fallback candidates."
|
||||
.to_string()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Publish every notice on the run's event stream.
|
||||
fn emit_all(notices: &[Self], emitter: &Emitter) {
|
||||
for notice in notices {
|
||||
emitter.notice(notice.level(), notice.code(), notice.message());
|
||||
}
|
||||
}
|
||||
fallbacks: ResolvedModelFallbacks,
|
||||
}
|
||||
|
||||
pub struct StartServices {
|
||||
|
|
@ -604,7 +478,7 @@ impl RunSession {
|
|||
llm: LlmSpec {
|
||||
model: llm.model.clone(),
|
||||
provider_id: llm.provider_id.clone(),
|
||||
fallback_chain: llm.fallbacks.targets,
|
||||
fallbacks: llm.fallbacks.policy,
|
||||
mcp_servers,
|
||||
model_controls: resolved.model.controls.clone(),
|
||||
dry_run: resolved.execution.mode == RunMode::DryRun,
|
||||
|
|
@ -743,8 +617,7 @@ fn resolve_start_llm(
|
|||
settings.model.provider.as_deref(),
|
||||
false,
|
||||
)?;
|
||||
let fallbacks =
|
||||
resolve_fallback_chain(catalog, &provider_id, &model, &settings.model, &eligible)?;
|
||||
let fallbacks = resolve_model_fallbacks(catalog, &eligible, &settings.model.fallbacks)?;
|
||||
|
||||
Ok(ResolvedStartLlm {
|
||||
model,
|
||||
|
|
@ -753,166 +626,6 @@ fn resolve_start_llm(
|
|||
})
|
||||
}
|
||||
|
||||
/// Resolve fallback candidates against the configured provider snapshot.
|
||||
///
|
||||
/// Candidates that cannot be used in this environment — an unconfigured
|
||||
/// provider, no compatible model, a target equal to the primary, or a duplicate
|
||||
/// — are dropped, and each drop records a [`ModelFallbackNotice`] that the run
|
||||
/// emits at startup. Remaining candidates keep their configured order.
|
||||
///
|
||||
/// A provider the catalog has never heard of is a different case: that is a
|
||||
/// typo rather than an environment difference, so it fails the run instead of
|
||||
/// being skipped. This is what keeps a chain portable without letting a
|
||||
/// misspelled provider silently disappear.
|
||||
fn resolve_fallback_chain(
|
||||
catalog: &Catalog,
|
||||
provider: &ProviderId,
|
||||
model: &str,
|
||||
settings: &ResolvedRunModelSettings,
|
||||
eligible: &HashSet<ProviderId>,
|
||||
) -> Result<ResolvedFallbackChain, Error> {
|
||||
if settings.fallbacks.is_empty() {
|
||||
return Ok(ResolvedFallbackChain::default());
|
||||
}
|
||||
|
||||
let primary_model = catalog.get_on_provider(provider, model);
|
||||
let primary = FallbackTarget::new(provider, model);
|
||||
let mut resolution = ResolvedFallbackChain::default();
|
||||
|
||||
for model_ref in &settings.fallbacks {
|
||||
let target = match resolve_fallback_candidate(
|
||||
catalog,
|
||||
&primary,
|
||||
primary_model,
|
||||
eligible,
|
||||
model_ref,
|
||||
)? {
|
||||
FallbackCandidate::Skipped(notice) => {
|
||||
resolution.notices.push(notice);
|
||||
continue;
|
||||
}
|
||||
FallbackCandidate::Target(target) => target,
|
||||
};
|
||||
|
||||
let reference = model_ref.clone();
|
||||
if target == primary {
|
||||
resolution
|
||||
.notices
|
||||
.push(ModelFallbackNotice::MatchesPrimary { reference, target });
|
||||
} else if resolution.targets.contains(&target) {
|
||||
resolution
|
||||
.notices
|
||||
.push(ModelFallbackNotice::Duplicate { reference, target });
|
||||
} else {
|
||||
resolution.targets.push(target);
|
||||
}
|
||||
}
|
||||
|
||||
if resolution.targets.is_empty() {
|
||||
resolution.notices.push(ModelFallbackNotice::ChainEmpty);
|
||||
}
|
||||
|
||||
Ok(resolution)
|
||||
}
|
||||
|
||||
/// The outcome of resolving one fallback candidate: either a dispatchable
|
||||
/// target or the reason the candidate cannot be used.
|
||||
enum FallbackCandidate {
|
||||
Target(FallbackTarget),
|
||||
Skipped(ModelFallbackNotice),
|
||||
}
|
||||
|
||||
/// Resolve one fallback reference against the configured provider snapshot.
|
||||
///
|
||||
/// `primary_model` is the primary's catalog entry, used to pick the closest
|
||||
/// capability match when a candidate names a provider but no model. It is
|
||||
/// `None` when the primary is itself a passthrough selector.
|
||||
///
|
||||
/// A selector the catalog does not know passes through verbatim so a model
|
||||
/// newer than the catalog still dispatches. When the candidate named a
|
||||
/// provider, it passes through on that provider; when it did not, it passes
|
||||
/// through on the primary's provider, which means such a fallback gives no
|
||||
/// cross-provider failover.
|
||||
fn resolve_fallback_candidate(
|
||||
catalog: &Catalog,
|
||||
primary: &FallbackTarget,
|
||||
primary_model: Option<&Model>,
|
||||
eligible: &HashSet<ProviderId>,
|
||||
model_ref: &ModelRef,
|
||||
) -> Result<FallbackCandidate, Error> {
|
||||
let reference = model_ref.clone();
|
||||
|
||||
Ok(match model_ref.resolve(catalog)? {
|
||||
ResolvedModelRef::Provider(provider_name) => {
|
||||
let provider = catalog.provider_id(&provider_name)?;
|
||||
if !eligible.contains(&provider) {
|
||||
return Ok(FallbackCandidate::Skipped(
|
||||
ModelFallbackNotice::ProviderUnconfigured {
|
||||
reference,
|
||||
provider,
|
||||
},
|
||||
));
|
||||
}
|
||||
// Without a catalog entry for the primary there is no capability
|
||||
// profile to match against, which is not the provider's fault.
|
||||
let Some(primary_model) = primary_model else {
|
||||
return Ok(FallbackCandidate::Skipped(
|
||||
ModelFallbackNotice::PrimaryNotInCatalog {
|
||||
reference,
|
||||
primary: primary.clone(),
|
||||
},
|
||||
));
|
||||
};
|
||||
match catalog.closest(&provider, primary_model) {
|
||||
Some(model) => FallbackCandidate::Target(FallbackTarget::new(provider, &model.id)),
|
||||
None => FallbackCandidate::Skipped(ModelFallbackNotice::NoCompatibleModel {
|
||||
reference,
|
||||
provider,
|
||||
}),
|
||||
}
|
||||
}
|
||||
ResolvedModelRef::Model {
|
||||
provider: Some(provider_name),
|
||||
selector,
|
||||
} => {
|
||||
let provider = catalog.provider_id(&provider_name)?;
|
||||
if !eligible.contains(&provider) {
|
||||
return Ok(FallbackCandidate::Skipped(
|
||||
ModelFallbackNotice::ProviderUnconfigured {
|
||||
reference,
|
||||
provider,
|
||||
},
|
||||
));
|
||||
}
|
||||
match catalog.resolve_on_provider(&provider, &selector) {
|
||||
Ok(info) => {
|
||||
FallbackCandidate::Target(FallbackTarget::new(&info.provider, &info.id))
|
||||
}
|
||||
Err(ModelSelectionError::UnknownSelectorOnProvider { .. }) => {
|
||||
FallbackCandidate::Target(FallbackTarget::new(provider, selector))
|
||||
}
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
}
|
||||
ResolvedModelRef::Model {
|
||||
provider: None,
|
||||
selector,
|
||||
} => match catalog.select(&selector, None, eligible) {
|
||||
Ok(info) => FallbackCandidate::Target(FallbackTarget::new(&info.provider, &info.id)),
|
||||
Err(ModelSelectionError::NoEligibleOffering { providers, .. }) => {
|
||||
FallbackCandidate::Skipped(ModelFallbackNotice::NoConfiguredOffering {
|
||||
reference,
|
||||
providers,
|
||||
})
|
||||
}
|
||||
Err(ModelSelectionError::UnknownSelector { .. }) => {
|
||||
FallbackCandidate::Target(FallbackTarget::new(&primary.provider, selector))
|
||||
}
|
||||
Err(error) => return Err(error.into()),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
/// Build the launch-time MCP config from resolved settings. Secret tokens in
|
||||
/// the transport (`command`/`url`/`env`/`headers`) resolve from the vault at
|
||||
/// the run boundary. Unsupported tokens fail.
|
||||
|
|
@ -1035,7 +748,10 @@ impl RunSession {
|
|||
store_progress_logger.register(self.emitter.as_ref());
|
||||
// Emit after the logger is registered so the notices reach the run
|
||||
// store, and before `run.started` so they read as launch-time context.
|
||||
ModelFallbackNotice::emit_all(&self.fallback_notices, self.emitter.as_ref());
|
||||
for notice in &self.fallback_notices {
|
||||
self.emitter
|
||||
.notice(notice.level(), notice.code(), notice.message());
|
||||
}
|
||||
|
||||
let init_options = InitOptions {
|
||||
run_store: self.run_store.clone(),
|
||||
|
|
@ -1334,11 +1050,11 @@ mod tests {
|
|||
RunEnvironmentLayer, RunExecutionLayer, RunLayer, StickyMap, WorkflowSettingsBuilder,
|
||||
};
|
||||
use fabro_store::Database;
|
||||
use fabro_types::settings::InterpString;
|
||||
use fabro_types::settings::run::{
|
||||
McpTransport as ResolvedMcpTransport, PreparedStep, PreparedStepRun, RunMode,
|
||||
RunPrepareSettings,
|
||||
};
|
||||
use fabro_types::settings::{InterpString, ModelRef};
|
||||
use fabro_types::{
|
||||
BilledModelUsage, ManifestPath, StageTiming, WorkflowSettings, fixtures, test_support,
|
||||
};
|
||||
|
|
@ -1505,517 +1221,6 @@ reasoning = false
|
|||
Catalog::from_settings(&settings).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_start_llm_infers_primary_and_filters_global_fallbacks() {
|
||||
let catalog = portable_model_catalog();
|
||||
let mut settings = ResolvedRunSettings::default();
|
||||
settings.model.name = Some("gpt-56-sol".to_string());
|
||||
settings.model.fallbacks = vec![
|
||||
"openai:gpt-56-sol".parse::<ModelRef>().unwrap(),
|
||||
"openrouter:gpt-56-sol".parse::<ModelRef>().unwrap(),
|
||||
"openrouter:openai/gpt-5.6-sol".parse::<ModelRef>().unwrap(),
|
||||
];
|
||||
|
||||
let resolved = resolve_start_llm(
|
||||
&catalog,
|
||||
&[ProviderId::new("openrouter"), ProviderId::openai()],
|
||||
&settings,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resolved.provider_id, ProviderId::openai());
|
||||
assert_eq!(resolved.model, "gpt-5.6-sol");
|
||||
assert_eq!(resolved.fallbacks.targets, vec![FallbackTarget {
|
||||
provider: "openrouter".to_string(),
|
||||
model: "gpt-5.6-sol".to_string(),
|
||||
}]);
|
||||
assert_eq!(resolved.fallbacks.notices, vec![
|
||||
ModelFallbackNotice::MatchesPrimary {
|
||||
reference: "openai:gpt-56-sol".parse().unwrap(),
|
||||
target: FallbackTarget {
|
||||
provider: "openai".to_string(),
|
||||
model: "gpt-5.6-sol".to_string(),
|
||||
},
|
||||
},
|
||||
ModelFallbackNotice::Duplicate {
|
||||
reference: "openrouter:openai/gpt-5.6-sol".parse().unwrap(),
|
||||
target: FallbackTarget {
|
||||
provider: "openrouter".to_string(),
|
||||
model: "gpt-5.6-sol".to_string(),
|
||||
},
|
||||
},
|
||||
]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_fallback_chain_skips_unconfigured_provider_and_preserves_order() {
|
||||
let catalog = test_catalog();
|
||||
let settings = ResolvedRunModelSettings {
|
||||
fallbacks: vec![
|
||||
"gemini".parse::<ModelRef>().unwrap(),
|
||||
"gemini:unused".parse::<ModelRef>().unwrap(),
|
||||
"openai:gpt-5.4-mini".parse::<ModelRef>().unwrap(),
|
||||
"anthropic:claude-fable-5".parse::<ModelRef>().unwrap(),
|
||||
],
|
||||
..ResolvedRunModelSettings::default()
|
||||
};
|
||||
|
||||
let resolution = resolve_fallback_chain(
|
||||
catalog.as_ref(),
|
||||
&ProviderId::anthropic(),
|
||||
"claude-opus-4-6",
|
||||
&settings,
|
||||
&HashSet::from([ProviderId::anthropic(), ProviderId::openai()]),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resolution.targets, vec![
|
||||
FallbackTarget {
|
||||
provider: "openai".to_string(),
|
||||
model: "gpt-5.4-mini".to_string(),
|
||||
},
|
||||
FallbackTarget {
|
||||
provider: "anthropic".to_string(),
|
||||
model: "claude-fable-5".to_string(),
|
||||
},
|
||||
]);
|
||||
assert_eq!(resolution.notices, vec![
|
||||
ModelFallbackNotice::ProviderUnconfigured {
|
||||
reference: "gemini".parse().unwrap(),
|
||||
provider: ProviderId::gemini(),
|
||||
},
|
||||
ModelFallbackNotice::ProviderUnconfigured {
|
||||
reference: "gemini:unused".parse().unwrap(),
|
||||
provider: ProviderId::gemini(),
|
||||
},
|
||||
]);
|
||||
assert_eq!(resolution.notices[0].level(), RunNoticeLevel::Warn);
|
||||
assert_eq!(
|
||||
resolution.notices[0].code(),
|
||||
RunNoticeCode::ModelFallbackSkipped
|
||||
);
|
||||
}
|
||||
|
||||
/// The resolver builds notices before the run's event sink exists, so this
|
||||
/// covers the hand-off: each notice must reach the event stream as a
|
||||
/// `run.notice` carrying its own level, code, and rendered message.
|
||||
#[test]
|
||||
fn fallback_notices_reach_the_event_stream() {
|
||||
let emitter = Arc::new(Emitter::new(fixtures::RUN_1));
|
||||
let captured = Arc::new(Mutex::new(Vec::new()));
|
||||
let sink = Arc::clone(&captured);
|
||||
emitter.on_event(move |event| sink.lock().unwrap().push(event.clone()));
|
||||
|
||||
let notices = vec![
|
||||
ModelFallbackNotice::ProviderUnconfigured {
|
||||
reference: "gemini".parse().unwrap(),
|
||||
provider: ProviderId::gemini(),
|
||||
},
|
||||
ModelFallbackNotice::MatchesPrimary {
|
||||
reference: "openai:gpt-5.6-sol".parse().unwrap(),
|
||||
target: FallbackTarget::new("openai", "gpt-5.6-sol"),
|
||||
},
|
||||
ModelFallbackNotice::ChainEmpty,
|
||||
];
|
||||
|
||||
ModelFallbackNotice::emit_all(¬ices, emitter.as_ref());
|
||||
|
||||
let events = captured.lock().unwrap();
|
||||
let emitted = events
|
||||
.iter()
|
||||
.map(|event| match &event.body {
|
||||
EventBody::RunNotice(props) => {
|
||||
(props.level, props.code.clone(), props.message.clone())
|
||||
}
|
||||
other => panic!("expected run.notice body, got {other:?}"),
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
assert_eq!(emitted, vec![
|
||||
(
|
||||
RunNoticeLevel::Warn,
|
||||
RunNoticeCode::ModelFallbackSkipped.to_string(),
|
||||
"Model fallback `gemini` was skipped because provider `gemini` is not configured."
|
||||
.to_string(),
|
||||
),
|
||||
(
|
||||
RunNoticeLevel::Info,
|
||||
RunNoticeCode::ModelFallbackSkipped.to_string(),
|
||||
"Model fallback `openai:gpt-5.6-sol` was skipped because it resolves to the primary target `openai:gpt-5.6-sol`."
|
||||
.to_string(),
|
||||
),
|
||||
(
|
||||
RunNoticeLevel::Warn,
|
||||
RunNoticeCode::ModelFallbackChainEmpty.to_string(),
|
||||
"No usable model fallbacks remain after filtering the configured fallback candidates."
|
||||
.to_string(),
|
||||
),
|
||||
]);
|
||||
}
|
||||
|
||||
/// A provider-only fallback cannot be matched when the primary model is a
|
||||
/// passthrough selector, because there is no capability profile to compare
|
||||
/// against. The notice must name that cause rather than blaming the
|
||||
/// provider, which may well have compatible models.
|
||||
#[test]
|
||||
fn resolve_fallback_chain_blames_missing_primary_not_the_fallback_provider() {
|
||||
let catalog = portable_model_catalog();
|
||||
let settings = ResolvedRunModelSettings {
|
||||
fallbacks: vec!["openrouter".parse::<ModelRef>().unwrap()],
|
||||
..ResolvedRunModelSettings::default()
|
||||
};
|
||||
|
||||
let resolution = resolve_fallback_chain(
|
||||
&catalog,
|
||||
&ProviderId::openai(),
|
||||
"gpt-5.9-not-in-catalog",
|
||||
&settings,
|
||||
&HashSet::from([ProviderId::openai(), ProviderId::new("openrouter")]),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert!(resolution.targets.is_empty());
|
||||
assert_eq!(resolution.notices, vec![
|
||||
ModelFallbackNotice::PrimaryNotInCatalog {
|
||||
reference: "openrouter".parse().unwrap(),
|
||||
primary: FallbackTarget::new("openai", "gpt-5.9-not-in-catalog"),
|
||||
},
|
||||
ModelFallbackNotice::ChainEmpty,
|
||||
]);
|
||||
let message = resolution.notices[0].message();
|
||||
assert!(
|
||||
message.contains("primary model `openai:gpt-5.9-not-in-catalog` is not in the catalog"),
|
||||
"notice should name the missing primary: {message}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_fallback_chain_skips_model_without_configured_offering() {
|
||||
let catalog = portable_model_catalog();
|
||||
let settings = ResolvedRunModelSettings {
|
||||
fallbacks: vec!["mini".parse::<ModelRef>().unwrap()],
|
||||
..ResolvedRunModelSettings::default()
|
||||
};
|
||||
|
||||
let resolution = resolve_fallback_chain(
|
||||
&catalog,
|
||||
&ProviderId::new("openrouter"),
|
||||
"gpt-5.6-sol",
|
||||
&settings,
|
||||
&HashSet::from([ProviderId::new("openrouter")]),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert!(resolution.targets.is_empty());
|
||||
assert_eq!(resolution.notices, vec![
|
||||
ModelFallbackNotice::NoConfiguredOffering {
|
||||
reference: "mini".parse().unwrap(),
|
||||
providers: vec![ProviderId::openai()],
|
||||
},
|
||||
ModelFallbackNotice::ChainEmpty,
|
||||
]);
|
||||
assert_eq!(
|
||||
resolution.notices[1].code(),
|
||||
RunNoticeCode::ModelFallbackChainEmpty
|
||||
);
|
||||
assert_eq!(resolution.notices[1].level(), RunNoticeLevel::Warn);
|
||||
assert!(
|
||||
resolution.notices[0]
|
||||
.message()
|
||||
.contains("offered by: openai"),
|
||||
"notice should name the providers that offer the model: {}",
|
||||
resolution.notices[0].message()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_fallback_chain_rejects_unknown_qualified_provider() {
|
||||
let catalog = portable_model_catalog();
|
||||
let settings = ResolvedRunModelSettings {
|
||||
fallbacks: vec!["missing/model".parse::<ModelRef>().unwrap()],
|
||||
..ResolvedRunModelSettings::default()
|
||||
};
|
||||
|
||||
let error = resolve_fallback_chain(
|
||||
&catalog,
|
||||
&ProviderId::openai(),
|
||||
"gpt-5.6-sol",
|
||||
&settings,
|
||||
&catalog.all_provider_ids(),
|
||||
)
|
||||
.unwrap_err();
|
||||
|
||||
assert!(matches!(
|
||||
error,
|
||||
Error::ModelSelection(ModelSelectionError::UnknownProvider { provider })
|
||||
if provider == ProviderId::new("missing")
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_fallback_chain_resolves_provider_fallbacks() {
|
||||
let catalog = test_catalog();
|
||||
let settings = ResolvedRunModelSettings {
|
||||
fallbacks: vec!["openai".parse::<ModelRef>().unwrap()],
|
||||
..ResolvedRunModelSettings::default()
|
||||
};
|
||||
|
||||
let chain = resolve_fallback_chain(
|
||||
catalog.as_ref(),
|
||||
&ProviderId::anthropic(),
|
||||
"claude-opus-4-6",
|
||||
&settings,
|
||||
&catalog.all_provider_ids(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(chain.targets, vec![FallbackTarget {
|
||||
provider: "openai".to_string(),
|
||||
model: "gpt-5.5".to_string(),
|
||||
}]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_fallback_chain_resolves_explicit_model_fallbacks() {
|
||||
let catalog = test_catalog();
|
||||
let settings = ResolvedRunModelSettings {
|
||||
fallbacks: vec!["openai:gpt-5.4-mini".parse::<ModelRef>().unwrap()],
|
||||
..ResolvedRunModelSettings::default()
|
||||
};
|
||||
|
||||
let chain = resolve_fallback_chain(
|
||||
catalog.as_ref(),
|
||||
&ProviderId::anthropic(),
|
||||
"claude-opus-4-6",
|
||||
&settings,
|
||||
&catalog.all_provider_ids(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(chain.targets, vec![FallbackTarget {
|
||||
provider: "openai".to_string(),
|
||||
model: "gpt-5.4-mini".to_string(),
|
||||
}]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_fallback_chain_selects_shared_bare_alias_from_ready_providers() {
|
||||
let catalog = portable_model_catalog();
|
||||
let settings = ResolvedRunModelSettings {
|
||||
fallbacks: vec!["gpt-56-sol".parse::<ModelRef>().unwrap()],
|
||||
..ResolvedRunModelSettings::default()
|
||||
};
|
||||
|
||||
let chain = resolve_fallback_chain(
|
||||
&catalog,
|
||||
&ProviderId::openai(),
|
||||
"gpt-5.6-sol",
|
||||
&settings,
|
||||
&HashSet::from([ProviderId::new("openrouter")]),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(chain.targets, vec![FallbackTarget {
|
||||
provider: "openrouter".to_string(),
|
||||
model: "gpt-5.6-sol".to_string(),
|
||||
}]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_fallback_chain_resolves_provider_qualified_shared_alias() {
|
||||
let catalog = portable_model_catalog();
|
||||
let settings = ResolvedRunModelSettings {
|
||||
fallbacks: vec!["openrouter:gpt-56-sol".parse::<ModelRef>().unwrap()],
|
||||
..ResolvedRunModelSettings::default()
|
||||
};
|
||||
|
||||
let chain = resolve_fallback_chain(
|
||||
&catalog,
|
||||
&ProviderId::openai(),
|
||||
"gpt-5.6-sol",
|
||||
&settings,
|
||||
&catalog.all_provider_ids(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(chain.targets, vec![FallbackTarget {
|
||||
provider: "openrouter".to_string(),
|
||||
model: "gpt-5.6-sol".to_string(),
|
||||
}]);
|
||||
}
|
||||
|
||||
/// A qualified fallback resolves to the same offering whether the selector
|
||||
/// is the canonical model ID or the provider's API ID. The trailing bare
|
||||
/// alias still goes through ready-provider priority selection.
|
||||
#[test]
|
||||
fn resolve_fallback_chain_resolves_qualified_model_id_and_api_id_alike() {
|
||||
let overrides: fabro_model::catalog::LlmCatalogSettings = toml::from_str(
|
||||
r"
|
||||
[providers.openrouter]
|
||||
enabled = true
|
||||
",
|
||||
)
|
||||
.unwrap();
|
||||
let catalog = Catalog::from_builtin_with_overrides(&overrides).unwrap();
|
||||
|
||||
for selector in ["openrouter:kimi-k3", "openrouter:moonshotai/kimi-k3"] {
|
||||
let settings = ResolvedRunModelSettings {
|
||||
fallbacks: vec![
|
||||
selector.parse::<ModelRef>().unwrap(),
|
||||
"gpt-terra".parse::<ModelRef>().unwrap(),
|
||||
],
|
||||
..ResolvedRunModelSettings::default()
|
||||
};
|
||||
|
||||
let chain = resolve_fallback_chain(
|
||||
&catalog,
|
||||
&ProviderId::new("kimi"),
|
||||
"kimi-k3",
|
||||
&settings,
|
||||
&HashSet::from([
|
||||
ProviderId::new("kimi"),
|
||||
ProviderId::new("openrouter"),
|
||||
ProviderId::openai(),
|
||||
]),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
chain.targets,
|
||||
vec![
|
||||
FallbackTarget {
|
||||
provider: "openrouter".to_string(),
|
||||
model: "kimi-k3".to_string(),
|
||||
},
|
||||
FallbackTarget {
|
||||
provider: "openai".to_string(),
|
||||
model: "gpt-5.6-terra".to_string(),
|
||||
},
|
||||
],
|
||||
"{selector}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// A colon in a model ID does not make it provider-qualified, so an
|
||||
/// unknown colon-bearing selector still passes through to a provider
|
||||
/// instead of failing the run with an unknown-provider error.
|
||||
#[test]
|
||||
fn resolve_fallback_chain_passes_through_colon_bearing_model_ids() {
|
||||
let catalog = portable_model_catalog();
|
||||
let settings = ResolvedRunModelSettings {
|
||||
fallbacks: vec!["future-model:latest".parse::<ModelRef>().unwrap()],
|
||||
..ResolvedRunModelSettings::default()
|
||||
};
|
||||
|
||||
let chain = resolve_fallback_chain(
|
||||
&catalog,
|
||||
&ProviderId::openai(),
|
||||
"gpt-5.6-sol",
|
||||
&settings,
|
||||
&HashSet::from([ProviderId::openai(), ProviderId::new("openrouter")]),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(chain.targets, vec![FallbackTarget {
|
||||
provider: ProviderId::openai().to_string(),
|
||||
model: "future-model:latest".to_string(),
|
||||
}]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_fallback_chain_keeps_qualified_legacy_references_as_provider_pins() {
|
||||
let catalog = test_catalog();
|
||||
let settings = ResolvedRunModelSettings {
|
||||
fallbacks: vec![
|
||||
"openai/gpt-5.6-sol".parse::<ModelRef>().unwrap(),
|
||||
"anthropic/claude-fable-5".parse::<ModelRef>().unwrap(),
|
||||
],
|
||||
..ResolvedRunModelSettings::default()
|
||||
};
|
||||
|
||||
let chain = resolve_fallback_chain(
|
||||
catalog.as_ref(),
|
||||
&ProviderId::anthropic(),
|
||||
"claude-opus-4-6",
|
||||
&settings,
|
||||
&catalog.all_provider_ids(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(chain.targets, vec![
|
||||
FallbackTarget {
|
||||
provider: "openai".to_string(),
|
||||
model: "gpt-5.6-sol".to_string(),
|
||||
},
|
||||
FallbackTarget {
|
||||
provider: "anthropic".to_string(),
|
||||
model: "claude-fable-5".to_string(),
|
||||
},
|
||||
]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_fallback_chain_propagates_provider_model_ambiguity() {
|
||||
let settings: fabro_model::catalog::LlmCatalogSettings = toml::from_str(
|
||||
r#"
|
||||
[providers.shared]
|
||||
display_name = "Shared Provider"
|
||||
adapter = "openai"
|
||||
agent_profile = "openai"
|
||||
|
||||
[providers.shared.models.default]
|
||||
display_name = "Default"
|
||||
family = "test"
|
||||
default = true
|
||||
|
||||
[providers.shared.models.default.limits]
|
||||
context_window = 1000
|
||||
|
||||
[providers.shared.models.default.features]
|
||||
tools = false
|
||||
vision = false
|
||||
reasoning = false
|
||||
|
||||
[providers.other]
|
||||
display_name = "Other"
|
||||
adapter = "openai"
|
||||
agent_profile = "openai"
|
||||
|
||||
[providers.other.models.model]
|
||||
display_name = "Shared Alias"
|
||||
family = "test"
|
||||
aliases = ["shared"]
|
||||
default = true
|
||||
|
||||
[providers.other.models.model.limits]
|
||||
context_window = 1000
|
||||
|
||||
[providers.other.models.model.features]
|
||||
tools = false
|
||||
vision = false
|
||||
reasoning = false
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
let catalog = Catalog::from_settings(&settings).unwrap();
|
||||
let run_model = ResolvedRunModelSettings {
|
||||
fallbacks: vec!["shared".parse::<ModelRef>().unwrap()],
|
||||
..ResolvedRunModelSettings::default()
|
||||
};
|
||||
|
||||
let error = resolve_fallback_chain(
|
||||
&catalog,
|
||||
&ProviderId::new("other"),
|
||||
"model",
|
||||
&run_model,
|
||||
&catalog.all_provider_ids(),
|
||||
)
|
||||
.unwrap_err();
|
||||
|
||||
assert!(matches!(error, Error::ModelReference(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn materialized_provider_pin_is_not_reselected_when_readiness_changes() {
|
||||
let catalog = portable_model_catalog();
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ use crate::error::Error;
|
|||
use crate::event::{Emitter, Event, StoreProgressLogger, append_event};
|
||||
use crate::handler::start::StartHandler;
|
||||
use crate::handler::{Handler as HandlerTrait, HandlerRegistry};
|
||||
use crate::model_fallback::ModelFallbackPolicy;
|
||||
use crate::outcome::{Outcome, OutcomeExt, StageOutcome};
|
||||
use crate::pipeline::initialize;
|
||||
use crate::pipeline::types::{InitOptions, LlmSpec, Persisted, ResumeState, SandboxEnvSpec};
|
||||
|
|
@ -267,7 +268,7 @@ async fn execute_test_run_with_options(
|
|||
llm: LlmSpec {
|
||||
model: "test-model".to_string(),
|
||||
provider_id: fabro_model::ProviderId::anthropic(),
|
||||
fallback_chain: Vec::new(),
|
||||
fallbacks: ModelFallbackPolicy::default(),
|
||||
mcp_servers: Vec::new(),
|
||||
model_controls: RunModelControls::default(),
|
||||
dry_run: true,
|
||||
|
|
@ -327,7 +328,7 @@ async fn execute_runs_start_to_exit_and_returns_final_context() {
|
|||
llm: LlmSpec {
|
||||
model: "test-model".to_string(),
|
||||
provider_id: fabro_model::ProviderId::anthropic(),
|
||||
fallback_chain: Vec::new(),
|
||||
fallbacks: ModelFallbackPolicy::default(),
|
||||
mcp_servers: Vec::new(),
|
||||
model_controls: RunModelControls::default(),
|
||||
dry_run: true,
|
||||
|
|
@ -468,7 +469,7 @@ async fn resumed_in_flight_node_starts_a_new_stage_execution() {
|
|||
llm: LlmSpec {
|
||||
model: "test-model".to_string(),
|
||||
provider_id: fabro_model::ProviderId::anthropic(),
|
||||
fallback_chain: Vec::new(),
|
||||
fallbacks: ModelFallbackPolicy::default(),
|
||||
mcp_servers: Vec::new(),
|
||||
model_controls: RunModelControls::default(),
|
||||
dry_run: true,
|
||||
|
|
@ -582,7 +583,7 @@ async fn run_with_lifecycle(
|
|||
llm: LlmSpec {
|
||||
model: "test-model".to_string(),
|
||||
provider_id: fabro_model::ProviderId::anthropic(),
|
||||
fallback_chain: Vec::new(),
|
||||
fallbacks: ModelFallbackPolicy::default(),
|
||||
mcp_servers: Vec::new(),
|
||||
model_controls: RunModelControls::default(),
|
||||
dry_run: true,
|
||||
|
|
|
|||
|
|
@ -25,6 +25,8 @@ use crate::git::GitAuthor;
|
|||
use crate::github_token_source::{AppIatMinter, GitHubTokenSource};
|
||||
use crate::handler::llm::{AgentAcpBackend, AgentApiBackend, BackendRouter, routing};
|
||||
use crate::handler::{HandlerRegistry, default_registry};
|
||||
#[cfg(test)]
|
||||
use crate::model_fallback::ModelFallbackPolicy;
|
||||
use crate::run_metadata::{RunMetadataRuntime, build_metadata_writer, metadata_branch_name};
|
||||
use crate::run_options::{GitCheckpointOptions, RunOptions};
|
||||
use crate::sandbox_git_runtime::SandboxGitRuntime;
|
||||
|
|
@ -166,7 +168,7 @@ async fn build_registry(
|
|||
let build_llm_registry = || {
|
||||
let model = spec.model.clone();
|
||||
let provider_id = spec.provider_id.clone();
|
||||
let fallback_chain = spec.fallback_chain.clone();
|
||||
let fallbacks = spec.fallbacks.clone();
|
||||
let mcp_servers = spec.mcp_servers.clone();
|
||||
let model_controls = spec.model_controls.clone();
|
||||
let tool_secrets_for_api = tool_secrets.clone();
|
||||
|
|
@ -180,7 +182,7 @@ async fn build_registry(
|
|||
let mut api = AgentApiBackend::new_with_catalog(
|
||||
model.clone(),
|
||||
provider_id.clone(),
|
||||
fallback_chain.clone(),
|
||||
fallbacks.clone(),
|
||||
Arc::clone(&llm_source_for_api),
|
||||
Arc::clone(&steering_hub_for_api),
|
||||
Arc::clone(&catalog_for_api),
|
||||
|
|
@ -830,7 +832,7 @@ mod tests {
|
|||
llm: LlmSpec {
|
||||
model: "test-model".to_string(),
|
||||
provider_id: fabro_model::ProviderId::anthropic(),
|
||||
fallback_chain: Vec::new(),
|
||||
fallbacks: ModelFallbackPolicy::default(),
|
||||
mcp_servers: Vec::new(),
|
||||
model_controls: RunModelControls::default(),
|
||||
dry_run: true,
|
||||
|
|
@ -911,7 +913,7 @@ mod tests {
|
|||
llm: LlmSpec {
|
||||
model: "test-model".to_string(),
|
||||
provider_id: fabro_model::ProviderId::anthropic(),
|
||||
fallback_chain: Vec::new(),
|
||||
fallbacks: ModelFallbackPolicy::default(),
|
||||
mcp_servers: Vec::new(),
|
||||
model_controls: RunModelControls::default(),
|
||||
dry_run: true,
|
||||
|
|
@ -1010,7 +1012,7 @@ mod tests {
|
|||
&LlmSpec {
|
||||
model: "claude-opus-4-6".to_string(),
|
||||
provider_id: fabro_model::ProviderId::anthropic(),
|
||||
fallback_chain: Vec::new(),
|
||||
fallbacks: ModelFallbackPolicy::default(),
|
||||
mcp_servers: Vec::new(),
|
||||
model_controls: RunModelControls::default(),
|
||||
dry_run: false,
|
||||
|
|
@ -1132,7 +1134,7 @@ mod tests {
|
|||
llm: LlmSpec {
|
||||
model: "fake-acp".to_string(),
|
||||
provider_id: fabro_model::ProviderId::openai(),
|
||||
fallback_chain: Vec::new(),
|
||||
fallbacks: ModelFallbackPolicy::default(),
|
||||
mcp_servers: Vec::new(),
|
||||
model_controls: RunModelControls::default(),
|
||||
dry_run: false,
|
||||
|
|
@ -1227,7 +1229,7 @@ mod tests {
|
|||
llm: LlmSpec {
|
||||
model: "test-model".to_string(),
|
||||
provider_id: fabro_model::ProviderId::anthropic(),
|
||||
fallback_chain: Vec::new(),
|
||||
fallbacks: ModelFallbackPolicy::default(),
|
||||
mcp_servers: Vec::new(),
|
||||
model_controls: RunModelControls::default(),
|
||||
dry_run: true,
|
||||
|
|
@ -1369,7 +1371,7 @@ mod tests {
|
|||
llm: LlmSpec {
|
||||
model: "test-model".to_string(),
|
||||
provider_id: fabro_model::ProviderId::anthropic(),
|
||||
fallback_chain: Vec::new(),
|
||||
fallbacks: ModelFallbackPolicy::default(),
|
||||
mcp_servers: Vec::new(),
|
||||
model_controls: RunModelControls::default(),
|
||||
dry_run: true,
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ use std::sync::Arc;
|
|||
use fabro_graphviz::graph::Graph;
|
||||
use fabro_interview::Interviewer;
|
||||
use fabro_mcp::config::McpServerSettings;
|
||||
use fabro_model::{Catalog, FallbackTarget, ProviderId};
|
||||
use fabro_model::{Catalog, ProviderId};
|
||||
use fabro_sandbox::SandboxSpec;
|
||||
use fabro_template::TemplateContext;
|
||||
use fabro_types::settings::run::{PullRequestSettings, RunModelControls};
|
||||
|
|
@ -20,6 +20,7 @@ use crate::error::Error;
|
|||
use crate::event::Emitter;
|
||||
use crate::file_resolver::FileResolver;
|
||||
use crate::handler::HandlerRegistry;
|
||||
use crate::model_fallback::ModelFallbackPolicy;
|
||||
use crate::outcome::Outcome;
|
||||
use crate::records::{Checkpoint, Conclusion, RunSpec};
|
||||
use crate::run_control::RunControlState;
|
||||
|
|
@ -236,7 +237,7 @@ impl Persisted {
|
|||
pub struct LlmSpec {
|
||||
pub model: String,
|
||||
pub provider_id: ProviderId,
|
||||
pub fallback_chain: Vec<FallbackTarget>,
|
||||
pub fallbacks: ModelFallbackPolicy,
|
||||
pub mcp_servers: Vec<McpServerSettings>,
|
||||
pub model_controls: RunModelControls,
|
||||
pub dry_run: bool,
|
||||
|
|
|
|||
|
|
@ -50,6 +50,7 @@ use fabro_workflow::handler::manager_loop::SubWorkflowHandler;
|
|||
use fabro_workflow::handler::start::StartHandler;
|
||||
use fabro_workflow::handler::wait::WaitHandler;
|
||||
use fabro_workflow::handler::{Handler, HandlerRegistry};
|
||||
use fabro_workflow::model_fallback::ModelFallbackPolicy;
|
||||
use fabro_workflow::outcome::{Outcome, OutcomeExt, StageOutcome};
|
||||
use fabro_workflow::records::{Checkpoint, CheckpointExt};
|
||||
use fabro_workflow::run_options::{GitCheckpointOptions, RunOptions};
|
||||
|
|
@ -2332,7 +2333,7 @@ reasoning = false
|
|||
let backend = AgentApiBackend::new_with_catalog(
|
||||
"compact-model".to_string(),
|
||||
ProviderId::from("compact"),
|
||||
Vec::new(),
|
||||
ModelFallbackPolicy::default(),
|
||||
source,
|
||||
Arc::new(SteeringHub::new(Arc::new(Emitter::default()))),
|
||||
catalog,
|
||||
|
|
@ -2487,7 +2488,7 @@ base_url = "{}"
|
|||
let backend = AgentApiBackend::new_with_catalog(
|
||||
"openai/gpt-5.4".to_string(),
|
||||
ProviderId::from("openrouter"),
|
||||
Vec::new(),
|
||||
ModelFallbackPolicy::default(),
|
||||
source,
|
||||
Arc::new(SteeringHub::new(Arc::new(Emitter::default()))),
|
||||
catalog,
|
||||
|
|
|
|||
|
|
@ -746,7 +746,7 @@ command = ["demo-mcp"]
|
|||
model: Some(RunModelLayer {
|
||||
provider: Some("openai".to_string()),
|
||||
name: Some("gpt-5".to_string()),
|
||||
fallbacks: Vec::new(),
|
||||
fallbacks: crate::MergeMap::default(),
|
||||
controls: None,
|
||||
}),
|
||||
execution: Some(RunExecutionLayer {
|
||||
|
|
|
|||
|
|
@ -148,14 +148,19 @@ pub struct RunModelLayer {
|
|||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
#[option(value_type = "string")]
|
||||
pub name: Option<String>,
|
||||
/// Ordered fallback references: bare providers, bare model IDs or aliases,
|
||||
/// or provider-qualified `provider:selector` values. A qualified selector
|
||||
/// may be a model ID, alias, or provider API ID. Legacy `provider/model`
|
||||
/// values remain accepted. Supports the `...` splice marker at layering
|
||||
/// time — see [`super::splice_array`].
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
#[option(default = "[]", value_type = "array<string>")]
|
||||
pub fallbacks: Vec<ModelRefOrSplice>,
|
||||
/// Model-keyed fallback chains. Each value is an ordered list of bare
|
||||
/// providers, bare model IDs or aliases, or provider-qualified
|
||||
/// `provider:selector` values. A qualified selector may be a model ID,
|
||||
/// alias, or provider API ID. Legacy `provider/model` values remain
|
||||
/// accepted. Each list supports the `...` splice marker at layering time.
|
||||
/// Fabro selects one chain from the original requested model. A fallback
|
||||
/// target never activates another model's chain.
|
||||
///
|
||||
/// Model keys stay unresolved in this sparse layer because `fabro
|
||||
/// validate` is offline and has no server model catalog.
|
||||
#[serde(default, skip_serializing_if = "MergeMap::is_empty")]
|
||||
#[option(default = "{}", value_type = "table<string, array<string>>")]
|
||||
pub fallbacks: MergeMap<Vec<ModelRefOrSplice>>,
|
||||
/// Run-level default values for typed model controls. Node attributes
|
||||
/// and style-applied attributes still win over these defaults.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
use std::collections::HashMap;
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
|
||||
use fabro_types::settings::InterpString;
|
||||
use fabro_types::settings::run::{
|
||||
|
|
@ -123,11 +123,17 @@ fn resolve_model(model: Option<&RunModelLayer>) -> RunModelSettings {
|
|||
fallbacks: model
|
||||
.fallbacks
|
||||
.iter()
|
||||
.filter_map(|entry| match entry {
|
||||
ModelRefOrSplice::ModelRef(model_ref) => Some(model_ref.clone()),
|
||||
ModelRefOrSplice::Splice => None,
|
||||
.map(|(requested_model, chain)| {
|
||||
let chain = chain
|
||||
.iter()
|
||||
.filter_map(|entry| match entry {
|
||||
ModelRefOrSplice::ModelRef(model_ref) => Some(model_ref.clone()),
|
||||
ModelRefOrSplice::Splice => None,
|
||||
})
|
||||
.collect();
|
||||
(requested_model.clone(), chain)
|
||||
})
|
||||
.collect(),
|
||||
.collect::<BTreeMap<_, _>>(),
|
||||
controls: model
|
||||
.controls
|
||||
.as_ref()
|
||||
|
|
|
|||
|
|
@ -100,21 +100,46 @@ script = "higher-1"
|
|||
fn run_model_fallbacks_splice_inserts_inherited() {
|
||||
let lower = parse(
|
||||
r#"
|
||||
[run.model]
|
||||
fallbacks = ["openrouter:moonshotai/kimi-k3", "gpt-terra"]
|
||||
[run.model.fallbacks]
|
||||
"kimi-k3" = ["openrouter:moonshotai/kimi-k3", "gpt-terra"]
|
||||
"gpt-sol" = ["claude-opus"]
|
||||
"#,
|
||||
);
|
||||
let higher = parse(
|
||||
r#"
|
||||
[run.model]
|
||||
fallbacks = ["anthropic", "..."]
|
||||
[run.model.fallbacks]
|
||||
"kimi-k3" = ["anthropic", "..."]
|
||||
"claude-fable" = ["gpt-sol", "claude-opus"]
|
||||
"#,
|
||||
);
|
||||
let merged = higher.combine(lower);
|
||||
let fallbacks = merged.run.unwrap().model.unwrap().fallbacks;
|
||||
assert_eq!(
|
||||
serde_json::to_value(&fallbacks).unwrap(),
|
||||
serde_json::json!(["anthropic", "openrouter:moonshotai/kimi-k3", "gpt-terra",])
|
||||
serde_json::to_value(fallbacks.get("kimi-k3").unwrap()).unwrap(),
|
||||
serde_json::json!(["anthropic", "openrouter:moonshotai/kimi-k3", "gpt-terra"])
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::to_value(fallbacks.get("gpt-sol").unwrap()).unwrap(),
|
||||
serde_json::json!(["claude-opus"])
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::to_value(fallbacks.get("claude-fable").unwrap()).unwrap(),
|
||||
serde_json::json!(["gpt-sol", "claude-opus"])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn run_model_fallbacks_rejects_the_removed_array_shape() {
|
||||
let error = r#"
|
||||
[run.model]
|
||||
fallbacks = ["anthropic"]
|
||||
"#
|
||||
.parse::<SettingsLayer>()
|
||||
.expect_err("the global fallback array must stay unsupported");
|
||||
|
||||
assert!(
|
||||
error.to_string().contains("invalid type: sequence"),
|
||||
"unexpected parse error: {error}"
|
||||
);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -109,7 +109,9 @@ permissions = "read-write""#,
|
|||
r#"[run.model]
|
||||
provider = "anthropic"
|
||||
name = "claude-sonnet-4-5"
|
||||
fallbacks = ["openrouter:kimi-k3", "gpt-terra"]"#,
|
||||
|
||||
[run.model.fallbacks]
|
||||
"claude-sonnet-4-5" = ["openrouter:kimi-k3", "gpt-terra"]"#,
|
||||
),
|
||||
Section::of::<fabro_config::CliLoggingLayer>(
|
||||
"[cli.logging]",
|
||||
|
|
|
|||
|
|
@ -1482,36 +1482,6 @@ impl Catalog {
|
|||
.unwrap_or(std::cmp::Ordering::Equal)
|
||||
})
|
||||
}
|
||||
|
||||
/// Build an ordered fallback chain for a primary provider/model.
|
||||
///
|
||||
/// For each fallback provider, finds the closest matching model. Providers
|
||||
/// where no capability match exists (or the provider string doesn't
|
||||
/// parse) are skipped.
|
||||
#[must_use]
|
||||
pub fn build_fallback_chain(
|
||||
&self,
|
||||
primary: &ProviderId,
|
||||
model: &str,
|
||||
fallbacks: &HashMap<String, Vec<String>>,
|
||||
) -> Vec<FallbackTarget> {
|
||||
let Some(reference) = self.get_on_provider(primary, model) else {
|
||||
return Vec::new();
|
||||
};
|
||||
|
||||
let Some(fallback_providers) = fallbacks.get(primary.as_str()) else {
|
||||
return Vec::new();
|
||||
};
|
||||
|
||||
fallback_providers
|
||||
.iter()
|
||||
.filter_map(|provider_str| {
|
||||
let provider = ProviderId::from(provider_str.clone());
|
||||
self.closest(&provider, reference)
|
||||
.map(|m| FallbackTarget::new(provider_str, &m.id))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
type ModelIndexes = (
|
||||
|
|
@ -4200,71 +4170,6 @@ enabled = true
|
|||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builtin_build_fallback_chain() {
|
||||
let fallbacks = HashMap::from([("anthropic".to_string(), vec![
|
||||
"gemini".to_string(),
|
||||
"openai".to_string(),
|
||||
])]);
|
||||
let chain = Catalog::builtin().build_fallback_chain(
|
||||
&ProviderId::anthropic(),
|
||||
"claude-opus-4-6",
|
||||
&fallbacks,
|
||||
);
|
||||
assert_eq!(chain.len(), 2);
|
||||
assert_eq!(chain[0].provider, "gemini");
|
||||
assert_eq!(chain[0].model, "gemini-3.1-pro-preview");
|
||||
assert_eq!(chain[1].provider, "openai");
|
||||
assert_eq!(chain[1].model, "gpt-5.5");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builtin_build_fallback_chain_unknown_model() {
|
||||
let fallbacks = HashMap::from([("anthropic".to_string(), vec!["gemini".to_string()])]);
|
||||
let chain = Catalog::builtin().build_fallback_chain(
|
||||
&ProviderId::anthropic(),
|
||||
"unknown-xyz",
|
||||
&fallbacks,
|
||||
);
|
||||
assert!(chain.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builtin_build_fallback_chain_provider_not_in_map() {
|
||||
let fallbacks = HashMap::from([("openai".to_string(), vec!["anthropic".to_string()])]);
|
||||
let chain = Catalog::builtin().build_fallback_chain(
|
||||
&ProviderId::anthropic(),
|
||||
"claude-opus-4-6",
|
||||
&fallbacks,
|
||||
);
|
||||
assert!(chain.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builtin_build_fallback_chain_skips_no_capability_match() {
|
||||
let fallbacks = HashMap::from([("anthropic".to_string(), vec![
|
||||
"openai".to_string(),
|
||||
"kimi".to_string(),
|
||||
])]);
|
||||
let chain = Catalog::builtin().build_fallback_chain(
|
||||
&ProviderId::anthropic(),
|
||||
"claude-haiku-4-5",
|
||||
&fallbacks,
|
||||
);
|
||||
assert!(chain.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builtin_build_fallback_chain_empty_map() {
|
||||
let fallbacks = HashMap::new();
|
||||
let chain = Catalog::builtin().build_fallback_chain(
|
||||
&ProviderId::anthropic(),
|
||||
"claude-opus-4-6",
|
||||
&fallbacks,
|
||||
);
|
||||
assert!(chain.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builtin_catalog_is_loaded_from_provider_toml_settings() {
|
||||
let catalog = Catalog::builtin();
|
||||
|
|
|
|||
|
|
@ -4,6 +4,8 @@
|
|||
//! validation, OpenAPI replacement types, and the LLM client all share one
|
||||
//! enum so that adding a new effort value remains a Rust change.
|
||||
|
||||
use std::cmp::Reverse;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(
|
||||
|
|
@ -37,6 +39,24 @@ impl ReasoningEffort {
|
|||
pub fn variants() -> &'static [Self] {
|
||||
<Self as strum::VariantArray>::VARIANTS
|
||||
}
|
||||
|
||||
/// Select the supported effort nearest to this value.
|
||||
///
|
||||
/// The enum declaration defines the ordered progression. When two values
|
||||
/// are equally distant, the higher effort wins.
|
||||
#[must_use]
|
||||
pub fn closest_supported(self, supported: &[Self]) -> Option<Self> {
|
||||
let variants = Self::variants();
|
||||
let requested_rank = variants.iter().position(|effort| *effort == self)?;
|
||||
|
||||
supported.iter().copied().min_by_key(|effort| {
|
||||
let rank = variants
|
||||
.iter()
|
||||
.position(|candidate| candidate == effort)
|
||||
.expect("supported reasoning effort must be an enum variant");
|
||||
(requested_rank.abs_diff(rank), Reverse(rank))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
@ -91,6 +111,49 @@ mod tests {
|
|||
assert_eq!(v[v.len() - 1], ReasoningEffort::Max);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn closest_supported_uses_exact_match() {
|
||||
assert_eq!(
|
||||
ReasoningEffort::High.closest_supported(&[
|
||||
ReasoningEffort::Low,
|
||||
ReasoningEffort::High,
|
||||
ReasoningEffort::Max,
|
||||
]),
|
||||
Some(ReasoningEffort::High)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn closest_supported_rounds_equal_distance_up() {
|
||||
let kimi = [
|
||||
ReasoningEffort::Low,
|
||||
ReasoningEffort::High,
|
||||
ReasoningEffort::Max,
|
||||
];
|
||||
assert_eq!(
|
||||
ReasoningEffort::Medium.closest_supported(&kimi),
|
||||
Some(ReasoningEffort::High)
|
||||
);
|
||||
assert_eq!(
|
||||
ReasoningEffort::XHigh.closest_supported(&kimi),
|
||||
Some(ReasoningEffort::Max)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn closest_supported_uses_nearest_lower_value_when_needed() {
|
||||
assert_eq!(
|
||||
ReasoningEffort::Max
|
||||
.closest_supported(&[ReasoningEffort::High, ReasoningEffort::XHigh]),
|
||||
Some(ReasoningEffort::XHigh)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn closest_supported_returns_none_for_unsupported_control() {
|
||||
assert_eq!(ReasoningEffort::High.closest_supported(&[]), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn round_trip_through_json() {
|
||||
let json = serde_json::to_string(&ReasoningEffort::High).unwrap();
|
||||
|
|
|
|||
|
|
@ -187,11 +187,21 @@ pub struct SshAccessReadyProps {
|
|||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct FailoverProps {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub original_provider: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub original_model: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub attempt: Option<u32>,
|
||||
pub from_provider: String,
|
||||
pub from_model: String,
|
||||
pub to_provider: String,
|
||||
pub to_model: String,
|
||||
pub error: String,
|
||||
pub from_model: String,
|
||||
pub to_provider: String,
|
||||
pub to_model: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub requested_reasoning_effort: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub effective_reasoning_effort: Option<String>,
|
||||
pub error: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
|
|
|
|||
|
|
@ -1013,6 +1013,33 @@ mod tests {
|
|||
assert!(matches!(parsed.body, EventBody::RunCreated(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn historical_failover_event_defaults_new_route_context() {
|
||||
let line = json!({
|
||||
"id": "evt_failover",
|
||||
"ts": "2026-04-04T12:00:00.000Z",
|
||||
"run_id": fixtures::RUN_1,
|
||||
"event": "agent.failover",
|
||||
"properties": {
|
||||
"from_provider": "anthropic",
|
||||
"from_model": "claude-fable-5",
|
||||
"to_provider": "openai",
|
||||
"to_model": "gpt-5.6-sol",
|
||||
"error": "provider unavailable"
|
||||
}
|
||||
});
|
||||
|
||||
let parsed = RunEvent::from_value(line).unwrap();
|
||||
let EventBody::Failover(props) = parsed.body else {
|
||||
panic!("expected agent.failover");
|
||||
};
|
||||
assert_eq!(props.original_provider, None);
|
||||
assert_eq!(props.original_model, None);
|
||||
assert_eq!(props.attempt, None);
|
||||
assert_eq!(props.requested_reasoning_effort, None);
|
||||
assert_eq!(props.effective_reasoning_effort, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn run_created_round_trip_preserves_manifest_blob() {
|
||||
let line = json!({
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
//! Model references for `run.model.fallbacks`.
|
||||
//!
|
||||
//! Each entry is one of:
|
||||
//! Each fallback-chain entry is one of:
|
||||
//!
|
||||
//! - a bare token such as `openai` or `gpt-5.4` — the parser cannot tell alone
|
||||
//! whether the token is a provider name or a model alias
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@
|
|||
//! notifications, interviews, agent knobs, hooks, SCM targeting, pull-request
|
||||
//! behavior, and artifact collection.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
use std::path::PathBuf;
|
||||
use std::time::Duration as StdDuration;
|
||||
|
||||
|
|
@ -684,7 +684,12 @@ pub enum RunGoal {
|
|||
pub struct RunModelSettings {
|
||||
pub provider: Option<String>,
|
||||
pub name: Option<String>,
|
||||
pub fallbacks: Vec<ModelRef>,
|
||||
/// Ordered fallback references keyed by the originally requested model.
|
||||
///
|
||||
/// Keys remain raw selectors during offline configuration resolution.
|
||||
/// The server canonicalizes them against its model catalog before a run
|
||||
/// starts.
|
||||
pub fallbacks: BTreeMap<String, Vec<ModelRef>>,
|
||||
/// Run-level default values for typed model controls
|
||||
/// (`reasoning_effort`, `speed`). Node and style attributes still win
|
||||
/// over these defaults.
|
||||
|
|
|
|||
|
|
@ -17,5 +17,8 @@
|
|||
export interface RunModelSettings {
|
||||
'provider': string | null;
|
||||
'name': string | null;
|
||||
'fallbacks': Array<string>;
|
||||
/**
|
||||
* Ordered fallback targets keyed by the originally requested model. Each chain is independent; selecting a fallback target does not activate that target model\'s own chain.
|
||||
*/
|
||||
'fallbacks': { [key: string]: Array<string>; };
|
||||
}
|
||||
|
|
|
|||
10
test/offline-fallbacks/workflow.fabro
Normal file
10
test/offline-fallbacks/workflow.fabro
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
digraph OfflineFallbacks {
|
||||
graph [goal="Validate a future model fallback offline"]
|
||||
|
||||
start [shape=Mdiamond, label="Start"]
|
||||
exit [shape=Msquare, label="Exit"]
|
||||
|
||||
work [label="Work", prompt="Do work", model="future-primary"]
|
||||
|
||||
start -> work -> exit
|
||||
}
|
||||
7
test/offline-fallbacks/workflow.toml
Normal file
7
test/offline-fallbacks/workflow.toml
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
_version = 1
|
||||
|
||||
[workflow]
|
||||
graph = "workflow.fabro"
|
||||
|
||||
[run.model.fallbacks]
|
||||
"future-primary" = ["future-provider:future-model"]
|
||||
Loading…
Add table
Reference in a new issue