scriptoriumadmin/fabro
1
0
Fork 0
mirror of https://github.com/fabro-sh/fabro.git synced 2026-10-07 03:00:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

test(model): prepare catalog-only providers (#264)

Browse source 
## Summary

Prepares the model catalog tests for catalog-only built-in providers so
a future provider can be added with just its catalog TOML.

## Changes

- Replaces the closed-enum round-trip guardrail with a catalog metadata
guardrail, allowing built-in TOML providers that do not have `Provider`
enum variants.
- Makes the all-model `fabro model` CLI tests assert stable table
structure instead of snapshotting every built-in catalog row.
- Renames synthetic custom-provider and missing-provider fixtures away
from provider names that can become real catalog entries.

## Verification

- `cargo +nightly-2026-04-14 fmt --check --all`
- `cargo nextest run -p fabro-model -p fabro-auth -p fabro-llm -p
fabro-server -p fabro-workflow -p fabro-config`
- `cargo nextest run -p fabro-cli cmd::model`

---

[![Compound
Engineering](https://img.shields.io/badge/Compound_Engineering-6366f1)](https://github.com/EveryInc/compound-engineering-plugin)
🤖 Generated with GPT-5 (unknown context, medium reasoning) via
[Codex](https://openai.com/codex)

---------

Co-authored-by: Jesse <606+jesseproudman@users.noreply.github.com>
This commit is contained in:
Bryan Helmkamp 2026-05-14 18:51:18 -07:00 • committed by GitHub
parent e7e4fb5ca1
commit 78941a2b84
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 217 additions and 257 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
lib/crates/fabro-auth/src/env_source.rs
View file

@ -266,44 +266,44 @@ mod tests {
async fn resolve_registers_custom_env_backed_provider() {
let catalog = catalog_with(
r#"
[providers.venice]
display_name = "Venice"
[providers.acme]
display_name = "Acme"
adapter = "openai_compatible"
base_url = "https://api.venice.ai/api/v1"
credentials = ["env:VENICE_API_KEY"]
base_url = "https://api.acme.test/v1"
credentials = ["env:ACME_API_KEY"]
[models."venice-large"]
provider = "venice"
display_name = "Venice Large"
family = "venice"
[models."acme-large"]
provider = "acme"
display_name = "Acme Large"
family = "acme"
default = true
[models."venice-large".limits]
[models."acme-large".limits]
context_window = 128000
[models."venice-large".features]
[models."acme-large".features]
tools = true
vision = false
reasoning = false
effort = false
"#,
);
let source = test_source(&[("VENICE_API_KEY", "venice-key")]);
let source = test_source(&[("ACME_API_KEY", "acme-key")]);
let resolved = source.resolve(&catalog).await.unwrap();
let credential = resolved
.credentials
.iter()
.find(|credential| credential.provider == ProviderId::new("venice"))
.find(|credential| credential.provider == ProviderId::new("acme"))
.expect("custom provider should resolve from the supplied catalog");
assert_eq!(
credential.auth_header.as_ref().unwrap(),
&crate::ApiKeyHeader::Bearer("venice-key".to_string(),)
&crate::ApiKeyHeader::Bearer("acme-key".to_string(),)
);
assert_eq!(
credential.base_url.as_deref(),
Some("https://api.venice.ai/api/v1")
Some("https://api.acme.test/v1")
);
}

37
lib/crates/fabro-auth/src/resolve.rs
View file

@ -952,22 +952,22 @@ effort = false
async fn resolve_uses_custom_vault_backed_provider() {
let catalog = catalog_with(
r#"
[providers.venice]
display_name = "Venice"
[providers.acme]
display_name = "Acme"
adapter = "openai_compatible"
base_url = "https://api.venice.ai/api/v1"
credentials = ["credential:venice"]
base_url = "https://api.acme.test/v1"
credentials = ["credential:acme"]
[models."venice-large"]
provider = "venice"
display_name = "Venice Large"
family = "venice"
[models."acme-large"]
provider = "acme"
display_name = "Acme Large"
family = "acme"
default = true
[models."venice-large".limits]
[models."acme-large".limits]
context_window = 128000
[models."venice-large".features]
[models."acme-large".features]
tools = true
vision = false
reasoning = false
@ -976,10 +976,10 @@ effort = false
);
let dir = tempfile::tempdir().unwrap();
let mut vault = Vault::load(dir.path().join("secrets.json")).unwrap();
vault_set_credential(&mut vault, "venice", &AuthCredential {
provider: ProviderId::new("venice"),
vault_set_credential(&mut vault, "acme", &AuthCredential {
provider: ProviderId::new("acme"),
details: AuthDetails::ApiKey {
key: "venice-key".to_string(),
key: "acme-key".to_string(),
},
})
.unwrap();
@ -987,7 +987,7 @@ effort = false
let resolved = resolver
.resolve(
ProviderId::new("venice"),
ProviderId::new("acme"),
CredentialUsage::ApiRequest,
&catalog,
)
@ -997,15 +997,12 @@ effort = false
let ResolvedCredential::Api(api) = resolved else {
panic!("expected api credential");
};
assert_eq!(api.provider, ProviderId::new("venice"));
assert_eq!(api.provider, ProviderId::new("acme"));
assert_eq!(
api.auth_header,
Some(ApiKeyHeader::Bearer("venice-key".to_string()))
);
assert_eq!(
api.base_url.as_deref(),
Some("https://api.venice.ai/api/v1")
Some(ApiKeyHeader::Bearer("acme-key".to_string()))
);
assert_eq!(api.base_url.as_deref(), Some("https://api.acme.test/v1"));
}
#[tokio::test]

88
lib/crates/fabro-cli/tests/it/cmd/model.rs
View file

@ -6,6 +6,30 @@
use fabro_test::{fabro_snapshot, test_context};
use httpmock::MockServer;
fn successful_stdout(mut cmd: assert_cmd::Command) -> String {
let assert = cmd.assert().success().stderr("");
String::from_utf8(assert.get_output().stdout.clone())
.expect("model list stdout should be UTF-8")
}
fn assert_model_list_table(stdout: &str) {
let mut lines = stdout.lines();
let header = lines.next().expect("model list should render a header");
assert!(header.contains("MODEL"), "missing MODEL column: {header}");
assert!(
header.contains("PROVIDER"),
"missing PROVIDER column: {header}"
);
assert!(
header.contains("CONTEXT"),
"missing CONTEXT column: {header}"
);
assert!(
lines.any(|line| !line.trim().is_empty()),
"model list should render at least one model row"
);
}
#[test]
fn help() {
let context = test_context!();
@ -38,36 +62,8 @@ fn help() {
#[test]
fn bare() {
let context = test_context!();
fabro_snapshot!(context.filters(), context.model(), @"
success: true
exit_code: 0
----- stdout -----
MODEL PROVIDER ALIASES CONTEXT COST SPEED
claude-haiku-4-5 anthropic haiku, claude-haiku 200k $0.8 / $4.0 100 tok/s
claude-opus-4-6 anthropic 1m $5.0 / $25.0 25 tok/s
claude-opus-4-7 anthropic opus, claude-opus 1m $5.0 / $25.0 25 tok/s
claude-sonnet-4-5 anthropic 200k $3.0 / $15.0 50 tok/s
claude-sonnet-4-6 anthropic sonnet, claude-sonnet 200k $3.0 / $15.0 50 tok/s
gemini-3-flash-preview gemini gemini-flash 1m $0.5 / $3.0 150 tok/s
gemini-3.1-flash-lite-preview gemini gemini-flash-lite 1m $0.2 / $1.5 200 tok/s
gemini-3.1-pro-preview gemini gemini-pro 1m $2.0 / $12.0 85 tok/s
gemini-3.1-pro-preview-customtools gemini gemini-customtools 1m $2.0 / $12.0 85 tok/s
mercury-2 inception mercury 131k $0.2 / $0.8 1000 tok/s
kimi-k2.5 kimi kimi 262k $0.6 / $3.0 50 tok/s
minimax-m2.5 minimax minimax 197k $0.3 / $1.2 45 tok/s
gpt-5-mini openai gpt5-mini 1m $0.2 / $2.0 70 tok/s
gpt-5.2 openai gpt5 1m $1.8 / $14.0 65 tok/s
gpt-5.2-codex openai 1m $1.8 / $14.0 100 tok/s
gpt-5.3-codex openai codex 1m $1.8 / $14.0 100 tok/s
gpt-5.3-codex-spark openai codex-spark 131k - / - 1000 tok/s
gpt-5.4 openai gpt54, gpt-54 1m $2.5 / $15.0 70 tok/s
gpt-5.4-mini openai gpt54-mini, gpt-54-mini 400k $0.8 / $4.5 140 tok/s
gpt-5.4-pro openai gpt54-pro, gpt-54-pro 1m $30.0 / $180.0 20 tok/s
gpt-5.5 openai gpt55, gpt-55 1m $5.0 / $30.0 70 tok/s
gpt-5.5-pro openai gpt55-pro, gpt-55-pro 1m $30.0 / $180.0 20 tok/s
glm-4.7 zai glm, glm4 203k $0.6 / $2.2 100 tok/s
----- stderr -----
");
let stdout = successful_stdout(context.model());
assert_model_list_table(&stdout);
}
#[test]
@ -75,36 +71,8 @@ fn list() {
let context = test_context!();
let mut cmd = context.model();
cmd.arg("list");
fabro_snapshot!(context.filters(), cmd, @"
success: true
exit_code: 0
----- stdout -----
MODEL PROVIDER ALIASES CONTEXT COST SPEED
claude-haiku-4-5 anthropic haiku, claude-haiku 200k $0.8 / $4.0 100 tok/s
claude-opus-4-6 anthropic 1m $5.0 / $25.0 25 tok/s
claude-opus-4-7 anthropic opus, claude-opus 1m $5.0 / $25.0 25 tok/s
claude-sonnet-4-5 anthropic 200k $3.0 / $15.0 50 tok/s
claude-sonnet-4-6 anthropic sonnet, claude-sonnet 200k $3.0 / $15.0 50 tok/s
gemini-3-flash-preview gemini gemini-flash 1m $0.5 / $3.0 150 tok/s
gemini-3.1-flash-lite-preview gemini gemini-flash-lite 1m $0.2 / $1.5 200 tok/s
gemini-3.1-pro-preview gemini gemini-pro 1m $2.0 / $12.0 85 tok/s
gemini-3.1-pro-preview-customtools gemini gemini-customtools 1m $2.0 / $12.0 85 tok/s
mercury-2 inception mercury 131k $0.2 / $0.8 1000 tok/s
kimi-k2.5 kimi kimi 262k $0.6 / $3.0 50 tok/s
minimax-m2.5 minimax minimax 197k $0.3 / $1.2 45 tok/s
gpt-5-mini openai gpt5-mini 1m $0.2 / $2.0 70 tok/s
gpt-5.2 openai gpt5 1m $1.8 / $14.0 65 tok/s
gpt-5.2-codex openai 1m $1.8 / $14.0 100 tok/s
gpt-5.3-codex openai codex 1m $1.8 / $14.0 100 tok/s
gpt-5.3-codex-spark openai codex-spark 131k - / - 1000 tok/s
gpt-5.4 openai gpt54, gpt-54 1m $2.5 / $15.0 70 tok/s
gpt-5.4-mini openai gpt54-mini, gpt-54-mini 400k $0.8 / $4.5 140 tok/s
gpt-5.4-pro openai gpt54-pro, gpt-54-pro 1m $30.0 / $180.0 20 tok/s
gpt-5.5 openai gpt55, gpt-55 1m $5.0 / $30.0 70 tok/s
gpt-5.5-pro openai gpt55-pro, gpt-55-pro 1m $30.0 / $180.0 20 tok/s
glm-4.7 zai glm, glm4 203k $0.6 / $2.2 100 tok/s
----- stderr -----
");
let stdout = successful_stdout(cmd);
assert_model_list_table(&stdout);
}
#[test]

24
lib/crates/fabro-config/src/builders.rs
View file

@ -686,22 +686,22 @@ _version = 1
[server.auth]
methods = ["dev-token"]
[llm.providers.venice]
display_name = "Venice"
[llm.providers.acme]
display_name = "Acme"
adapter = "openai_compatible"
base_url = "https://api.venice.ai/api/v1"
credentials = ["env:VENICE_API_KEY"]
base_url = "https://api.acme.test/v1"
credentials = ["env:ACME_API_KEY"]
[llm.models."venice-large"]
provider = "venice"
display_name = "Venice Large"
family = "venice"
[llm.models."acme-large"]
provider = "acme"
display_name = "Acme Large"
family = "acme"
default = true
[llm.models."venice-large".limits]
[llm.models."acme-large".limits]
context_window = 128000
[llm.models."venice-large".features]
[llm.models."acme-large".features]
tools = true
vision = false
reasoning = false
@ -718,9 +718,9 @@ effort = false
assert_eq!(
catalog
.get("venice-large")
.get("acme-large")
.map(|model| model.provider.clone()),
Some(fabro_model::ProviderId::new("venice"))
Some(fabro_model::ProviderId::new("acme"))
);
}
}

68
lib/crates/fabro-llm/src/client.rs
View file

@ -714,8 +714,8 @@ mod tests {
let catalog = catalog_with("");
let result = Client::from_credentials(
vec![ApiCredential {
provider: fabro_model::ProviderId::new("venice"),
auth_header: Some(ApiKeyHeader::Bearer("venice-key".to_string())),
provider: fabro_model::ProviderId::new("custom"),
auth_header: Some(ApiKeyHeader::Bearer("custom-key".to_string())),
extra_headers: HashMap::new(),
base_url: None,
codex_mode: false,
@ -734,7 +734,7 @@ mod tests {
Error::Configuration {
ref message,
..
} if message == "Provider \"venice\" is not supported by credential-only registration"
} if message == "Provider \"custom\" is not supported by credential-only registration"
));
}
@ -765,23 +765,23 @@ mod tests {
async fn from_credentials_registers_custom_openai_compatible_provider() {
let catalog = catalog_with(
r#"
[providers.venice]
display_name = "Venice"
[providers.acme]
display_name = "Acme"
adapter = "openai_compatible"
base_url = "https://api.venice.ai/api/v1"
credentials = ["env:VENICE_API_KEY"]
aliases = ["venice-ai"]
base_url = "https://api.acme.test/v1"
credentials = ["env:ACME_API_KEY"]
aliases = ["acme-ai"]
[models."venice-large"]
provider = "venice"
display_name = "Venice Large"
family = "venice"
[models."acme-large"]
provider = "acme"
display_name = "Acme Large"
family = "acme"
default = true
[models."venice-large".limits]
[models."acme-large".limits]
context_window = 128000
[models."venice-large".features]
[models."acme-large".features]
tools = true
vision = false
reasoning = false
@ -791,8 +791,8 @@ effort = false
let client = Client::from_credentials(
vec![ApiCredential {
provider: fabro_model::ProviderId::new("venice"),
auth_header: Some(ApiKeyHeader::Bearer("venice-key".to_string())),
provider: fabro_model::ProviderId::new("acme"),
auth_header: Some(ApiKeyHeader::Bearer("acme-key".to_string())),
extra_headers: HashMap::new(),
base_url: None,
codex_mode: false,
@ -804,32 +804,32 @@ effort = false
.await
.unwrap();
assert_eq!(client.provider_names(), vec!["venice"]);
assert!(client.has_provider("venice"));
assert!(client.has_provider("venice-ai"));
assert_eq!(client.provider_names(), vec!["acme"]);
assert!(client.has_provider("acme"));
assert!(client.has_provider("acme-ai"));
}
#[tokio::test]
async fn resolve_provider_accepts_catalog_provider_alias() {
let catalog = catalog_with(
r#"
[providers.venice]
display_name = "Venice"
[providers.acme]
display_name = "Acme"
adapter = "openai_compatible"
base_url = "https://api.venice.ai/api/v1"
credentials = ["env:VENICE_API_KEY"]
aliases = ["venice-ai"]
base_url = "https://api.acme.test/v1"
credentials = ["env:ACME_API_KEY"]
aliases = ["acme-ai"]
[models."venice-large"]
provider = "venice"
display_name = "Venice Large"
family = "venice"
[models."acme-large"]
provider = "acme"
display_name = "Acme Large"
family = "acme"
default = true
[models."venice-large".limits]
[models."acme-large".limits]
context_window = 128000
[models."venice-large".features]
[models."acme-large".features]
tools = true
vision = false
reasoning = false
@ -839,8 +839,8 @@ effort = false
let client = Client::from_credentials(
vec![ApiCredential {
provider: fabro_model::ProviderId::new("venice"),
auth_header: Some(ApiKeyHeader::Bearer("venice-key".to_string())),
provider: fabro_model::ProviderId::new("acme"),
auth_header: Some(ApiKeyHeader::Bearer("acme-key".to_string())),
extra_headers: HashMap::new(),
base_url: None,
codex_mode: false,
@ -852,11 +852,11 @@ effort = false
.await
.unwrap();
let mut request = test_request();
request.provider = Some("venice-ai".to_string());
request.provider = Some("acme-ai".to_string());
let provider = client.resolve_provider(&request).unwrap();
assert_eq!(provider.name(), "venice");
assert_eq!(provider.name(), "acme");
}
#[tokio::test]

30
lib/crates/fabro-llm/src/providers/openai_compatible.rs
View file

@ -1358,23 +1358,23 @@ mod tests {
fn catalog_api_id_is_used_for_provider_request_body() {
let settings: LlmCatalogSettings = toml::from_str(
r#"
[providers.venice]
display_name = "Venice"
[providers.acme]
display_name = "Acme"
adapter = "openai_compatible"
base_url = "https://api.venice.ai/api/v1"
credentials = ["env:VENICE_API_KEY"]
base_url = "https://api.acme.test/v1"
credentials = ["env:ACME_API_KEY"]
[models."venice-large"]
provider = "venice"
api_id = "venice/model-large"
display_name = "Venice Large"
family = "venice"
[models."acme-large"]
provider = "acme"
api_id = "acme/model-large"
display_name = "Acme Large"
family = "acme"
default = true
[models."venice-large".limits]
[models."acme-large".limits]
context_window = 128000
[models."venice-large".features]
[models."acme-large".features]
tools = true
vision = false
reasoning = false
@ -1384,12 +1384,12 @@ effort = false
.unwrap();
let catalog = Catalog::from_builtin_with_overrides(&settings).unwrap();
let mut request = minimal_request();
request.model = "venice-large".to_string();
request.model = "acme-large".to_string();
let body = build_api_request_with_catalog(&request, None, "venice", Some(&catalog));
let body = build_api_request_with_catalog(&request, None, "acme", Some(&catalog));
assert_eq!(request.model, "venice-large");
assert_eq!(body["model"], "venice/model-large");
assert_eq!(request.model, "acme-large");
assert_eq!(body["model"], "acme/model-large");
}
#[test]

59
lib/crates/fabro-model/src/catalog.rs
View file

@ -1558,25 +1558,25 @@ enabled = false
fn builtin_overrides_add_custom_openai_compatible_provider_and_model() {
let catalog = Catalog::from_builtin_with_overrides(&minimal_settings(
r#"
[providers.venice]
display_name = "Venice"
[providers.acme]
display_name = "Acme"
adapter = "openai_compatible"
base_url = "https://api.venice.ai/api/v1"
credentials = ["env:VENICE_API_KEY"]
base_url = "https://api.acme.test/v1"
credentials = ["env:ACME_API_KEY"]
priority = 120
aliases = ["venice-ai"]
aliases = ["acme-ai"]
[models."venice-large"]
provider = "venice"
display_name = "Venice Large"
family = "venice"
[models."acme-large"]
provider = "acme"
display_name = "Acme Large"
family = "acme"
default = true
aliases = ["vl"]
aliases = ["al"]
[models."venice-large".limits]
[models."acme-large".limits]
context_window = 128000
[models."venice-large".features]
[models."acme-large".features]
tools = true
vision = false
reasoning = false
@ -1586,14 +1586,14 @@ effort = false
.expect("custom provider overlay should build");
let provider = catalog
.provider(&ProviderId::new("venice-ai"))
.provider(&ProviderId::new("acme-ai"))
.expect("provider alias should resolve");
assert_eq!(provider.id, ProviderId::new("venice"));
assert_eq!(provider.id, ProviderId::new("acme"));
assert_eq!(provider.adapter, "openai_compatible");
let model = catalog.get("vl").expect("model alias should resolve");
assert_eq!(model.id, "venice-large");
assert_eq!(model.provider, ProviderId::new("venice"));
let model = catalog.get("al").expect("model alias should resolve");
assert_eq!(model.id, "acme-large");
assert_eq!(model.provider, ProviderId::new("acme"));
}
#[test]
@ -1849,8 +1849,8 @@ effort = false
fn catalog_from_settings_rejects_unknown_adapter() {
let layer = minimal_settings(
r#"
[providers.venice]
display_name = "Venice"
[providers.test-provider]
display_name = "Test Provider"
adapter = "not_real"
enabled = true
"#,
@ -1861,7 +1861,7 @@ enabled = true
assert!(matches!(
err,
CatalogBuildError::UnknownAdapter { provider, adapter }
if provider == ProviderId::new("venice") && adapter == "not_real"
if provider == ProviderId::new("test-provider") && adapter == "not_real"
));
}
@ -2464,19 +2464,14 @@ reasoning_effort = "levels"
}
#[test]
fn catalog_providers_roundtrip_through_static_str() {
for model in Catalog::builtin().list(None) {
let roundtripped = Provider::from_id(&model.provider);
assert_eq!(
roundtripped,
Some(
model
.builtin_provider()
.expect("catalog model provider should be a built-in provider")
),
"catalog model '{}' provider {:?} does not roundtrip through ProviderId",
fn every_catalog_model_provider_has_catalog_provider() {
let catalog = Catalog::builtin();
for model in catalog.list(None) {
assert!(
catalog.provider(&model.provider).is_some(),
"catalog model '{}' provider {:?} has no provider metadata",
model.id,
model.provider
model.provider,
);
}
}

6
lib/crates/fabro-model/src/types.rs
View file

@ -194,9 +194,9 @@ mod tests {
}
#[test]
fn all_catalog_providers_are_valid() {
for model in Catalog::builtin().list(None) {
assert!(model.builtin_provider().is_some());
fn builtin_provider_matches_known_static_provider_ids() {
for info in Catalog::builtin().list(None) {
assert_eq!(info.builtin_provider(), Provider::from_id(info.provider()));
}
}
}

34
lib/crates/fabro-server/src/run_manifest.rs
View file

@ -2158,7 +2158,7 @@ digraph Demo {
digraph Demo {
start [shape=Mdiamond]
exit [shape=Msquare]
work [prompt="Do work", model="venice-model", provider="venice"]
work [prompt="Do work", model="missing-model", provider="missing-provider"]
start -> work -> exit
}
"#
@ -2179,18 +2179,18 @@ digraph Demo {
let llm_check = response.checks.sections[0]
.checks
.iter()
.find(|check| check.name == "LLM" && check.summary == "venice-model")
.find(|check| check.name == "LLM" && check.summary == "missing-model")
.expect("preflight should include the requested custom LLM provider");
assert_eq!(llm_check.status, types::PreflightCheckResultStatus::Warning);
assert_eq!(
llm_check.remediation.as_deref(),
Some("Provider \"venice\" is not configured")
Some("Provider \"missing-provider\" is not configured")
);
assert!(
llm_check
.details
.iter()
.any(|detail| detail.text == "Provider: venice")
.any(|detail| detail.text == "Provider: missing-provider")
);
}
@ -2198,23 +2198,23 @@ digraph Demo {
async fn preflight_resolves_model_aliases_from_app_state_catalog() {
let llm_catalog_settings: fabro_model::catalog::LlmCatalogSettings = toml::from_str(
r#"
[providers.venice]
display_name = "Venice"
[providers.acme]
display_name = "Acme"
adapter = "openai_compatible"
base_url = "https://api.venice.ai/api/v1"
credentials = ["env:VENICE_API_KEY"]
base_url = "https://api.acme.test/v1"
credentials = ["env:ACME_API_KEY"]
[models."venice-large"]
provider = "venice"
display_name = "Venice Large"
family = "venice"
[models."acme-large"]
provider = "acme"
display_name = "Acme Large"
family = "acme"
default = true
aliases = ["vl"]
[models."venice-large".limits]
[models."acme-large".limits]
context_window = 128000
[models."venice-large".features]
[models."acme-large".features]
tools = true
vision = false
reasoning = false
@ -2251,18 +2251,18 @@ digraph Demo {
let llm_check = response.checks.sections[0]
.checks
.iter()
.find(|check| check.name == "LLM" && check.summary == "venice-large")
.find(|check| check.name == "LLM" && check.summary == "acme-large")
.expect("preflight should resolve the catalog alias");
assert_eq!(llm_check.status, types::PreflightCheckResultStatus::Warning);
assert_eq!(
llm_check.remediation.as_deref(),
Some("Provider \"venice\" is not configured")
Some("Provider \"acme\" is not configured")
);
assert!(
llm_check
.details
.iter()
.any(|detail| detail.text == "Provider: venice")
.any(|detail| detail.text == "Provider: acme")
);
}

76
lib/crates/fabro-server/src/server/tests.rs
View file

@ -2263,22 +2263,22 @@ async fn validate_endpoint_returns_workflow_summary_without_preflight_checks() {
async fn validate_endpoint_uses_app_state_catalog_for_model_diagnostics() {
let llm_catalog_settings: LlmCatalogSettings = toml::from_str(
r#"
[providers.venice]
display_name = "Venice"
[providers.acme]
display_name = "Acme"
adapter = "openai_compatible"
base_url = "https://api.venice.ai/api/v1"
credentials = ["env:VENICE_API_KEY"]
base_url = "https://api.acme.test/v1"
credentials = ["env:ACME_API_KEY"]
[models."venice-large"]
provider = "venice"
display_name = "Venice Large"
family = "venice"
[models."acme-large"]
provider = "acme"
display_name = "Acme Large"
family = "acme"
default = true
[models."venice-large".limits]
[models."acme-large".limits]
context_window = 128000
[models."venice-large".features]
[models."acme-large".features]
tools = true
vision = false
reasoning = false
@ -2293,7 +2293,7 @@ effort = false
let dot = r#"digraph Test {
graph [goal="Test"]
start [shape=Mdiamond]
work [model="venice-large", provider="venice", prompt="Do it"]
work [model="acme-large", provider="acme", prompt="Do it"]
exit [shape=Msquare]
start -> work -> exit
}"#;
@ -3998,7 +3998,7 @@ async fn list_models_unknown_provider_returns_empty_page() {
let req = Request::builder()
.method("GET")
.uri(api("/models?provider=venice"))
.uri(api("/models?provider=missing-provider"))
.body(Body::empty())
.unwrap();
@ -4012,23 +4012,23 @@ async fn list_models_unknown_provider_returns_empty_page() {
async fn list_models_uses_app_state_catalog_overrides() {
let llm_catalog_settings: LlmCatalogSettings = toml::from_str(
r#"
[providers.venice]
display_name = "Venice"
[providers.acme]
display_name = "Acme"
adapter = "openai_compatible"
base_url = "https://api.venice.ai/api/v1"
credentials = ["env:VENICE_API_KEY"]
base_url = "https://api.acme.test/v1"
credentials = ["env:ACME_API_KEY"]
priority = 120
[models."venice-large"]
provider = "venice"
display_name = "Venice Large"
family = "venice"
[models."acme-large"]
provider = "acme"
display_name = "Acme Large"
family = "acme"
default = true
[models."venice-large".limits]
[models."acme-large".limits]
context_window = 128000
[models."venice-large".features]
[models."acme-large".features]
tools = true
vision = false
reasoning = false
@ -4043,7 +4043,7 @@ effort = false
let req = Request::builder()
.method("GET")
.uri(api("/models?provider=venice"))
.uri(api("/models?provider=acme"))
.body(Body::empty())
.unwrap();
@ -4051,8 +4051,8 @@ effort = false
let body = response_json!(response, StatusCode::OK).await;
let models = body["data"].as_array().unwrap();
assert_eq!(models.len(), 1);
assert_eq!(models[0]["id"], "venice-large");
assert_eq!(models[0]["provider"], "venice");
assert_eq!(models[0]["id"], "acme-large");
assert_eq!(models[0]["provider"], "acme");
}
#[tokio::test]
@ -9444,7 +9444,7 @@ async fn create_completion_unknown_provider_returns_clear_error() {
.header("content-type", "application/json")
.body(Body::from(
serde_json::json!({
"provider": "venice",
"provider": "missing-provider",
"model": "gpt-5.4",
"stream": false,
"messages": [
@ -9462,7 +9462,7 @@ async fn create_completion_unknown_provider_returns_clear_error() {
let body = response_json!(response, StatusCode::BAD_REQUEST).await;
assert_eq!(
body["errors"][0]["detail"],
"Provider \"venice\" is not configured"
"Provider \"missing-provider\" is not configured"
);
}
@ -9470,23 +9470,23 @@ async fn create_completion_unknown_provider_returns_clear_error() {
async fn create_completion_default_model_uses_app_state_catalog() {
let llm_catalog_settings: LlmCatalogSettings = toml::from_str(
r#"
[providers.venice]
display_name = "Venice"
[providers.acme]
display_name = "Acme"
adapter = "openai_compatible"
base_url = "https://api.venice.ai/api/v1"
credentials = ["env:VENICE_API_KEY"]
base_url = "https://api.acme.test/v1"
credentials = ["env:ACME_API_KEY"]
priority = 120
[models."venice-large"]
provider = "venice"
display_name = "Venice Large"
family = "venice"
[models."acme-large"]
provider = "acme"
display_name = "Acme Large"
family = "acme"
default = true
[models."venice-large".limits]
[models."acme-large".limits]
context_window = 128000
[models."venice-large".features]
[models."acme-large".features]
tools = true
vision = false
reasoning = false
@ -9523,7 +9523,7 @@ effort = false
body["errors"][0]["detail"]
.as_str()
.unwrap()
.contains("Provider 'venice' not registered"),
.contains("Provider 'acme' not registered"),
"unexpected error body: {body:?}"
);
}

24
lib/crates/fabro-workflow/src/handler/llm/api.rs
View file

@ -1471,23 +1471,23 @@ mod tests {
fn api_backend_resolves_custom_catalog_provider_profile() {
let settings: LlmCatalogSettings = toml::from_str(
r#"
[providers.venice]
[providers.acme]
adapter = "openai_compatible"
base_url = "https://api.venice.ai/api/v1"
credentials = ["env:VENICE_API_KEY"]
base_url = "https://api.acme.test/v1"
credentials = ["env:ACME_API_KEY"]
[models.venice-llama]
provider = "venice"
display_name = "Venice Llama"
[models.acme-llama]
provider = "acme"
display_name = "Acme Llama"
family = "llama"
training = "2026-01"
default = true
[models.venice-llama.limits]
[models.acme-llama.limits]
context_window = 131072
max_output = 8192
[models.venice-llama.features]
[models.acme-llama.features]
tools = true
vision = false
reasoning = false
@ -1497,9 +1497,9 @@ effort = false
.unwrap();
let catalog = Arc::new(Catalog::from_builtin_with_overrides(&settings).unwrap());
let backend = AgentApiBackend::new_with_catalog(
"venice-llama".to_string(),
"acme-llama".to_string(),
Provider::OpenAiCompatible,
ProviderId::from("venice"),
ProviderId::from("acme"),
AgentProfileKind::OpenAi,
Vec::new(),
Arc::new(EnvCredentialSource::new()),
@ -1508,10 +1508,10 @@ effort = false
);
let provider = backend
.resolve_provider_context("venice-llama", None)
.resolve_provider_context("acme-llama", None)
.unwrap();
assert_eq!(provider.provider_id, ProviderId::from("venice"));
assert_eq!(provider.provider_id, ProviderId::from("acme"));
assert_eq!(provider.profile_kind, AgentProfileKind::OpenAi);
assert_eq!(provider.provider, Provider::OpenAiCompatible);
}