From 78941a2b84aa73fb67159e3aeee27d29bb6e7c32 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp <19+brynary@users.noreply.github.com> Date: Thu, 14 May 2026 18:51:18 -0700 Subject: [PATCH] test(model): prepare catalog-only providers (#264) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Prepares the model catalog tests for catalog-only built-in providers so a future provider can be added with just its catalog TOML. ## Changes - Replaces the closed-enum round-trip guardrail with a catalog metadata guardrail, allowing built-in TOML providers that do not have `Provider` enum variants. - Makes the all-model `fabro model` CLI tests assert stable table structure instead of snapshotting every built-in catalog row. - Renames synthetic custom-provider and missing-provider fixtures away from provider names that can become real catalog entries. ## Verification - `cargo +nightly-2026-04-14 fmt --check --all` - `cargo nextest run -p fabro-model -p fabro-auth -p fabro-llm -p fabro-server -p fabro-workflow -p fabro-config` - `cargo nextest run -p fabro-cli cmd::model` --- [![Compound Engineering](https://img.shields.io/badge/Compound_Engineering-6366f1)](https://github.com/EveryInc/compound-engineering-plugin) 🤖 Generated with GPT-5 (unknown context, medium reasoning) via [Codex](https://openai.com/codex) --------- Co-authored-by: Jesse <606+jesseproudman@users.noreply.github.com> --- lib/crates/fabro-auth/src/env_source.rs | 28 +++--- lib/crates/fabro-auth/src/resolve.rs | 37 ++++---- lib/crates/fabro-cli/tests/it/cmd/model.rs | 88 ++++++------------- lib/crates/fabro-config/src/builders.rs | 24 ++--- lib/crates/fabro-llm/src/client.rs | 68 +++++++------- .../src/providers/openai_compatible.rs | 30 +++---- lib/crates/fabro-model/src/catalog.rs | 59 ++++++------- lib/crates/fabro-model/src/types.rs | 6 +- lib/crates/fabro-server/src/run_manifest.rs | 34 +++---- lib/crates/fabro-server/src/server/tests.rs | 76 ++++++++-------- .../fabro-workflow/src/handler/llm/api.rs | 24 ++--- 11 files changed, 217 insertions(+), 257 deletions(-) diff --git a/lib/crates/fabro-auth/src/env_source.rs b/lib/crates/fabro-auth/src/env_source.rs index 290babbd2..dc755767f 100644 --- a/lib/crates/fabro-auth/src/env_source.rs +++ b/lib/crates/fabro-auth/src/env_source.rs @@ -266,44 +266,44 @@ mod tests { async fn resolve_registers_custom_env_backed_provider() { let catalog = catalog_with( r#" -[providers.venice] -display_name = "Venice" +[providers.acme] +display_name = "Acme" adapter = "openai_compatible" -base_url = "https://api.venice.ai/api/v1" -credentials = ["env:VENICE_API_KEY"] +base_url = "https://api.acme.test/v1" +credentials = ["env:ACME_API_KEY"] -[models."venice-large"] -provider = "venice" -display_name = "Venice Large" -family = "venice" +[models."acme-large"] +provider = "acme" +display_name = "Acme Large" +family = "acme" default = true -[models."venice-large".limits] +[models."acme-large".limits] context_window = 128000 -[models."venice-large".features] +[models."acme-large".features] tools = true vision = false reasoning = false effort = false "#, ); - let source = test_source(&[("VENICE_API_KEY", "venice-key")]); + let source = test_source(&[("ACME_API_KEY", "acme-key")]); let resolved = source.resolve(&catalog).await.unwrap(); let credential = resolved .credentials .iter() - .find(|credential| credential.provider == ProviderId::new("venice")) + .find(|credential| credential.provider == ProviderId::new("acme")) .expect("custom provider should resolve from the supplied catalog"); assert_eq!( credential.auth_header.as_ref().unwrap(), - &crate::ApiKeyHeader::Bearer("venice-key".to_string(),) + &crate::ApiKeyHeader::Bearer("acme-key".to_string(),) ); assert_eq!( credential.base_url.as_deref(), - Some("https://api.venice.ai/api/v1") + Some("https://api.acme.test/v1") ); } diff --git a/lib/crates/fabro-auth/src/resolve.rs b/lib/crates/fabro-auth/src/resolve.rs index 91aff3d43..28f5232c1 100644 --- a/lib/crates/fabro-auth/src/resolve.rs +++ b/lib/crates/fabro-auth/src/resolve.rs @@ -952,22 +952,22 @@ effort = false async fn resolve_uses_custom_vault_backed_provider() { let catalog = catalog_with( r#" -[providers.venice] -display_name = "Venice" +[providers.acme] +display_name = "Acme" adapter = "openai_compatible" -base_url = "https://api.venice.ai/api/v1" -credentials = ["credential:venice"] +base_url = "https://api.acme.test/v1" +credentials = ["credential:acme"] -[models."venice-large"] -provider = "venice" -display_name = "Venice Large" -family = "venice" +[models."acme-large"] +provider = "acme" +display_name = "Acme Large" +family = "acme" default = true -[models."venice-large".limits] +[models."acme-large".limits] context_window = 128000 -[models."venice-large".features] +[models."acme-large".features] tools = true vision = false reasoning = false @@ -976,10 +976,10 @@ effort = false ); let dir = tempfile::tempdir().unwrap(); let mut vault = Vault::load(dir.path().join("secrets.json")).unwrap(); - vault_set_credential(&mut vault, "venice", &AuthCredential { - provider: ProviderId::new("venice"), + vault_set_credential(&mut vault, "acme", &AuthCredential { + provider: ProviderId::new("acme"), details: AuthDetails::ApiKey { - key: "venice-key".to_string(), + key: "acme-key".to_string(), }, }) .unwrap(); @@ -987,7 +987,7 @@ effort = false let resolved = resolver .resolve( - ProviderId::new("venice"), + ProviderId::new("acme"), CredentialUsage::ApiRequest, &catalog, ) @@ -997,15 +997,12 @@ effort = false let ResolvedCredential::Api(api) = resolved else { panic!("expected api credential"); }; - assert_eq!(api.provider, ProviderId::new("venice")); + assert_eq!(api.provider, ProviderId::new("acme")); assert_eq!( api.auth_header, - Some(ApiKeyHeader::Bearer("venice-key".to_string())) - ); - assert_eq!( - api.base_url.as_deref(), - Some("https://api.venice.ai/api/v1") + Some(ApiKeyHeader::Bearer("acme-key".to_string())) ); + assert_eq!(api.base_url.as_deref(), Some("https://api.acme.test/v1")); } #[tokio::test] diff --git a/lib/crates/fabro-cli/tests/it/cmd/model.rs b/lib/crates/fabro-cli/tests/it/cmd/model.rs index 4525d2533..a725d42e2 100644 --- a/lib/crates/fabro-cli/tests/it/cmd/model.rs +++ b/lib/crates/fabro-cli/tests/it/cmd/model.rs @@ -6,6 +6,30 @@ use fabro_test::{fabro_snapshot, test_context}; use httpmock::MockServer; +fn successful_stdout(mut cmd: assert_cmd::Command) -> String { + let assert = cmd.assert().success().stderr(""); + String::from_utf8(assert.get_output().stdout.clone()) + .expect("model list stdout should be UTF-8") +} + +fn assert_model_list_table(stdout: &str) { + let mut lines = stdout.lines(); + let header = lines.next().expect("model list should render a header"); + assert!(header.contains("MODEL"), "missing MODEL column: {header}"); + assert!( + header.contains("PROVIDER"), + "missing PROVIDER column: {header}" + ); + assert!( + header.contains("CONTEXT"), + "missing CONTEXT column: {header}" + ); + assert!( + lines.any(|line| !line.trim().is_empty()), + "model list should render at least one model row" + ); +} + #[test] fn help() { let context = test_context!(); @@ -38,36 +62,8 @@ fn help() { #[test] fn bare() { let context = test_context!(); - fabro_snapshot!(context.filters(), context.model(), @" - success: true - exit_code: 0 - ----- stdout ----- - MODEL PROVIDER ALIASES CONTEXT COST SPEED - claude-haiku-4-5 anthropic haiku, claude-haiku 200k $0.8 / $4.0 100 tok/s - claude-opus-4-6 anthropic 1m $5.0 / $25.0 25 tok/s - claude-opus-4-7 anthropic opus, claude-opus 1m $5.0 / $25.0 25 tok/s - claude-sonnet-4-5 anthropic 200k $3.0 / $15.0 50 tok/s - claude-sonnet-4-6 anthropic sonnet, claude-sonnet 200k $3.0 / $15.0 50 tok/s - gemini-3-flash-preview gemini gemini-flash 1m $0.5 / $3.0 150 tok/s - gemini-3.1-flash-lite-preview gemini gemini-flash-lite 1m $0.2 / $1.5 200 tok/s - gemini-3.1-pro-preview gemini gemini-pro 1m $2.0 / $12.0 85 tok/s - gemini-3.1-pro-preview-customtools gemini gemini-customtools 1m $2.0 / $12.0 85 tok/s - mercury-2 inception mercury 131k $0.2 / $0.8 1000 tok/s - kimi-k2.5 kimi kimi 262k $0.6 / $3.0 50 tok/s - minimax-m2.5 minimax minimax 197k $0.3 / $1.2 45 tok/s - gpt-5-mini openai gpt5-mini 1m $0.2 / $2.0 70 tok/s - gpt-5.2 openai gpt5 1m $1.8 / $14.0 65 tok/s - gpt-5.2-codex openai 1m $1.8 / $14.0 100 tok/s - gpt-5.3-codex openai codex 1m $1.8 / $14.0 100 tok/s - gpt-5.3-codex-spark openai codex-spark 131k - / - 1000 tok/s - gpt-5.4 openai gpt54, gpt-54 1m $2.5 / $15.0 70 tok/s - gpt-5.4-mini openai gpt54-mini, gpt-54-mini 400k $0.8 / $4.5 140 tok/s - gpt-5.4-pro openai gpt54-pro, gpt-54-pro 1m $30.0 / $180.0 20 tok/s - gpt-5.5 openai gpt55, gpt-55 1m $5.0 / $30.0 70 tok/s - gpt-5.5-pro openai gpt55-pro, gpt-55-pro 1m $30.0 / $180.0 20 tok/s - glm-4.7 zai glm, glm4 203k $0.6 / $2.2 100 tok/s - ----- stderr ----- - "); + let stdout = successful_stdout(context.model()); + assert_model_list_table(&stdout); } #[test] @@ -75,36 +71,8 @@ fn list() { let context = test_context!(); let mut cmd = context.model(); cmd.arg("list"); - fabro_snapshot!(context.filters(), cmd, @" - success: true - exit_code: 0 - ----- stdout ----- - MODEL PROVIDER ALIASES CONTEXT COST SPEED - claude-haiku-4-5 anthropic haiku, claude-haiku 200k $0.8 / $4.0 100 tok/s - claude-opus-4-6 anthropic 1m $5.0 / $25.0 25 tok/s - claude-opus-4-7 anthropic opus, claude-opus 1m $5.0 / $25.0 25 tok/s - claude-sonnet-4-5 anthropic 200k $3.0 / $15.0 50 tok/s - claude-sonnet-4-6 anthropic sonnet, claude-sonnet 200k $3.0 / $15.0 50 tok/s - gemini-3-flash-preview gemini gemini-flash 1m $0.5 / $3.0 150 tok/s - gemini-3.1-flash-lite-preview gemini gemini-flash-lite 1m $0.2 / $1.5 200 tok/s - gemini-3.1-pro-preview gemini gemini-pro 1m $2.0 / $12.0 85 tok/s - gemini-3.1-pro-preview-customtools gemini gemini-customtools 1m $2.0 / $12.0 85 tok/s - mercury-2 inception mercury 131k $0.2 / $0.8 1000 tok/s - kimi-k2.5 kimi kimi 262k $0.6 / $3.0 50 tok/s - minimax-m2.5 minimax minimax 197k $0.3 / $1.2 45 tok/s - gpt-5-mini openai gpt5-mini 1m $0.2 / $2.0 70 tok/s - gpt-5.2 openai gpt5 1m $1.8 / $14.0 65 tok/s - gpt-5.2-codex openai 1m $1.8 / $14.0 100 tok/s - gpt-5.3-codex openai codex 1m $1.8 / $14.0 100 tok/s - gpt-5.3-codex-spark openai codex-spark 131k - / - 1000 tok/s - gpt-5.4 openai gpt54, gpt-54 1m $2.5 / $15.0 70 tok/s - gpt-5.4-mini openai gpt54-mini, gpt-54-mini 400k $0.8 / $4.5 140 tok/s - gpt-5.4-pro openai gpt54-pro, gpt-54-pro 1m $30.0 / $180.0 20 tok/s - gpt-5.5 openai gpt55, gpt-55 1m $5.0 / $30.0 70 tok/s - gpt-5.5-pro openai gpt55-pro, gpt-55-pro 1m $30.0 / $180.0 20 tok/s - glm-4.7 zai glm, glm4 203k $0.6 / $2.2 100 tok/s - ----- stderr ----- - "); + let stdout = successful_stdout(cmd); + assert_model_list_table(&stdout); } #[test] diff --git a/lib/crates/fabro-config/src/builders.rs b/lib/crates/fabro-config/src/builders.rs index 8f19d0792..e1d779c20 100644 --- a/lib/crates/fabro-config/src/builders.rs +++ b/lib/crates/fabro-config/src/builders.rs @@ -686,22 +686,22 @@ _version = 1 [server.auth] methods = ["dev-token"] -[llm.providers.venice] -display_name = "Venice" +[llm.providers.acme] +display_name = "Acme" adapter = "openai_compatible" -base_url = "https://api.venice.ai/api/v1" -credentials = ["env:VENICE_API_KEY"] +base_url = "https://api.acme.test/v1" +credentials = ["env:ACME_API_KEY"] -[llm.models."venice-large"] -provider = "venice" -display_name = "Venice Large" -family = "venice" +[llm.models."acme-large"] +provider = "acme" +display_name = "Acme Large" +family = "acme" default = true -[llm.models."venice-large".limits] +[llm.models."acme-large".limits] context_window = 128000 -[llm.models."venice-large".features] +[llm.models."acme-large".features] tools = true vision = false reasoning = false @@ -718,9 +718,9 @@ effort = false assert_eq!( catalog - .get("venice-large") + .get("acme-large") .map(|model| model.provider.clone()), - Some(fabro_model::ProviderId::new("venice")) + Some(fabro_model::ProviderId::new("acme")) ); } } diff --git a/lib/crates/fabro-llm/src/client.rs b/lib/crates/fabro-llm/src/client.rs index 598610295..f5235b746 100644 --- a/lib/crates/fabro-llm/src/client.rs +++ b/lib/crates/fabro-llm/src/client.rs @@ -714,8 +714,8 @@ mod tests { let catalog = catalog_with(""); let result = Client::from_credentials( vec![ApiCredential { - provider: fabro_model::ProviderId::new("venice"), - auth_header: Some(ApiKeyHeader::Bearer("venice-key".to_string())), + provider: fabro_model::ProviderId::new("custom"), + auth_header: Some(ApiKeyHeader::Bearer("custom-key".to_string())), extra_headers: HashMap::new(), base_url: None, codex_mode: false, @@ -734,7 +734,7 @@ mod tests { Error::Configuration { ref message, .. - } if message == "Provider \"venice\" is not supported by credential-only registration" + } if message == "Provider \"custom\" is not supported by credential-only registration" )); } @@ -765,23 +765,23 @@ mod tests { async fn from_credentials_registers_custom_openai_compatible_provider() { let catalog = catalog_with( r#" -[providers.venice] -display_name = "Venice" +[providers.acme] +display_name = "Acme" adapter = "openai_compatible" -base_url = "https://api.venice.ai/api/v1" -credentials = ["env:VENICE_API_KEY"] -aliases = ["venice-ai"] +base_url = "https://api.acme.test/v1" +credentials = ["env:ACME_API_KEY"] +aliases = ["acme-ai"] -[models."venice-large"] -provider = "venice" -display_name = "Venice Large" -family = "venice" +[models."acme-large"] +provider = "acme" +display_name = "Acme Large" +family = "acme" default = true -[models."venice-large".limits] +[models."acme-large".limits] context_window = 128000 -[models."venice-large".features] +[models."acme-large".features] tools = true vision = false reasoning = false @@ -791,8 +791,8 @@ effort = false let client = Client::from_credentials( vec![ApiCredential { - provider: fabro_model::ProviderId::new("venice"), - auth_header: Some(ApiKeyHeader::Bearer("venice-key".to_string())), + provider: fabro_model::ProviderId::new("acme"), + auth_header: Some(ApiKeyHeader::Bearer("acme-key".to_string())), extra_headers: HashMap::new(), base_url: None, codex_mode: false, @@ -804,32 +804,32 @@ effort = false .await .unwrap(); - assert_eq!(client.provider_names(), vec!["venice"]); - assert!(client.has_provider("venice")); - assert!(client.has_provider("venice-ai")); + assert_eq!(client.provider_names(), vec!["acme"]); + assert!(client.has_provider("acme")); + assert!(client.has_provider("acme-ai")); } #[tokio::test] async fn resolve_provider_accepts_catalog_provider_alias() { let catalog = catalog_with( r#" -[providers.venice] -display_name = "Venice" +[providers.acme] +display_name = "Acme" adapter = "openai_compatible" -base_url = "https://api.venice.ai/api/v1" -credentials = ["env:VENICE_API_KEY"] -aliases = ["venice-ai"] +base_url = "https://api.acme.test/v1" +credentials = ["env:ACME_API_KEY"] +aliases = ["acme-ai"] -[models."venice-large"] -provider = "venice" -display_name = "Venice Large" -family = "venice" +[models."acme-large"] +provider = "acme" +display_name = "Acme Large" +family = "acme" default = true -[models."venice-large".limits] +[models."acme-large".limits] context_window = 128000 -[models."venice-large".features] +[models."acme-large".features] tools = true vision = false reasoning = false @@ -839,8 +839,8 @@ effort = false let client = Client::from_credentials( vec![ApiCredential { - provider: fabro_model::ProviderId::new("venice"), - auth_header: Some(ApiKeyHeader::Bearer("venice-key".to_string())), + provider: fabro_model::ProviderId::new("acme"), + auth_header: Some(ApiKeyHeader::Bearer("acme-key".to_string())), extra_headers: HashMap::new(), base_url: None, codex_mode: false, @@ -852,11 +852,11 @@ effort = false .await .unwrap(); let mut request = test_request(); - request.provider = Some("venice-ai".to_string()); + request.provider = Some("acme-ai".to_string()); let provider = client.resolve_provider(&request).unwrap(); - assert_eq!(provider.name(), "venice"); + assert_eq!(provider.name(), "acme"); } #[tokio::test] diff --git a/lib/crates/fabro-llm/src/providers/openai_compatible.rs b/lib/crates/fabro-llm/src/providers/openai_compatible.rs index add8f025c..de9780def 100644 --- a/lib/crates/fabro-llm/src/providers/openai_compatible.rs +++ b/lib/crates/fabro-llm/src/providers/openai_compatible.rs @@ -1358,23 +1358,23 @@ mod tests { fn catalog_api_id_is_used_for_provider_request_body() { let settings: LlmCatalogSettings = toml::from_str( r#" -[providers.venice] -display_name = "Venice" +[providers.acme] +display_name = "Acme" adapter = "openai_compatible" -base_url = "https://api.venice.ai/api/v1" -credentials = ["env:VENICE_API_KEY"] +base_url = "https://api.acme.test/v1" +credentials = ["env:ACME_API_KEY"] -[models."venice-large"] -provider = "venice" -api_id = "venice/model-large" -display_name = "Venice Large" -family = "venice" +[models."acme-large"] +provider = "acme" +api_id = "acme/model-large" +display_name = "Acme Large" +family = "acme" default = true -[models."venice-large".limits] +[models."acme-large".limits] context_window = 128000 -[models."venice-large".features] +[models."acme-large".features] tools = true vision = false reasoning = false @@ -1384,12 +1384,12 @@ effort = false .unwrap(); let catalog = Catalog::from_builtin_with_overrides(&settings).unwrap(); let mut request = minimal_request(); - request.model = "venice-large".to_string(); + request.model = "acme-large".to_string(); - let body = build_api_request_with_catalog(&request, None, "venice", Some(&catalog)); + let body = build_api_request_with_catalog(&request, None, "acme", Some(&catalog)); - assert_eq!(request.model, "venice-large"); - assert_eq!(body["model"], "venice/model-large"); + assert_eq!(request.model, "acme-large"); + assert_eq!(body["model"], "acme/model-large"); } #[test] diff --git a/lib/crates/fabro-model/src/catalog.rs b/lib/crates/fabro-model/src/catalog.rs index 88ed22290..06dd0f054 100644 --- a/lib/crates/fabro-model/src/catalog.rs +++ b/lib/crates/fabro-model/src/catalog.rs @@ -1558,25 +1558,25 @@ enabled = false fn builtin_overrides_add_custom_openai_compatible_provider_and_model() { let catalog = Catalog::from_builtin_with_overrides(&minimal_settings( r#" -[providers.venice] -display_name = "Venice" +[providers.acme] +display_name = "Acme" adapter = "openai_compatible" -base_url = "https://api.venice.ai/api/v1" -credentials = ["env:VENICE_API_KEY"] +base_url = "https://api.acme.test/v1" +credentials = ["env:ACME_API_KEY"] priority = 120 -aliases = ["venice-ai"] +aliases = ["acme-ai"] -[models."venice-large"] -provider = "venice" -display_name = "Venice Large" -family = "venice" +[models."acme-large"] +provider = "acme" +display_name = "Acme Large" +family = "acme" default = true -aliases = ["vl"] +aliases = ["al"] -[models."venice-large".limits] +[models."acme-large".limits] context_window = 128000 -[models."venice-large".features] +[models."acme-large".features] tools = true vision = false reasoning = false @@ -1586,14 +1586,14 @@ effort = false .expect("custom provider overlay should build"); let provider = catalog - .provider(&ProviderId::new("venice-ai")) + .provider(&ProviderId::new("acme-ai")) .expect("provider alias should resolve"); - assert_eq!(provider.id, ProviderId::new("venice")); + assert_eq!(provider.id, ProviderId::new("acme")); assert_eq!(provider.adapter, "openai_compatible"); - let model = catalog.get("vl").expect("model alias should resolve"); - assert_eq!(model.id, "venice-large"); - assert_eq!(model.provider, ProviderId::new("venice")); + let model = catalog.get("al").expect("model alias should resolve"); + assert_eq!(model.id, "acme-large"); + assert_eq!(model.provider, ProviderId::new("acme")); } #[test] @@ -1849,8 +1849,8 @@ effort = false fn catalog_from_settings_rejects_unknown_adapter() { let layer = minimal_settings( r#" -[providers.venice] -display_name = "Venice" +[providers.test-provider] +display_name = "Test Provider" adapter = "not_real" enabled = true "#, @@ -1861,7 +1861,7 @@ enabled = true assert!(matches!( err, CatalogBuildError::UnknownAdapter { provider, adapter } - if provider == ProviderId::new("venice") && adapter == "not_real" + if provider == ProviderId::new("test-provider") && adapter == "not_real" )); } @@ -2464,19 +2464,14 @@ reasoning_effort = "levels" } #[test] - fn catalog_providers_roundtrip_through_static_str() { - for model in Catalog::builtin().list(None) { - let roundtripped = Provider::from_id(&model.provider); - assert_eq!( - roundtripped, - Some( - model - .builtin_provider() - .expect("catalog model provider should be a built-in provider") - ), - "catalog model '{}' provider {:?} does not roundtrip through ProviderId", + fn every_catalog_model_provider_has_catalog_provider() { + let catalog = Catalog::builtin(); + for model in catalog.list(None) { + assert!( + catalog.provider(&model.provider).is_some(), + "catalog model '{}' provider {:?} has no provider metadata", model.id, - model.provider + model.provider, ); } } diff --git a/lib/crates/fabro-model/src/types.rs b/lib/crates/fabro-model/src/types.rs index bbd653e89..8f754160a 100644 --- a/lib/crates/fabro-model/src/types.rs +++ b/lib/crates/fabro-model/src/types.rs @@ -194,9 +194,9 @@ mod tests { } #[test] - fn all_catalog_providers_are_valid() { - for model in Catalog::builtin().list(None) { - assert!(model.builtin_provider().is_some()); + fn builtin_provider_matches_known_static_provider_ids() { + for info in Catalog::builtin().list(None) { + assert_eq!(info.builtin_provider(), Provider::from_id(info.provider())); } } } diff --git a/lib/crates/fabro-server/src/run_manifest.rs b/lib/crates/fabro-server/src/run_manifest.rs index 8b1fe214e..342517c3c 100644 --- a/lib/crates/fabro-server/src/run_manifest.rs +++ b/lib/crates/fabro-server/src/run_manifest.rs @@ -2158,7 +2158,7 @@ digraph Demo { digraph Demo { start [shape=Mdiamond] exit [shape=Msquare] - work [prompt="Do work", model="venice-model", provider="venice"] + work [prompt="Do work", model="missing-model", provider="missing-provider"] start -> work -> exit } "# @@ -2179,18 +2179,18 @@ digraph Demo { let llm_check = response.checks.sections[0] .checks .iter() - .find(|check| check.name == "LLM" && check.summary == "venice-model") + .find(|check| check.name == "LLM" && check.summary == "missing-model") .expect("preflight should include the requested custom LLM provider"); assert_eq!(llm_check.status, types::PreflightCheckResultStatus::Warning); assert_eq!( llm_check.remediation.as_deref(), - Some("Provider \"venice\" is not configured") + Some("Provider \"missing-provider\" is not configured") ); assert!( llm_check .details .iter() - .any(|detail| detail.text == "Provider: venice") + .any(|detail| detail.text == "Provider: missing-provider") ); } @@ -2198,23 +2198,23 @@ digraph Demo { async fn preflight_resolves_model_aliases_from_app_state_catalog() { let llm_catalog_settings: fabro_model::catalog::LlmCatalogSettings = toml::from_str( r#" -[providers.venice] -display_name = "Venice" +[providers.acme] +display_name = "Acme" adapter = "openai_compatible" -base_url = "https://api.venice.ai/api/v1" -credentials = ["env:VENICE_API_KEY"] +base_url = "https://api.acme.test/v1" +credentials = ["env:ACME_API_KEY"] -[models."venice-large"] -provider = "venice" -display_name = "Venice Large" -family = "venice" +[models."acme-large"] +provider = "acme" +display_name = "Acme Large" +family = "acme" default = true aliases = ["vl"] -[models."venice-large".limits] +[models."acme-large".limits] context_window = 128000 -[models."venice-large".features] +[models."acme-large".features] tools = true vision = false reasoning = false @@ -2251,18 +2251,18 @@ digraph Demo { let llm_check = response.checks.sections[0] .checks .iter() - .find(|check| check.name == "LLM" && check.summary == "venice-large") + .find(|check| check.name == "LLM" && check.summary == "acme-large") .expect("preflight should resolve the catalog alias"); assert_eq!(llm_check.status, types::PreflightCheckResultStatus::Warning); assert_eq!( llm_check.remediation.as_deref(), - Some("Provider \"venice\" is not configured") + Some("Provider \"acme\" is not configured") ); assert!( llm_check .details .iter() - .any(|detail| detail.text == "Provider: venice") + .any(|detail| detail.text == "Provider: acme") ); } diff --git a/lib/crates/fabro-server/src/server/tests.rs b/lib/crates/fabro-server/src/server/tests.rs index 011954da0..e3d0304a9 100644 --- a/lib/crates/fabro-server/src/server/tests.rs +++ b/lib/crates/fabro-server/src/server/tests.rs @@ -2263,22 +2263,22 @@ async fn validate_endpoint_returns_workflow_summary_without_preflight_checks() { async fn validate_endpoint_uses_app_state_catalog_for_model_diagnostics() { let llm_catalog_settings: LlmCatalogSettings = toml::from_str( r#" -[providers.venice] -display_name = "Venice" +[providers.acme] +display_name = "Acme" adapter = "openai_compatible" -base_url = "https://api.venice.ai/api/v1" -credentials = ["env:VENICE_API_KEY"] +base_url = "https://api.acme.test/v1" +credentials = ["env:ACME_API_KEY"] -[models."venice-large"] -provider = "venice" -display_name = "Venice Large" -family = "venice" +[models."acme-large"] +provider = "acme" +display_name = "Acme Large" +family = "acme" default = true -[models."venice-large".limits] +[models."acme-large".limits] context_window = 128000 -[models."venice-large".features] +[models."acme-large".features] tools = true vision = false reasoning = false @@ -2293,7 +2293,7 @@ effort = false let dot = r#"digraph Test { graph [goal="Test"] start [shape=Mdiamond] - work [model="venice-large", provider="venice", prompt="Do it"] + work [model="acme-large", provider="acme", prompt="Do it"] exit [shape=Msquare] start -> work -> exit }"#; @@ -3998,7 +3998,7 @@ async fn list_models_unknown_provider_returns_empty_page() { let req = Request::builder() .method("GET") - .uri(api("/models?provider=venice")) + .uri(api("/models?provider=missing-provider")) .body(Body::empty()) .unwrap(); @@ -4012,23 +4012,23 @@ async fn list_models_unknown_provider_returns_empty_page() { async fn list_models_uses_app_state_catalog_overrides() { let llm_catalog_settings: LlmCatalogSettings = toml::from_str( r#" -[providers.venice] -display_name = "Venice" +[providers.acme] +display_name = "Acme" adapter = "openai_compatible" -base_url = "https://api.venice.ai/api/v1" -credentials = ["env:VENICE_API_KEY"] +base_url = "https://api.acme.test/v1" +credentials = ["env:ACME_API_KEY"] priority = 120 -[models."venice-large"] -provider = "venice" -display_name = "Venice Large" -family = "venice" +[models."acme-large"] +provider = "acme" +display_name = "Acme Large" +family = "acme" default = true -[models."venice-large".limits] +[models."acme-large".limits] context_window = 128000 -[models."venice-large".features] +[models."acme-large".features] tools = true vision = false reasoning = false @@ -4043,7 +4043,7 @@ effort = false let req = Request::builder() .method("GET") - .uri(api("/models?provider=venice")) + .uri(api("/models?provider=acme")) .body(Body::empty()) .unwrap(); @@ -4051,8 +4051,8 @@ effort = false let body = response_json!(response, StatusCode::OK).await; let models = body["data"].as_array().unwrap(); assert_eq!(models.len(), 1); - assert_eq!(models[0]["id"], "venice-large"); - assert_eq!(models[0]["provider"], "venice"); + assert_eq!(models[0]["id"], "acme-large"); + assert_eq!(models[0]["provider"], "acme"); } #[tokio::test] @@ -9444,7 +9444,7 @@ async fn create_completion_unknown_provider_returns_clear_error() { .header("content-type", "application/json") .body(Body::from( serde_json::json!({ - "provider": "venice", + "provider": "missing-provider", "model": "gpt-5.4", "stream": false, "messages": [ @@ -9462,7 +9462,7 @@ async fn create_completion_unknown_provider_returns_clear_error() { let body = response_json!(response, StatusCode::BAD_REQUEST).await; assert_eq!( body["errors"][0]["detail"], - "Provider \"venice\" is not configured" + "Provider \"missing-provider\" is not configured" ); } @@ -9470,23 +9470,23 @@ async fn create_completion_unknown_provider_returns_clear_error() { async fn create_completion_default_model_uses_app_state_catalog() { let llm_catalog_settings: LlmCatalogSettings = toml::from_str( r#" -[providers.venice] -display_name = "Venice" +[providers.acme] +display_name = "Acme" adapter = "openai_compatible" -base_url = "https://api.venice.ai/api/v1" -credentials = ["env:VENICE_API_KEY"] +base_url = "https://api.acme.test/v1" +credentials = ["env:ACME_API_KEY"] priority = 120 -[models."venice-large"] -provider = "venice" -display_name = "Venice Large" -family = "venice" +[models."acme-large"] +provider = "acme" +display_name = "Acme Large" +family = "acme" default = true -[models."venice-large".limits] +[models."acme-large".limits] context_window = 128000 -[models."venice-large".features] +[models."acme-large".features] tools = true vision = false reasoning = false @@ -9523,7 +9523,7 @@ effort = false body["errors"][0]["detail"] .as_str() .unwrap() - .contains("Provider 'venice' not registered"), + .contains("Provider 'acme' not registered"), "unexpected error body: {body:?}" ); } diff --git a/lib/crates/fabro-workflow/src/handler/llm/api.rs b/lib/crates/fabro-workflow/src/handler/llm/api.rs index 34db6f9ac..5f38a56e5 100644 --- a/lib/crates/fabro-workflow/src/handler/llm/api.rs +++ b/lib/crates/fabro-workflow/src/handler/llm/api.rs @@ -1471,23 +1471,23 @@ mod tests { fn api_backend_resolves_custom_catalog_provider_profile() { let settings: LlmCatalogSettings = toml::from_str( r#" -[providers.venice] +[providers.acme] adapter = "openai_compatible" -base_url = "https://api.venice.ai/api/v1" -credentials = ["env:VENICE_API_KEY"] +base_url = "https://api.acme.test/v1" +credentials = ["env:ACME_API_KEY"] -[models.venice-llama] -provider = "venice" -display_name = "Venice Llama" +[models.acme-llama] +provider = "acme" +display_name = "Acme Llama" family = "llama" training = "2026-01" default = true -[models.venice-llama.limits] +[models.acme-llama.limits] context_window = 131072 max_output = 8192 -[models.venice-llama.features] +[models.acme-llama.features] tools = true vision = false reasoning = false @@ -1497,9 +1497,9 @@ effort = false .unwrap(); let catalog = Arc::new(Catalog::from_builtin_with_overrides(&settings).unwrap()); let backend = AgentApiBackend::new_with_catalog( - "venice-llama".to_string(), + "acme-llama".to_string(), Provider::OpenAiCompatible, - ProviderId::from("venice"), + ProviderId::from("acme"), AgentProfileKind::OpenAi, Vec::new(), Arc::new(EnvCredentialSource::new()), @@ -1508,10 +1508,10 @@ effort = false ); let provider = backend - .resolve_provider_context("venice-llama", None) + .resolve_provider_context("acme-llama", None) .unwrap(); - assert_eq!(provider.provider_id, ProviderId::from("venice")); + assert_eq!(provider.provider_id, ProviderId::from("acme")); assert_eq!(provider.profile_kind, AgentProfileKind::OpenAi); assert_eq!(provider.provider, Provider::OpenAiCompatible); }