diff --git a/Cargo.lock b/Cargo.lock index 3762a99fd..b655ac6c6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2116,6 +2116,7 @@ dependencies = [ "fabro-client", "fabro-config", "fabro-manifest", + "fabro-model", "fabro-server", "fabro-types", "fabro-util", @@ -2506,6 +2507,7 @@ dependencies = [ "fabro-types", "serde", "thiserror 2.0.18", + "toml 0.8.23", ] [[package]] diff --git a/lib/crates/fabro-agent/src/agent_profile.rs b/lib/crates/fabro-agent/src/agent_profile.rs index 91f46e41c..048449dff 100644 --- a/lib/crates/fabro-agent/src/agent_profile.rs +++ b/lib/crates/fabro-agent/src/agent_profile.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use fabro_llm::types::ToolDefinition; -use fabro_model::{Catalog, Provider, ProviderId}; +use fabro_model::{Catalog, Model, Provider, ProviderId}; use tokio::sync::Mutex; use crate::profiles::EnvContext; @@ -19,6 +19,9 @@ pub trait AgentProfile: Send + Sync { self.provider().id() } fn model(&self) -> &str; + fn catalog(&self) -> Option<&Catalog> { + None + } fn tool_registry(&self) -> &ToolRegistry; fn tool_registry_mut(&mut self) -> &mut ToolRegistry; fn build_system_prompt( @@ -35,17 +38,25 @@ pub trait AgentProfile: Send + Sync { } fn knowledge_cutoff(&self) -> Option { - Catalog::builtin() - .get(self.model()) + self.catalog() + .and_then(|catalog| catalog.get(self.model())) .and_then(|m| m.knowledge_cutoff().map(str::to_string)) } + fn catalog_model(&self) -> Option<&Model> { + self.catalog().and_then(|catalog| catalog.get(self.model())) + } + fn context_window_size(&self) -> usize { - Catalog::builtin().get(self.model()).map_or(200_000, |m| { + self.catalog_model().map_or(200_000, |m| { usize::try_from(m.context_window()).unwrap_or(usize::MAX) }) } + fn max_output_tokens(&self) -> Option { + self.catalog_model().and_then(Model::max_output) + } + fn register_subagent_tools( &mut self, manager: Arc>, diff --git a/lib/crates/fabro-agent/src/cli.rs b/lib/crates/fabro-agent/src/cli.rs index 92486776f..b1ccc8060 100644 --- a/lib/crates/fabro-agent/src/cli.rs +++ b/lib/crates/fabro-agent/src/cli.rs @@ -18,6 +18,7 @@ use fabro_llm::middleware::{Middleware, NextFn, NextStreamFn}; use fabro_llm::provider::StreamEventStream; use fabro_llm::types::{Request, Response}; use fabro_mcp::config::McpServerSettings; +use fabro_model::catalog::LlmCatalogSettings; use fabro_model::{Catalog, ModelHandle, Provider}; use fabro_util::terminal::Styles; use fabro_vault::Vault; @@ -231,18 +232,27 @@ fn build_profile( provider: Provider, model: &str, summarizer: Option, + catalog: Arc, ) -> Box { match provider { - Provider::OpenAi => Box::new(OpenAiProfile::with_summarizer(model, summarizer)), + Provider::OpenAi => { + Box::new(OpenAiProfile::with_summarizer(model, summarizer).with_catalog(catalog)) + } Provider::Kimi | Provider::Zai | Provider::Minimax | Provider::Inception - | Provider::OpenAiCompatible => { - Box::new(OpenAiProfile::with_summarizer(model, summarizer).with_provider(provider)) + | Provider::OpenAiCompatible => Box::new( + OpenAiProfile::with_summarizer(model, summarizer) + .with_provider(provider) + .with_catalog(catalog), + ), + Provider::Gemini => { + Box::new(GeminiProfile::with_summarizer(model, summarizer).with_catalog(catalog)) + } + Provider::Anthropic => { + Box::new(AnthropicProfile::with_summarizer(model, summarizer).with_catalog(catalog)) } - Provider::Gemini => Box::new(GeminiProfile::with_summarizer(model, summarizer)), - Provider::Anthropic => Box::new(AnthropicProfile::with_summarizer(model, summarizer)), } } @@ -480,10 +490,14 @@ pub async fn run_with_args_and_client( } // Resolve model and build profile + let catalog = Arc::new( + Catalog::from_builtin_with_overrides(&LlmCatalogSettings::default()) + .context("failed to build standalone agent LLM catalog")?, + ); let model = if let Some(model) = args.model.clone() { model } else { - Catalog::builtin() + catalog .default_for_provider(&provider.id()) .map(|model| model.id.clone()) .ok_or_else(|| { @@ -497,6 +511,7 @@ pub async fn run_with_args_and_client( provider, &model, Some(build_summarizer(provider, client.clone())), + Arc::clone(&catalog), ); // Build sandbox @@ -530,32 +545,17 @@ pub async fn run_with_args_and_client( let manager_for_callback = manager.clone(); let factory_client = client.clone(); let factory_model = model.clone(); + let factory_catalog = Arc::clone(&catalog); let factory_env = Arc::clone(&env); let factory_hooks = config.tool_hooks.clone(); let factory: SessionFactory = Arc::new(move || { let child_summarizer = Some(build_summarizer(provider, factory_client.clone())); - let child_profile: Arc = match provider { - Provider::OpenAi => Arc::new(OpenAiProfile::with_summarizer( - &factory_model, - child_summarizer, - )), - Provider::Kimi - | Provider::Zai - | Provider::Minimax - | Provider::Inception - | Provider::OpenAiCompatible => Arc::new( - OpenAiProfile::with_summarizer(&factory_model, child_summarizer) - .with_provider(provider), - ), - Provider::Gemini => Arc::new(GeminiProfile::with_summarizer( - &factory_model, - child_summarizer, - )), - Provider::Anthropic => Arc::new(AnthropicProfile::with_summarizer( - &factory_model, - child_summarizer, - )), - }; + let child_profile: Arc = Arc::from(build_profile( + provider, + &factory_model, + child_summarizer, + Arc::clone(&factory_catalog), + )); Session::new( factory_client.clone(), child_profile, @@ -852,15 +852,19 @@ mod tests { // build_profile tests + fn test_catalog() -> Arc { + Arc::new(Catalog::from_builtin_with_overrides(&LlmCatalogSettings::default()).unwrap()) + } + #[test] fn build_profile_anthropic() { - let profile = build_profile(Provider::Anthropic, "model", None); + let profile = build_profile(Provider::Anthropic, "model", None, test_catalog()); assert_eq!(profile.provider(), Provider::Anthropic); } #[test] fn build_profile_openai() { - let profile = build_profile(Provider::OpenAi, "model", None); + let profile = build_profile(Provider::OpenAi, "model", None, test_catalog()); assert_eq!(profile.provider(), Provider::OpenAi); } @@ -876,7 +880,7 @@ mod tests { #[test] fn build_profile_gemini() { - let profile = build_profile(Provider::Gemini, "model", None); + let profile = build_profile(Provider::Gemini, "model", None, test_catalog()); assert_eq!(profile.provider(), Provider::Gemini); } @@ -884,7 +888,7 @@ mod tests { #[test] fn build_profile_can_register_subagent_tools() { - let mut profile = build_profile(Provider::Anthropic, "model", None); + let mut profile = build_profile(Provider::Anthropic, "model", None, test_catalog()); let manager = Arc::new(AsyncMutex::new(SubAgentManager::new(1))); let factory: SessionFactory = Arc::new(|| { panic!("factory should not be called in this test"); diff --git a/lib/crates/fabro-agent/src/profiles/anthropic.rs b/lib/crates/fabro-agent/src/profiles/anthropic.rs index 9e718ffa1..379fe7c07 100644 --- a/lib/crates/fabro-agent/src/profiles/anthropic.rs +++ b/lib/crates/fabro-agent/src/profiles/anthropic.rs @@ -1,4 +1,6 @@ -use fabro_model::{Provider, ProviderId}; +use std::sync::Arc; + +use fabro_model::{Catalog, Provider, ProviderId}; use super::EnvContext; use crate::agent_profile::AgentProfile; @@ -38,6 +40,7 @@ impl AnthropicProfile { provider: Provider::Anthropic, provider_id: Provider::Anthropic.id(), model: model.into(), + catalog: None, registry, }, } @@ -58,6 +61,12 @@ impl AnthropicProfile { self.base.provider_id = provider_id; self } + + #[must_use] + pub fn with_catalog(mut self, catalog: Arc) -> Self { + self.base.catalog = Some(catalog); + self + } } impl AgentProfile for AnthropicProfile { @@ -73,6 +82,10 @@ impl AgentProfile for AnthropicProfile { &self.base.model } + fn catalog(&self) -> Option<&Catalog> { + self.base.catalog.as_deref() + } + fn tool_registry(&self) -> &ToolRegistry { &self.base.registry } @@ -180,12 +193,17 @@ in the project. Keep changes minimal and focused on the task."; mod tests { use std::sync::Arc; + use fabro_model::catalog::LlmCatalogSettings; use tokio::sync::Mutex as AsyncMutex; use super::*; use crate::subagent::{SessionFactory, SubAgentManager}; use crate::test_support::MockSandbox; + fn test_catalog() -> Arc { + Arc::new(Catalog::from_builtin_with_overrides(&LlmCatalogSettings::default()).unwrap()) + } + #[test] fn anthropic_profile_identity() { let profile = AnthropicProfile::new("claude-sonnet-4-20250514"); @@ -195,16 +213,16 @@ mod tests { #[test] fn anthropic_context_window_from_catalog() { - let profile = AnthropicProfile::new("claude-opus-4-6"); + let profile = AnthropicProfile::new("claude-opus-4-6").with_catalog(test_catalog()); assert_eq!(profile.context_window_size(), 1_000_000); - let profile = AnthropicProfile::new("claude-sonnet-4-6"); + let profile = AnthropicProfile::new("claude-sonnet-4-6").with_catalog(test_catalog()); assert_eq!(profile.context_window_size(), 200_000); } #[test] fn anthropic_knowledge_cutoff_from_catalog() { - let profile = AnthropicProfile::new("claude-opus-4-6"); + let profile = AnthropicProfile::new("claude-opus-4-6").with_catalog(test_catalog()); assert_eq!(profile.knowledge_cutoff(), Some("May 2025".to_string())); } diff --git a/lib/crates/fabro-agent/src/profiles/gemini.rs b/lib/crates/fabro-agent/src/profiles/gemini.rs index 93ef9c0a0..9ba16df92 100644 --- a/lib/crates/fabro-agent/src/profiles/gemini.rs +++ b/lib/crates/fabro-agent/src/profiles/gemini.rs @@ -1,4 +1,6 @@ -use fabro_model::{Provider, ProviderId}; +use std::sync::Arc; + +use fabro_model::{Catalog, Provider, ProviderId}; use super::EnvContext; use crate::agent_profile::AgentProfile; @@ -40,6 +42,7 @@ impl GeminiProfile { provider: Provider::Gemini, provider_id: Provider::Gemini.id(), model: model.into(), + catalog: None, registry, }, } @@ -59,6 +62,12 @@ impl GeminiProfile { self.base.provider_id = provider_id; self } + + #[must_use] + pub fn with_catalog(mut self, catalog: Arc) -> Self { + self.base.catalog = Some(catalog); + self + } } impl AgentProfile for GeminiProfile { @@ -74,6 +83,10 @@ impl AgentProfile for GeminiProfile { &self.base.model } + fn catalog(&self) -> Option<&Catalog> { + self.base.catalog.as_deref() + } + fn tool_registry(&self) -> &ToolRegistry { &self.base.registry } @@ -222,12 +235,17 @@ in the project."; mod tests { use std::sync::Arc; + use fabro_model::catalog::LlmCatalogSettings; use tokio::sync::Mutex as AsyncMutex; use super::*; use crate::subagent::{SessionFactory, SubAgentManager}; use crate::test_support::MockSandbox; + fn test_catalog() -> Arc { + Arc::new(Catalog::from_builtin_with_overrides(&LlmCatalogSettings::default()).unwrap()) + } + #[test] fn gemini_profile_identity() { let profile = GeminiProfile::new("gemini-2.0-flash"); @@ -237,7 +255,7 @@ mod tests { #[test] fn gemini_context_window_from_catalog() { - let profile = GeminiProfile::new("gemini-3.1-pro-preview"); + let profile = GeminiProfile::new("gemini-3.1-pro-preview").with_catalog(test_catalog()); assert_eq!(profile.context_window_size(), 1_048_576); } diff --git a/lib/crates/fabro-agent/src/profiles/mod.rs b/lib/crates/fabro-agent/src/profiles/mod.rs index a4c068965..72c9cceb6 100644 --- a/lib/crates/fabro-agent/src/profiles/mod.rs +++ b/lib/crates/fabro-agent/src/profiles/mod.rs @@ -1,9 +1,12 @@ +use std::sync::Arc; + +use fabro_model::{Catalog, Provider, ProviderId}; + pub mod anthropic; pub mod gemini; pub mod openai; pub use anthropic::AnthropicProfile; -use fabro_model::{Provider, ProviderId}; pub use gemini::GeminiProfile; pub use openai::OpenAiProfile; @@ -19,6 +22,7 @@ pub struct BaseProfile { pub provider: Provider, pub provider_id: ProviderId, pub model: String, + pub catalog: Option>, pub registry: ToolRegistry, } diff --git a/lib/crates/fabro-agent/src/profiles/openai.rs b/lib/crates/fabro-agent/src/profiles/openai.rs index 4f2f10166..dc323c466 100644 --- a/lib/crates/fabro-agent/src/profiles/openai.rs +++ b/lib/crates/fabro-agent/src/profiles/openai.rs @@ -1,4 +1,6 @@ -use fabro_model::{Provider, ProviderId}; +use std::sync::Arc; + +use fabro_model::{Catalog, Provider, ProviderId}; use super::EnvContext; use crate::agent_profile::AgentProfile; @@ -36,6 +38,7 @@ impl OpenAiProfile { provider: Provider::OpenAi, provider_id: Provider::OpenAi.id(), model: model.into(), + catalog: None, registry, }, } @@ -57,6 +60,12 @@ impl OpenAiProfile { self } + #[must_use] + pub fn with_catalog(mut self, catalog: Arc) -> Self { + self.base.catalog = Some(catalog); + self + } + fn provider_display_name(&self) -> &str { match self.base.provider { Provider::OpenAi => "OpenAI", @@ -82,6 +91,10 @@ impl AgentProfile for OpenAiProfile { &self.base.model } + fn catalog(&self) -> Option<&Catalog> { + self.base.catalog.as_deref() + } + fn tool_registry(&self) -> &ToolRegistry { &self.base.registry } diff --git a/lib/crates/fabro-agent/src/session.rs b/lib/crates/fabro-agent/src/session.rs index 5798bae96..9420ea004 100644 --- a/lib/crates/fabro-agent/src/session.rs +++ b/lib/crates/fabro-agent/src/session.rs @@ -1553,11 +1553,10 @@ impl Session { response_format: None, temperature: None, top_p: None, - max_tokens: self.config.max_tokens.or_else(|| { - fabro_model::Catalog::builtin() - .get(self.provider_profile.model()) - .and_then(fabro_model::Model::max_output) - }), + max_tokens: self + .config + .max_tokens + .or_else(|| self.provider_profile.max_output_tokens()), stop_sequences: None, reasoning_effort: self.config.reasoning_effort, speed: self.config.speed, diff --git a/lib/crates/fabro-auth/src/env_source.rs b/lib/crates/fabro-auth/src/env_source.rs index 0d7a17e03..6511ff154 100644 --- a/lib/crates/fabro-auth/src/env_source.rs +++ b/lib/crates/fabro-auth/src/env_source.rs @@ -2,7 +2,9 @@ use std::sync::Arc; use async_trait::async_trait; use fabro_model::catalog::CatalogProvider; -use fabro_model::{Catalog, CredentialRef, HeaderValueRef, Provider, ProviderId, adapter}; +use fabro_model::{ + Catalog, CredentialRef, HeaderValueRef, Provider, ProviderId, adapter, bootstrap_catalog, +}; use fabro_static::EnvVars; use crate::credential_source::{CredentialSource, ResolvedCredentials}; @@ -135,7 +137,7 @@ impl Default for EnvCredentialSource { #[async_trait] impl CredentialSource for EnvCredentialSource { async fn resolve(&self) -> anyhow::Result { - self.resolve_for_catalog(Catalog::builtin()).await + self.resolve_for_catalog(bootstrap_catalog::catalog()).await } async fn resolve_for_catalog(&self, catalog: &Catalog) -> anyhow::Result { @@ -158,7 +160,7 @@ impl CredentialSource for EnvCredentialSource { } async fn configured_providers(&self) -> Vec { - self.configured_providers_for_catalog(Catalog::builtin()) + self.configured_providers_for_catalog(bootstrap_catalog::catalog()) .await } diff --git a/lib/crates/fabro-auth/src/resolve.rs b/lib/crates/fabro-auth/src/resolve.rs index 8b12ba2c0..4774347ff 100644 --- a/lib/crates/fabro-auth/src/resolve.rs +++ b/lib/crates/fabro-auth/src/resolve.rs @@ -4,6 +4,7 @@ use std::sync::Arc; use fabro_model::catalog::CatalogProvider; use fabro_model::{ ApiKeyHeaderPolicy, Catalog, CredentialRef, HeaderValueRef, Provider, ProviderId, adapter, + bootstrap_catalog, }; use fabro_static::EnvVars; use fabro_vault::Vault; @@ -49,7 +50,7 @@ impl ApiCredential { #[must_use] pub fn from_api_key(provider: impl Into, key: String) -> Self { let provider = provider.into(); - let auth_header = auth_header_for_provider(&provider, key); + let auth_header = default_auth_header_for_provider(&provider, key); Self { provider, auth_header: Some(auth_header), @@ -60,6 +61,33 @@ impl ApiCredential { project_id: None, } } + + /// Build an `ApiCredential` from an API key using the supplied catalog for + /// auth header policy and provider base URL. + #[must_use] + pub fn from_api_key_for_catalog( + provider: impl Into, + key: String, + catalog: &Catalog, + ) -> Self { + let provider_id = provider.into(); + let (auth_header, base_url) = match catalog.provider(&provider_id) { + Some(provider) => ( + auth_header_for_catalog_provider(provider, key), + provider.base_url.clone(), + ), + None => (default_auth_header_for_provider(&provider_id, key), None), + }; + Self { + provider: provider_id, + auth_header: Some(auth_header), + extra_headers: HashMap::new(), + base_url, + codex_mode: false, + org_id: None, + project_id: None, + } + } } #[must_use] @@ -73,8 +101,8 @@ pub fn build_api_key_header(policy: ApiKeyHeaderPolicy, key: String) -> ApiKeyHe } } -fn auth_header_for_provider(provider: &ProviderId, key: String) -> ApiKeyHeader { - let policy = Catalog::builtin() +fn default_auth_header_for_provider(provider: &ProviderId, key: String) -> ApiKeyHeader { + let policy = bootstrap_catalog::catalog() .provider(provider) .and_then(|provider| adapter::get(&provider.adapter)) .map_or_else( @@ -221,7 +249,7 @@ impl CredentialResolver { #[must_use] pub fn configured_providers(&self, vault: &Vault) -> Vec { - self.configured_providers_for_catalog(vault, Catalog::builtin()) + self.configured_providers_for_catalog(vault, bootstrap_catalog::catalog()) } pub fn configured_providers_for_catalog( @@ -253,7 +281,7 @@ impl CredentialResolver { } } - if let Some(catalog_provider) = Catalog::builtin().provider(provider) { + if let Some(catalog_provider) = bootstrap_catalog::catalog().provider(provider) { for credential_ref in &catalog_provider.credentials { if let Some(credential) = self.credential_from_ref(vault, provider, credential_ref) { @@ -389,7 +417,7 @@ impl CredentialResolver { vault: &Vault, credential: &AuthCredential, ) -> Result { - self.to_api_credential_for_catalog(vault, credential, Catalog::builtin()) + self.to_api_credential_for_catalog(vault, credential, bootstrap_catalog::catalog()) } fn to_api_credential_for_catalog( @@ -402,7 +430,7 @@ impl CredentialResolver { match &credential.details { AuthDetails::ApiKey { key } => { let auth_header = catalog.provider(&credential.provider).map_or_else( - || auth_header_for_provider(&credential.provider, key.clone()), + || default_auth_header_for_provider(&credential.provider, key.clone()), |provider| auth_header_for_catalog_provider(provider, key.clone()), ); let mut cred = ApiCredential { @@ -452,7 +480,7 @@ impl CredentialResolver { ) -> Result { let provider_id = provider.into(); let Some(catalog_provider) = catalog.provider(&provider_id) else { - return self.resolve(provider_id, usage).await; + return Err(ResolveError::NotConfigured(provider_id)); }; let initial_credential = { let vault = self.vault.read().await; @@ -565,7 +593,7 @@ pub async fn configured_providers_from_process_env( let guard = vault_arc.read().await; resolver.configured_providers(&guard) } - None => Catalog::builtin() + None => bootstrap_catalog::catalog() .providers() .iter() .filter(|provider| provider_has_process_env_api_key(&provider.id)) @@ -579,7 +607,7 @@ pub async fn configured_providers_from_process_env( reason = "Provider discovery intentionally checks documented API-key env names." )] fn provider_has_process_env_api_key(provider: &ProviderId) -> bool { - Catalog::builtin() + bootstrap_catalog::catalog() .provider(provider) .is_some_and(|catalog_provider| { catalog_provider.credentials.iter().any(|credential_ref| { @@ -589,7 +617,7 @@ fn provider_has_process_env_api_key(provider: &ProviderId) -> bool { } fn primary_api_key_env_var(provider: &ProviderId) -> Option<&'static str> { - Catalog::builtin() + bootstrap_catalog::catalog() .provider(provider)? .credentials .iter() @@ -613,6 +641,7 @@ mod tests { use std::os::unix::fs::PermissionsExt; use chrono::{Duration, Utc}; + use fabro_model::catalog::LlmCatalogSettings; use httpmock::Method::POST; use httpmock::MockServer; @@ -655,6 +684,11 @@ mod tests { CredentialResolver::with_env_lookup(Arc::new(AsyncRwLock::new(vault)), env_lookup) } + fn catalog_with(overrides: &str) -> Catalog { + let settings: LlmCatalogSettings = toml::from_str(overrides).unwrap(); + Catalog::from_builtin_with_overrides(&settings).unwrap() + } + #[tokio::test] async fn resolve_openai_api_request_prefers_typed_credential() { let dir = tempfile::tempdir().unwrap(); @@ -990,6 +1024,66 @@ mod tests { ]); } + #[tokio::test] + async fn resolve_for_catalog_uses_custom_vault_backed_provider() { + let catalog = catalog_with( + r#" +[providers.venice] +display_name = "Venice" +adapter = "openai_compatible" +base_url = "https://api.venice.ai/api/v1" +credentials = ["credential:venice"] + +[models."venice-large"] +provider = "venice" +display_name = "Venice Large" +family = "venice" +default = true + +[models."venice-large".limits] +context_window = 128000 + +[models."venice-large".features] +tools = true +vision = false +reasoning = false +effort = false +"#, + ); + let dir = tempfile::tempdir().unwrap(); + let mut vault = Vault::load(dir.path().join("secrets.json")).unwrap(); + vault_set_credential(&mut vault, "venice", &AuthCredential { + provider: ProviderId::new("venice"), + details: AuthDetails::ApiKey { + key: "venice-key".to_string(), + }, + }) + .unwrap(); + let resolver = test_resolver(vault, Arc::new(|_| None)); + + let resolved = resolver + .resolve_for_catalog( + ProviderId::new("venice"), + CredentialUsage::ApiRequest, + &catalog, + ) + .await + .unwrap(); + + let ResolvedCredential::Api(api) = resolved else { + panic!("expected api credential"); + }; + assert_eq!(api.provider, ProviderId::new("venice")); + assert_eq!( + api.auth_header, + Some(ApiKeyHeader::Bearer("venice-key".to_string())) + ); + assert_eq!( + api.base_url.as_deref(), + Some("https://api.venice.ai/api/v1") + ); + } + #[tokio::test] async fn configured_providers_returns_env_backed_provider() { let dir = tempfile::tempdir().unwrap(); diff --git a/lib/crates/fabro-auth/src/vault_source.rs b/lib/crates/fabro-auth/src/vault_source.rs index e61e156ad..de178bfad 100644 --- a/lib/crates/fabro-auth/src/vault_source.rs +++ b/lib/crates/fabro-auth/src/vault_source.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use async_trait::async_trait; -use fabro_model::{Catalog, ProviderId}; +use fabro_model::{Catalog, ProviderId, bootstrap_catalog}; use fabro_vault::Vault; use tokio::sync::RwLock as AsyncRwLock; @@ -42,7 +42,7 @@ impl std::fmt::Debug for VaultCredentialSource { #[async_trait] impl CredentialSource for VaultCredentialSource { async fn resolve(&self) -> anyhow::Result { - self.resolve_for_catalog(Catalog::builtin()).await + self.resolve_for_catalog(bootstrap_catalog::catalog()).await } async fn resolve_for_catalog(&self, catalog: &Catalog) -> anyhow::Result { @@ -79,7 +79,7 @@ impl CredentialSource for VaultCredentialSource { } async fn configured_providers(&self) -> Vec { - self.configured_providers_for_catalog(Catalog::builtin()) + self.configured_providers_for_catalog(bootstrap_catalog::catalog()) .await } diff --git a/lib/crates/fabro-cli/src/command_context.rs b/lib/crates/fabro-cli/src/command_context.rs index 5403c57ce..a30ec66f0 100644 --- a/lib/crates/fabro-cli/src/command_context.rs +++ b/lib/crates/fabro-cli/src/command_context.rs @@ -1,9 +1,10 @@ use std::path::{Path, PathBuf}; -use std::sync::Arc; +use std::sync::{Arc, OnceLock}; use anyhow::{Context as _, Result, bail}; use fabro_auth::{CredentialSource, EnvCredentialSource, VaultCredentialSource}; -use fabro_config::{CliLayer, Storage}; +use fabro_config::{CliLayer, Storage, load_llm_catalog_settings}; +use fabro_model::Catalog; use fabro_types::UserSettings; use fabro_types::settings::RunNamespace; use fabro_types::settings::cli::{OutputFormat, OutputVerbosity}; @@ -43,6 +44,7 @@ pub(crate) struct CommandContext { server_mode: ServerMode, server: OnceCell>, llm_source: OnceCell>, + catalog: OnceLock>, } struct ResolvedCommandSettings { @@ -70,6 +72,7 @@ impl CommandContext { server_mode: ServerMode::None, server: OnceCell::new(), llm_source: OnceCell::new(), + catalog: OnceLock::new(), }) } @@ -180,6 +183,28 @@ impl CommandContext { Ok(Arc::clone(source)) } + pub(crate) fn catalog(&self) -> Result> { + if let Some(catalog) = self.catalog.get() { + return Ok(Arc::clone(catalog)); + } + + let llm_catalog_settings = + load_llm_catalog_settings(None).context("loading LLM catalog")?; + let catalog = Arc::new( + Catalog::from_builtin_with_overrides(&llm_catalog_settings) + .context("building LLM catalog")?, + ); + if self.catalog.set(Arc::clone(&catalog)).is_ok() { + return Ok(catalog); + } + + Ok(Arc::clone( + self.catalog + .get() + .expect("catalog must exist after failed OnceLock set"), + )) + } + fn with_server_mode(&self, server_mode: ServerMode) -> Result { // Always reload settings for the requested derivation mode so the result // depends only on the requested mode, not on whichever derived context @@ -198,6 +223,7 @@ impl CommandContext { server_mode, server: OnceCell::new(), llm_source: OnceCell::new(), + catalog: OnceLock::new(), }) } } @@ -233,6 +259,7 @@ fn resolve_command_settings(loaded_settings: LoadedSettings) -> ResolvedCommandS #[cfg(test)] mod tests { use std::path::PathBuf; + use std::sync::OnceLock; use fabro_config::{CliLayer, CliOutputLayer}; use fabro_types::settings::cli::{OutputFormat, OutputVerbosity}; @@ -270,6 +297,7 @@ mod tests { server_mode: ServerMode::None, server: OnceCell::new(), llm_source: OnceCell::new(), + catalog: OnceLock::new(), } } diff --git a/lib/crates/fabro-cli/src/commands/model.rs b/lib/crates/fabro-cli/src/commands/model.rs index 0525038da..e7f284205 100644 --- a/lib/crates/fabro-cli/src/commands/model.rs +++ b/lib/crates/fabro-cli/src/commands/model.rs @@ -2,7 +2,7 @@ use anyhow::{Context, Result, bail}; use cli_table::format::{Border, Justify, Separator}; use cli_table::{Cell, CellStruct, Color, Style, Table}; use fabro_api::types as api_types; -use fabro_model::{Catalog, Model, ModelTestMode, Provider, ProviderId}; +use fabro_model::{Model, ModelTestMode, Provider, ProviderId}; use fabro_util::terminal::Styles; use futures::{StreamExt, stream}; use serde::Serialize; @@ -45,6 +45,13 @@ struct CompletedModelTest { status: String, } +fn find_model_by_id_or_alias(models: &[Model], id: &str) -> Option { + models + .iter() + .find(|model| model.id == id || model.aliases.iter().any(|alias| alias == id)) + .cloned() +} + pub(crate) async fn execute( command: Option, base_ctx: &CommandContext, @@ -244,6 +251,8 @@ async fn test_models_via_server( let mut skipped = 0u32; let mut skipped_providers: Vec = Vec::new(); if let Some(model_id) = model { + let listed_models = client.list_models(None, Some(model_id)).await?; + let listed_info = find_model_by_id_or_alias(&listed_models, model_id); if !json_output { eprint!("Testing {model_id}..."); } @@ -254,12 +263,9 @@ async fn test_models_via_server( let (info, result_color, status) = match result { Ok(resp) => { - let info = Catalog::builtin() - .get(&resp.model_id) - .cloned() - .with_context(|| { - format!("Unknown model returned by server: {}", resp.model_id) - })?; + let info = find_model_by_id_or_alias(&listed_models, &resp.model_id).with_context( + || format!("Unknown model returned by server: {}", resp.model_id), + )?; if resp.status == api_types::ModelTestResultStatus::Ok { (info, Color::Green, "ok".to_string()) } else if resp.status == api_types::ModelTestResultStatus::Skip { @@ -278,10 +284,7 @@ async fn test_models_via_server( bail!("Unknown model: {model_id}"); } Err(err) => { - let info = Catalog::builtin() - .get(model_id) - .cloned() - .with_context(|| format!("Unknown model: {model_id}"))?; + let info = listed_info.with_context(|| format!("Unknown model: {model_id}"))?; failures += 1; (info, Color::Red, format!("error: {err}")) } @@ -506,6 +509,39 @@ mod tests { .unwrap() } + fn custom_model_json(id: &str, provider: &str) -> serde_json::Value { + serde_json::to_value(Model { + id: id.to_string(), + provider: ProviderId::new(provider), + family: "test".to_string(), + display_name: format!("{id} display"), + limits: ModelLimits { + context_window: 128_000, + max_output: Some(4096), + }, + training: None, + knowledge_cutoff: None, + features: ModelFeatures { + tools: true, + vision: false, + reasoning: false, + reasoning_effort: ReasoningEffortFeature::None, + prompt_cache: false, + effort: false, + }, + costs: ModelCosts { + input_cost_per_mtok: Some(1.0), + output_cost_per_mtok: Some(2.0), + cache_input_cost_per_mtok: None, + }, + estimated_output_tps: Some(100.0), + aliases: vec![], + default: false, + configured: true, + }) + .unwrap() + } + #[test] fn format_context_window_millions() { assert_eq!(format_context_window(1_000_000), "1m"); @@ -665,6 +701,57 @@ mod tests { assert!(result.unwrap_err().to_string().contains("Model not found")); } + #[tokio::test] + async fn single_model_test_uses_server_model_metadata_for_custom_models() { + let server = httpmock::MockServer::start_async().await; + server + .mock_async(|when, then| { + when.method("GET") + .path("/api/v1/models") + .query_param("page[limit]", "100") + .query_param("page[offset]", "0") + .query_param("query", "venice-large"); + then.status(200) + .header("Content-Type", "application/json") + .body( + serde_json::json!({ + "data": [custom_model_json("venice-large", "venice")], + "meta": { "has_more": false } + }) + .to_string(), + ); + }) + .await; + server + .mock_async(|when, then| { + when.method("POST").path("/api/v1/models/venice-large/test"); + then.status(200) + .header("Content-Type", "application/json") + .body( + serde_json::json!({ + "model_id": "venice-large", + "status": "ok" + }) + .to_string(), + ); + }) + .await; + + let client = test_client(&server.url("")); + + test_models_via_server( + &client, + None, + Some("venice-large"), + false, + 1, + &Styles::new(false), + true, + ) + .await + .unwrap(); + } + #[tokio::test] async fn fetch_models_from_server_parses_response() { let server = httpmock::MockServer::start_async().await; diff --git a/lib/crates/fabro-cli/src/commands/provider/login.rs b/lib/crates/fabro-cli/src/commands/provider/login.rs index 32babd47d..352ddda05 100644 --- a/lib/crates/fabro-cli/src/commands/provider/login.rs +++ b/lib/crates/fabro-cli/src/commands/provider/login.rs @@ -17,11 +17,12 @@ pub(super) async fn login_command( let ctx = base_ctx.with_target(&args.target)?; let server = ctx.server().await?; let credential = if args.api_key_stdin { - provider_auth::authenticate_provider_with_api_key_source( + provider_auth::authenticate_provider_with_api_key_source_and_catalog( args.provider, provider_auth::ApiKeySource::Stdin, &s, printer, + ctx.catalog()?, ) .await? } else { diff --git a/lib/crates/fabro-cli/src/commands/run/create.rs b/lib/crates/fabro-cli/src/commands/run/create.rs index 6f4b15831..2357b5758 100644 --- a/lib/crates/fabro-cli/src/commands/run/create.rs +++ b/lib/crates/fabro-cli/src/commands/run/create.rs @@ -54,6 +54,7 @@ pub(crate) async fn create_run( &RunLayer::default(), &built.manifest, RenderMode::Strict, + ctx.catalog()?, )?; let diagnostics = api_diagnostics_to_local(&validation.workflow.diagnostics); if !quiet { diff --git a/lib/crates/fabro-cli/src/commands/run/runner.rs b/lib/crates/fabro-cli/src/commands/run/runner.rs index 281e379d6..985becf26 100644 --- a/lib/crates/fabro-cli/src/commands/run/runner.rs +++ b/lib/crates/fabro-cli/src/commands/run/runner.rs @@ -11,12 +11,11 @@ use std::time::Duration; use anyhow::{Context, Result, anyhow}; use async_trait::async_trait; -use fabro_config::{ServerSettingsBuilder, Storage}; +use fabro_config::{ServerSettingsBuilder, Storage, load_llm_catalog_settings}; use fabro_interview::{ AnswerSubmission, ControlInterviewer, WorkerControlEnvelope, WorkerControlMessage, }; use fabro_model::Catalog; -use fabro_model::catalog::LlmCatalogSettings; use fabro_store::{EventEnvelope, RunProjection, RunProjectionReducer}; use fabro_types::settings::InterpString; use fabro_types::settings::run::{RunMode, RunNamespace}; @@ -95,9 +94,11 @@ pub(crate) async fn execute( let run_control = RunControlState::new(); install_signal_handlers(Arc::clone(&run_control), cancel_token.clone())?; let vault = load_worker_vault(storage_dir.as_deref())?; + let llm_catalog_settings = + load_llm_catalog_settings(None).context("failed to load worker LLM catalog settings")?; let catalog = Arc::new( - Catalog::from_builtin_with_overrides(&LlmCatalogSettings::default()) - .context("failed to build default LLM catalog")?, + Catalog::from_builtin_with_overrides(&llm_catalog_settings) + .context("failed to build worker LLM catalog")?, ); let github_app = { let vault_guard = match &vault { diff --git a/lib/crates/fabro-cli/src/commands/validate.rs b/lib/crates/fabro-cli/src/commands/validate.rs index 7565f2aad..8d9c9e5c2 100644 --- a/lib/crates/fabro-cli/src/commands/validate.rs +++ b/lib/crates/fabro-cli/src/commands/validate.rs @@ -27,6 +27,7 @@ pub(crate) fn run( &RunLayer::default(), &built.manifest, RenderMode::Structural, + base_ctx.catalog()?, )?; let diagnostics = api_diagnostics_to_local(&response.workflow.diagnostics); diff --git a/lib/crates/fabro-cli/src/shared/provider_auth.rs b/lib/crates/fabro-cli/src/shared/provider_auth.rs index f5686b0f0..0a8a1e160 100644 --- a/lib/crates/fabro-cli/src/shared/provider_auth.rs +++ b/lib/crates/fabro-cli/src/shared/provider_auth.rs @@ -20,6 +20,7 @@ use fabro_auth::{ }; use fabro_llm::client::Client as LlmClient; use fabro_llm::generate::{GenerateParams, generate}; +use fabro_model::catalog::LlmCatalogSettings; use fabro_model::{Catalog, Provider}; use fabro_util::printer::Printer; use fabro_util::terminal::Styles; @@ -77,15 +78,30 @@ pub(crate) enum ApiKeySource { // API key validation // --------------------------------------------------------------------------- -pub(crate) async fn validate_api_key(provider: Provider, api_key: &str) -> Result<()> { - let client = LlmClient::from_credentials(vec![ApiCredential::from_api_key( - provider, - api_key.to_string(), - )]) +fn default_catalog_for_provider_auth() -> Result> { + Ok(Arc::new( + Catalog::from_builtin_with_overrides(&LlmCatalogSettings::default()) + .context("failed to build provider auth catalog")?, + )) +} + +pub(crate) async fn validate_api_key( + provider: Provider, + api_key: &str, + catalog: Arc, +) -> Result<()> { + let client = LlmClient::from_credentials_with_catalog( + vec![ApiCredential::from_api_key_for_catalog( + provider, + api_key.to_string(), + catalog.as_ref(), + )], + Arc::clone(&catalog), + ) .await .context("failed to create LLM client")?; - let probe_model = Catalog::builtin() + let probe_model = catalog .probe_for_provider(provider) .map_or_else(|| format!("unknown-{provider}"), |model| model.id.clone()); @@ -145,12 +161,13 @@ async fn read_and_validate_api_key( env_var: &str, s: &Styles, printer: Printer, + catalog: Arc, ) -> Result { loop { let key = read_api_key_from_source(source, env_var).await?; fabro_util::printerr!(printer, " {}", s.dim.apply_to("Validating API key...")); - match validate_api_key(provider, &key).await { + match validate_api_key(provider, &key, Arc::clone(&catalog)).await { Ok(()) => { fabro_util::printerr!(printer, " {} API key is valid", s.green.apply_to("✔")); return Ok(key); @@ -201,11 +218,28 @@ pub(crate) async fn authenticate_provider_with_api_key_source( source: ApiKeySource, s: &Styles, printer: Printer, +) -> Result { + authenticate_provider_with_api_key_source_and_catalog( + provider, + source, + s, + printer, + default_catalog_for_provider_auth()?, + ) + .await +} + +pub(crate) async fn authenticate_provider_with_api_key_source_and_catalog( + provider: Provider, + source: ApiKeySource, + s: &Styles, + printer: Printer, + catalog: Arc, ) -> Result { let mut strategy = strategy_for(provider, AuthMethod::ApiKey); let request = strategy.init().await?; present_to_user(&request, s, printer); - let response = await_user_response_from_source(&request, &source, s, printer).await?; + let response = await_user_response_from_source(&request, &source, s, printer, catalog).await?; strategy.complete(response).await } @@ -218,8 +252,14 @@ pub(crate) async fn authenticate_provider_with_method( let mut strategy = strategy_for(provider, method); let request = strategy.init().await?; present_to_user(&request, s, printer); - let response = - await_user_response_from_source(&request, &ApiKeySource::Prompt, s, printer).await?; + let response = await_user_response_from_source( + &request, + &ApiKeySource::Prompt, + s, + printer, + default_catalog_for_provider_auth()?, + ) + .await?; strategy.complete(response).await } @@ -269,6 +309,7 @@ async fn await_user_response_from_source( source: &ApiKeySource, s: &Styles, printer: Printer, + catalog: Arc, ) -> Result { match request { AuthContextRequest::ApiKey { @@ -276,7 +317,8 @@ async fn await_user_response_from_source( env_var_names, } => { let env_var = env_var_names.first().map_or("API_KEY", String::as_str); - let key = read_and_validate_api_key(*provider, source, env_var, s, printer).await?; + let key = + read_and_validate_api_key(*provider, source, env_var, s, printer, catalog).await?; Ok(AuthContextResponse::ApiKey { key }) } AuthContextRequest::DeviceCode { .. } => { @@ -319,7 +361,12 @@ mod tests { #[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))] async fn validate_api_key_rejects_invalid_key() { - let result = validate_api_key(Provider::Anthropic, "sk-invalid-key-12345").await; + let result = validate_api_key( + Provider::Anthropic, + "sk-invalid-key-12345", + default_catalog_for_provider_auth().unwrap(), + ) + .await; assert!(result.is_err(), "expected invalid key to be rejected"); } diff --git a/lib/crates/fabro-config/src/builders.rs b/lib/crates/fabro-config/src/builders.rs index d9896ef09..25c5ad81f 100644 --- a/lib/crates/fabro-config/src/builders.rs +++ b/lib/crates/fabro-config/src/builders.rs @@ -225,6 +225,14 @@ pub fn load_server_runtime_settings( resolve_server_runtime_settings(layer, run_overrides, server_overrides) } +pub fn load_llm_catalog_settings(path: Option<&Path>) -> Result { + let layer = match path { + Some(path) => load_settings_path(path)?, + None => load_settings_config(None)?, + }; + Ok(llm_catalog_settings_from_layer(&layer)) +} + #[cfg(test)] pub fn server_runtime_settings_from_toml( source: &str, diff --git a/lib/crates/fabro-config/src/lib.rs b/lib/crates/fabro-config/src/lib.rs index 9fd4d33d1..2a59359ac 100644 --- a/lib/crates/fabro-config/src/lib.rs +++ b/lib/crates/fabro-config/src/lib.rs @@ -31,7 +31,8 @@ use std::path::Path; pub use builders::{ ResolveErrors, RunSettingsBuilder, ServerRuntimeSettings, ServerSettingsBuilder, - UserSettingsBuilder, WorkflowSettingsBuilder, load_server_runtime_settings, + UserSettingsBuilder, WorkflowSettingsBuilder, load_llm_catalog_settings, + load_server_runtime_settings, }; pub use error::{Error, Result}; pub use fabro_util::path::expand_tilde; diff --git a/lib/crates/fabro-llm/src/providers/anthropic.rs b/lib/crates/fabro-llm/src/providers/anthropic.rs index f8b36a92b..d7469a3f2 100644 --- a/lib/crates/fabro-llm/src/providers/anthropic.rs +++ b/lib/crates/fabro-llm/src/providers/anthropic.rs @@ -1128,8 +1128,7 @@ async fn build_api_request( request.tools.as_ref().map(|t| translate_tools(t)) }; - let model_info = common::catalog_model(adapter.catalog.as_deref(), &request.model) - .or_else(|| Catalog::builtin().get(&request.model)); + let model_info = common::catalog_model(adapter.catalog.as_deref(), &request.model); let supports_prompt_cache = model_info.is_some_and(|m| m.features.prompt_cache); let auto_cache = supports_prompt_cache && is_auto_cache_enabled(request.provider_options.as_ref()); @@ -2364,6 +2363,36 @@ prompt_cache = false ); } + #[tokio::test] + async fn build_api_request_without_injected_catalog_does_not_use_builtin_model_metadata() { + let adapter = Adapter::new("test-key"); + let request = Request { + model: "claude-sonnet-4-5".to_string(), + messages: vec![ + Message::system("Do not infer cache support from built-ins."), + Message::user("Hello"), + ], + provider_options: Some(serde_json::json!({ + "anthropic": {"auto_cache": true} + })), + ..make_base_request() + }; + + let (api_request, req_builder) = build_api_request(&adapter, &request, false).await; + assert_eq!( + api_request.system, + Some(serde_json::Value::String( + "Do not infer cache support from built-ins.".to_string() + )) + ); + let built = req_builder.build().expect("should build request"); + let beta = built.headers().get("anthropic-beta"); + assert!( + beta.is_none_or(|value| !value.to_str().unwrap().contains(CACHE_BETA_HEADER)), + "cache beta header must require injected model metadata" + ); + } + #[tokio::test] async fn build_api_request_enables_prompt_cache_when_model_feature_is_true() { let adapter = Adapter::new("test-key").with_catalog(catalog_with_anthropic_model( @@ -2397,10 +2426,15 @@ prompt_cache = true } #[tokio::test] - async fn build_api_request_uses_adaptive_thinking_for_opus_4_7_without_forced_tools() { - let adapter = Adapter::new("test-key"); + async fn build_api_request_uses_adaptive_thinking_for_injected_effort_model_without_forced_tools() + { + let adapter = Adapter::new("test-key").with_catalog(catalog_with_anthropic_model( + r#" +reasoning_effort = "levels" +"#, + )); let request = Request { - model: "claude-opus-4-7".to_string(), + model: "test-claude".to_string(), ..make_base_request() }; @@ -2589,9 +2623,9 @@ prompt_cache = true #[tokio::test] async fn build_api_request_falls_back_to_thinking_budget_for_non_effort_model() { - let adapter = Adapter::new("test-key"); + let adapter = Adapter::new("test-key").with_catalog(catalog_with_anthropic_model("")); let request = Request { - model: "claude-sonnet-4-5".to_string(), + model: "test-claude".to_string(), max_tokens: Some(16_000), reasoning_effort: Some(ReasoningEffort::XHigh), ..make_base_request() diff --git a/lib/crates/fabro-mcp-server/Cargo.toml b/lib/crates/fabro-mcp-server/Cargo.toml index 15f481431..2aad3c067 100644 --- a/lib/crates/fabro-mcp-server/Cargo.toml +++ b/lib/crates/fabro-mcp-server/Cargo.toml @@ -19,6 +19,7 @@ fabro-api = { path = "../fabro-api" } fabro-client = { path = "../fabro-client" } fabro-manifest = { path = "../fabro-manifest" } fabro-config = { path = "../fabro-config" } +fabro-model = { path = "../fabro-model" } fabro-server = { path = "../fabro-server" } fabro-types = { path = "../fabro-types" } fabro-util = { path = "../fabro-util" } diff --git a/lib/crates/fabro-mcp-server/src/run_tools/manifest.rs b/lib/crates/fabro-mcp-server/src/run_tools/manifest.rs index a9da087e8..fe7a10c56 100644 --- a/lib/crates/fabro-mcp-server/src/run_tools/manifest.rs +++ b/lib/crates/fabro-mcp-server/src/run_tools/manifest.rs @@ -1,8 +1,9 @@ use std::path::{Path, PathBuf}; use fabro_api::types; -use fabro_config::{CliLayer, RunLayer}; +use fabro_config::{CliLayer, RunLayer, load_llm_catalog_settings}; use fabro_manifest::{self, ManifestBuildInput, RunOverrideInput}; +use fabro_model::Catalog; use fabro_server::manifest_validation::{self, RenderMode}; use serde_json::Value; @@ -25,10 +26,17 @@ pub(super) fn build_mcp_run_manifest( user_settings_path: Some(user_settings_path.to_path_buf()), }) .map_err(|err| ToolError::from_anyhow(&err))?; + let llm_catalog_settings = load_llm_catalog_settings(Some(user_settings_path)) + .map_err(|err| ToolError::message(err.to_string()))?; + let catalog = std::sync::Arc::new( + Catalog::from_builtin_with_overrides(&llm_catalog_settings) + .map_err(|err| ToolError::message(err.to_string()))?, + ); let validation = manifest_validation::validate_manifest( &RunLayer::default(), &built.manifest, RenderMode::Strict, + catalog, ) .map_err(|err| ToolError::from_anyhow(&err))?; if !validation.ok { diff --git a/lib/crates/fabro-server/src/manifest_validation.rs b/lib/crates/fabro-server/src/manifest_validation.rs index 14b539413..2ea7de08c 100644 --- a/lib/crates/fabro-server/src/manifest_validation.rs +++ b/lib/crates/fabro-server/src/manifest_validation.rs @@ -1,6 +1,9 @@ +use std::sync::Arc; + use anyhow::Result; use fabro_api::types; use fabro_config::RunLayer; +use fabro_model::Catalog; pub use fabro_workflow::operations::RenderMode; use crate::run_manifest; @@ -9,9 +12,10 @@ pub fn validate_manifest( manifest_run_defaults: &RunLayer, manifest: &types::RunManifest, mode: RenderMode, + catalog: Arc, ) -> Result { let prepared = run_manifest::prepare_manifest(manifest_run_defaults, manifest)?; - let validated = - run_manifest::validate_prepared_manifest(&prepared, mode).map_err(anyhow::Error::new)?; + let validated = run_manifest::validate_prepared_manifest(&prepared, mode, catalog) + .map_err(anyhow::Error::new)?; Ok(run_manifest::validate_response(&prepared, &validated)) } diff --git a/lib/crates/fabro-server/src/run_manifest.rs b/lib/crates/fabro-server/src/run_manifest.rs index bd7d90357..a600234d4 100644 --- a/lib/crates/fabro-server/src/run_manifest.rs +++ b/lib/crates/fabro-server/src/run_manifest.rs @@ -157,12 +157,14 @@ pub(crate) fn prepare_manifest( pub(crate) fn validate_prepared_manifest( prepared: &PreparedManifest, mode: RenderMode, + catalog: Arc, ) -> Result { validate(ValidateInput { workflow: WorkflowInput::Bundled(prepared.workflow_input.clone()), settings: prepared.settings.clone(), cwd: prepared.cwd.clone(), custom_transforms: Vec::new(), + catalog, mode, }) } @@ -1282,6 +1284,7 @@ fn report_to_api(report: &CheckReport) -> types::PreflightCheckReport { #[cfg(test)] mod tests { use fabro_model::Provider; + use fabro_model::catalog::LlmCatalogSettings; use super::*; @@ -1335,6 +1338,10 @@ mod tests { RunLayer::default() } + fn test_catalog() -> Arc { + Arc::new(Catalog::from_builtin_with_overrides(&LlmCatalogSettings::default()).unwrap()) + } + fn manifest_workflow() -> types::ManifestWorkflow { types::ManifestWorkflow { config: None, @@ -1393,7 +1400,8 @@ enabled = {clone_enabled} &manifest, ) .unwrap(); - let validated = validate_prepared_manifest(&prepared, RenderMode::Strict).unwrap(); + let validated = + validate_prepared_manifest(&prepared, RenderMode::Strict, test_catalog()).unwrap(); let resolved = materialize_run( prepared.settings.clone(), validated.graph(), @@ -1772,7 +1780,8 @@ app_id = "fixture-app-id" &invalid_manifest(), ) .unwrap(); - let validated = validate_prepared_manifest(&prepared, RenderMode::Strict).unwrap(); + let validated = + validate_prepared_manifest(&prepared, RenderMode::Strict, test_catalog()).unwrap(); assert!(validated.has_errors()); @@ -1817,7 +1826,8 @@ issues = "read" &manifest, ) .unwrap(); - let validated = validate_prepared_manifest(&prepared, RenderMode::Strict).unwrap(); + let validated = + validate_prepared_manifest(&prepared, RenderMode::Strict, test_catalog()).unwrap(); assert!(!validated.has_errors()); let (response, _ok) = run_preflight(state.as_ref(), &prepared, &validated) @@ -1865,7 +1875,8 @@ provider = "local" &manifest, ) .unwrap(); - let validated = validate_prepared_manifest(&prepared, RenderMode::Strict).unwrap(); + let validated = + validate_prepared_manifest(&prepared, RenderMode::Strict, test_catalog()).unwrap(); assert!(!validated.has_errors()); @@ -1906,7 +1917,8 @@ provider = "daytona" &manifest, ) .unwrap(); - let validated = validate_prepared_manifest(&prepared, RenderMode::Strict).unwrap(); + let validated = + validate_prepared_manifest(&prepared, RenderMode::Strict, test_catalog()).unwrap(); let (response, _ok) = run_preflight(state.as_ref(), &prepared, &validated) .await @@ -1979,7 +1991,8 @@ digraph Demo { &manifest, ) .unwrap(); - let validated = validate_prepared_manifest(&prepared, RenderMode::Strict).unwrap(); + let validated = + validate_prepared_manifest(&prepared, RenderMode::Strict, test_catalog()).unwrap(); let (response, ok) = run_preflight(state.as_ref(), &prepared, &validated) .await @@ -2020,7 +2033,8 @@ digraph Demo { &manifest, ) .unwrap(); - let validated = validate_prepared_manifest(&prepared, RenderMode::Strict).unwrap(); + let validated = + validate_prepared_manifest(&prepared, RenderMode::Strict, test_catalog()).unwrap(); let (response, ok) = run_preflight(state.as_ref(), &prepared, &validated) .await @@ -2091,7 +2105,8 @@ digraph Demo { &manifest, ) .unwrap(); - let validated = validate_prepared_manifest(&prepared, RenderMode::Strict).unwrap(); + let validated = + validate_prepared_manifest(&prepared, RenderMode::Strict, test_catalog()).unwrap(); let (response, ok) = run_preflight(state.as_ref(), &prepared, &validated) .await diff --git a/lib/crates/fabro-server/src/serve.rs b/lib/crates/fabro-server/src/serve.rs index dc1075868..95638826b 100644 --- a/lib/crates/fabro-server/src/serve.rs +++ b/lib/crates/fabro-server/src/serve.rs @@ -7,6 +7,7 @@ use std::time::Duration; use anyhow::{Context, bail}; use clap::Args; use fabro_config::bind::{self, Bind, BindRequest}; +use fabro_config::user::active_settings_path; use fabro_config::{ RunLayer, RunModelLayer, RunSandboxLayer, ServerLayer, ServerWebLayer, Storage, load_config_file, load_server_runtime_settings, @@ -610,6 +611,16 @@ fn resolve_interp_path(value: &InterpString) -> anyhow::Result { Ok(PathBuf::from(resolve_interp(value)?)) } +fn absolute_path(path: PathBuf) -> anyhow::Result { + if path.is_absolute() { + Ok(path) + } else { + Ok(std::env::current_dir() + .context("resolving current directory for config path")? + .join(path)) + } +} + fn load_server_secrets_for_settings(settings: &ServerNamespace) -> anyhow::Result { let storage_root = resolve_interp_path(&settings.storage.root)?; let server_env_path = Storage::new(&storage_root).runtime_directory().env_path(); @@ -687,6 +698,7 @@ where #[cfg(debug_assertions)] let watch_web = args.watch_web; let config_path = args.config.clone(); + let active_config_path = absolute_path(active_settings_path(config_path.as_deref()))?; let disk_document: toml::Table = load_config_file(config_path.as_deref(), "settings.toml")?; let (run_overrides, server_overrides) = serve_overrides(&args); let mut runtime_settings = load_server_runtime_settings( @@ -789,6 +801,7 @@ where server_secrets, env_lookup, github_api_base_url: None, + active_config_path, http_client: None, shutdown: shutdown.clone(), })?; diff --git a/lib/crates/fabro-server/src/server.rs b/lib/crates/fabro-server/src/server.rs index 017dec794..b2e2c9bd1 100644 --- a/lib/crates/fabro-server/src/server.rs +++ b/lib/crates/fabro-server/src/server.rs @@ -531,6 +531,7 @@ pub struct AppState { catalog: RwLock>, pub(crate) env_lookup: EnvLookup, pub(crate) github_api_base_url: String, + active_config_path: PathBuf, http_client: Option, shutdown: CancellationToken, shutting_down: AtomicBool, @@ -595,6 +596,7 @@ pub(crate) struct AppStateConfig { pub(crate) server_secrets: ServerSecrets, pub(crate) env_lookup: EnvLookup, pub(crate) github_api_base_url: Option, + pub(crate) active_config_path: PathBuf, pub(crate) http_client: Option, pub(crate) shutdown: CancellationToken, } @@ -664,6 +666,10 @@ impl AppState { Arc::clone(&self.catalog.read().expect("catalog lock poisoned")) } + pub(crate) fn active_config_path(&self) -> &std::path::Path { + &self.active_config_path + } + pub(crate) fn manifest_run_settings(&self) -> std::result::Result { self.manifest_run_settings .read() @@ -1514,6 +1520,7 @@ pub(crate) fn build_app_state(config: AppStateConfig) -> anyhow::Result anyhow::Result prepared, Err(err) => return ApiError::bad_request(err.to_string()).into_response(), }; - let validated = - match run_manifest::validate_prepared_manifest(&prepared, RenderMode::Structural) { - Ok(validated) => validated, - Err(err) => return ApiError::bad_request(err.to_string()).into_response(), - }; + let validated = match run_manifest::validate_prepared_manifest( + &prepared, + RenderMode::Structural, + state.catalog(), + ) { + Ok(validated) => validated, + Err(err) => return ApiError::bad_request(err.to_string()).into_response(), + }; if validated.has_errors() { return ApiError::bad_request("Validation failed").into_response(); } diff --git a/lib/crates/fabro-server/src/server/handler/runs.rs b/lib/crates/fabro-server/src/server/handler/runs.rs index d7d108aa1..f4fb41e2b 100644 --- a/lib/crates/fabro-server/src/server/handler/runs.rs +++ b/lib/crates/fabro-server/src/server/handler/runs.rs @@ -518,7 +518,11 @@ async fn run_preflight( Ok(prepared) => prepared, Err(err) => return ApiError::bad_request(err.to_string()).into_response(), }; - let validated = match run_manifest::validate_prepared_manifest(&prepared, RenderMode::Strict) { + let validated = match run_manifest::validate_prepared_manifest( + &prepared, + RenderMode::Strict, + state.catalog(), + ) { Ok(validated) => validated, Err(WorkflowError::Parse(_)) => { return ApiError::bad_request("Validation failed").into_response(); @@ -545,14 +549,17 @@ async fn validate_run_manifest( Ok(prepared) => prepared, Err(err) => return ApiError::bad_request(err.to_string()).into_response(), }; - let validated = - match run_manifest::validate_prepared_manifest(&prepared, RenderMode::Structural) { - Ok(validated) => validated, - Err(WorkflowError::Parse(_)) => { - return ApiError::bad_request("Validation failed").into_response(); - } - Err(err) => return ApiError::bad_request(err.to_string()).into_response(), - }; + let validated = match run_manifest::validate_prepared_manifest( + &prepared, + RenderMode::Structural, + state.catalog(), + ) { + Ok(validated) => validated, + Err(WorkflowError::Parse(_)) => { + return ApiError::bad_request("Validation failed").into_response(); + } + Err(err) => return ApiError::bad_request(err.to_string()).into_response(), + }; ( StatusCode::OK, Json(run_manifest::validate_response(&prepared, &validated)), diff --git a/lib/crates/fabro-server/src/server/tests.rs b/lib/crates/fabro-server/src/server/tests.rs index ec827967e..8a946eed2 100644 --- a/lib/crates/fabro-server/src/server/tests.rs +++ b/lib/crates/fabro-server/src/server/tests.rs @@ -1692,6 +1692,44 @@ destination = "stdout" ); } +#[cfg(unix)] +#[test] +fn worker_command_sets_fabro_config_to_active_absolute_config_path() { + let storage_dir = tempfile::tempdir().unwrap(); + let config_dir = tempfile::tempdir().unwrap(); + let active_config_path = config_dir.path().join("settings.toml"); + let state = worker_command_test_state_with_active_config_path( + storage_dir.path(), + &["dev-token"], + Some(TEST_DEV_TOKEN), + active_config_path.clone(), + ); + let run_id = RunId::new(); + + let cmd = worker_command( + state.as_ref(), + run_id, + RunExecutionMode::Start, + storage_dir.path(), + ) + .unwrap(); + + assert!(active_config_path.is_absolute()); + assert_eq!( + command_env_value(&cmd, EnvVars::FABRO_CONFIG), + EnvOverride::Set(active_config_path.display().to_string()) + ); + let worker_args = cmd + .as_std() + .get_args() + .map(|arg| arg.to_string_lossy().into_owned()) + .collect::>(); + assert!( + !worker_args.iter().any(|arg| arg == "--config"), + "__run-worker argument contract should not grow hidden config args: {worker_args:?}" + ); +} + #[cfg(unix)] #[test] fn worker_command_env_log_destination_overrides_server_logging_config() { @@ -1781,6 +1819,7 @@ methods = ["dev-token"] server_secrets: ServerSecrets::load(server_env_path, HashMap::new()).unwrap(), env_lookup: default_env_lookup(), github_api_base_url: None, + active_config_path: tempfile::tempdir().unwrap().path().join("settings.toml"), http_client: Some(fabro_http::test_http_client().expect("test HTTP client should build")), shutdown: tokio_util::sync::CancellationToken::new(), }) else { @@ -1823,6 +1862,43 @@ fn worker_command_test_state_with_extra_config_and_env_lookup( extra_config: &str, extra_server_secrets: &[(&str, &str)], env_lookup: impl Fn(&str) -> Option + Send + Sync + 'static, +) -> Arc { + worker_command_test_state_inner( + storage_dir, + methods, + dev_token, + extra_config, + extra_server_secrets, + env_lookup, + None, + ) +} + +fn worker_command_test_state_with_active_config_path( + storage_dir: &Path, + methods: &[&str], + dev_token: Option<&str>, + active_config_path: PathBuf, +) -> Arc { + worker_command_test_state_inner( + storage_dir, + methods, + dev_token, + "", + &[], + |_| None, + Some(active_config_path), + ) +} + +fn worker_command_test_state_inner( + storage_dir: &Path, + methods: &[&str], + dev_token: Option<&str>, + extra_config: &str, + extra_server_secrets: &[(&str, &str)], + env_lookup: impl Fn(&str) -> Option + Send + Sync + 'static, + active_config_path: Option, ) -> Arc { let dev_token = dev_token.map(str::to_owned); std::fs::create_dir_all(storage_dir).unwrap(); @@ -1862,13 +1938,18 @@ allowed_usernames = ["octocat"] for (key, value) in extra_server_secrets { server_secret_env.insert((*key).to_string(), (*value).to_string()); } - test_app_state_with_env_lookup_and_server_secret_env( - server_settings_from_toml(&source), - manifest_run_defaults_from_toml(&source), - 5, - env_lookup, - &server_secret_env, - ) + let mut builder = TestAppStateBuilder::new() + .runtime_settings( + server_settings_from_toml(&source), + manifest_run_defaults_from_toml(&source), + ) + .max_concurrent_runs(5) + .env_lookup(env_lookup) + .server_secret_env(server_secret_env); + if let Some(active_config_path) = active_config_path { + builder = builder.active_config_path(active_config_path); + } + builder.build() } #[cfg(unix)] @@ -2170,6 +2251,67 @@ async fn validate_endpoint_returns_workflow_summary_without_preflight_checks() { assert!(body.get("checks").is_none()); } +#[tokio::test] +async fn validate_endpoint_uses_app_state_catalog_for_model_diagnostics() { + let llm_catalog_settings: LlmCatalogSettings = toml::from_str( + r#" +[providers.venice] +display_name = "Venice" +adapter = "openai_compatible" +base_url = "https://api.venice.ai/api/v1" +credentials = ["env:VENICE_API_KEY"] + +[models."venice-large"] +provider = "venice" +display_name = "Venice Large" +family = "venice" +default = true + +[models."venice-large".limits] +context_window = 128000 + +[models."venice-large".features] +tools = true +vision = false +reasoning = false +effort = false +"#, + ) + .expect("catalog fixture should parse"); + let state = TestAppStateBuilder::new() + .llm_catalog_settings(llm_catalog_settings) + .build(); + let app = crate::test_support::build_test_router(state); + let dot = r#"digraph Test { + graph [goal="Test"] + start [shape=Mdiamond] + work [model="venice-large", provider="venice", prompt="Do it"] + exit [shape=Msquare] + start -> work -> exit + }"#; + + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri(api("/validate")) + .header("content-type", "application/json") + .body(manifest_body(dot)) + .unwrap(), + ) + .await + .unwrap(); + let body = response_json!(response, StatusCode::OK).await; + let diagnostics = body["workflow"]["diagnostics"].as_array().unwrap(); + + assert!( + diagnostics + .iter() + .all(|diagnostic| diagnostic["rule"] != "node_model_known"), + "custom model/provider should validate against app-state catalog: {body}" + ); +} + async fn create_run_for_target(app: &Router, target_path: &str, dot_source: &str) -> String { let req = Request::builder() .method("POST") @@ -3484,6 +3626,7 @@ fn create_github_token_app_state_with_env_lookup( server_secrets: load_test_server_secrets(server_env_path, HashMap::new()), env_lookup: Arc::new(env_lookup), github_api_base_url, + active_config_path: tempfile::tempdir().unwrap().path().join("settings.toml"), http_client: Some(fabro_http::test_http_client().expect("test HTTP client should build")), shutdown: tokio_util::sync::CancellationToken::new(), }; diff --git a/lib/crates/fabro-server/src/test_support.rs b/lib/crates/fabro-server/src/test_support.rs index 04d17f386..4e8f4d710 100644 --- a/lib/crates/fabro-server/src/test_support.rs +++ b/lib/crates/fabro-server/src/test_support.rs @@ -63,6 +63,7 @@ pub struct TestAppStateBuilder { store_bundle: Option<(Arc, ArtifactStore)>, vault_path: Option, server_env_path: Option, + active_config_path: Option, server_secret_env: HashMap, env_lookup: EnvLookup, llm_catalog_settings: LlmCatalogSettings, @@ -78,6 +79,7 @@ impl Default for TestAppStateBuilder { store_bundle: None, vault_path: None, server_env_path: None, + active_config_path: None, server_secret_env: HashMap::new(), env_lookup: default_env_lookup(), llm_catalog_settings: LlmCatalogSettings::default(), @@ -149,12 +151,20 @@ impl TestAppStateBuilder { self } + pub fn active_config_path(mut self, active_config_path: PathBuf) -> Self { + self.active_config_path = Some(active_config_path); + self + } + pub fn build(self) -> Arc { let (store, artifact_store) = self.store_bundle.unwrap_or_else(test_store_bundle); let vault_path = self.vault_path.unwrap_or_else(test_secret_store_path); let server_env_path = self .server_env_path .unwrap_or_else(|| vault_path.with_file_name("server.env")); + let active_config_path = self.active_config_path.unwrap_or_else(|| { + std::env::temp_dir().join(format!("fabro-test-settings-{}.toml", Ulid::new())) + }); build_app_state(AppStateConfig { resolved_settings: resolved_runtime_settings_for_tests( self.server_settings, @@ -169,6 +179,7 @@ impl TestAppStateBuilder { server_secrets: load_test_server_secrets(server_env_path, self.server_secret_env), env_lookup: self.env_lookup, github_api_base_url: None, + active_config_path, http_client: Some( fabro_http::test_http_client().expect("test HTTP client should build"), ), diff --git a/lib/crates/fabro-validate/Cargo.toml b/lib/crates/fabro-validate/Cargo.toml index 2ea44ae2d..d2a1252f0 100644 --- a/lib/crates/fabro-validate/Cargo.toml +++ b/lib/crates/fabro-validate/Cargo.toml @@ -18,3 +18,6 @@ fabro-model = { path = "../fabro-model" } fabro-types = { path = "../fabro-types" } serde = { workspace = true } thiserror = { workspace = true } + +[dev-dependencies] +toml = { workspace = true } diff --git a/lib/crates/fabro-validate/src/lib.rs b/lib/crates/fabro-validate/src/lib.rs index 733a3a605..5b342d518 100644 --- a/lib/crates/fabro-validate/src/lib.rs +++ b/lib/crates/fabro-validate/src/lib.rs @@ -1,6 +1,7 @@ pub mod rules; use fabro_graphviz::graph::Graph; +use fabro_model::Catalog; use serde::{Deserialize, Serialize}; /// Severity level for validation diagnostics. @@ -47,6 +48,25 @@ pub fn validate(graph: &Graph, extra_rules: &[&dyn LintRule]) -> Vec diagnostics } +/// Run all built-in catalog-free lint rules, caller-supplied model catalog +/// rules, and any extra rules against the graph. +#[must_use] +pub fn validate_with_catalog( + graph: &Graph, + catalog: &Catalog, + extra_rules: &[&dyn LintRule], +) -> Vec { + let mut diagnostics = validate(graph, &[]); + let catalog_rules = rules::catalog_rules(catalog); + for rule in &catalog_rules { + diagnostics.extend(rule.apply(graph)); + } + for rule in extra_rules { + diagnostics.extend(rule.apply(graph)); + } + diagnostics +} + /// If any Error-severity diagnostics are present, return `ValidationError`. /// /// # Errors @@ -80,9 +100,26 @@ pub fn validate_or_raise( Ok(diagnostics) } +/// Run catalog-aware validation and return `ValidationError` if any +/// Error-severity diagnostics are found. +/// +/// # Errors +/// Returns `ValidationError` if any Error-severity diagnostics are found. +pub fn validate_with_catalog_or_raise( + graph: &Graph, + catalog: &Catalog, + extra_rules: &[&dyn LintRule], +) -> Result, ValidationError> { + let diagnostics = validate_with_catalog(graph, catalog, extra_rules); + raise_on_errors(&diagnostics)?; + Ok(diagnostics) +} + #[cfg(test)] mod tests { use fabro_graphviz::graph::{AttrValue, Edge, Graph, Node}; + use fabro_model::catalog::LlmCatalogSettings; + use fabro_model::{Catalog, ProviderId}; use super::*; @@ -106,6 +143,65 @@ mod tests { g } + fn graph_with_model_and_provider(model: &str, provider: &str) -> Graph { + let mut g = Graph::new("test"); + let mut start = Node::new("start"); + start.attrs.insert( + "shape".to_string(), + AttrValue::String("Mdiamond".to_string()), + ); + g.nodes.insert("start".to_string(), start); + + let mut work = Node::new("work"); + work.attrs + .insert("model".to_string(), AttrValue::String(model.to_string())); + work.attrs.insert( + "provider".to_string(), + AttrValue::String(provider.to_string()), + ); + g.nodes.insert("work".to_string(), work); + + let mut exit = Node::new("exit"); + exit.attrs.insert( + "shape".to_string(), + AttrValue::String("Msquare".to_string()), + ); + g.nodes.insert("exit".to_string(), exit); + + g.edges.push(Edge::new("start", "work")); + g.edges.push(Edge::new("work", "exit")); + g + } + + fn custom_catalog() -> Catalog { + let settings: LlmCatalogSettings = toml::from_str( + r#" +[providers.venice] +display_name = "Venice" +adapter = "openai_compatible" +base_url = "https://api.venice.ai/api/v1" +credentials = ["env:VENICE_API_KEY"] + +[models."venice-large"] +provider = "venice" +display_name = "Venice Large" +family = "venice" +default = true + +[models."venice-large".limits] +context_window = 128000 + +[models."venice-large".features] +tools = true +vision = false +reasoning = false +effort = false +"#, + ) + .unwrap(); + Catalog::from_settings(&settings).unwrap() + } + #[test] fn validate_minimal_valid_graph_has_no_errors() { let g = minimal_valid_graph(); @@ -178,6 +274,56 @@ mod tests { assert_eq!(custom.len(), 1); } + #[test] + fn validate_does_not_run_catalog_aware_model_rules() { + let g = graph_with_model_and_provider("not-in-any-catalog", "not-a-provider"); + + let diagnostics = validate(&g, &[]); + + assert!( + diagnostics + .iter() + .all(|d| d.rule != "node_model_known" && d.rule != "stylesheet_model_known"), + "catalog-free validation should not emit model/provider diagnostics: {diagnostics:?}" + ); + } + + #[test] + fn validate_with_catalog_accepts_custom_catalog_entries() { + let g = graph_with_model_and_provider("venice-large", "venice"); + let catalog = custom_catalog(); + + let diagnostics = validate_with_catalog(&g, &catalog, &[]); + + assert!( + diagnostics + .iter() + .all(|d| d.rule != "node_model_known" && d.rule != "stylesheet_model_known"), + "custom catalog entries should validate cleanly: {diagnostics:?}" + ); + } + + #[test] + fn validate_with_catalog_warns_for_unknown_model_and_provider() { + let g = graph_with_model_and_provider("missing-model", "missing-provider"); + let catalog = custom_catalog(); + + let diagnostics = validate_with_catalog(&g, &catalog, &[]); + + assert!( + diagnostics + .iter() + .any(|d| d.rule == "node_model_known" && d.message.contains("missing-model")), + "missing model diagnostic not found: {diagnostics:?}" + ); + assert!( + diagnostics.iter().any(|d| d.rule == "node_model_known" + && d.message.contains("missing-provider") + && d.message.contains(ProviderId::new("venice").as_str())), + "missing provider diagnostic not found: {diagnostics:?}" + ); + } + #[test] fn diagnostic_severity_eq() { assert_eq!(Severity::Error, Severity::Error); diff --git a/lib/crates/fabro-validate/src/rules/mod.rs b/lib/crates/fabro-validate/src/rules/mod.rs index 208fe0bcf..98ece95af 100644 --- a/lib/crates/fabro-validate/src/rules/mod.rs +++ b/lib/crates/fabro-validate/src/rules/mod.rs @@ -55,8 +55,6 @@ pub fn built_in_rules() -> Vec> { all_conditional_edges::rule(), orphan_custom_outcome::rule(), script_absolute_cd::rule(), - stylesheet_model_known::rule(), - node_model_known::rule(), import_error::rule(), unresolved_file_ref::rule(), thread_id_requires_fidelity_full::rule(), @@ -64,3 +62,12 @@ pub fn built_in_rules() -> Vec> { random_selection_no_conditions::rule(), ] } + +/// Returns lint rules that require the caller's resolved model catalog. +#[must_use] +pub fn catalog_rules(catalog: &fabro_model::Catalog) -> Vec> { + vec![ + stylesheet_model_known::rule(catalog), + node_model_known::rule(catalog), + ] +} diff --git a/lib/crates/fabro-validate/src/rules/model_support.rs b/lib/crates/fabro-validate/src/rules/model_support.rs index fcf505c7e..f6749e899 100644 --- a/lib/crates/fabro-validate/src/rules/model_support.rs +++ b/lib/crates/fabro-validate/src/rules/model_support.rs @@ -1,14 +1,13 @@ -use std::str::FromStr; - use crate::{Diagnostic, Severity}; pub(super) fn check_model_known( rule_name: &str, + catalog: &fabro_model::Catalog, model: &str, context: &str, node_id: Option, ) -> Option { - if fabro_model::Catalog::builtin().get(model).is_some() { + if catalog.get(model).is_some() { return None; } Some(Diagnostic { @@ -25,16 +24,21 @@ pub(super) fn check_model_known( pub(super) fn check_provider_known( rule_name: &str, + catalog: &fabro_model::Catalog, provider: &str, context: &str, node_id: Option, ) -> Option { - if fabro_model::Provider::from_str(provider).is_ok() { + if catalog + .provider(&fabro_model::ProviderId::new(provider)) + .is_some() + { return None; } - let valid: Vec<&str> = fabro_model::Provider::ALL + let valid: Vec<&str> = catalog + .providers() .iter() - .map(|&p| <&'static str>::from(p)) + .map(|provider| provider.id.as_str()) .collect(); let valid_str = valid.join(", "); Some(Diagnostic { diff --git a/lib/crates/fabro-validate/src/rules/node_model_known.rs b/lib/crates/fabro-validate/src/rules/node_model_known.rs index cc3b1baa0..64ba03eac 100644 --- a/lib/crates/fabro-validate/src/rules/node_model_known.rs +++ b/lib/crates/fabro-validate/src/rules/node_model_known.rs @@ -1,15 +1,18 @@ use fabro_graphviz::graph::Graph; +use fabro_model::Catalog; use super::model_support::{check_model_known, check_provider_known}; use crate::{Diagnostic, LintRule}; -pub(super) fn rule() -> Box { - Box::new(Rule) +pub(super) fn rule(catalog: &Catalog) -> Box { + Box::new(Rule { catalog }) } -struct Rule; +struct Rule<'a> { + catalog: &'a Catalog, +} -impl LintRule for Rule { +impl LintRule for Rule<'_> { fn name(&self) -> &'static str { "node_model_known" } @@ -20,14 +23,20 @@ impl LintRule for Rule { let context = format!("on node '{}'", node.id); let node_id = Some(node.id.clone()); if let Some(model) = node.model() { - if let Some(d) = check_model_known(self.name(), model, &context, node_id.clone()) { + if let Some(d) = + check_model_known(self.name(), self.catalog, model, &context, node_id.clone()) + { diagnostics.push(d); } } if let Some(provider) = node.provider() { - if let Some(d) = - check_provider_known(self.name(), provider, &context, node_id.clone()) - { + if let Some(d) = check_provider_known( + self.name(), + self.catalog, + provider, + &context, + node_id.clone(), + ) { diagnostics.push(d); } } @@ -39,6 +48,7 @@ impl LintRule for Rule { #[cfg(test)] mod tests { use fabro_graphviz::graph::{AttrValue, Node}; + use fabro_model::Catalog; use super::Rule; use crate::rules::test_support::minimal_graph; @@ -53,7 +63,9 @@ mod tests { AttrValue::String("claude-sonnet-4-5".to_string()), ); g.nodes.insert("work".to_string(), node); - let rule = Rule; + let rule = Rule { + catalog: Catalog::builtin(), + }; let d = rule.apply(&g); assert!(d.is_empty()); } @@ -67,7 +79,9 @@ mod tests { AttrValue::String("nonexistent-model-xyz".to_string()), ); g.nodes.insert("work".to_string(), node); - let rule = Rule; + let rule = Rule { + catalog: Catalog::builtin(), + }; let d = rule.apply(&g); assert_eq!(d.len(), 1); assert_eq!(d[0].severity, Severity::Warning); @@ -82,7 +96,9 @@ mod tests { node.attrs .insert("model".to_string(), AttrValue::String("opus".to_string())); g.nodes.insert("work".to_string(), node); - let rule = Rule; + let rule = Rule { + catalog: Catalog::builtin(), + }; let d = rule.apply(&g); assert!(d.is_empty()); } @@ -96,7 +112,9 @@ mod tests { AttrValue::String("google".to_string()), ); g.nodes.insert("work".to_string(), node); - let rule = Rule; + let rule = Rule { + catalog: Catalog::builtin(), + }; let d = rule.apply(&g); assert_eq!(d.len(), 1); assert_eq!(d[0].severity, Severity::Warning); @@ -107,7 +125,9 @@ mod tests { #[test] fn node_model_known_rule_no_model_no_provider() { let g = minimal_graph(); - let rule = Rule; + let rule = Rule { + catalog: Catalog::builtin(), + }; let d = rule.apply(&g); assert!(d.is_empty()); } diff --git a/lib/crates/fabro-validate/src/rules/stylesheet_model_known.rs b/lib/crates/fabro-validate/src/rules/stylesheet_model_known.rs index c436dbe37..655da0dd6 100644 --- a/lib/crates/fabro-validate/src/rules/stylesheet_model_known.rs +++ b/lib/crates/fabro-validate/src/rules/stylesheet_model_known.rs @@ -1,16 +1,19 @@ use fabro_graphviz::graph::Graph; use fabro_graphviz::stylesheet::{Selector, parse_stylesheet}; +use fabro_model::Catalog; use super::model_support::{check_model_known, check_provider_known}; use crate::{Diagnostic, LintRule}; -pub(super) fn rule() -> Box { - Box::new(Rule) +pub(super) fn rule(catalog: &Catalog) -> Box { + Box::new(Rule { catalog }) } -struct Rule; +struct Rule<'a> { + catalog: &'a Catalog, +} -impl Rule { +impl Rule<'_> { fn selector_label(selector: &Selector) -> String { match selector { Selector::Universal => "*".to_string(), @@ -21,7 +24,7 @@ impl Rule { } } -impl LintRule for Rule { +impl LintRule for Rule<'_> { fn name(&self) -> &'static str { "stylesheet_model_known" } @@ -42,15 +45,24 @@ impl LintRule for Rule { let context = format!("in stylesheet rule '{label}'"); match decl.property.as_str() { "model" => { - if let Some(d) = check_model_known(self.name(), &decl.value, &context, None) - { + if let Some(d) = check_model_known( + self.name(), + self.catalog, + &decl.value, + &context, + None, + ) { diagnostics.push(d); } } "provider" => { - if let Some(d) = - check_provider_known(self.name(), &decl.value, &context, None) - { + if let Some(d) = check_provider_known( + self.name(), + self.catalog, + &decl.value, + &context, + None, + ) { diagnostics.push(d); } } @@ -65,6 +77,7 @@ impl LintRule for Rule { #[cfg(test)] mod tests { use fabro_graphviz::graph::AttrValue; + use fabro_model::Catalog; use super::Rule; use crate::rules::test_support::minimal_graph; @@ -77,7 +90,9 @@ mod tests { "model_stylesheet".to_string(), AttrValue::String("* { model: claude-sonnet-4-5; provider: anthropic; }".to_string()), ); - let rule = Rule; + let rule = Rule { + catalog: Catalog::builtin(), + }; let d = rule.apply(&g); assert!(d.is_empty()); } @@ -89,7 +104,9 @@ mod tests { "model_stylesheet".to_string(), AttrValue::String("#opus { model: claude-opus-4-5; }".to_string()), ); - let rule = Rule; + let rule = Rule { + catalog: Catalog::builtin(), + }; let d = rule.apply(&g); assert_eq!(d.len(), 1); assert_eq!(d[0].severity, Severity::Warning); @@ -104,7 +121,9 @@ mod tests { "model_stylesheet".to_string(), AttrValue::String("* { provider: google; }".to_string()), ); - let rule = Rule; + let rule = Rule { + catalog: Catalog::builtin(), + }; let d = rule.apply(&g); assert_eq!(d.len(), 1); assert_eq!(d[0].severity, Severity::Warning); @@ -118,7 +137,9 @@ mod tests { "model_stylesheet".to_string(), AttrValue::String("* { model: opus; }".to_string()), ); - let rule = Rule; + let rule = Rule { + catalog: Catalog::builtin(), + }; let d = rule.apply(&g); assert!(d.is_empty()); } @@ -126,7 +147,9 @@ mod tests { #[test] fn stylesheet_model_known_rule_no_stylesheet() { let g = minimal_graph(); - let rule = Rule; + let rule = Rule { + catalog: Catalog::builtin(), + }; let d = rule.apply(&g); assert!(d.is_empty()); } diff --git a/lib/crates/fabro-workflow/src/handler/llm/api.rs b/lib/crates/fabro-workflow/src/handler/llm/api.rs index 4c216e98d..115a371cf 100644 --- a/lib/crates/fabro-workflow/src/handler/llm/api.rs +++ b/lib/crates/fabro-workflow/src/handler/llm/api.rs @@ -172,22 +172,26 @@ fn build_profile( provider: Provider, provider_id: ProviderId, profile_kind: AgentProfileKind, + catalog: Arc, ) -> Box { match profile_kind { AgentProfileKind::OpenAi => Box::new( OpenAiProfile::new(model) .with_provider(provider) - .with_provider_id(provider_id), + .with_provider_id(provider_id) + .with_catalog(catalog), ), AgentProfileKind::Gemini => Box::new( GeminiProfile::new(model) .with_provider(provider) - .with_provider_id(provider_id), + .with_provider_id(provider_id) + .with_catalog(catalog), ), AgentProfileKind::Anthropic => Box::new( AnthropicProfile::new(model) .with_provider(provider) - .with_provider_id(provider_id), + .with_provider_id(provider_id) + .with_catalog(catalog), ), } } @@ -573,6 +577,7 @@ impl AgentApiBackend { provider.provider, provider.provider_id.clone(), provider.profile_kind, + Arc::clone(&catalog), ); let config = SessionOptions { @@ -593,6 +598,7 @@ impl AgentApiBackend { let factory_client = client.clone(); let factory_model = model.to_string(); let factory_provider = provider.clone(); + let factory_catalog = Arc::clone(&catalog); let factory_env = Arc::clone(sandbox); let factory_tool_env = tool_env.cloned(); let factory: SessionFactory = Arc::new(move || { @@ -601,6 +607,7 @@ impl AgentApiBackend { factory_provider.provider, factory_provider.provider_id.clone(), factory_provider.profile_kind, + Arc::clone(&factory_catalog), )); let mut session = Session::new( factory_client.clone(), @@ -1433,6 +1440,7 @@ mod tests { Provider::Anthropic, Provider::Anthropic.id(), AgentProfileKind::Anthropic, + Arc::new(Catalog::from_builtin_with_overrides(&LlmCatalogSettings::default()).unwrap()), ); let manager = Arc::new(TokioMutex::new(SubAgentManager::new(1))); let factory: SessionFactory = Arc::new(|| { diff --git a/lib/crates/fabro-workflow/src/handler/manager_loop.rs b/lib/crates/fabro-workflow/src/handler/manager_loop.rs index e8e6669eb..d22331ad0 100644 --- a/lib/crates/fabro-workflow/src/handler/manager_loop.rs +++ b/lib/crates/fabro-workflow/src/handler/manager_loop.rs @@ -75,6 +75,7 @@ fn parse_child_graph(node: &Node, services: &EngineServices) -> Result Result, goal_override: Option<&str>, render_mode: RenderMode, + catalog: &Arc, ) -> Result { let inputs = run_inputs(settings); let template_ctx = TemplateContext::for_input_scan(inputs.clone()); @@ -372,8 +374,9 @@ pub(super) fn preprocess_and_validate( file_resolver, inputs, custom_transforms, + catalog: Arc::clone(catalog), })?; - let mut validated = pipeline::validate(transformed, &[]); + let mut validated = pipeline::validate(transformed, catalog.as_ref(), &[]); if !template_diagnostics.is_empty() { validated.prepend_diagnostics(template_diagnostics); } @@ -527,6 +530,10 @@ mod tests { .expect("default settings should resolve") } + fn test_catalog() -> Arc { + Arc::new(Catalog::from_builtin_with_overrides(&LlmCatalogSettings::default()).unwrap()) + } + fn validate_dot(dot_source: &str, settings: WorkflowSettings) -> Validated { validate(ValidateInput { workflow: WorkflowInput::DotSource { @@ -536,6 +543,7 @@ mod tests { settings, cwd: PathBuf::from("."), custom_transforms: Vec::new(), + catalog: test_catalog(), mode: RenderMode::Structural, }) .unwrap() @@ -599,6 +607,7 @@ mod tests { Some(&WorkflowSettings::default()), None, RenderMode::Strict, + &test_catalog(), ); let Err(err) = result else { @@ -713,6 +722,7 @@ mod tests { settings: WorkflowSettings::default(), cwd: PathBuf::from("."), custom_transforms: Vec::new(), + catalog: test_catalog(), mode: RenderMode::Strict, }); assert!(result.is_err()); @@ -757,6 +767,7 @@ mod tests { settings: WorkflowSettings::default(), cwd: PathBuf::from("."), custom_transforms: vec![Box::new(TagTransform)], + catalog: test_catalog(), mode: RenderMode::Strict, }) .unwrap(); @@ -791,6 +802,7 @@ mod tests { settings: WorkflowSettings::default(), cwd: dir.path().to_path_buf(), custom_transforms: Vec::new(), + catalog: test_catalog(), mode: RenderMode::Strict, }) .unwrap(); @@ -832,6 +844,7 @@ mod tests { settings: WorkflowSettings::default(), cwd: PathBuf::from("."), custom_transforms: Vec::new(), + catalog: test_catalog(), mode: RenderMode::Strict, }) .unwrap(); diff --git a/lib/crates/fabro-workflow/src/operations/validate.rs b/lib/crates/fabro-workflow/src/operations/validate.rs index 5415ce8b2..046944428 100644 --- a/lib/crates/fabro-workflow/src/operations/validate.rs +++ b/lib/crates/fabro-workflow/src/operations/validate.rs @@ -1,5 +1,7 @@ use std::path::PathBuf; +use std::sync::Arc; +use fabro_model::Catalog; use fabro_types::WorkflowSettings; use super::create::{RenderMode, preprocess_and_validate}; @@ -13,6 +15,7 @@ pub struct ValidateInput { pub settings: WorkflowSettings, pub cwd: PathBuf, pub custom_transforms: Vec>, + pub catalog: Arc, /// How undefined template inputs are treated. Validate-style callers /// (`fabro validate`, the `/validate` API) pass [`RenderMode::Structural`] /// so unbound inputs surface as warning diagnostics. Run-style callers @@ -41,5 +44,6 @@ pub fn validate(input: ValidateInput) -> Result { Some(&resolved.settings), resolved.goal_override.as_deref(), input.mode, + &input.catalog, ) } diff --git a/lib/crates/fabro-workflow/src/pipeline/transform.rs b/lib/crates/fabro-workflow/src/pipeline/transform.rs index f2ff3c459..5bba82788 100644 --- a/lib/crates/fabro-workflow/src/pipeline/transform.rs +++ b/lib/crates/fabro-workflow/src/pipeline/transform.rs @@ -41,7 +41,7 @@ pub fn transform(parsed: Parsed, options: &TransformOptions) -> Result Arc { + Arc::new(Catalog::from_builtin_with_overrides(&LlmCatalogSettings::default()).unwrap()) + } + + fn transform_options() -> TransformOptions { + TransformOptions { + current_dir: None, + file_resolver: None, + inputs: HashMap::new(), + custom_transforms: vec![], + catalog: test_catalog(), + } + } + #[test] fn transform_applies_variable_expansion() { let dot = r#"digraph Test { @@ -82,13 +98,7 @@ mod tests { start -> work -> exit }"#; let parsed = parse(dot).unwrap(); - let transformed = transform(parsed, &TransformOptions { - current_dir: None, - file_resolver: None, - inputs: HashMap::new(), - custom_transforms: vec![], - }) - .unwrap(); + let transformed = transform(parsed, &transform_options()).unwrap(); let prompt = transformed.graph.nodes["work"] .attrs .get("prompt") @@ -107,13 +117,7 @@ mod tests { start -> work -> exit }"#; let parsed = parse(dot).unwrap(); - let transformed = transform(parsed, &TransformOptions { - current_dir: None, - file_resolver: None, - inputs: HashMap::new(), - custom_transforms: vec![], - }) - .unwrap(); + let transformed = transform(parsed, &transform_options()).unwrap(); assert_eq!( transformed.graph.nodes["work"].attrs.get("model"), Some(&AttrValue::String("claude-sonnet-4-6".into())) @@ -140,6 +144,7 @@ mod tests { file_resolver: Some(Arc::new(FilesystemFileResolver::new(None))), inputs: HashMap::new(), custom_transforms: vec![], + catalog: test_catalog(), }) .unwrap(); @@ -187,6 +192,7 @@ mod tests { toml::Value::String("Launch".to_string()), )]), custom_transforms: vec![], + catalog: test_catalog(), }) .unwrap(); diff --git a/lib/crates/fabro-workflow/src/pipeline/types.rs b/lib/crates/fabro-workflow/src/pipeline/types.rs index 79622e6fb..98825bd4d 100644 --- a/lib/crates/fabro-workflow/src/pipeline/types.rs +++ b/lib/crates/fabro-workflow/src/pipeline/types.rs @@ -321,6 +321,7 @@ pub struct TransformOptions { pub file_resolver: Option>, pub inputs: HashMap, pub custom_transforms: Vec>, + pub catalog: Arc, } /// Options for the FINALIZE phase. diff --git a/lib/crates/fabro-workflow/src/pipeline/validate.rs b/lib/crates/fabro-workflow/src/pipeline/validate.rs index 83c140ac7..abedb0709 100644 --- a/lib/crates/fabro-workflow/src/pipeline/validate.rs +++ b/lib/crates/fabro-workflow/src/pipeline/validate.rs @@ -1,3 +1,4 @@ +use fabro_model::Catalog; use fabro_validate::LintRule; use super::types::{Transformed, Validated}; @@ -6,29 +7,44 @@ use super::types::{Transformed, Validated}; /// /// **Infallible.** Always returns `Validated` with diagnostics. Caller decides /// whether to fail via `validated.raise_on_errors()`. -pub fn validate(transformed: Transformed, extra_rules: &[&dyn LintRule]) -> Validated { +pub fn validate( + transformed: Transformed, + catalog: &Catalog, + extra_rules: &[&dyn LintRule], +) -> Validated { let Transformed { graph, source } = transformed; - let diagnostics = fabro_validate::validate(&graph, extra_rules); + let diagnostics = fabro_validate::validate_with_catalog(&graph, catalog, extra_rules); Validated::new(graph, source, diagnostics) } #[cfg(test)] mod tests { + use fabro_model::Catalog; + use fabro_model::catalog::LlmCatalogSettings; + use super::*; use crate::pipeline::parse::parse; use crate::pipeline::transform; use crate::pipeline::types::TransformOptions; + fn test_catalog() -> std::sync::Arc { + std::sync::Arc::new( + Catalog::from_builtin_with_overrides(&LlmCatalogSettings::default()).unwrap(), + ) + } + fn run_pipeline(dot: &str) -> Validated { + let catalog = test_catalog(); let parsed = parse(dot).unwrap(); let transformed = transform::transform(parsed, &TransformOptions { current_dir: None, file_resolver: None, inputs: std::collections::HashMap::new(), custom_transforms: vec![], + catalog: std::sync::Arc::clone(&catalog), }) .unwrap(); - validate(transformed, &[]) + validate(transformed, catalog.as_ref(), &[]) } #[test] diff --git a/lib/crates/fabro-workflow/src/transforms/model_resolution.rs b/lib/crates/fabro-workflow/src/transforms/model_resolution.rs index 59d9c5e65..e389005bc 100644 --- a/lib/crates/fabro-workflow/src/transforms/model_resolution.rs +++ b/lib/crates/fabro-workflow/src/transforms/model_resolution.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + use fabro_graphviz::graph::{AttrValue, Graph}; use fabro_model::Catalog; @@ -6,7 +8,16 @@ use crate::error::Error; /// Resolves model aliases to canonical IDs and infers the provider from the /// model catalog. -pub struct ModelResolutionTransform; +pub struct ModelResolutionTransform { + catalog: Arc, +} + +impl ModelResolutionTransform { + #[must_use] + pub fn new(catalog: Arc) -> Self { + Self { catalog } + } +} impl Transform for ModelResolutionTransform { fn apply(&self, graph: Graph) -> Result { @@ -18,7 +29,7 @@ impl Transform for ModelResolutionTransform { .and_then(AttrValue::as_str) .map(String::from); if let Some(model) = model { - if let Some(info) = Catalog::builtin().get(&model) { + if let Some(info) = self.catalog.get(&model) { let canonical_id = info.id.clone(); let provider = info.provider.to_string(); // Resolve alias to canonical model ID @@ -40,10 +51,48 @@ impl Transform for ModelResolutionTransform { #[cfg(test)] mod tests { + use std::sync::Arc; + use fabro_graphviz::graph::{AttrValue, Graph, Node}; + use fabro_model::catalog::LlmCatalogSettings; use super::*; + fn custom_catalog() -> Arc { + let settings: LlmCatalogSettings = toml::from_str( + r#" +[providers.venice] +display_name = "Venice" +adapter = "openai_compatible" +base_url = "https://api.venice.ai/api/v1" +credentials = ["env:VENICE_API_KEY"] + +[models."venice-large"] +provider = "venice" +display_name = "Venice Large" +family = "venice" +default = true +aliases = ["vl"] + +[models."venice-large".limits] +context_window = 128000 + +[models."venice-large".features] +tools = true +vision = false +reasoning = false +effort = false +"#, + ) + .unwrap(); + Arc::new(Catalog::from_settings(&settings).unwrap()) + } + + fn builtin_transform() -> ModelResolutionTransform { + let catalog = Catalog::from_builtin_with_overrides(&LlmCatalogSettings::default()).unwrap(); + ModelResolutionTransform::new(Arc::new(catalog)) + } + #[test] fn provider_inference_sets_provider_from_catalog() { let mut graph = Graph::new("test"); @@ -54,7 +103,7 @@ mod tests { ); graph.nodes.insert("a".to_string(), node); - let graph = ModelResolutionTransform.apply(graph).unwrap(); + let graph = builtin_transform().apply(graph).unwrap(); assert_eq!( graph.nodes["a"] @@ -79,7 +128,7 @@ mod tests { ); graph.nodes.insert("a".to_string(), node); - let graph = ModelResolutionTransform.apply(graph).unwrap(); + let graph = builtin_transform().apply(graph).unwrap(); assert_eq!( graph.nodes["a"] @@ -100,7 +149,7 @@ mod tests { ); graph.nodes.insert("a".to_string(), node); - let graph = ModelResolutionTransform.apply(graph).unwrap(); + let graph = builtin_transform().apply(graph).unwrap(); assert_eq!(graph.nodes["a"].attrs.get("provider"), None); } @@ -111,7 +160,7 @@ mod tests { let node = Node::new("a"); graph.nodes.insert("a".to_string(), node); - let graph = ModelResolutionTransform.apply(graph).unwrap(); + let graph = builtin_transform().apply(graph).unwrap(); assert_eq!(graph.nodes["a"].attrs.get("provider"), None); } @@ -124,7 +173,7 @@ mod tests { .insert("model".to_string(), AttrValue::String("gpt-54".to_string())); graph.nodes.insert("a".to_string(), node); - let graph = ModelResolutionTransform.apply(graph).unwrap(); + let graph = builtin_transform().apply(graph).unwrap(); assert_eq!( graph.nodes["a"] @@ -152,7 +201,7 @@ mod tests { ); graph.nodes.insert("a".to_string(), node); - let graph = ModelResolutionTransform.apply(graph).unwrap(); + let graph = builtin_transform().apply(graph).unwrap(); assert_eq!( graph.nodes["a"] @@ -162,4 +211,32 @@ mod tests { Some("gpt-5.4") ); } + + #[test] + fn model_resolution_uses_injected_catalog_for_alias_and_provider() { + let mut graph = Graph::new("test"); + let mut node = Node::new("a"); + node.attrs + .insert("model".to_string(), AttrValue::String("vl".to_string())); + graph.nodes.insert("a".to_string(), node); + + let graph = ModelResolutionTransform::new(custom_catalog()) + .apply(graph) + .unwrap(); + + assert_eq!( + graph.nodes["a"] + .attrs + .get("model") + .and_then(AttrValue::as_str), + Some("venice-large") + ); + assert_eq!( + graph.nodes["a"] + .attrs + .get("provider") + .and_then(AttrValue::as_str), + Some("venice") + ); + } } diff --git a/lib/crates/fabro-workflow/tests/it/integration.rs b/lib/crates/fabro-workflow/tests/it/integration.rs index 832430e76..4fcb20b76 100644 --- a/lib/crates/fabro-workflow/tests/it/integration.rs +++ b/lib/crates/fabro-workflow/tests/it/integration.rs @@ -4272,6 +4272,12 @@ async fn import_e2e_through_engine() { use fabro_workflow::pipeline::{TransformOptions, transform, validate}; let dir = tempfile::tempdir().unwrap(); + let catalog = std::sync::Arc::new( + fabro_model::Catalog::from_builtin_with_overrides( + &fabro_model::catalog::LlmCatalogSettings::default(), + ) + .unwrap(), + ); std::fs::write( dir.path().join("val.fabro"), r#"digraph validate { @@ -4313,9 +4319,10 @@ async fn import_e2e_through_engine() { )), inputs: std::collections::HashMap::new(), custom_transforms: vec![], + catalog: std::sync::Arc::clone(&catalog), }) .unwrap(); - let validated = validate(transformed, &[]); + let validated = validate(transformed, catalog.as_ref(), &[]); validated .raise_on_errors() .expect("validation should pass after imports expand");