scriptoriumadmin/fabro
1
0
Fork 0
mirror of https://github.com/fabro-sh/fabro.git synced 2026-09-23 00:41:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

feat(llm): use server catalog for workflow execution

Browse source 
Thread the resolved server LLM catalog through workflow validation,
model resolution, credential lookup, request construction, and worker
startup so request-serving paths no longer depend on the builtin catalog.
This commit is contained in:
Bryan Helmkamp 2026-05-13 12:49:45 -04:00
parent 68ab756040
commit de5c27ca20
No known key found for this signature in database
46 changed files with 1128 additions and 203 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Cargo.lock generated
View file

@ -2116,6 +2116,7 @@ dependencies = [
"fabro-client",
"fabro-config",
"fabro-manifest",
"fabro-model",
"fabro-server",
"fabro-types",
"fabro-util",
@ -2506,6 +2507,7 @@ dependencies = [
"fabro-types",
"serde",
"thiserror 2.0.18",
"toml 0.8.23",
]
[[package]]

19
lib/crates/fabro-agent/src/agent_profile.rs
View file

@ -1,7 +1,7 @@
use std::sync::Arc;
use fabro_llm::types::ToolDefinition;
use fabro_model::{Catalog, Provider, ProviderId};
use fabro_model::{Catalog, Model, Provider, ProviderId};
use tokio::sync::Mutex;
use crate::profiles::EnvContext;
@ -19,6 +19,9 @@ pub trait AgentProfile: Send + Sync {
self.provider().id()
}
fn model(&self) -> &str;
fn catalog(&self) -> Option<&Catalog> {
None
}
fn tool_registry(&self) -> &ToolRegistry;
fn tool_registry_mut(&mut self) -> &mut ToolRegistry;
fn build_system_prompt(
@ -35,17 +38,25 @@ pub trait AgentProfile: Send + Sync {
}
fn knowledge_cutoff(&self) -> Option<String> {
Catalog::builtin()
.get(self.model())
self.catalog()
.and_then(|catalog| catalog.get(self.model()))
.and_then(|m| m.knowledge_cutoff().map(str::to_string))
}
fn catalog_model(&self) -> Option<&Model> {
self.catalog().and_then(|catalog| catalog.get(self.model()))
}
fn context_window_size(&self) -> usize {
Catalog::builtin().get(self.model()).map_or(200_000, |m| {
self.catalog_model().map_or(200_000, |m| {
usize::try_from(m.context_window()).unwrap_or(usize::MAX)
})
}
fn max_output_tokens(&self) -> Option<i64> {
self.catalog_model().and_then(Model::max_output)
}
fn register_subagent_tools(
&mut self,
manager: Arc<Mutex<SubAgentManager>>,

68
lib/crates/fabro-agent/src/cli.rs
View file

@ -18,6 +18,7 @@ use fabro_llm::middleware::{Middleware, NextFn, NextStreamFn};
use fabro_llm::provider::StreamEventStream;
use fabro_llm::types::{Request, Response};
use fabro_mcp::config::McpServerSettings;
use fabro_model::catalog::LlmCatalogSettings;
use fabro_model::{Catalog, ModelHandle, Provider};
use fabro_util::terminal::Styles;
use fabro_vault::Vault;
@ -231,18 +232,27 @@ fn build_profile(
provider: Provider,
model: &str,
summarizer: Option<WebFetchSummarizer>,
catalog: Arc<Catalog>,
) -> Box<dyn AgentProfile> {
match provider {
Provider::OpenAi => Box::new(OpenAiProfile::with_summarizer(model, summarizer)),
Provider::OpenAi => {
Box::new(OpenAiProfile::with_summarizer(model, summarizer).with_catalog(catalog))
}
Provider::Kimi
| Provider::Zai
| Provider::Minimax
| Provider::Inception
| Provider::OpenAiCompatible => {
Box::new(OpenAiProfile::with_summarizer(model, summarizer).with_provider(provider))
| Provider::OpenAiCompatible => Box::new(
OpenAiProfile::with_summarizer(model, summarizer)
.with_provider(provider)
.with_catalog(catalog),
),
Provider::Gemini => {
Box::new(GeminiProfile::with_summarizer(model, summarizer).with_catalog(catalog))
}
Provider::Anthropic => {
Box::new(AnthropicProfile::with_summarizer(model, summarizer).with_catalog(catalog))
}
Provider::Gemini => Box::new(GeminiProfile::with_summarizer(model, summarizer)),
Provider::Anthropic => Box::new(AnthropicProfile::with_summarizer(model, summarizer)),
}
}
@ -480,10 +490,14 @@ pub async fn run_with_args_and_client(
}
// Resolve model and build profile
let catalog = Arc::new(
Catalog::from_builtin_with_overrides(&LlmCatalogSettings::default())
.context("failed to build standalone agent LLM catalog")?,
);
let model = if let Some(model) = args.model.clone() {
model
} else {
Catalog::builtin()
catalog
.default_for_provider(&provider.id())
.map(|model| model.id.clone())
.ok_or_else(|| {
@ -497,6 +511,7 @@ pub async fn run_with_args_and_client(
provider,
&model,
Some(build_summarizer(provider, client.clone())),
Arc::clone(&catalog),
);
// Build sandbox
@ -530,32 +545,17 @@ pub async fn run_with_args_and_client(
let manager_for_callback = manager.clone();
let factory_client = client.clone();
let factory_model = model.clone();
let factory_catalog = Arc::clone(&catalog);
let factory_env = Arc::clone(&env);
let factory_hooks = config.tool_hooks.clone();
let factory: SessionFactory = Arc::new(move || {
let child_summarizer = Some(build_summarizer(provider, factory_client.clone()));
let child_profile: Arc<dyn AgentProfile> = match provider {
Provider::OpenAi => Arc::new(OpenAiProfile::with_summarizer(
&factory_model,
child_summarizer,
)),
Provider::Kimi
| Provider::Zai
| Provider::Minimax
| Provider::Inception
| Provider::OpenAiCompatible => Arc::new(
OpenAiProfile::with_summarizer(&factory_model, child_summarizer)
.with_provider(provider),
),
Provider::Gemini => Arc::new(GeminiProfile::with_summarizer(
&factory_model,
child_summarizer,
)),
Provider::Anthropic => Arc::new(AnthropicProfile::with_summarizer(
&factory_model,
child_summarizer,
)),
};
let child_profile: Arc<dyn AgentProfile> = Arc::from(build_profile(
provider,
&factory_model,
child_summarizer,
Arc::clone(&factory_catalog),
));
Session::new(
factory_client.clone(),
child_profile,
@ -852,15 +852,19 @@ mod tests {
// build_profile tests
fn test_catalog() -> Arc<Catalog> {
Arc::new(Catalog::from_builtin_with_overrides(&LlmCatalogSettings::default()).unwrap())
}
#[test]
fn build_profile_anthropic() {
let profile = build_profile(Provider::Anthropic, "model", None);
let profile = build_profile(Provider::Anthropic, "model", None, test_catalog());
assert_eq!(profile.provider(), Provider::Anthropic);
}
#[test]
fn build_profile_openai() {
let profile = build_profile(Provider::OpenAi, "model", None);
let profile = build_profile(Provider::OpenAi, "model", None, test_catalog());
assert_eq!(profile.provider(), Provider::OpenAi);
}
@ -876,7 +880,7 @@ mod tests {
#[test]
fn build_profile_gemini() {
let profile = build_profile(Provider::Gemini, "model", None);
let profile = build_profile(Provider::Gemini, "model", None, test_catalog());
assert_eq!(profile.provider(), Provider::Gemini);
}
@ -884,7 +888,7 @@ mod tests {
#[test]
fn build_profile_can_register_subagent_tools() {
let mut profile = build_profile(Provider::Anthropic, "model", None);
let mut profile = build_profile(Provider::Anthropic, "model", None, test_catalog());
let manager = Arc::new(AsyncMutex::new(SubAgentManager::new(1)));
let factory: SessionFactory = Arc::new(|| {
panic!("factory should not be called in this test");

26
lib/crates/fabro-agent/src/profiles/anthropic.rs
View file

@ -1,4 +1,6 @@
use fabro_model::{Provider, ProviderId};
use std::sync::Arc;
use fabro_model::{Catalog, Provider, ProviderId};
use super::EnvContext;
use crate::agent_profile::AgentProfile;
@ -38,6 +40,7 @@ impl AnthropicProfile {
provider: Provider::Anthropic,
provider_id: Provider::Anthropic.id(),
model: model.into(),
catalog: None,
registry,
},
}
@ -58,6 +61,12 @@ impl AnthropicProfile {
self.base.provider_id = provider_id;
self
}
#[must_use]
pub fn with_catalog(mut self, catalog: Arc<Catalog>) -> Self {
self.base.catalog = Some(catalog);
self
}
}
impl AgentProfile for AnthropicProfile {
@ -73,6 +82,10 @@ impl AgentProfile for AnthropicProfile {
&self.base.model
}
fn catalog(&self) -> Option<&Catalog> {
self.base.catalog.as_deref()
}
fn tool_registry(&self) -> &ToolRegistry {
&self.base.registry
}
@ -180,12 +193,17 @@ in the project. Keep changes minimal and focused on the task.";
mod tests {
use std::sync::Arc;
use fabro_model::catalog::LlmCatalogSettings;
use tokio::sync::Mutex as AsyncMutex;
use super::*;
use crate::subagent::{SessionFactory, SubAgentManager};
use crate::test_support::MockSandbox;
fn test_catalog() -> Arc<Catalog> {
Arc::new(Catalog::from_builtin_with_overrides(&LlmCatalogSettings::default()).unwrap())
}
#[test]
fn anthropic_profile_identity() {
let profile = AnthropicProfile::new("claude-sonnet-4-20250514");
@ -195,16 +213,16 @@ mod tests {
#[test]
fn anthropic_context_window_from_catalog() {
let profile = AnthropicProfile::new("claude-opus-4-6");
let profile = AnthropicProfile::new("claude-opus-4-6").with_catalog(test_catalog());
assert_eq!(profile.context_window_size(), 1_000_000);
let profile = AnthropicProfile::new("claude-sonnet-4-6");
let profile = AnthropicProfile::new("claude-sonnet-4-6").with_catalog(test_catalog());
assert_eq!(profile.context_window_size(), 200_000);
}
#[test]
fn anthropic_knowledge_cutoff_from_catalog() {
let profile = AnthropicProfile::new("claude-opus-4-6");
let profile = AnthropicProfile::new("claude-opus-4-6").with_catalog(test_catalog());
assert_eq!(profile.knowledge_cutoff(), Some("May 2025".to_string()));
}

22
lib/crates/fabro-agent/src/profiles/gemini.rs
View file

@ -1,4 +1,6 @@
use fabro_model::{Provider, ProviderId};
use std::sync::Arc;
use fabro_model::{Catalog, Provider, ProviderId};
use super::EnvContext;
use crate::agent_profile::AgentProfile;
@ -40,6 +42,7 @@ impl GeminiProfile {
provider: Provider::Gemini,
provider_id: Provider::Gemini.id(),
model: model.into(),
catalog: None,
registry,
},
}
@ -59,6 +62,12 @@ impl GeminiProfile {
self.base.provider_id = provider_id;
self
}
#[must_use]
pub fn with_catalog(mut self, catalog: Arc<Catalog>) -> Self {
self.base.catalog = Some(catalog);
self
}
}
impl AgentProfile for GeminiProfile {
@ -74,6 +83,10 @@ impl AgentProfile for GeminiProfile {
&self.base.model
}
fn catalog(&self) -> Option<&Catalog> {
self.base.catalog.as_deref()
}
fn tool_registry(&self) -> &ToolRegistry {
&self.base.registry
}
@ -222,12 +235,17 @@ in the project.";
mod tests {
use std::sync::Arc;
use fabro_model::catalog::LlmCatalogSettings;
use tokio::sync::Mutex as AsyncMutex;
use super::*;
use crate::subagent::{SessionFactory, SubAgentManager};
use crate::test_support::MockSandbox;
fn test_catalog() -> Arc<Catalog> {
Arc::new(Catalog::from_builtin_with_overrides(&LlmCatalogSettings::default()).unwrap())
}
#[test]
fn gemini_profile_identity() {
let profile = GeminiProfile::new("gemini-2.0-flash");
@ -237,7 +255,7 @@ mod tests {
#[test]
fn gemini_context_window_from_catalog() {
let profile = GeminiProfile::new("gemini-3.1-pro-preview");
let profile = GeminiProfile::new("gemini-3.1-pro-preview").with_catalog(test_catalog());
assert_eq!(profile.context_window_size(), 1_048_576);
}

6
lib/crates/fabro-agent/src/profiles/mod.rs
View file

@ -1,9 +1,12 @@
use std::sync::Arc;
use fabro_model::{Catalog, Provider, ProviderId};
pub mod anthropic;
pub mod gemini;
pub mod openai;
pub use anthropic::AnthropicProfile;
use fabro_model::{Provider, ProviderId};
pub use gemini::GeminiProfile;
pub use openai::OpenAiProfile;
@ -19,6 +22,7 @@ pub struct BaseProfile {
pub provider: Provider,
pub provider_id: ProviderId,
pub model: String,
pub catalog: Option<Arc<Catalog>>,
pub registry: ToolRegistry,
}

15
lib/crates/fabro-agent/src/profiles/openai.rs
View file

@ -1,4 +1,6 @@
use fabro_model::{Provider, ProviderId};
use std::sync::Arc;
use fabro_model::{Catalog, Provider, ProviderId};
use super::EnvContext;
use crate::agent_profile::AgentProfile;
@ -36,6 +38,7 @@ impl OpenAiProfile {
provider: Provider::OpenAi,
provider_id: Provider::OpenAi.id(),
model: model.into(),
catalog: None,
registry,
},
}
@ -57,6 +60,12 @@ impl OpenAiProfile {
self
}
#[must_use]
pub fn with_catalog(mut self, catalog: Arc<Catalog>) -> Self {
self.base.catalog = Some(catalog);
self
}
fn provider_display_name(&self) -> &str {
match self.base.provider {
Provider::OpenAi => "OpenAI",
@ -82,6 +91,10 @@ impl AgentProfile for OpenAiProfile {
&self.base.model
}
fn catalog(&self) -> Option<&Catalog> {
self.base.catalog.as_deref()
}
fn tool_registry(&self) -> &ToolRegistry {
&self.base.registry
}

9
lib/crates/fabro-agent/src/session.rs
View file

@ -1553,11 +1553,10 @@ impl Session {
response_format: None,
temperature: None,
top_p: None,
max_tokens: self.config.max_tokens.or_else(|| {
fabro_model::Catalog::builtin()
.get(self.provider_profile.model())
.and_then(fabro_model::Model::max_output)
}),
max_tokens: self
.config
.max_tokens
.or_else(|| self.provider_profile.max_output_tokens()),
stop_sequences: None,
reasoning_effort: self.config.reasoning_effort,
speed: self.config.speed,

8
lib/crates/fabro-auth/src/env_source.rs
View file

@ -2,7 +2,9 @@ use std::sync::Arc;
use async_trait::async_trait;
use fabro_model::catalog::CatalogProvider;
use fabro_model::{Catalog, CredentialRef, HeaderValueRef, Provider, ProviderId, adapter};
use fabro_model::{
Catalog, CredentialRef, HeaderValueRef, Provider, ProviderId, adapter, bootstrap_catalog,
};
use fabro_static::EnvVars;
use crate::credential_source::{CredentialSource, ResolvedCredentials};
@ -135,7 +137,7 @@ impl Default for EnvCredentialSource {
#[async_trait]
impl CredentialSource for EnvCredentialSource {
async fn resolve(&self) -> anyhow::Result<ResolvedCredentials> {
self.resolve_for_catalog(Catalog::builtin()).await
self.resolve_for_catalog(bootstrap_catalog::catalog()).await
}
async fn resolve_for_catalog(&self, catalog: &Catalog) -> anyhow::Result<ResolvedCredentials> {
@ -158,7 +160,7 @@ impl CredentialSource for EnvCredentialSource {
}
async fn configured_providers(&self) -> Vec<ProviderId> {
self.configured_providers_for_catalog(Catalog::builtin())
self.configured_providers_for_catalog(bootstrap_catalog::catalog())
.await
}

116
lib/crates/fabro-auth/src/resolve.rs
View file

@ -4,6 +4,7 @@ use std::sync::Arc;
use fabro_model::catalog::CatalogProvider;
use fabro_model::{
ApiKeyHeaderPolicy, Catalog, CredentialRef, HeaderValueRef, Provider, ProviderId, adapter,
bootstrap_catalog,
};
use fabro_static::EnvVars;
use fabro_vault::Vault;
@ -49,7 +50,7 @@ impl ApiCredential {
#[must_use]
pub fn from_api_key(provider: impl Into<ProviderId>, key: String) -> Self {
let provider = provider.into();
let auth_header = auth_header_for_provider(&provider, key);
let auth_header = default_auth_header_for_provider(&provider, key);
Self {
provider,
auth_header: Some(auth_header),
@ -60,6 +61,33 @@ impl ApiCredential {
project_id: None,
}
}
/// Build an `ApiCredential` from an API key using the supplied catalog for
/// auth header policy and provider base URL.
#[must_use]
pub fn from_api_key_for_catalog(
provider: impl Into<ProviderId>,
key: String,
catalog: &Catalog,
) -> Self {
let provider_id = provider.into();
let (auth_header, base_url) = match catalog.provider(&provider_id) {
Some(provider) => (
auth_header_for_catalog_provider(provider, key),
provider.base_url.clone(),
),
None => (default_auth_header_for_provider(&provider_id, key), None),
};
Self {
provider: provider_id,
auth_header: Some(auth_header),
extra_headers: HashMap::new(),
base_url,
codex_mode: false,
org_id: None,
project_id: None,
}
}
}
#[must_use]
@ -73,8 +101,8 @@ pub fn build_api_key_header(policy: ApiKeyHeaderPolicy, key: String) -> ApiKeyHe
}
}
fn auth_header_for_provider(provider: &ProviderId, key: String) -> ApiKeyHeader {
let policy = Catalog::builtin()
fn default_auth_header_for_provider(provider: &ProviderId, key: String) -> ApiKeyHeader {
let policy = bootstrap_catalog::catalog()
.provider(provider)
.and_then(|provider| adapter::get(&provider.adapter))
.map_or_else(
@ -221,7 +249,7 @@ impl CredentialResolver {
#[must_use]
pub fn configured_providers(&self, vault: &Vault) -> Vec<ProviderId> {
self.configured_providers_for_catalog(vault, Catalog::builtin())
self.configured_providers_for_catalog(vault, bootstrap_catalog::catalog())
}
pub fn configured_providers_for_catalog(
@ -253,7 +281,7 @@ impl CredentialResolver {
}
}
if let Some(catalog_provider) = Catalog::builtin().provider(provider) {
if let Some(catalog_provider) = bootstrap_catalog::catalog().provider(provider) {
for credential_ref in &catalog_provider.credentials {
if let Some(credential) = self.credential_from_ref(vault, provider, credential_ref)
{
@ -389,7 +417,7 @@ impl CredentialResolver {
vault: &Vault,
credential: &AuthCredential,
) -> Result<ApiCredential, ResolveError> {
self.to_api_credential_for_catalog(vault, credential, Catalog::builtin())
self.to_api_credential_for_catalog(vault, credential, bootstrap_catalog::catalog())
}
fn to_api_credential_for_catalog(
@ -402,7 +430,7 @@ impl CredentialResolver {
match &credential.details {
AuthDetails::ApiKey { key } => {
let auth_header = catalog.provider(&credential.provider).map_or_else(
|| auth_header_for_provider(&credential.provider, key.clone()),
|| default_auth_header_for_provider(&credential.provider, key.clone()),
|provider| auth_header_for_catalog_provider(provider, key.clone()),
);
let mut cred = ApiCredential {
@ -452,7 +480,7 @@ impl CredentialResolver {
) -> Result<ResolvedCredential, ResolveError> {
let provider_id = provider.into();
let Some(catalog_provider) = catalog.provider(&provider_id) else {
return self.resolve(provider_id, usage).await;
return Err(ResolveError::NotConfigured(provider_id));
};
let initial_credential = {
let vault = self.vault.read().await;
@ -565,7 +593,7 @@ pub async fn configured_providers_from_process_env(
let guard = vault_arc.read().await;
resolver.configured_providers(&guard)
}
None => Catalog::builtin()
None => bootstrap_catalog::catalog()
.providers()
.iter()
.filter(|provider| provider_has_process_env_api_key(&provider.id))
@ -579,7 +607,7 @@ pub async fn configured_providers_from_process_env(
reason = "Provider discovery intentionally checks documented API-key env names."
)]
fn provider_has_process_env_api_key(provider: &ProviderId) -> bool {
Catalog::builtin()
bootstrap_catalog::catalog()
.provider(provider)
.is_some_and(|catalog_provider| {
catalog_provider.credentials.iter().any(|credential_ref| {
@ -589,7 +617,7 @@ fn provider_has_process_env_api_key(provider: &ProviderId) -> bool {
}
fn primary_api_key_env_var(provider: &ProviderId) -> Option<&'static str> {
Catalog::builtin()
bootstrap_catalog::catalog()
.provider(provider)?
.credentials
.iter()
@ -613,6 +641,7 @@ mod tests {
use std::os::unix::fs::PermissionsExt;
use chrono::{Duration, Utc};
use fabro_model::catalog::LlmCatalogSettings;
use httpmock::Method::POST;
use httpmock::MockServer;
@ -655,6 +684,11 @@ mod tests {
CredentialResolver::with_env_lookup(Arc::new(AsyncRwLock::new(vault)), env_lookup)
}
fn catalog_with(overrides: &str) -> Catalog {
let settings: LlmCatalogSettings = toml::from_str(overrides).unwrap();
Catalog::from_builtin_with_overrides(&settings).unwrap()
}
#[tokio::test]
async fn resolve_openai_api_request_prefers_typed_credential() {
let dir = tempfile::tempdir().unwrap();
@ -990,6 +1024,66 @@ mod tests {
]);
}
#[tokio::test]
async fn resolve_for_catalog_uses_custom_vault_backed_provider() {
let catalog = catalog_with(
r#"
[providers.venice]
display_name = "Venice"
adapter = "openai_compatible"
base_url = "https://api.venice.ai/api/v1"
credentials = ["credential:venice"]
[models."venice-large"]
provider = "venice"
display_name = "Venice Large"
family = "venice"
default = true
[models."venice-large".limits]
context_window = 128000
[models."venice-large".features]
tools = true
vision = false
reasoning = false
effort = false
"#,
);
let dir = tempfile::tempdir().unwrap();
let mut vault = Vault::load(dir.path().join("secrets.json")).unwrap();
vault_set_credential(&mut vault, "venice", &AuthCredential {
provider: ProviderId::new("venice"),
details: AuthDetails::ApiKey {
key: "venice-key".to_string(),
},
})
.unwrap();
let resolver = test_resolver(vault, Arc::new(|_| None));
let resolved = resolver
.resolve_for_catalog(
ProviderId::new("venice"),
CredentialUsage::ApiRequest,
&catalog,
)
.await
.unwrap();
let ResolvedCredential::Api(api) = resolved else {
panic!("expected api credential");
};
assert_eq!(api.provider, ProviderId::new("venice"));
assert_eq!(
api.auth_header,
Some(ApiKeyHeader::Bearer("venice-key".to_string()))
);
assert_eq!(
api.base_url.as_deref(),
Some("https://api.venice.ai/api/v1")
);
}
#[tokio::test]
async fn configured_providers_returns_env_backed_provider() {
let dir = tempfile::tempdir().unwrap();

6
lib/crates/fabro-auth/src/vault_source.rs
View file

@ -1,7 +1,7 @@
use std::sync::Arc;
use async_trait::async_trait;
use fabro_model::{Catalog, ProviderId};
use fabro_model::{Catalog, ProviderId, bootstrap_catalog};
use fabro_vault::Vault;
use tokio::sync::RwLock as AsyncRwLock;
@ -42,7 +42,7 @@ impl std::fmt::Debug for VaultCredentialSource {
#[async_trait]
impl CredentialSource for VaultCredentialSource {
async fn resolve(&self) -> anyhow::Result<ResolvedCredentials> {
self.resolve_for_catalog(Catalog::builtin()).await
self.resolve_for_catalog(bootstrap_catalog::catalog()).await
}
async fn resolve_for_catalog(&self, catalog: &Catalog) -> anyhow::Result<ResolvedCredentials> {
@ -79,7 +79,7 @@ impl CredentialSource for VaultCredentialSource {
}
async fn configured_providers(&self) -> Vec<ProviderId> {
self.configured_providers_for_catalog(Catalog::builtin())
self.configured_providers_for_catalog(bootstrap_catalog::catalog())
.await
}

32
lib/crates/fabro-cli/src/command_context.rs
View file

@ -1,9 +1,10 @@
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::{Arc, OnceLock};
use anyhow::{Context as _, Result, bail};
use fabro_auth::{CredentialSource, EnvCredentialSource, VaultCredentialSource};
use fabro_config::{CliLayer, Storage};
use fabro_config::{CliLayer, Storage, load_llm_catalog_settings};
use fabro_model::Catalog;
use fabro_types::UserSettings;
use fabro_types::settings::RunNamespace;
use fabro_types::settings::cli::{OutputFormat, OutputVerbosity};
@ -43,6 +44,7 @@ pub(crate) struct CommandContext {
server_mode: ServerMode,
server: OnceCell<Arc<Client>>,
llm_source: OnceCell<Arc<dyn CredentialSource>>,
catalog: OnceLock<Arc<Catalog>>,
}
struct ResolvedCommandSettings {
@ -70,6 +72,7 @@ impl CommandContext {
server_mode: ServerMode::None,
server: OnceCell::new(),
llm_source: OnceCell::new(),
catalog: OnceLock::new(),
})
}
@ -180,6 +183,28 @@ impl CommandContext {
Ok(Arc::clone(source))
}
pub(crate) fn catalog(&self) -> Result<Arc<Catalog>> {
if let Some(catalog) = self.catalog.get() {
return Ok(Arc::clone(catalog));
}
let llm_catalog_settings =
load_llm_catalog_settings(None).context("loading LLM catalog")?;
let catalog = Arc::new(
Catalog::from_builtin_with_overrides(&llm_catalog_settings)
.context("building LLM catalog")?,
);
if self.catalog.set(Arc::clone(&catalog)).is_ok() {
return Ok(catalog);
}
Ok(Arc::clone(
self.catalog
.get()
.expect("catalog must exist after failed OnceLock set"),
))
}
fn with_server_mode(&self, server_mode: ServerMode) -> Result<Self> {
// Always reload settings for the requested derivation mode so the result
// depends only on the requested mode, not on whichever derived context
@ -198,6 +223,7 @@ impl CommandContext {
server_mode,
server: OnceCell::new(),
llm_source: OnceCell::new(),
catalog: OnceLock::new(),
})
}
}
@ -233,6 +259,7 @@ fn resolve_command_settings(loaded_settings: LoadedSettings) -> ResolvedCommandS
#[cfg(test)]
mod tests {
use std::path::PathBuf;
use std::sync::OnceLock;
use fabro_config::{CliLayer, CliOutputLayer};
use fabro_types::settings::cli::{OutputFormat, OutputVerbosity};
@ -270,6 +297,7 @@ mod tests {
server_mode: ServerMode::None,
server: OnceCell::new(),
llm_source: OnceCell::new(),
catalog: OnceLock::new(),
}
}

109
lib/crates/fabro-cli/src/commands/model.rs
View file

@ -2,7 +2,7 @@ use anyhow::{Context, Result, bail};
use cli_table::format::{Border, Justify, Separator};
use cli_table::{Cell, CellStruct, Color, Style, Table};
use fabro_api::types as api_types;
use fabro_model::{Catalog, Model, ModelTestMode, Provider, ProviderId};
use fabro_model::{Model, ModelTestMode, Provider, ProviderId};
use fabro_util::terminal::Styles;
use futures::{StreamExt, stream};
use serde::Serialize;
@ -45,6 +45,13 @@ struct CompletedModelTest {
status: String,
}
fn find_model_by_id_or_alias(models: &[Model], id: &str) -> Option<Model> {
models
.iter()
.find(|model| model.id == id || model.aliases.iter().any(|alias| alias == id))
.cloned()
}
pub(crate) async fn execute(
command: Option<ModelsCommand>,
base_ctx: &CommandContext,
@ -244,6 +251,8 @@ async fn test_models_via_server(
let mut skipped = 0u32;
let mut skipped_providers: Vec<String> = Vec::new();
if let Some(model_id) = model {
let listed_models = client.list_models(None, Some(model_id)).await?;
let listed_info = find_model_by_id_or_alias(&listed_models, model_id);
if !json_output {
eprint!("Testing {model_id}...");
}
@ -254,12 +263,9 @@ async fn test_models_via_server(
let (info, result_color, status) = match result {
Ok(resp) => {
let info = Catalog::builtin()
.get(&resp.model_id)
.cloned()
.with_context(|| {
format!("Unknown model returned by server: {}", resp.model_id)
})?;
let info = find_model_by_id_or_alias(&listed_models, &resp.model_id).with_context(
|| format!("Unknown model returned by server: {}", resp.model_id),
)?;
if resp.status == api_types::ModelTestResultStatus::Ok {
(info, Color::Green, "ok".to_string())
} else if resp.status == api_types::ModelTestResultStatus::Skip {
@ -278,10 +284,7 @@ async fn test_models_via_server(
bail!("Unknown model: {model_id}");
}
Err(err) => {
let info = Catalog::builtin()
.get(model_id)
.cloned()
.with_context(|| format!("Unknown model: {model_id}"))?;
let info = listed_info.with_context(|| format!("Unknown model: {model_id}"))?;
failures += 1;
(info, Color::Red, format!("error: {err}"))
}
@ -506,6 +509,39 @@ mod tests {
.unwrap()
}
fn custom_model_json(id: &str, provider: &str) -> serde_json::Value {
serde_json::to_value(Model {
id: id.to_string(),
provider: ProviderId::new(provider),
family: "test".to_string(),
display_name: format!("{id} display"),
limits: ModelLimits {
context_window: 128_000,
max_output: Some(4096),
},
training: None,
knowledge_cutoff: None,
features: ModelFeatures {
tools: true,
vision: false,
reasoning: false,
reasoning_effort: ReasoningEffortFeature::None,
prompt_cache: false,
effort: false,
},
costs: ModelCosts {
input_cost_per_mtok: Some(1.0),
output_cost_per_mtok: Some(2.0),
cache_input_cost_per_mtok: None,
},
estimated_output_tps: Some(100.0),
aliases: vec![],
default: false,
configured: true,
})
.unwrap()
}
#[test]
fn format_context_window_millions() {
assert_eq!(format_context_window(1_000_000), "1m");
@ -665,6 +701,57 @@ mod tests {
assert!(result.unwrap_err().to_string().contains("Model not found"));
}
#[tokio::test]
async fn single_model_test_uses_server_model_metadata_for_custom_models() {
let server = httpmock::MockServer::start_async().await;
server
.mock_async(|when, then| {
when.method("GET")
.path("/api/v1/models")
.query_param("page[limit]", "100")
.query_param("page[offset]", "0")
.query_param("query", "venice-large");
then.status(200)
.header("Content-Type", "application/json")
.body(
serde_json::json!({
"data": [custom_model_json("venice-large", "venice")],
"meta": { "has_more": false }
})
.to_string(),
);
})
.await;
server
.mock_async(|when, then| {
when.method("POST").path("/api/v1/models/venice-large/test");
then.status(200)
.header("Content-Type", "application/json")
.body(
serde_json::json!({
"model_id": "venice-large",
"status": "ok"
})
.to_string(),
);
})
.await;
let client = test_client(&server.url(""));
test_models_via_server(
&client,
None,
Some("venice-large"),
false,
1,
&Styles::new(false),
true,
)
.await
.unwrap();
}
#[tokio::test]
async fn fetch_models_from_server_parses_response() {
let server = httpmock::MockServer::start_async().await;

3
lib/crates/fabro-cli/src/commands/provider/login.rs
View file

@ -17,11 +17,12 @@ pub(super) async fn login_command(
let ctx = base_ctx.with_target(&args.target)?;
let server = ctx.server().await?;
let credential = if args.api_key_stdin {
provider_auth::authenticate_provider_with_api_key_source(
provider_auth::authenticate_provider_with_api_key_source_and_catalog(
args.provider,
provider_auth::ApiKeySource::Stdin,
&s,
printer,
ctx.catalog()?,
)
.await?
} else {

1
lib/crates/fabro-cli/src/commands/run/create.rs
View file

@ -54,6 +54,7 @@ pub(crate) async fn create_run(
&RunLayer::default(),
&built.manifest,
RenderMode::Strict,
ctx.catalog()?,
)?;
let diagnostics = api_diagnostics_to_local(&validation.workflow.diagnostics);
if !quiet {

9
lib/crates/fabro-cli/src/commands/run/runner.rs
View file

@ -11,12 +11,11 @@ use std::time::Duration;
use anyhow::{Context, Result, anyhow};
use async_trait::async_trait;
use fabro_config::{ServerSettingsBuilder, Storage};
use fabro_config::{ServerSettingsBuilder, Storage, load_llm_catalog_settings};
use fabro_interview::{
AnswerSubmission, ControlInterviewer, WorkerControlEnvelope, WorkerControlMessage,
};
use fabro_model::Catalog;
use fabro_model::catalog::LlmCatalogSettings;
use fabro_store::{EventEnvelope, RunProjection, RunProjectionReducer};
use fabro_types::settings::InterpString;
use fabro_types::settings::run::{RunMode, RunNamespace};
@ -95,9 +94,11 @@ pub(crate) async fn execute(
let run_control = RunControlState::new();
install_signal_handlers(Arc::clone(&run_control), cancel_token.clone())?;
let vault = load_worker_vault(storage_dir.as_deref())?;
let llm_catalog_settings =
load_llm_catalog_settings(None).context("failed to load worker LLM catalog settings")?;
let catalog = Arc::new(
Catalog::from_builtin_with_overrides(&LlmCatalogSettings::default())
.context("failed to build default LLM catalog")?,
Catalog::from_builtin_with_overrides(&llm_catalog_settings)
.context("failed to build worker LLM catalog")?,
);
let github_app = {
let vault_guard = match &vault {

1
lib/crates/fabro-cli/src/commands/validate.rs
View file

@ -27,6 +27,7 @@ pub(crate) fn run(
&RunLayer::default(),
&built.manifest,
RenderMode::Structural,
base_ctx.catalog()?,
)?;
let diagnostics = api_diagnostics_to_local(&response.workflow.diagnostics);

71
lib/crates/fabro-cli/src/shared/provider_auth.rs
View file

@ -20,6 +20,7 @@ use fabro_auth::{
};
use fabro_llm::client::Client as LlmClient;
use fabro_llm::generate::{GenerateParams, generate};
use fabro_model::catalog::LlmCatalogSettings;
use fabro_model::{Catalog, Provider};
use fabro_util::printer::Printer;
use fabro_util::terminal::Styles;
@ -77,15 +78,30 @@ pub(crate) enum ApiKeySource {
// API key validation
// ---------------------------------------------------------------------------
pub(crate) async fn validate_api_key(provider: Provider, api_key: &str) -> Result<()> {
let client = LlmClient::from_credentials(vec![ApiCredential::from_api_key(
provider,
api_key.to_string(),
)])
fn default_catalog_for_provider_auth() -> Result<Arc<Catalog>> {
Ok(Arc::new(
Catalog::from_builtin_with_overrides(&LlmCatalogSettings::default())
.context("failed to build provider auth catalog")?,
))
}
pub(crate) async fn validate_api_key(
provider: Provider,
api_key: &str,
catalog: Arc<Catalog>,
) -> Result<()> {
let client = LlmClient::from_credentials_with_catalog(
vec![ApiCredential::from_api_key_for_catalog(
provider,
api_key.to_string(),
catalog.as_ref(),
)],
Arc::clone(&catalog),
)
.await
.context("failed to create LLM client")?;
let probe_model = Catalog::builtin()
let probe_model = catalog
.probe_for_provider(provider)
.map_or_else(|| format!("unknown-{provider}"), |model| model.id.clone());
@ -145,12 +161,13 @@ async fn read_and_validate_api_key(
env_var: &str,
s: &Styles,
printer: Printer,
catalog: Arc<Catalog>,
) -> Result<String> {
loop {
let key = read_api_key_from_source(source, env_var).await?;
fabro_util::printerr!(printer, " {}", s.dim.apply_to("Validating API key..."));
match validate_api_key(provider, &key).await {
match validate_api_key(provider, &key, Arc::clone(&catalog)).await {
Ok(()) => {
fabro_util::printerr!(printer, " {} API key is valid", s.green.apply_to("✔"));
return Ok(key);
@ -201,11 +218,28 @@ pub(crate) async fn authenticate_provider_with_api_key_source(
source: ApiKeySource,
s: &Styles,
printer: Printer,
) -> Result<AuthCredential> {
authenticate_provider_with_api_key_source_and_catalog(
provider,
source,
s,
printer,
default_catalog_for_provider_auth()?,
)
.await
}
pub(crate) async fn authenticate_provider_with_api_key_source_and_catalog(
provider: Provider,
source: ApiKeySource,
s: &Styles,
printer: Printer,
catalog: Arc<Catalog>,
) -> Result<AuthCredential> {
let mut strategy = strategy_for(provider, AuthMethod::ApiKey);
let request = strategy.init().await?;
present_to_user(&request, s, printer);
let response = await_user_response_from_source(&request, &source, s, printer).await?;
let response = await_user_response_from_source(&request, &source, s, printer, catalog).await?;
strategy.complete(response).await
}
@ -218,8 +252,14 @@ pub(crate) async fn authenticate_provider_with_method(
let mut strategy = strategy_for(provider, method);
let request = strategy.init().await?;
present_to_user(&request, s, printer);
let response =
await_user_response_from_source(&request, &ApiKeySource::Prompt, s, printer).await?;
let response = await_user_response_from_source(
&request,
&ApiKeySource::Prompt,
s,
printer,
default_catalog_for_provider_auth()?,
)
.await?;
strategy.complete(response).await
}
@ -269,6 +309,7 @@ async fn await_user_response_from_source(
source: &ApiKeySource,
s: &Styles,
printer: Printer,
catalog: Arc<Catalog>,
) -> Result<AuthContextResponse> {
match request {
AuthContextRequest::ApiKey {
@ -276,7 +317,8 @@ async fn await_user_response_from_source(
env_var_names,
} => {
let env_var = env_var_names.first().map_or("API_KEY", String::as_str);
let key = read_and_validate_api_key(*provider, source, env_var, s, printer).await?;
let key =
read_and_validate_api_key(*provider, source, env_var, s, printer, catalog).await?;
Ok(AuthContextResponse::ApiKey { key })
}
AuthContextRequest::DeviceCode { .. } => {
@ -319,7 +361,12 @@ mod tests {
#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))]
async fn validate_api_key_rejects_invalid_key() {
let result = validate_api_key(Provider::Anthropic, "sk-invalid-key-12345").await;
let result = validate_api_key(
Provider::Anthropic,
"sk-invalid-key-12345",
default_catalog_for_provider_auth().unwrap(),
)
.await;
assert!(result.is_err(), "expected invalid key to be rejected");
}

8
lib/crates/fabro-config/src/builders.rs
View file

@ -225,6 +225,14 @@ pub fn load_server_runtime_settings(
resolve_server_runtime_settings(layer, run_overrides, server_overrides)
}
pub fn load_llm_catalog_settings(path: Option<&Path>) -> Result<model_catalog::LlmCatalogSettings> {
let layer = match path {
Some(path) => load_settings_path(path)?,
None => load_settings_config(None)?,
};
Ok(llm_catalog_settings_from_layer(&layer))
}
#[cfg(test)]
pub fn server_runtime_settings_from_toml(
source: &str,

3
lib/crates/fabro-config/src/lib.rs
View file

@ -31,7 +31,8 @@ use std::path::Path;
pub use builders::{
ResolveErrors, RunSettingsBuilder, ServerRuntimeSettings, ServerSettingsBuilder,
UserSettingsBuilder, WorkflowSettingsBuilder, load_server_runtime_settings,
UserSettingsBuilder, WorkflowSettingsBuilder, load_llm_catalog_settings,
load_server_runtime_settings,
};
pub use error::{Error, Result};
pub use fabro_util::path::expand_tilde;

48
lib/crates/fabro-llm/src/providers/anthropic.rs
View file

@ -1128,8 +1128,7 @@ async fn build_api_request(
request.tools.as_ref().map(|t| translate_tools(t))
};
let model_info = common::catalog_model(adapter.catalog.as_deref(), &request.model)
.or_else(|| Catalog::builtin().get(&request.model));
let model_info = common::catalog_model(adapter.catalog.as_deref(), &request.model);
let supports_prompt_cache = model_info.is_some_and(|m| m.features.prompt_cache);
let auto_cache =
supports_prompt_cache && is_auto_cache_enabled(request.provider_options.as_ref());
@ -2364,6 +2363,36 @@ prompt_cache = false
);
}
#[tokio::test]
async fn build_api_request_without_injected_catalog_does_not_use_builtin_model_metadata() {
let adapter = Adapter::new("test-key");
let request = Request {
model: "claude-sonnet-4-5".to_string(),
messages: vec![
Message::system("Do not infer cache support from built-ins."),
Message::user("Hello"),
],
provider_options: Some(serde_json::json!({
"anthropic": {"auto_cache": true}
})),
..make_base_request()
};
let (api_request, req_builder) = build_api_request(&adapter, &request, false).await;
assert_eq!(
api_request.system,
Some(serde_json::Value::String(
"Do not infer cache support from built-ins.".to_string()
))
);
let built = req_builder.build().expect("should build request");
let beta = built.headers().get("anthropic-beta");
assert!(
beta.is_none_or(|value| !value.to_str().unwrap().contains(CACHE_BETA_HEADER)),
"cache beta header must require injected model metadata"
);
}
#[tokio::test]
async fn build_api_request_enables_prompt_cache_when_model_feature_is_true() {
let adapter = Adapter::new("test-key").with_catalog(catalog_with_anthropic_model(
@ -2397,10 +2426,15 @@ prompt_cache = true
}
#[tokio::test]
async fn build_api_request_uses_adaptive_thinking_for_opus_4_7_without_forced_tools() {
let adapter = Adapter::new("test-key");
async fn build_api_request_uses_adaptive_thinking_for_injected_effort_model_without_forced_tools()
{
let adapter = Adapter::new("test-key").with_catalog(catalog_with_anthropic_model(
r#"
reasoning_effort = "levels"
"#,
));
let request = Request {
model: "claude-opus-4-7".to_string(),
model: "test-claude".to_string(),
..make_base_request()
};
@ -2589,9 +2623,9 @@ prompt_cache = true
#[tokio::test]
async fn build_api_request_falls_back_to_thinking_budget_for_non_effort_model() {
let adapter = Adapter::new("test-key");
let adapter = Adapter::new("test-key").with_catalog(catalog_with_anthropic_model(""));
let request = Request {
model: "claude-sonnet-4-5".to_string(),
model: "test-claude".to_string(),
max_tokens: Some(16_000),
reasoning_effort: Some(ReasoningEffort::XHigh),
..make_base_request()

1
lib/crates/fabro-mcp-server/Cargo.toml
View file

@ -19,6 +19,7 @@ fabro-api = { path = "../fabro-api" }
fabro-client = { path = "../fabro-client" }
fabro-manifest = { path = "../fabro-manifest" }
fabro-config = { path = "../fabro-config" }
fabro-model = { path = "../fabro-model" }
fabro-server = { path = "../fabro-server" }
fabro-types = { path = "../fabro-types" }
fabro-util = { path = "../fabro-util" }

10
lib/crates/fabro-mcp-server/src/run_tools/manifest.rs
View file

@ -1,8 +1,9 @@
use std::path::{Path, PathBuf};
use fabro_api::types;
use fabro_config::{CliLayer, RunLayer};
use fabro_config::{CliLayer, RunLayer, load_llm_catalog_settings};
use fabro_manifest::{self, ManifestBuildInput, RunOverrideInput};
use fabro_model::Catalog;
use fabro_server::manifest_validation::{self, RenderMode};
use serde_json::Value;
@ -25,10 +26,17 @@ pub(super) fn build_mcp_run_manifest(
user_settings_path: Some(user_settings_path.to_path_buf()),
})
.map_err(|err| ToolError::from_anyhow(&err))?;
let llm_catalog_settings = load_llm_catalog_settings(Some(user_settings_path))
.map_err(|err| ToolError::message(err.to_string()))?;
let catalog = std::sync::Arc::new(
Catalog::from_builtin_with_overrides(&llm_catalog_settings)
.map_err(|err| ToolError::message(err.to_string()))?,
);
let validation = manifest_validation::validate_manifest(
&RunLayer::default(),
&built.manifest,
RenderMode::Strict,
catalog,
)
.map_err(|err| ToolError::from_anyhow(&err))?;
if !validation.ok {

8
lib/crates/fabro-server/src/manifest_validation.rs
View file

@ -1,6 +1,9 @@
use std::sync::Arc;
use anyhow::Result;
use fabro_api::types;
use fabro_config::RunLayer;
use fabro_model::Catalog;
pub use fabro_workflow::operations::RenderMode;
use crate::run_manifest;
@ -9,9 +12,10 @@ pub fn validate_manifest(
manifest_run_defaults: &RunLayer,
manifest: &types::RunManifest,
mode: RenderMode,
catalog: Arc<Catalog>,
) -> Result<types::ValidateResponse> {
let prepared = run_manifest::prepare_manifest(manifest_run_defaults, manifest)?;
let validated =
run_manifest::validate_prepared_manifest(&prepared, mode).map_err(anyhow::Error::new)?;
let validated = run_manifest::validate_prepared_manifest(&prepared, mode, catalog)
.map_err(anyhow::Error::new)?;
Ok(run_manifest::validate_response(&prepared, &validated))
}

31
lib/crates/fabro-server/src/run_manifest.rs
View file

@ -157,12 +157,14 @@ pub(crate) fn prepare_manifest(
pub(crate) fn validate_prepared_manifest(
prepared: &PreparedManifest,
mode: RenderMode,
catalog: Arc<Catalog>,
) -> Result<Validated, WorkflowError> {
validate(ValidateInput {
workflow: WorkflowInput::Bundled(prepared.workflow_input.clone()),
settings: prepared.settings.clone(),
cwd: prepared.cwd.clone(),
custom_transforms: Vec::new(),
catalog,
mode,
})
}
@ -1282,6 +1284,7 @@ fn report_to_api(report: &CheckReport) -> types::PreflightCheckReport {
#[cfg(test)]
mod tests {
use fabro_model::Provider;
use fabro_model::catalog::LlmCatalogSettings;
use super::*;
@ -1335,6 +1338,10 @@ mod tests {
RunLayer::default()
}
fn test_catalog() -> Arc<Catalog> {
Arc::new(Catalog::from_builtin_with_overrides(&LlmCatalogSettings::default()).unwrap())
}
fn manifest_workflow() -> types::ManifestWorkflow {
types::ManifestWorkflow {
config: None,
@ -1393,7 +1400,8 @@ enabled = {clone_enabled}
&manifest,
)
.unwrap();
let validated = validate_prepared_manifest(&prepared, RenderMode::Strict).unwrap();
let validated =
validate_prepared_manifest(&prepared, RenderMode::Strict, test_catalog()).unwrap();
let resolved = materialize_run(
prepared.settings.clone(),
validated.graph(),
@ -1772,7 +1780,8 @@ app_id = "fixture-app-id"
&invalid_manifest(),
)
.unwrap();
let validated = validate_prepared_manifest(&prepared, RenderMode::Strict).unwrap();
let validated =
validate_prepared_manifest(&prepared, RenderMode::Strict, test_catalog()).unwrap();
assert!(validated.has_errors());
@ -1817,7 +1826,8 @@ issues = "read"
&manifest,
)
.unwrap();
let validated = validate_prepared_manifest(&prepared, RenderMode::Strict).unwrap();
let validated =
validate_prepared_manifest(&prepared, RenderMode::Strict, test_catalog()).unwrap();
assert!(!validated.has_errors());
let (response, _ok) = run_preflight(state.as_ref(), &prepared, &validated)
@ -1865,7 +1875,8 @@ provider = "local"
&manifest,
)
.unwrap();
let validated = validate_prepared_manifest(&prepared, RenderMode::Strict).unwrap();
let validated =
validate_prepared_manifest(&prepared, RenderMode::Strict, test_catalog()).unwrap();
assert!(!validated.has_errors());
@ -1906,7 +1917,8 @@ provider = "daytona"
&manifest,
)
.unwrap();
let validated = validate_prepared_manifest(&prepared, RenderMode::Strict).unwrap();
let validated =
validate_prepared_manifest(&prepared, RenderMode::Strict, test_catalog()).unwrap();
let (response, _ok) = run_preflight(state.as_ref(), &prepared, &validated)
.await
@ -1979,7 +1991,8 @@ digraph Demo {
&manifest,
)
.unwrap();
let validated = validate_prepared_manifest(&prepared, RenderMode::Strict).unwrap();
let validated =
validate_prepared_manifest(&prepared, RenderMode::Strict, test_catalog()).unwrap();
let (response, ok) = run_preflight(state.as_ref(), &prepared, &validated)
.await
@ -2020,7 +2033,8 @@ digraph Demo {
&manifest,
)
.unwrap();
let validated = validate_prepared_manifest(&prepared, RenderMode::Strict).unwrap();
let validated =
validate_prepared_manifest(&prepared, RenderMode::Strict, test_catalog()).unwrap();
let (response, ok) = run_preflight(state.as_ref(), &prepared, &validated)
.await
@ -2091,7 +2105,8 @@ digraph Demo {
&manifest,
)
.unwrap();
let validated = validate_prepared_manifest(&prepared, RenderMode::Strict).unwrap();
let validated =
validate_prepared_manifest(&prepared, RenderMode::Strict, test_catalog()).unwrap();
let (response, ok) = run_preflight(state.as_ref(), &prepared, &validated)
.await

13
lib/crates/fabro-server/src/serve.rs
View file

@ -7,6 +7,7 @@ use std::time::Duration;
use anyhow::{Context, bail};
use clap::Args;
use fabro_config::bind::{self, Bind, BindRequest};
use fabro_config::user::active_settings_path;
use fabro_config::{
RunLayer, RunModelLayer, RunSandboxLayer, ServerLayer, ServerWebLayer, Storage,
load_config_file, load_server_runtime_settings,
@ -610,6 +611,16 @@ fn resolve_interp_path(value: &InterpString) -> anyhow::Result<PathBuf> {
Ok(PathBuf::from(resolve_interp(value)?))
}
fn absolute_path(path: PathBuf) -> anyhow::Result<PathBuf> {
if path.is_absolute() {
Ok(path)
} else {
Ok(std::env::current_dir()
.context("resolving current directory for config path")?
.join(path))
}
}
fn load_server_secrets_for_settings(settings: &ServerNamespace) -> anyhow::Result<ServerSecrets> {
let storage_root = resolve_interp_path(&settings.storage.root)?;
let server_env_path = Storage::new(&storage_root).runtime_directory().env_path();
@ -687,6 +698,7 @@ where
#[cfg(debug_assertions)]
let watch_web = args.watch_web;
let config_path = args.config.clone();
let active_config_path = absolute_path(active_settings_path(config_path.as_deref()))?;
let disk_document: toml::Table = load_config_file(config_path.as_deref(), "settings.toml")?;
let (run_overrides, server_overrides) = serve_overrides(&args);
let mut runtime_settings = load_server_runtime_settings(
@ -789,6 +801,7 @@ where
server_secrets,
env_lookup,
github_api_base_url: None,
active_config_path,
http_client: None,
shutdown: shutdown.clone(),
})?;

9
lib/crates/fabro-server/src/server.rs
View file

@ -531,6 +531,7 @@ pub struct AppState {
catalog: RwLock<Arc<Catalog>>,
pub(crate) env_lookup: EnvLookup,
pub(crate) github_api_base_url: String,
active_config_path: PathBuf,
http_client: Option<fabro_http::HttpClient>,
shutdown: CancellationToken,
shutting_down: AtomicBool,
@ -595,6 +596,7 @@ pub(crate) struct AppStateConfig {
pub(crate) server_secrets: ServerSecrets,
pub(crate) env_lookup: EnvLookup,
pub(crate) github_api_base_url: Option<String>,
pub(crate) active_config_path: PathBuf,
pub(crate) http_client: Option<fabro_http::HttpClient>,
pub(crate) shutdown: CancellationToken,
}
@ -664,6 +666,10 @@ impl AppState {
Arc::clone(&self.catalog.read().expect("catalog lock poisoned"))
}
pub(crate) fn active_config_path(&self) -> &std::path::Path {
&self.active_config_path
}
pub(crate) fn manifest_run_settings(&self) -> std::result::Result<RunNamespace, SharedError> {
self.manifest_run_settings
.read()
@ -1514,6 +1520,7 @@ pub(crate) fn build_app_state(config: AppStateConfig) -> anyhow::Result<Arc<AppS
server_secrets,
env_lookup,
github_api_base_url,
active_config_path,
http_client,
shutdown,
} = config;
@ -1581,6 +1588,7 @@ pub(crate) fn build_app_state(config: AppStateConfig) -> anyhow::Result<Arc<AppS
catalog: RwLock::new(current_catalog),
env_lookup: Arc::clone(&env_lookup),
github_api_base_url,
active_config_path,
http_client,
shutdown,
shutting_down: AtomicBool::new(false),
@ -2601,6 +2609,7 @@ fn worker_command(
}
let value: &'static str = server_destination.into();
cmd.env(EnvVars::FABRO_LOG_DESTINATION, value);
cmd.env(EnvVars::FABRO_CONFIG, state.active_config_path());
cmd.env_remove(EnvVars::FABRO_WORKER_TOKEN);
cmd.env(EnvVars::FABRO_WORKER_TOKEN, worker_token);
if let Some(pem) = state.server_secret(EnvVars::GITHUB_APP_PRIVATE_KEY) {

13
lib/crates/fabro-server/src/server/handler/graph.rs
View file

@ -48,11 +48,14 @@ async fn render_graph_from_manifest(
Ok(prepared) => prepared,
Err(err) => return ApiError::bad_request(err.to_string()).into_response(),
};
let validated =
match run_manifest::validate_prepared_manifest(&prepared, RenderMode::Structural) {
Ok(validated) => validated,
Err(err) => return ApiError::bad_request(err.to_string()).into_response(),
};
let validated = match run_manifest::validate_prepared_manifest(
&prepared,
RenderMode::Structural,
state.catalog(),
) {
Ok(validated) => validated,
Err(err) => return ApiError::bad_request(err.to_string()).into_response(),
};
if validated.has_errors() {
return ApiError::bad_request("Validation failed").into_response();
}

25
lib/crates/fabro-server/src/server/handler/runs.rs
View file

@ -518,7 +518,11 @@ async fn run_preflight(
Ok(prepared) => prepared,
Err(err) => return ApiError::bad_request(err.to_string()).into_response(),
};
let validated = match run_manifest::validate_prepared_manifest(&prepared, RenderMode::Strict) {
let validated = match run_manifest::validate_prepared_manifest(
&prepared,
RenderMode::Strict,
state.catalog(),
) {
Ok(validated) => validated,
Err(WorkflowError::Parse(_)) => {
return ApiError::bad_request("Validation failed").into_response();
@ -545,14 +549,17 @@ async fn validate_run_manifest(
Ok(prepared) => prepared,
Err(err) => return ApiError::bad_request(err.to_string()).into_response(),
};
let validated =
match run_manifest::validate_prepared_manifest(&prepared, RenderMode::Structural) {
Ok(validated) => validated,
Err(WorkflowError::Parse(_)) => {
return ApiError::bad_request("Validation failed").into_response();
}
Err(err) => return ApiError::bad_request(err.to_string()).into_response(),
};
let validated = match run_manifest::validate_prepared_manifest(
&prepared,
RenderMode::Structural,
state.catalog(),
) {
Ok(validated) => validated,
Err(WorkflowError::Parse(_)) => {
return ApiError::bad_request("Validation failed").into_response();
}
Err(err) => return ApiError::bad_request(err.to_string()).into_response(),
};
(
StatusCode::OK,
Json(run_manifest::validate_response(&prepared, &validated)),

157
lib/crates/fabro-server/src/server/tests.rs
View file

@ -1692,6 +1692,44 @@ destination = "stdout"
);
}
#[cfg(unix)]
#[test]
fn worker_command_sets_fabro_config_to_active_absolute_config_path() {
let storage_dir = tempfile::tempdir().unwrap();
let config_dir = tempfile::tempdir().unwrap();
let active_config_path = config_dir.path().join("settings.toml");
let state = worker_command_test_state_with_active_config_path(
storage_dir.path(),
&["dev-token"],
Some(TEST_DEV_TOKEN),
active_config_path.clone(),
);
let run_id = RunId::new();
let cmd = worker_command(
state.as_ref(),
run_id,
RunExecutionMode::Start,
storage_dir.path(),
)
.unwrap();
assert!(active_config_path.is_absolute());
assert_eq!(
command_env_value(&cmd, EnvVars::FABRO_CONFIG),
EnvOverride::Set(active_config_path.display().to_string())
);
let worker_args = cmd
.as_std()
.get_args()
.map(|arg| arg.to_string_lossy().into_owned())
.collect::<Vec<_>>();
assert!(
!worker_args.iter().any(|arg| arg == "--config"),
"__run-worker argument contract should not grow hidden config args: {worker_args:?}"
);
}
#[cfg(unix)]
#[test]
fn worker_command_env_log_destination_overrides_server_logging_config() {
@ -1781,6 +1819,7 @@ methods = ["dev-token"]
server_secrets: ServerSecrets::load(server_env_path, HashMap::new()).unwrap(),
env_lookup: default_env_lookup(),
github_api_base_url: None,
active_config_path: tempfile::tempdir().unwrap().path().join("settings.toml"),
http_client: Some(fabro_http::test_http_client().expect("test HTTP client should build")),
shutdown: tokio_util::sync::CancellationToken::new(),
}) else {
@ -1823,6 +1862,43 @@ fn worker_command_test_state_with_extra_config_and_env_lookup(
extra_config: &str,
extra_server_secrets: &[(&str, &str)],
env_lookup: impl Fn(&str) -> Option<String> + Send + Sync + 'static,
) -> Arc<AppState> {
worker_command_test_state_inner(
storage_dir,
methods,
dev_token,
extra_config,
extra_server_secrets,
env_lookup,
None,
)
}
fn worker_command_test_state_with_active_config_path(
storage_dir: &Path,
methods: &[&str],
dev_token: Option<&str>,
active_config_path: PathBuf,
) -> Arc<AppState> {
worker_command_test_state_inner(
storage_dir,
methods,
dev_token,
"",
&[],
|_| None,
Some(active_config_path),
)
}
fn worker_command_test_state_inner(
storage_dir: &Path,
methods: &[&str],
dev_token: Option<&str>,
extra_config: &str,
extra_server_secrets: &[(&str, &str)],
env_lookup: impl Fn(&str) -> Option<String> + Send + Sync + 'static,
active_config_path: Option<PathBuf>,
) -> Arc<AppState> {
let dev_token = dev_token.map(str::to_owned);
std::fs::create_dir_all(storage_dir).unwrap();
@ -1862,13 +1938,18 @@ allowed_usernames = ["octocat"]
for (key, value) in extra_server_secrets {
server_secret_env.insert((*key).to_string(), (*value).to_string());
}
test_app_state_with_env_lookup_and_server_secret_env(
server_settings_from_toml(&source),
manifest_run_defaults_from_toml(&source),
5,
env_lookup,
&server_secret_env,
)
let mut builder = TestAppStateBuilder::new()
.runtime_settings(
server_settings_from_toml(&source),
manifest_run_defaults_from_toml(&source),
)
.max_concurrent_runs(5)
.env_lookup(env_lookup)
.server_secret_env(server_secret_env);
if let Some(active_config_path) = active_config_path {
builder = builder.active_config_path(active_config_path);
}
builder.build()
}
#[cfg(unix)]
@ -2170,6 +2251,67 @@ async fn validate_endpoint_returns_workflow_summary_without_preflight_checks() {
assert!(body.get("checks").is_none());
}
#[tokio::test]
async fn validate_endpoint_uses_app_state_catalog_for_model_diagnostics() {
let llm_catalog_settings: LlmCatalogSettings = toml::from_str(
r#"
[providers.venice]
display_name = "Venice"
adapter = "openai_compatible"
base_url = "https://api.venice.ai/api/v1"
credentials = ["env:VENICE_API_KEY"]
[models."venice-large"]
provider = "venice"
display_name = "Venice Large"
family = "venice"
default = true
[models."venice-large".limits]
context_window = 128000
[models."venice-large".features]
tools = true
vision = false
reasoning = false
effort = false
"#,
)
.expect("catalog fixture should parse");
let state = TestAppStateBuilder::new()
.llm_catalog_settings(llm_catalog_settings)
.build();
let app = crate::test_support::build_test_router(state);
let dot = r#"digraph Test {
graph [goal="Test"]
start [shape=Mdiamond]
work [model="venice-large", provider="venice", prompt="Do it"]
exit [shape=Msquare]
start -> work -> exit
}"#;
let response = app
.oneshot(
Request::builder()
.method("POST")
.uri(api("/validate"))
.header("content-type", "application/json")
.body(manifest_body(dot))
.unwrap(),
)
.await
.unwrap();
let body = response_json!(response, StatusCode::OK).await;
let diagnostics = body["workflow"]["diagnostics"].as_array().unwrap();
assert!(
diagnostics
.iter()
.all(|diagnostic| diagnostic["rule"] != "node_model_known"),
"custom model/provider should validate against app-state catalog: {body}"
);
}
async fn create_run_for_target(app: &Router, target_path: &str, dot_source: &str) -> String {
let req = Request::builder()
.method("POST")
@ -3484,6 +3626,7 @@ fn create_github_token_app_state_with_env_lookup(
server_secrets: load_test_server_secrets(server_env_path, HashMap::new()),
env_lookup: Arc::new(env_lookup),
github_api_base_url,
active_config_path: tempfile::tempdir().unwrap().path().join("settings.toml"),
http_client: Some(fabro_http::test_http_client().expect("test HTTP client should build")),
shutdown: tokio_util::sync::CancellationToken::new(),
};

11
lib/crates/fabro-server/src/test_support.rs
View file

@ -63,6 +63,7 @@ pub struct TestAppStateBuilder {
store_bundle: Option<(Arc<Database>, ArtifactStore)>,
vault_path: Option<PathBuf>,
server_env_path: Option<PathBuf>,
active_config_path: Option<PathBuf>,
server_secret_env: HashMap<String, String>,
env_lookup: EnvLookup,
llm_catalog_settings: LlmCatalogSettings,
@ -78,6 +79,7 @@ impl Default for TestAppStateBuilder {
store_bundle: None,
vault_path: None,
server_env_path: None,
active_config_path: None,
server_secret_env: HashMap::new(),
env_lookup: default_env_lookup(),
llm_catalog_settings: LlmCatalogSettings::default(),
@ -149,12 +151,20 @@ impl TestAppStateBuilder {
self
}
pub fn active_config_path(mut self, active_config_path: PathBuf) -> Self {
self.active_config_path = Some(active_config_path);
self
}
pub fn build(self) -> Arc<AppState> {
let (store, artifact_store) = self.store_bundle.unwrap_or_else(test_store_bundle);
let vault_path = self.vault_path.unwrap_or_else(test_secret_store_path);
let server_env_path = self
.server_env_path
.unwrap_or_else(|| vault_path.with_file_name("server.env"));
let active_config_path = self.active_config_path.unwrap_or_else(|| {
std::env::temp_dir().join(format!("fabro-test-settings-{}.toml", Ulid::new()))
});
build_app_state(AppStateConfig {
resolved_settings: resolved_runtime_settings_for_tests(
self.server_settings,
@ -169,6 +179,7 @@ impl TestAppStateBuilder {
server_secrets: load_test_server_secrets(server_env_path, self.server_secret_env),
env_lookup: self.env_lookup,
github_api_base_url: None,
active_config_path,
http_client: Some(
fabro_http::test_http_client().expect("test HTTP client should build"),
),

3
lib/crates/fabro-validate/Cargo.toml
View file

@ -18,3 +18,6 @@ fabro-model = { path = "../fabro-model" }
fabro-types = { path = "../fabro-types" }
serde = { workspace = true }
thiserror = { workspace = true }
[dev-dependencies]
toml = { workspace = true }

146
lib/crates/fabro-validate/src/lib.rs
View file

@ -1,6 +1,7 @@
pub mod rules;
use fabro_graphviz::graph::Graph;
use fabro_model::Catalog;
use serde::{Deserialize, Serialize};
/// Severity level for validation diagnostics.
@ -47,6 +48,25 @@ pub fn validate(graph: &Graph, extra_rules: &[&dyn LintRule]) -> Vec<Diagnostic>
diagnostics
}
/// Run all built-in catalog-free lint rules, caller-supplied model catalog
/// rules, and any extra rules against the graph.
#[must_use]
pub fn validate_with_catalog(
graph: &Graph,
catalog: &Catalog,
extra_rules: &[&dyn LintRule],
) -> Vec<Diagnostic> {
let mut diagnostics = validate(graph, &[]);
let catalog_rules = rules::catalog_rules(catalog);
for rule in &catalog_rules {
diagnostics.extend(rule.apply(graph));
}
for rule in extra_rules {
diagnostics.extend(rule.apply(graph));
}
diagnostics
}
/// If any Error-severity diagnostics are present, return `ValidationError`.
///
/// # Errors
@ -80,9 +100,26 @@ pub fn validate_or_raise(
Ok(diagnostics)
}
/// Run catalog-aware validation and return `ValidationError` if any
/// Error-severity diagnostics are found.
///
/// # Errors
/// Returns `ValidationError` if any Error-severity diagnostics are found.
pub fn validate_with_catalog_or_raise(
graph: &Graph,
catalog: &Catalog,
extra_rules: &[&dyn LintRule],
) -> Result<Vec<Diagnostic>, ValidationError> {
let diagnostics = validate_with_catalog(graph, catalog, extra_rules);
raise_on_errors(&diagnostics)?;
Ok(diagnostics)
}
#[cfg(test)]
mod tests {
use fabro_graphviz::graph::{AttrValue, Edge, Graph, Node};
use fabro_model::catalog::LlmCatalogSettings;
use fabro_model::{Catalog, ProviderId};
use super::*;
@ -106,6 +143,65 @@ mod tests {
g
}
fn graph_with_model_and_provider(model: &str, provider: &str) -> Graph {
let mut g = Graph::new("test");
let mut start = Node::new("start");
start.attrs.insert(
"shape".to_string(),
AttrValue::String("Mdiamond".to_string()),
);
g.nodes.insert("start".to_string(), start);
let mut work = Node::new("work");
work.attrs
.insert("model".to_string(), AttrValue::String(model.to_string()));
work.attrs.insert(
"provider".to_string(),
AttrValue::String(provider.to_string()),
);
g.nodes.insert("work".to_string(), work);
let mut exit = Node::new("exit");
exit.attrs.insert(
"shape".to_string(),
AttrValue::String("Msquare".to_string()),
);
g.nodes.insert("exit".to_string(), exit);
g.edges.push(Edge::new("start", "work"));
g.edges.push(Edge::new("work", "exit"));
g
}
fn custom_catalog() -> Catalog {
let settings: LlmCatalogSettings = toml::from_str(
r#"
[providers.venice]
display_name = "Venice"
adapter = "openai_compatible"
base_url = "https://api.venice.ai/api/v1"
credentials = ["env:VENICE_API_KEY"]
[models."venice-large"]
provider = "venice"
display_name = "Venice Large"
family = "venice"
default = true
[models."venice-large".limits]
context_window = 128000
[models."venice-large".features]
tools = true
vision = false
reasoning = false
effort = false
"#,
)
.unwrap();
Catalog::from_settings(&settings).unwrap()
}
#[test]
fn validate_minimal_valid_graph_has_no_errors() {
let g = minimal_valid_graph();
@ -178,6 +274,56 @@ mod tests {
assert_eq!(custom.len(), 1);
}
#[test]
fn validate_does_not_run_catalog_aware_model_rules() {
let g = graph_with_model_and_provider("not-in-any-catalog", "not-a-provider");
let diagnostics = validate(&g, &[]);
assert!(
diagnostics
.iter()
.all(|d| d.rule != "node_model_known" && d.rule != "stylesheet_model_known"),
"catalog-free validation should not emit model/provider diagnostics: {diagnostics:?}"
);
}
#[test]
fn validate_with_catalog_accepts_custom_catalog_entries() {
let g = graph_with_model_and_provider("venice-large", "venice");
let catalog = custom_catalog();
let diagnostics = validate_with_catalog(&g, &catalog, &[]);
assert!(
diagnostics
.iter()
.all(|d| d.rule != "node_model_known" && d.rule != "stylesheet_model_known"),
"custom catalog entries should validate cleanly: {diagnostics:?}"
);
}
#[test]
fn validate_with_catalog_warns_for_unknown_model_and_provider() {
let g = graph_with_model_and_provider("missing-model", "missing-provider");
let catalog = custom_catalog();
let diagnostics = validate_with_catalog(&g, &catalog, &[]);
assert!(
diagnostics
.iter()
.any(|d| d.rule == "node_model_known" && d.message.contains("missing-model")),
"missing model diagnostic not found: {diagnostics:?}"
);
assert!(
diagnostics.iter().any(|d| d.rule == "node_model_known"
&& d.message.contains("missing-provider")
&& d.message.contains(ProviderId::new("venice").as_str())),
"missing provider diagnostic not found: {diagnostics:?}"
);
}
#[test]
fn diagnostic_severity_eq() {
assert_eq!(Severity::Error, Severity::Error);

11
lib/crates/fabro-validate/src/rules/mod.rs
View file

@ -55,8 +55,6 @@ pub fn built_in_rules() -> Vec<Box<dyn LintRule>> {
all_conditional_edges::rule(),
orphan_custom_outcome::rule(),
script_absolute_cd::rule(),
stylesheet_model_known::rule(),
node_model_known::rule(),
import_error::rule(),
unresolved_file_ref::rule(),
thread_id_requires_fidelity_full::rule(),
@ -64,3 +62,12 @@ pub fn built_in_rules() -> Vec<Box<dyn LintRule>> {
random_selection_no_conditions::rule(),
]
}
/// Returns lint rules that require the caller's resolved model catalog.
#[must_use]
pub fn catalog_rules(catalog: &fabro_model::Catalog) -> Vec<Box<dyn LintRule + '_>> {
vec![
stylesheet_model_known::rule(catalog),
node_model_known::rule(catalog),
]
}

16
lib/crates/fabro-validate/src/rules/model_support.rs
View file

@ -1,14 +1,13 @@
use std::str::FromStr;
use crate::{Diagnostic, Severity};
pub(super) fn check_model_known(
rule_name: &str,
catalog: &fabro_model::Catalog,
model: &str,
context: &str,
node_id: Option<String>,
) -> Option<Diagnostic> {
if fabro_model::Catalog::builtin().get(model).is_some() {
if catalog.get(model).is_some() {
return None;
}
Some(Diagnostic {
@ -25,16 +24,21 @@ pub(super) fn check_model_known(
pub(super) fn check_provider_known(
rule_name: &str,
catalog: &fabro_model::Catalog,
provider: &str,
context: &str,
node_id: Option<String>,
) -> Option<Diagnostic> {
if fabro_model::Provider::from_str(provider).is_ok() {
if catalog
.provider(&fabro_model::ProviderId::new(provider))
.is_some()
{
return None;
}
let valid: Vec<&str> = fabro_model::Provider::ALL
let valid: Vec<&str> = catalog
.providers()
.iter()
.map(|&p| <&'static str>::from(p))
.map(|provider| provider.id.as_str())
.collect();
let valid_str = valid.join(", ");
Some(Diagnostic {

46
lib/crates/fabro-validate/src/rules/node_model_known.rs
View file

@ -1,15 +1,18 @@
use fabro_graphviz::graph::Graph;
use fabro_model::Catalog;
use super::model_support::{check_model_known, check_provider_known};
use crate::{Diagnostic, LintRule};
pub(super) fn rule() -> Box<dyn LintRule> {
Box::new(Rule)
pub(super) fn rule(catalog: &Catalog) -> Box<dyn LintRule + '_> {
Box::new(Rule { catalog })
}
struct Rule;
struct Rule<'a> {
catalog: &'a Catalog,
}
impl LintRule for Rule {
impl LintRule for Rule<'_> {
fn name(&self) -> &'static str {
"node_model_known"
}
@ -20,14 +23,20 @@ impl LintRule for Rule {
let context = format!("on node '{}'", node.id);
let node_id = Some(node.id.clone());
if let Some(model) = node.model() {
if let Some(d) = check_model_known(self.name(), model, &context, node_id.clone()) {
if let Some(d) =
check_model_known(self.name(), self.catalog, model, &context, node_id.clone())
{
diagnostics.push(d);
}
}
if let Some(provider) = node.provider() {
if let Some(d) =
check_provider_known(self.name(), provider, &context, node_id.clone())
{
if let Some(d) = check_provider_known(
self.name(),
self.catalog,
provider,
&context,
node_id.clone(),
) {
diagnostics.push(d);
}
}
@ -39,6 +48,7 @@ impl LintRule for Rule {
#[cfg(test)]
mod tests {
use fabro_graphviz::graph::{AttrValue, Node};
use fabro_model::Catalog;
use super::Rule;
use crate::rules::test_support::minimal_graph;
@ -53,7 +63,9 @@ mod tests {
AttrValue::String("claude-sonnet-4-5".to_string()),
);
g.nodes.insert("work".to_string(), node);
let rule = Rule;
let rule = Rule {
catalog: Catalog::builtin(),
};
let d = rule.apply(&g);
assert!(d.is_empty());
}
@ -67,7 +79,9 @@ mod tests {
AttrValue::String("nonexistent-model-xyz".to_string()),
);
g.nodes.insert("work".to_string(), node);
let rule = Rule;
let rule = Rule {
catalog: Catalog::builtin(),
};
let d = rule.apply(&g);
assert_eq!(d.len(), 1);
assert_eq!(d[0].severity, Severity::Warning);
@ -82,7 +96,9 @@ mod tests {
node.attrs
.insert("model".to_string(), AttrValue::String("opus".to_string()));
g.nodes.insert("work".to_string(), node);
let rule = Rule;
let rule = Rule {
catalog: Catalog::builtin(),
};
let d = rule.apply(&g);
assert!(d.is_empty());
}
@ -96,7 +112,9 @@ mod tests {
AttrValue::String("google".to_string()),
);
g.nodes.insert("work".to_string(), node);
let rule = Rule;
let rule = Rule {
catalog: Catalog::builtin(),
};
let d = rule.apply(&g);
assert_eq!(d.len(), 1);
assert_eq!(d[0].severity, Severity::Warning);
@ -107,7 +125,9 @@ mod tests {
#[test]
fn node_model_known_rule_no_model_no_provider() {
let g = minimal_graph();
let rule = Rule;
let rule = Rule {
catalog: Catalog::builtin(),
};
let d = rule.apply(&g);
assert!(d.is_empty());
}

53
lib/crates/fabro-validate/src/rules/stylesheet_model_known.rs
View file

@ -1,16 +1,19 @@
use fabro_graphviz::graph::Graph;
use fabro_graphviz::stylesheet::{Selector, parse_stylesheet};
use fabro_model::Catalog;
use super::model_support::{check_model_known, check_provider_known};
use crate::{Diagnostic, LintRule};
pub(super) fn rule() -> Box<dyn LintRule> {
Box::new(Rule)
pub(super) fn rule(catalog: &Catalog) -> Box<dyn LintRule + '_> {
Box::new(Rule { catalog })
}
struct Rule;
struct Rule<'a> {
catalog: &'a Catalog,
}
impl Rule {
impl Rule<'_> {
fn selector_label(selector: &Selector) -> String {
match selector {
Selector::Universal => "*".to_string(),
@ -21,7 +24,7 @@ impl Rule {
}
}
impl LintRule for Rule {
impl LintRule for Rule<'_> {
fn name(&self) -> &'static str {
"stylesheet_model_known"
}
@ -42,15 +45,24 @@ impl LintRule for Rule {
let context = format!("in stylesheet rule '{label}'");
match decl.property.as_str() {
"model" => {
if let Some(d) = check_model_known(self.name(), &decl.value, &context, None)
{
if let Some(d) = check_model_known(
self.name(),
self.catalog,
&decl.value,
&context,
None,
) {
diagnostics.push(d);
}
}
"provider" => {
if let Some(d) =
check_provider_known(self.name(), &decl.value, &context, None)
{
if let Some(d) = check_provider_known(
self.name(),
self.catalog,
&decl.value,
&context,
None,
) {
diagnostics.push(d);
}
}
@ -65,6 +77,7 @@ impl LintRule for Rule {
#[cfg(test)]
mod tests {
use fabro_graphviz::graph::AttrValue;
use fabro_model::Catalog;
use super::Rule;
use crate::rules::test_support::minimal_graph;
@ -77,7 +90,9 @@ mod tests {
"model_stylesheet".to_string(),
AttrValue::String("* { model: claude-sonnet-4-5; provider: anthropic; }".to_string()),
);
let rule = Rule;
let rule = Rule {
catalog: Catalog::builtin(),
};
let d = rule.apply(&g);
assert!(d.is_empty());
}
@ -89,7 +104,9 @@ mod tests {
"model_stylesheet".to_string(),
AttrValue::String("#opus { model: claude-opus-4-5; }".to_string()),
);
let rule = Rule;
let rule = Rule {
catalog: Catalog::builtin(),
};
let d = rule.apply(&g);
assert_eq!(d.len(), 1);
assert_eq!(d[0].severity, Severity::Warning);
@ -104,7 +121,9 @@ mod tests {
"model_stylesheet".to_string(),
AttrValue::String("* { provider: google; }".to_string()),
);
let rule = Rule;
let rule = Rule {
catalog: Catalog::builtin(),
};
let d = rule.apply(&g);
assert_eq!(d.len(), 1);
assert_eq!(d[0].severity, Severity::Warning);
@ -118,7 +137,9 @@ mod tests {
"model_stylesheet".to_string(),
AttrValue::String("* { model: opus; }".to_string()),
);
let rule = Rule;
let rule = Rule {
catalog: Catalog::builtin(),
};
let d = rule.apply(&g);
assert!(d.is_empty());
}
@ -126,7 +147,9 @@ mod tests {
#[test]
fn stylesheet_model_known_rule_no_stylesheet() {
let g = minimal_graph();
let rule = Rule;
let rule = Rule {
catalog: Catalog::builtin(),
};
let d = rule.apply(&g);
assert!(d.is_empty());
}

14
lib/crates/fabro-workflow/src/handler/llm/api.rs
View file

@ -172,22 +172,26 @@ fn build_profile(
provider: Provider,
provider_id: ProviderId,
profile_kind: AgentProfileKind,
catalog: Arc<Catalog>,
) -> Box<dyn AgentProfile> {
match profile_kind {
AgentProfileKind::OpenAi => Box::new(
OpenAiProfile::new(model)
.with_provider(provider)
.with_provider_id(provider_id),
.with_provider_id(provider_id)
.with_catalog(catalog),
),
AgentProfileKind::Gemini => Box::new(
GeminiProfile::new(model)
.with_provider(provider)
.with_provider_id(provider_id),
.with_provider_id(provider_id)
.with_catalog(catalog),
),
AgentProfileKind::Anthropic => Box::new(
AnthropicProfile::new(model)
.with_provider(provider)
.with_provider_id(provider_id),
.with_provider_id(provider_id)
.with_catalog(catalog),
),
}
}
@ -573,6 +577,7 @@ impl AgentApiBackend {
provider.provider,
provider.provider_id.clone(),
provider.profile_kind,
Arc::clone(&catalog),
);
let config = SessionOptions {
@ -593,6 +598,7 @@ impl AgentApiBackend {
let factory_client = client.clone();
let factory_model = model.to_string();
let factory_provider = provider.clone();
let factory_catalog = Arc::clone(&catalog);
let factory_env = Arc::clone(sandbox);
let factory_tool_env = tool_env.cloned();
let factory: SessionFactory = Arc::new(move || {
@ -601,6 +607,7 @@ impl AgentApiBackend {
factory_provider.provider,
factory_provider.provider_id.clone(),
factory_provider.profile_kind,
Arc::clone(&factory_catalog),
));
let mut session = Session::new(
factory_client.clone(),
@ -1433,6 +1440,7 @@ mod tests {
Provider::Anthropic,
Provider::Anthropic.id(),
AgentProfileKind::Anthropic,
Arc::new(Catalog::from_builtin_with_overrides(&LlmCatalogSettings::default()).unwrap()),
);
let manager = Arc::new(TokioMutex::new(SubAgentManager::new(1)));
let factory: SessionFactory = Arc::new(|| {

2
lib/crates/fabro-workflow/src/handler/manager_loop.rs
View file

@ -75,6 +75,7 @@ fn parse_child_graph(node: &Node, services: &EngineServices) -> Result<ParsedChi
settings: WorkflowSettings::default(),
cwd: cwd.clone(),
custom_transforms: Vec::new(),
catalog: Arc::clone(&services.run.catalog),
mode: RenderMode::Strict,
})?;
validated.raise_on_errors()?;
@ -117,6 +118,7 @@ fn parse_child_graph(node: &Node, services: &EngineServices) -> Result<ParsedChi
settings: WorkflowSettings::default(),
cwd,
custom_transforms: Vec::new(),
catalog: Arc::clone(&services.run.catalog),
mode: RenderMode::Strict,
})?;
validated.raise_on_errors()?;

15
lib/crates/fabro-workflow/src/operations/create.rs
View file

@ -322,6 +322,7 @@ fn create_from_source(
Some(&options.settings),
goal_override,
RenderMode::Strict,
&options.catalog,
)?;
if validated.has_errors() {
@ -341,6 +342,7 @@ pub(super) fn preprocess_and_validate(
settings: Option<&WorkflowSettings>,
goal_override: Option<&str>,
render_mode: RenderMode,
catalog: &Arc<Catalog>,
) -> Result<Validated, Error> {
let inputs = run_inputs(settings);
let template_ctx = TemplateContext::for_input_scan(inputs.clone());
@ -372,8 +374,9 @@ pub(super) fn preprocess_and_validate(
file_resolver,
inputs,
custom_transforms,
catalog: Arc::clone(catalog),
})?;
let mut validated = pipeline::validate(transformed, &[]);
let mut validated = pipeline::validate(transformed, catalog.as_ref(), &[]);
if !template_diagnostics.is_empty() {
validated.prepend_diagnostics(template_diagnostics);
}
@ -527,6 +530,10 @@ mod tests {
.expect("default settings should resolve")
}
fn test_catalog() -> Arc<Catalog> {
Arc::new(Catalog::from_builtin_with_overrides(&LlmCatalogSettings::default()).unwrap())
}
fn validate_dot(dot_source: &str, settings: WorkflowSettings) -> Validated {
validate(ValidateInput {
workflow: WorkflowInput::DotSource {
@ -536,6 +543,7 @@ mod tests {
settings,
cwd: PathBuf::from("."),
custom_transforms: Vec::new(),
catalog: test_catalog(),
mode: RenderMode::Structural,
})
.unwrap()
@ -599,6 +607,7 @@ mod tests {
Some(&WorkflowSettings::default()),
None,
RenderMode::Strict,
&test_catalog(),
);
let Err(err) = result else {
@ -713,6 +722,7 @@ mod tests {
settings: WorkflowSettings::default(),
cwd: PathBuf::from("."),
custom_transforms: Vec::new(),
catalog: test_catalog(),
mode: RenderMode::Strict,
});
assert!(result.is_err());
@ -757,6 +767,7 @@ mod tests {
settings: WorkflowSettings::default(),
cwd: PathBuf::from("."),
custom_transforms: vec![Box::new(TagTransform)],
catalog: test_catalog(),
mode: RenderMode::Strict,
})
.unwrap();
@ -791,6 +802,7 @@ mod tests {
settings: WorkflowSettings::default(),
cwd: dir.path().to_path_buf(),
custom_transforms: Vec::new(),
catalog: test_catalog(),
mode: RenderMode::Strict,
})
.unwrap();
@ -832,6 +844,7 @@ mod tests {
settings: WorkflowSettings::default(),
cwd: PathBuf::from("."),
custom_transforms: Vec::new(),
catalog: test_catalog(),
mode: RenderMode::Strict,
})
.unwrap();

4
lib/crates/fabro-workflow/src/operations/validate.rs
View file

@ -1,5 +1,7 @@
use std::path::PathBuf;
use std::sync::Arc;
use fabro_model::Catalog;
use fabro_types::WorkflowSettings;
use super::create::{RenderMode, preprocess_and_validate};
@ -13,6 +15,7 @@ pub struct ValidateInput {
pub settings: WorkflowSettings,
pub cwd: PathBuf,
pub custom_transforms: Vec<Box<dyn Transform>>,
pub catalog: Arc<Catalog>,
/// How undefined template inputs are treated. Validate-style callers
/// (`fabro validate`, the `/validate` API) pass [`RenderMode::Structural`]
/// so unbound inputs surface as warning diagnostics. Run-style callers
@ -41,5 +44,6 @@ pub fn validate(input: ValidateInput) -> Result<Validated, Error> {
Some(&resolved.settings),
resolved.goal_override.as_deref(),
input.mode,
&input.catalog,
)
}

36
lib/crates/fabro-workflow/src/pipeline/transform.rs
View file

@ -41,7 +41,7 @@ pub fn transform(parsed: Parsed, options: &TransformOptions) -> Result<Transform
}
.apply(graph)?;
let graph = StylesheetApplicationTransform.apply(graph)?;
let graph = ModelResolutionTransform.apply(graph)?;
let graph = ModelResolutionTransform::new(Arc::clone(&options.catalog)).apply(graph)?;
// Custom transforms
let graph = options
@ -60,6 +60,8 @@ mod tests {
use std::sync::Arc;
use fabro_graphviz::graph::AttrValue;
use fabro_model::Catalog;
use fabro_model::catalog::LlmCatalogSettings;
use super::*;
use crate::file_resolver::FilesystemFileResolver;
@ -72,6 +74,20 @@ mod tests {
std::fs::write(path, contents).unwrap();
}
fn test_catalog() -> Arc<Catalog> {
Arc::new(Catalog::from_builtin_with_overrides(&LlmCatalogSettings::default()).unwrap())
}
fn transform_options() -> TransformOptions {
TransformOptions {
current_dir: None,
file_resolver: None,
inputs: HashMap::new(),
custom_transforms: vec![],
catalog: test_catalog(),
}
}
#[test]
fn transform_applies_variable_expansion() {
let dot = r#"digraph Test {
@ -82,13 +98,7 @@ mod tests {
start -> work -> exit
}"#;
let parsed = parse(dot).unwrap();
let transformed = transform(parsed, &TransformOptions {
current_dir: None,
file_resolver: None,
inputs: HashMap::new(),
custom_transforms: vec![],
})
.unwrap();
let transformed = transform(parsed, &transform_options()).unwrap();
let prompt = transformed.graph.nodes["work"]
.attrs
.get("prompt")
@ -107,13 +117,7 @@ mod tests {
start -> work -> exit
}"#;
let parsed = parse(dot).unwrap();
let transformed = transform(parsed, &TransformOptions {
current_dir: None,
file_resolver: None,
inputs: HashMap::new(),
custom_transforms: vec![],
})
.unwrap();
let transformed = transform(parsed, &transform_options()).unwrap();
assert_eq!(
transformed.graph.nodes["work"].attrs.get("model"),
Some(&AttrValue::String("claude-sonnet-4-6".into()))
@ -140,6 +144,7 @@ mod tests {
file_resolver: Some(Arc::new(FilesystemFileResolver::new(None))),
inputs: HashMap::new(),
custom_transforms: vec![],
catalog: test_catalog(),
})
.unwrap();
@ -187,6 +192,7 @@ mod tests {
toml::Value::String("Launch".to_string()),
)]),
custom_transforms: vec![],
catalog: test_catalog(),
})
.unwrap();

1
lib/crates/fabro-workflow/src/pipeline/types.rs
View file

@ -321,6 +321,7 @@ pub struct TransformOptions {
pub file_resolver: Option<Arc<dyn FileResolver>>,
pub inputs: HashMap<String, toml::Value>,
pub custom_transforms: Vec<Box<dyn Transform>>,
pub catalog: Arc<fabro_model::Catalog>,
}
/// Options for the FINALIZE phase.

22
lib/crates/fabro-workflow/src/pipeline/validate.rs
View file

@ -1,3 +1,4 @@
use fabro_model::Catalog;
use fabro_validate::LintRule;
use super::types::{Transformed, Validated};
@ -6,29 +7,44 @@ use super::types::{Transformed, Validated};
///
/// **Infallible.** Always returns `Validated` with diagnostics. Caller decides
/// whether to fail via `validated.raise_on_errors()`.
pub fn validate(transformed: Transformed, extra_rules: &[&dyn LintRule]) -> Validated {
pub fn validate(
transformed: Transformed,
catalog: &Catalog,
extra_rules: &[&dyn LintRule],
) -> Validated {
let Transformed { graph, source } = transformed;
let diagnostics = fabro_validate::validate(&graph, extra_rules);
let diagnostics = fabro_validate::validate_with_catalog(&graph, catalog, extra_rules);
Validated::new(graph, source, diagnostics)
}
#[cfg(test)]
mod tests {
use fabro_model::Catalog;
use fabro_model::catalog::LlmCatalogSettings;
use super::*;
use crate::pipeline::parse::parse;
use crate::pipeline::transform;
use crate::pipeline::types::TransformOptions;
fn test_catalog() -> std::sync::Arc<Catalog> {
std::sync::Arc::new(
Catalog::from_builtin_with_overrides(&LlmCatalogSettings::default()).unwrap(),
)
}
fn run_pipeline(dot: &str) -> Validated {
let catalog = test_catalog();
let parsed = parse(dot).unwrap();
let transformed = transform::transform(parsed, &TransformOptions {
current_dir: None,
file_resolver: None,
inputs: std::collections::HashMap::new(),
custom_transforms: vec![],
catalog: std::sync::Arc::clone(&catalog),
})
.unwrap();
validate(transformed, &[])
validate(transformed, catalog.as_ref(), &[])
}
#[test]

93
lib/crates/fabro-workflow/src/transforms/model_resolution.rs
View file

@ -1,3 +1,5 @@
use std::sync::Arc;
use fabro_graphviz::graph::{AttrValue, Graph};
use fabro_model::Catalog;
@ -6,7 +8,16 @@ use crate::error::Error;
/// Resolves model aliases to canonical IDs and infers the provider from the
/// model catalog.
pub struct ModelResolutionTransform;
pub struct ModelResolutionTransform {
catalog: Arc<Catalog>,
}
impl ModelResolutionTransform {
#[must_use]
pub fn new(catalog: Arc<Catalog>) -> Self {
Self { catalog }
}
}
impl Transform for ModelResolutionTransform {
fn apply(&self, graph: Graph) -> Result<Graph, Error> {
@ -18,7 +29,7 @@ impl Transform for ModelResolutionTransform {
.and_then(AttrValue::as_str)
.map(String::from);
if let Some(model) = model {
if let Some(info) = Catalog::builtin().get(&model) {
if let Some(info) = self.catalog.get(&model) {
let canonical_id = info.id.clone();
let provider = info.provider.to_string();
// Resolve alias to canonical model ID
@ -40,10 +51,48 @@ impl Transform for ModelResolutionTransform {
#[cfg(test)]
mod tests {
use std::sync::Arc;
use fabro_graphviz::graph::{AttrValue, Graph, Node};
use fabro_model::catalog::LlmCatalogSettings;
use super::*;
fn custom_catalog() -> Arc<Catalog> {
let settings: LlmCatalogSettings = toml::from_str(
r#"
[providers.venice]
display_name = "Venice"
adapter = "openai_compatible"
base_url = "https://api.venice.ai/api/v1"
credentials = ["env:VENICE_API_KEY"]
[models."venice-large"]
provider = "venice"
display_name = "Venice Large"
family = "venice"
default = true
aliases = ["vl"]
[models."venice-large".limits]
context_window = 128000
[models."venice-large".features]
tools = true
vision = false
reasoning = false
effort = false
"#,
)
.unwrap();
Arc::new(Catalog::from_settings(&settings).unwrap())
}
fn builtin_transform() -> ModelResolutionTransform {
let catalog = Catalog::from_builtin_with_overrides(&LlmCatalogSettings::default()).unwrap();
ModelResolutionTransform::new(Arc::new(catalog))
}
#[test]
fn provider_inference_sets_provider_from_catalog() {
let mut graph = Graph::new("test");
@ -54,7 +103,7 @@ mod tests {
);
graph.nodes.insert("a".to_string(), node);
let graph = ModelResolutionTransform.apply(graph).unwrap();
let graph = builtin_transform().apply(graph).unwrap();
assert_eq!(
graph.nodes["a"]
@ -79,7 +128,7 @@ mod tests {
);
graph.nodes.insert("a".to_string(), node);
let graph = ModelResolutionTransform.apply(graph).unwrap();
let graph = builtin_transform().apply(graph).unwrap();
assert_eq!(
graph.nodes["a"]
@ -100,7 +149,7 @@ mod tests {
);
graph.nodes.insert("a".to_string(), node);
let graph = ModelResolutionTransform.apply(graph).unwrap();
let graph = builtin_transform().apply(graph).unwrap();
assert_eq!(graph.nodes["a"].attrs.get("provider"), None);
}
@ -111,7 +160,7 @@ mod tests {
let node = Node::new("a");
graph.nodes.insert("a".to_string(), node);
let graph = ModelResolutionTransform.apply(graph).unwrap();
let graph = builtin_transform().apply(graph).unwrap();
assert_eq!(graph.nodes["a"].attrs.get("provider"), None);
}
@ -124,7 +173,7 @@ mod tests {
.insert("model".to_string(), AttrValue::String("gpt-54".to_string()));
graph.nodes.insert("a".to_string(), node);
let graph = ModelResolutionTransform.apply(graph).unwrap();
let graph = builtin_transform().apply(graph).unwrap();
assert_eq!(
graph.nodes["a"]
@ -152,7 +201,7 @@ mod tests {
);
graph.nodes.insert("a".to_string(), node);
let graph = ModelResolutionTransform.apply(graph).unwrap();
let graph = builtin_transform().apply(graph).unwrap();
assert_eq!(
graph.nodes["a"]
@ -162,4 +211,32 @@ mod tests {
Some("gpt-5.4")
);
}
#[test]
fn model_resolution_uses_injected_catalog_for_alias_and_provider() {
let mut graph = Graph::new("test");
let mut node = Node::new("a");
node.attrs
.insert("model".to_string(), AttrValue::String("vl".to_string()));
graph.nodes.insert("a".to_string(), node);
let graph = ModelResolutionTransform::new(custom_catalog())
.apply(graph)
.unwrap();
assert_eq!(
graph.nodes["a"]
.attrs
.get("model")
.and_then(AttrValue::as_str),
Some("venice-large")
);
assert_eq!(
graph.nodes["a"]
.attrs
.get("provider")
.and_then(AttrValue::as_str),
Some("venice")
);
}
}

9
lib/crates/fabro-workflow/tests/it/integration.rs
View file

@ -4272,6 +4272,12 @@ async fn import_e2e_through_engine() {
use fabro_workflow::pipeline::{TransformOptions, transform, validate};
let dir = tempfile::tempdir().unwrap();
let catalog = std::sync::Arc::new(
fabro_model::Catalog::from_builtin_with_overrides(
&fabro_model::catalog::LlmCatalogSettings::default(),
)
.unwrap(),
);
std::fs::write(
dir.path().join("val.fabro"),
r#"digraph validate {
@ -4313,9 +4319,10 @@ async fn import_e2e_through_engine() {
)),
inputs: std::collections::HashMap::new(),
custom_transforms: vec![],
catalog: std::sync::Arc::clone(&catalog),
})
.unwrap();
let validated = validate(transformed, &[]);
let validated = validate(transformed, catalog.as_ref(), &[]);
validated
.raise_on_errors()
.expect("validation should pass after imports expand");