From 82bcafcfca45ee56f8fb6b85144b4b24c5229f8e Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Wed, 9 Sep 2026 17:26:57 -0600 Subject: [PATCH] Port fabro-workflow, hooks, validate, and store to the lithos types Workflow LLM handlers build lithos requests, bill from lithos usage and cost, and classify failures from lithos `ErrorKind`. Model resolution and fallback use the fabro-llm selection and catalog helpers. Validation rules read the lithos catalog, and store fixtures use the new `BilledModelUsage` shape. Co-Authored-By: Claude Fable 5.1 --- lib/components/fabro-hooks/Cargo.toml | 1 - lib/components/fabro-hooks/src/bridge.rs | 4 +- lib/components/fabro-hooks/src/executor.rs | 173 ++-- lib/components/fabro-hooks/src/runner.rs | 12 +- .../fabro-hooks/tests/host_command_hooks.rs | 4 +- lib/components/fabro-manifest/src/lib.rs | 4 - lib/components/fabro-store/src/run_state.rs | 70 +- .../tests/serializable_projection.rs | 16 +- lib/components/fabro-validate/Cargo.toml | 3 +- lib/components/fabro-validate/src/lib.rs | 45 +- .../fabro-validate/src/rules/mod.rs | 3 +- .../fabro-validate/src/rules/model_support.rs | 19 +- .../src/rules/node_model_known.rs | 35 +- .../src/rules/stylesheet_model_known.rs | 35 +- lib/components/fabro-workflow/Cargo.toml | 2 +- .../fabro-workflow/src/billing_rollup.rs | 49 +- lib/components/fabro-workflow/src/error.rs | 194 ++-- .../fabro-workflow/src/event/convert.rs | 91 +- .../fabro-workflow/src/event/events.rs | 9 +- .../fabro-workflow/src/event/redaction.rs | 16 +- .../fabro-workflow/src/event/sink.rs | 16 +- .../fabro-workflow/src/handler/agent.rs | 3 +- .../src/handler/llm/activation_lease.rs | 3 +- .../fabro-workflow/src/handler/llm/api.rs | 927 +++++++++--------- .../src/handler/llm/preamble.rs | 19 +- .../fabro-workflow/src/handler/llm/router.rs | 2 +- .../fabro-workflow/src/handler/llm/routing.rs | 46 +- .../fabro-workflow/src/handler/prompt.rs | 108 +- .../src/handler/structured_output.rs | 29 +- .../fabro-workflow/src/lifecycle/git.rs | 5 +- .../fabro-workflow/src/model_fallback.rs | 90 +- .../fabro-workflow/src/operations/create.rs | 84 +- .../fabro-workflow/src/operations/start.rs | 148 +-- .../fabro-workflow/src/operations/validate.rs | 4 +- lib/components/fabro-workflow/src/outcome.rs | 243 ++--- .../src/pipeline/execute/tests.rs | 17 +- .../fabro-workflow/src/pipeline/finalize.rs | 13 +- .../fabro-workflow/src/pipeline/initialize.rs | 93 +- .../fabro-workflow/src/pipeline/publish.rs | 2 +- .../src/pipeline/pull_request.rs | 235 +++-- .../fabro-workflow/src/pipeline/transform.rs | 4 +- .../fabro-workflow/src/pipeline/types.rs | 4 +- .../fabro-workflow/src/pipeline/validate.rs | 2 +- .../fabro-workflow/src/run_materialization.rs | 16 +- lib/components/fabro-workflow/src/services.rs | 25 +- .../fabro-workflow/src/test_support.rs | 45 +- .../src/transforms/model_resolution.rs | 82 +- .../fabro-workflow/tests/it/integration.rs | 191 ++-- .../fabro-workflow/tests/materialize_run.rs | 12 +- 49 files changed, 1455 insertions(+), 1798 deletions(-) diff --git a/lib/components/fabro-hooks/Cargo.toml b/lib/components/fabro-hooks/Cargo.toml index 2c1a68422..fe9659c76 100644 --- a/lib/components/fabro-hooks/Cargo.toml +++ b/lib/components/fabro-hooks/Cargo.toml @@ -16,7 +16,6 @@ workspace = true fabro-agent = { path = "../fabro-agent" } fabro-auth = { path = "../../foundation/fabro-auth" } fabro-llm = { path = "../fabro-llm" } -fabro-model = { path = "../../foundation/fabro-model" } fabro-redact.workspace = true fabro-types = { path = "../../foundation/fabro-types" } fabro-util = { path = "../../foundation/fabro-util" } diff --git a/lib/components/fabro-hooks/src/bridge.rs b/lib/components/fabro-hooks/src/bridge.rs index 6aa7e60f6..6f49a6a00 100644 --- a/lib/components/fabro-hooks/src/bridge.rs +++ b/lib/components/fabro-hooks/src/bridge.rs @@ -78,7 +78,7 @@ mod tests { use std::path::PathBuf; use std::sync::Mutex; - use fabro_model::Catalog; + use fabro_llm::lithos_catalog::Catalog; use fabro_types::fixtures; use super::*; @@ -100,7 +100,7 @@ mod tests { context: &HookContext, _sandbox: Arc, execution_context: &HookExecutionContext, - _llm_source: &dyn fabro_auth::CredentialSource, + _llm_source: Arc, _catalog: Arc, ) -> HookResult { self.captured_contexts.lock().unwrap().push(context.clone()); diff --git a/lib/components/fabro-hooks/src/executor.rs b/lib/components/fabro-hooks/src/executor.rs index 4aa87d168..3d208ed7e 100644 --- a/lib/components/fabro-hooks/src/executor.rs +++ b/lib/components/fabro-hooks/src/executor.rs @@ -7,12 +7,11 @@ use async_trait::async_trait; use fabro_agent::Sandbox; use fabro_agent::tool_registry::ToolContext; use fabro_auth::CredentialSource; -use fabro_llm::client::Client as LlmClient; -use fabro_llm::generate::{GenerateParams, generate_object}; -use fabro_llm::types::{Message, Request, ToolResult}; -use fabro_model::Catalog; +use fabro_llm::lithos_catalog::Catalog; +use fabro_llm::{Client, ClientOptions, Request, structured}; use fabro_redact::redacted_url_for_log; use fabro_types::settings::{InterpString, ResolveCtx, ResolveError}; +use fabro_types::{Message, Role, ToolCall, tool_call_arguments, tool_result_from_json}; use tokio::process::Command as TokioCommand; use tokio::time::timeout as tokio_timeout; use tokio_util::sync::CancellationToken; @@ -49,7 +48,7 @@ pub trait HookExecutor: Send + Sync { context: &HookContext, sandbox: Arc, execution_context: &HookExecutionContext, - llm_source: &dyn CredentialSource, + llm_source: Arc, catalog: Arc, ) -> HookResult; } @@ -282,7 +281,7 @@ impl HookExecutorImpl { prompt: &InterpString, model: Option<&InterpString>, context: &HookContext, - llm_source: &dyn CredentialSource, + llm_source: Arc, catalog: Arc, ) -> HookDecision { let (prompt, model) = match Self::resolve_prompt_and_model(prompt, model) { @@ -299,33 +298,48 @@ impl HookExecutorImpl { let user_msg = Self::build_hook_user_message(&prompt, context); Self::execute_llm_with_timeout(definition.timeout(), "prompt", || async move { - let client = match LlmClient::from_source(llm_source, catalog).await { - Ok(client) => Arc::new(client), + let client = match Self::build_client(catalog, llm_source).await { + Ok(client) => client, Err(e) => { tracing::warn!(error = %e, "prompt hook client creation failed, proceeding"); return HookDecision::Proceed; } }; - let params = GenerateParams::new(&resolved_model, client) + let request = Request::builder() + .model(&resolved_model) .system(HOOK_EVALUATOR_SYSTEM_PROMPT) - .prompt(user_msg) - .max_tokens(1024); + .user(user_msg) + .max_output_tokens(1024) + .build(); + let request = match request { + Ok(request) => request, + Err(e) => { + tracing::warn!(error = %e, "prompt hook request invalid, proceeding"); + return HookDecision::Proceed; + } + }; - match generate_object(params, HOOK_RESPONSE_SCHEMA.clone()).await { - Ok(result) => if let Some(obj) = result.output { match serde_json::from_value::(obj) { - Ok(resp) if resp.ok => HookDecision::Proceed, - Ok(resp) => HookDecision::Block { - reason: resp.reason, - }, - Err(e) => { - tracing::warn!(error = %e, "prompt hook response deserialize failed, proceeding"); - HookDecision::Proceed + match structured::complete_object( + &client, + request, + "hook_response", + HOOK_RESPONSE_SCHEMA.clone(), + ) + .await + { + Ok(completion) => { + match serde_json::from_value::(completion.object) { + Ok(resp) if resp.ok => HookDecision::Proceed, + Ok(resp) => HookDecision::Block { + reason: resp.reason, + }, + Err(e) => { + tracing::warn!(error = %e, "prompt hook response deserialize failed, proceeding"); + HookDecision::Proceed + } } - } } else { - tracing::warn!("prompt hook returned no structured output, proceeding"); - HookDecision::Proceed - }, + } Err(e) => { tracing::warn!(error = %e, "prompt hook LLM call failed, proceeding"); HookDecision::Proceed @@ -347,7 +361,7 @@ impl HookExecutorImpl { max_tool_rounds: Option, context: &HookContext, sandbox: Arc, - llm_source: &dyn CredentialSource, + llm_source: Arc, catalog: Arc, ) -> HookDecision { let (prompt, model) = match Self::resolve_prompt_and_model(prompt, model) { @@ -364,7 +378,7 @@ impl HookExecutorImpl { let user_msg = Self::build_hook_user_message(&prompt, context); Self::execute_llm_with_timeout(definition.timeout(), "agent", || async move { - let client = match LlmClient::from_source(llm_source, catalog).await { + let client = match Self::build_client(catalog, llm_source).await { Ok(c) => c, Err(e) => { tracing::warn!(error = %e, "agent hook client creation failed, proceeding"); @@ -378,32 +392,30 @@ impl HookExecutorImpl { let tool_defs = registry.definitions(); let mut messages = vec![ - Message::system(HOOK_EVALUATOR_SYSTEM_PROMPT), - Message::user(user_msg), + Message::text(Role::System, HOOK_EVALUATOR_SYSTEM_PROMPT), + Message::text(Role::User, user_msg), ]; let rounds = max_tool_rounds.unwrap_or(50); let cancel = CancellationToken::new(); for _ in 0..rounds { - let request = Request { - model: resolved_model.clone(), - messages: messages.clone(), - provider: None, - tools: Some(tool_defs.clone()), - tool_choice: None, - response_format: None, - temperature: None, - top_p: None, - max_tokens: None, - stop_sequences: None, - reasoning_effort: None, - speed: None, - metadata: None, - provider_options: None, + let mut builder = Request::builder().model(&resolved_model); + for message in &messages { + builder = builder.message(message.clone()); + } + for tool in &tool_defs { + builder = builder.tool(tool.clone()); + } + let request = match builder.build() { + Ok(request) => request, + Err(e) => { + tracing::warn!(error = %e, "agent hook request invalid, proceeding"); + return HookDecision::Proceed; + } }; - let response = match client.complete(&request).await { + let response = match client.complete(request).await { Ok(r) => r, Err(e) => { tracing::warn!(error = %e, "agent hook LLM call failed, proceeding"); @@ -411,13 +423,14 @@ impl HookExecutorImpl { } }; - let tool_calls = response.tool_calls(); + let tool_calls: Vec = response.tool_calls().cloned().collect(); if tool_calls.is_empty() { return Self::parse_prompt_response(&response.text()); } - messages.push(response.message.clone()); + messages.push(response.into_message()); + let mut results = Vec::with_capacity(tool_calls.len()); for tc in &tool_calls { let tool = registry.get(&tc.name).cloned(); let ctx = ToolContext { @@ -430,22 +443,27 @@ impl HookExecutorImpl { agent_event_emitter: None, }; let result = match tool { - Some(t) => match (t.executor)(tc.arguments.clone(), ctx).await { - Ok(output) => { - ToolResult::success(tc.id.clone(), serde_json::json!(output)) - } - Err(err) => ToolResult::error(tc.id.clone(), err), + Some(t) => match (t.executor)(tool_call_arguments(tc), ctx).await { + Ok(output) => tool_result_from_json( + tc.id.clone(), + serde_json::Value::String(output), + false, + ), + Err(err) => tool_result_from_json( + tc.id.clone(), + serde_json::Value::String(err), + true, + ), }, - None => { - ToolResult::error(tc.id.clone(), format!("Unknown tool: {}", tc.name)) - } + None => tool_result_from_json( + tc.id.clone(), + serde_json::Value::String(format!("Unknown tool: {}", tc.name)), + true, + ), }; - messages.push(Message::tool_result( - result.tool_call_id, - result.content, - result.is_error, - )); + results.push(fabro_types::ContentPart::ToolResult(result)); } + messages.push(Message::new(Role::Tool, results)); } tracing::warn!("agent hook exhausted max tool rounds, proceeding"); @@ -454,6 +472,21 @@ impl HookExecutorImpl { .await } + /// The LLM client hooks dispatch through: every provider the source can + /// serve, with standard retries. + async fn build_client( + catalog: Arc, + llm_source: Arc, + ) -> Result { + fabro_llm::build_client( + Catalog::clone(&catalog), + llm_source, + ClientOptions::standard(), + ) + .await + .map(|built| built.client) + } + /// Build an HTTP client for the given TLS mode. fn build_http_client(tls: TlsMode) -> fabro_http::HttpClient { let accept_invalid = matches!(tls, TlsMode::NoVerify | TlsMode::Off); @@ -629,7 +662,7 @@ impl HookExecutor for HookExecutorImpl { context: &HookContext, sandbox: Arc, execution_context: &HookExecutionContext, - llm_source: &dyn CredentialSource, + llm_source: Arc, catalog: Arc, ) -> HookResult { use std::sync::OnceLock; @@ -751,7 +784,7 @@ mod tests { } fn test_catalog() -> Arc { - Arc::new(Catalog::from_builtin().unwrap()) + Arc::new(fabro_llm::default_catalog()) } fn test_http_client() -> fabro_http::HttpClient { @@ -841,7 +874,7 @@ mod tests { &ctx, sandbox, &HookExecutionContext::default(), - source.as_ref(), + Arc::clone(&source), test_catalog(), ) .await; @@ -862,7 +895,7 @@ mod tests { &ctx, sandbox, &HookExecutionContext::default(), - source.as_ref(), + Arc::clone(&source), test_catalog(), ) .await; @@ -882,7 +915,7 @@ mod tests { &ctx, sandbox, &HookExecutionContext::default(), - source.as_ref(), + Arc::clone(&source), test_catalog(), ) .await; @@ -902,7 +935,7 @@ mod tests { &ctx, sandbox, &HookExecutionContext::default(), - source.as_ref(), + Arc::clone(&source), test_catalog(), ) .await; @@ -926,7 +959,7 @@ mod tests { &ctx, sandbox, &HookExecutionContext::default(), - source.as_ref(), + Arc::clone(&source), test_catalog(), ) .await; @@ -955,7 +988,7 @@ mod tests { &ctx, sandbox, &HookExecutionContext::default(), - source.as_ref(), + Arc::clone(&source), test_catalog(), ) .await; @@ -1441,7 +1474,7 @@ mod tests { &ctx, sandbox, &HookExecutionContext::default(), - source.as_ref(), + Arc::clone(&source), test_catalog(), ) .await; @@ -1475,7 +1508,7 @@ mod tests { &interp("{{ env.MISSING_HOOK_VALUE }}"), None, &make_context(), - test_llm_source().as_ref(), + test_llm_source(), test_catalog(), ) .await; @@ -1503,7 +1536,7 @@ mod tests { Some(1), &make_context(), make_sandbox(), - test_llm_source().as_ref(), + test_llm_source(), test_catalog(), ) .await; diff --git a/lib/components/fabro-hooks/src/runner.rs b/lib/components/fabro-hooks/src/runner.rs index c3ec53987..489edd0dd 100644 --- a/lib/components/fabro-hooks/src/runner.rs +++ b/lib/components/fabro-hooks/src/runner.rs @@ -5,7 +5,7 @@ use fabro_agent::Sandbox; use fabro_auth::CredentialSource; #[cfg(test)] use fabro_auth::test_support; -use fabro_model::Catalog; +use fabro_llm::lithos_catalog::Catalog; use crate::config::{HookDefinition, HookSettings}; use crate::executor::{HookExecutor, HookExecutorImpl}; @@ -47,7 +47,7 @@ impl HookRunner { config, executor, llm_source: test_support::vault_only_credential_source(), - catalog: Arc::new(Catalog::from_builtin().expect("default catalog should build")), + catalog: Arc::new(fabro_llm::default_catalog()), compiled_matchers, } } @@ -158,7 +158,7 @@ impl HookRunner { context, sandbox.clone(), execution_context, - self.llm_source.as_ref(), + Arc::clone(&self.llm_source), Arc::clone(&self.catalog), ) .await; @@ -213,7 +213,7 @@ impl HookRunner { context, sandbox.clone(), execution_context, - self.llm_source.as_ref(), + Arc::clone(&self.llm_source), Arc::clone(&self.catalog), ) .await; @@ -256,7 +256,7 @@ mod tests { _context: &HookContext, _sandbox: Arc, _execution_context: &HookExecutionContext, - _llm_source: &dyn CredentialSource, + _llm_source: Arc, _catalog: Arc, ) -> HookResult { HookResult { @@ -282,7 +282,7 @@ mod tests { } fn test_catalog() -> Arc { - Arc::new(Catalog::from_builtin().expect("default catalog should build")) + Arc::new(fabro_llm::default_catalog()) } fn make_hook(event: HookEvent, name: &str) -> HookDefinition { diff --git a/lib/components/fabro-hooks/tests/host_command_hooks.rs b/lib/components/fabro-hooks/tests/host_command_hooks.rs index fe73371dd..74cacd33c 100644 --- a/lib/components/fabro-hooks/tests/host_command_hooks.rs +++ b/lib/components/fabro-hooks/tests/host_command_hooks.rs @@ -7,7 +7,7 @@ use fabro_hooks::{ HookContext, HookDecision, HookDefinition, HookEvent, HookExecutionContext, HookRunner, HookSettings, InterpString, }; -use fabro_model::Catalog; +use fabro_llm::lithos_catalog::Catalog; use fabro_types::RunId; use tokio::fs; @@ -16,7 +16,7 @@ fn test_llm_source() -> Arc { } fn test_catalog() -> Arc { - Arc::new(Catalog::from_builtin().expect("default catalog should build")) + Arc::new(fabro_llm::default_catalog()) } fn local_sandbox() -> Arc { diff --git a/lib/components/fabro-manifest/src/lib.rs b/lib/components/fabro-manifest/src/lib.rs index f1f4bea15..db317abb1 100644 --- a/lib/components/fabro-manifest/src/lib.rs +++ b/lib/components/fabro-manifest/src/lib.rs @@ -557,10 +557,6 @@ fn remotely_available_sha( /// Resolve a workflow reference and reject it when neither its config nor /// its graph exists on disk. /// A missing workflow surfaces as `fabro_config::Error::WorkflowNotFound`. -#[expect( - clippy::result_large_err, - reason = "callers match on the concrete config error to classify missing workflows" -)] fn resolve_existing_workflow_location( workflow: &Path, cwd: &Path, diff --git a/lib/components/fabro-store/src/run_state.rs b/lib/components/fabro-store/src/run_state.rs index b9a9fdacf..b2df4d2af 100644 --- a/lib/components/fabro-store/src/run_state.rs +++ b/lib/components/fabro-store/src/run_state.rs @@ -1564,8 +1564,8 @@ fn conclusion_from_completed( props: &RunCompletedProps, timestamp: DateTime, ) -> Result { - let (stages, total_retries) = billing_rollup::billing_rollup_from_projection(projection, None) - .conclusion_stages(projection); + let (stages, total_retries) = + billing_rollup::billing_rollup_from_projection(projection).conclusion_stages(projection); Ok(Conclusion { timestamp, status: StageOutcome::from_str(&props.status) @@ -1588,8 +1588,8 @@ fn conclusion_from_failed( props: &RunFailedProps, timestamp: DateTime, ) -> Conclusion { - let (stages, total_retries) = billing_rollup::billing_rollup_from_projection(projection, None) - .conclusion_stages(projection); + let (stages, total_retries) = + billing_rollup::billing_rollup_from_projection(projection).conclusion_stages(projection); Conclusion { timestamp, status: StageOutcome::Failed { @@ -1759,8 +1759,8 @@ mod tests { AgentBackend, AgentControlState, AttrValue, AutomationRef, BilledModelUsage, BilledTokenCounts, BlobHash, BlockedReason, Checkpoint, CheckpointRecord, CommandTermination, EventBody, FailureCategory, FailureDetail, FailureReason, Graph, - McpServerStatus, Node, Outcome, ParallelBranchId, PendingReason, PermissionLevel, - PullRequestCreationStatus, PullRequestLink, QuestionType, ReasoningEffort, + McpServerStatus, ModelId, Node, Outcome, ParallelBranchId, PendingReason, PermissionLevel, + ProviderId, PullRequestCreationStatus, PullRequestLink, QuestionType, ReasoningEffort, RunApprovalState, RunBillingSummary, RunControlAction, RunDiff, RunEvent, RunSize, RunSpec, RunStatus, Speed, StageContextWindowBreakdownItem, StageContextWindowCategory, StageContextWindowCountMethod, StageContextWindowProjection, StageContextWindowStaleness, @@ -1812,11 +1812,11 @@ mod tests { fn llm_started() -> EventBody { EventBody::AgentLlmStarted(AgentLlmStartedProps { - requested_model: ModelRef { - provider: "anthropic".parse().unwrap(), - model_id: "claude-fable-5".into(), - speed: Some(Speed::Fast), - }, + requested_model: ModelRef::new( + ProviderId::new("anthropic"), + ModelId::new("claude-fable-5"), + ) + .with_speed(Some(Speed::Fast)), visit: 1, }) } @@ -2324,18 +2324,10 @@ mod tests { fn test_usage(model_id: &str, input_tokens: i64, output_tokens: i64) -> BilledModelUsage { serde_json::from_value(json!({ - "input": { - "usage": { - "model": { - "provider": "openai", - "model_id": model_id - }, - "tokens": { - "input_tokens": input_tokens, - "output_tokens": output_tokens - } - }, - "facts": { "algorithm": "openai" } + "model": { "provider": "openai", "model_id": model_id }, + "tokens": { + "input": input_tokens, + "output": output_tokens }, "total_usd_micros": input_tokens + output_tokens })) @@ -5388,21 +5380,13 @@ mod tests { fn billed_usage() -> BilledModelUsage { serde_json::from_value(json!({ - "input": { - "usage": { - "model": { - "provider": "openai", - "model_id": "gpt-test" - }, - "tokens": { - "input_tokens": 10, - "output_tokens": 5, - "reasoning_tokens": 2, - "cache_read_tokens": 3, - "cache_write_tokens": 4 - } - }, - "facts": { "algorithm": "openai" } + "model": { "provider": "openai", "model_id": "gpt-test" }, + "tokens": { + "input": 10, + "output": 5, + "reasoning": 2, + "cache_read": 3, + "cache_write": 4 }, "total_usd_micros": 123 })) @@ -7622,11 +7606,11 @@ mod tests { fn started() -> EventBody { EventBody::AgentLlmStarted(AgentLlmStartedProps { - requested_model: ModelRef { - provider: "anthropic".parse().unwrap(), - model_id: "claude-fable-5".into(), - speed: Some(Speed::Fast), - }, + requested_model: ModelRef::new( + ProviderId::new("anthropic"), + ModelId::new("claude-fable-5"), + ) + .with_speed(Some(Speed::Fast)), visit: 1, }) } diff --git a/lib/components/fabro-store/tests/serializable_projection.rs b/lib/components/fabro-store/tests/serializable_projection.rs index d6fdcc682..d16a0e952 100644 --- a/lib/components/fabro-store/tests/serializable_projection.rs +++ b/lib/components/fabro-store/tests/serializable_projection.rs @@ -49,18 +49,10 @@ fn sample_checkpoint() -> Checkpoint { fn sample_usage() -> BilledModelUsage { serde_json::from_value(json!({ - "input": { - "usage": { - "model": { - "provider": "openai", - "model_id": "gpt-5.2" - }, - "tokens": { - "input_tokens": 123, - "output_tokens": 45 - } - }, - "facts": { "algorithm": "openai" } + "model": { "provider": "openai", "model_id": "gpt-5.2" }, + "tokens": { + "input": 123, + "output": 45 }, "total_usd_micros": 168 })) diff --git a/lib/components/fabro-validate/Cargo.toml b/lib/components/fabro-validate/Cargo.toml index 4287fd1f1..4ffd06a2a 100644 --- a/lib/components/fabro-validate/Cargo.toml +++ b/lib/components/fabro-validate/Cargo.toml @@ -15,10 +15,11 @@ workspace = true [dependencies] fabro-acp = { path = "../fabro-acp", default-features = false } fabro-graphviz = { path = "../fabro-graphviz" } -fabro-model = { path = "../../foundation/fabro-model" } +fabro-llm = { path = "../fabro-llm" } fabro-types = { path = "../../foundation/fabro-types" } serde = { workspace = true } thiserror = { workspace = true } [dev-dependencies] +fabro-llm = { path = "../fabro-llm", features = ["test-support"] } toml = { workspace = true } diff --git a/lib/components/fabro-validate/src/lib.rs b/lib/components/fabro-validate/src/lib.rs index 3747b677c..b86bc5954 100644 --- a/lib/components/fabro-validate/src/lib.rs +++ b/lib/components/fabro-validate/src/lib.rs @@ -1,7 +1,7 @@ pub mod rules; use fabro_graphviz::graph::Graph; -use fabro_model::Catalog; +use fabro_llm::lithos_catalog::Catalog; use serde::{Deserialize, Serialize}; /// Severity level for validation diagnostics. @@ -153,8 +153,8 @@ pub fn validate_with_catalog_or_raise( mod tests { use fabro_graphviz::graph::{AttrValue, Edge, Graph, Node}; use fabro_graphviz::parser; - use fabro_model::catalog::LlmCatalogSettings; - use fabro_model::{Catalog, ProviderId}; + use fabro_llm::lithos_catalog::Catalog; + use fabro_llm::test_support::test_catalog_with_overlay; use super::*; @@ -208,35 +208,30 @@ mod tests { g } + /// An operator-defined provider layered over the built-ins, the shape an + /// `[llm]` overlay produces. fn custom_catalog() -> Catalog { - let settings: LlmCatalogSettings = toml::from_str( + test_catalog_with_overlay( r#" -[providers.venice] -display_name = "Venice" -adapter = "openai_compatible" -agent_profile = "openai" +[providers.acme-venice] +display_name = "Acme Venice" +adapter = "openai-compatible" +codec = "openai-chat" base_url = "https://api.venice.ai/api/v1" +auth = { type = "bearer" } +default_model = "venice-large" -[providers.venice.auth] +[providers.acme-venice.metadata.fabro] +agent_profile = "openai" credentials = ["env:VENICE_API_KEY"] -[models."venice-large"] -provider = "venice" +[providers.acme-venice.models.venice-large] display_name = "Venice Large" -family = "venice" -default = true - -[models."venice-large".limits] -context_window = 128000 - -[models."venice-large".features] -tools = true -vision = false -reasoning = false +api_model = "venice-large" +limits = { context_tokens = 128000, max_output_tokens = 8192 } +capabilities = { text = true, tools = true } "#, ) - .unwrap(); - Catalog::from_settings(&settings).unwrap() } #[test] @@ -377,7 +372,7 @@ reasoning = false #[test] fn validate_with_catalog_accepts_custom_catalog_entries() { - let g = graph_with_model_and_provider("venice-large", "venice"); + let g = graph_with_model_and_provider("venice-large", "acme-venice"); let catalog = custom_catalog(); let diagnostics = validate_with_catalog(&g, &catalog, &[]); @@ -406,7 +401,7 @@ reasoning = false assert!( diagnostics.iter().any(|d| d.rule == "node_model_known" && d.message.contains("missing-provider") - && d.message.contains(ProviderId::new("venice").as_str())), + && d.message.contains("acme-venice")), "missing provider diagnostic not found: {diagnostics:?}" ); } diff --git a/lib/components/fabro-validate/src/rules/mod.rs b/lib/components/fabro-validate/src/rules/mod.rs index c73657718..dc9270b87 100644 --- a/lib/components/fabro-validate/src/rules/mod.rs +++ b/lib/components/fabro-validate/src/rules/mod.rs @@ -1,3 +1,4 @@ +use fabro_llm::lithos_catalog::Catalog; mod all_conditional_edges; mod auto_status_deprecated; mod backend_valid; @@ -82,7 +83,7 @@ pub fn built_in_rules() -> Vec> { /// Returns lint rules that require the caller's resolved model catalog. #[must_use] -pub fn catalog_rules(catalog: &fabro_model::Catalog) -> Vec> { +pub fn catalog_rules(catalog: &Catalog) -> Vec> { vec![ stylesheet_model_known::rule(catalog), node_model_known::rule(catalog), diff --git a/lib/components/fabro-validate/src/rules/model_support.rs b/lib/components/fabro-validate/src/rules/model_support.rs index 7bab56a87..3c4f3544a 100644 --- a/lib/components/fabro-validate/src/rules/model_support.rs +++ b/lib/components/fabro-validate/src/rules/model_support.rs @@ -1,13 +1,16 @@ +use fabro_llm::catalog; +use fabro_llm::lithos_catalog::Catalog; + use crate::{Diagnostic, Severity}; pub(super) fn check_model_known( rule_name: &str, - catalog: &fabro_model::Catalog, + catalog: &Catalog, model: &str, context: &str, node_id: Option, ) -> Option { - if catalog.is_model_selector(model) { + if catalog::is_model_selector(catalog, model) { return None; } Some(Diagnostic { @@ -26,21 +29,17 @@ pub(super) fn check_model_known( pub(super) fn check_provider_known( rule_name: &str, - catalog: &fabro_model::Catalog, + catalog: &Catalog, provider: &str, context: &str, node_id: Option, ) -> Option { - if catalog - .provider(&fabro_model::ProviderId::new(provider)) - .is_some() - { + if catalog::is_provider_selector(catalog, provider) { return None; } - let valid: Vec<&str> = catalog - .providers() + let valid: Vec = catalog::listed_providers(catalog) .iter() - .map(|provider| provider.id.as_str()) + .map(|entry| entry.provider.id().to_string()) .collect(); let valid_str = valid.join(", "); Some(Diagnostic { diff --git a/lib/components/fabro-validate/src/rules/node_model_known.rs b/lib/components/fabro-validate/src/rules/node_model_known.rs index 64ba03eac..0cfb13841 100644 --- a/lib/components/fabro-validate/src/rules/node_model_known.rs +++ b/lib/components/fabro-validate/src/rules/node_model_known.rs @@ -1,5 +1,5 @@ use fabro_graphviz::graph::Graph; -use fabro_model::Catalog; +use fabro_llm::lithos_catalog::Catalog; use super::model_support::{check_model_known, check_provider_known}; use crate::{Diagnostic, LintRule}; @@ -48,7 +48,7 @@ impl LintRule for Rule<'_> { #[cfg(test)] mod tests { use fabro_graphviz::graph::{AttrValue, Node}; - use fabro_model::Catalog; + use fabro_llm::test_support::test_catalog; use super::Rule; use crate::rules::test_support::minimal_graph; @@ -60,12 +60,11 @@ mod tests { let mut node = Node::new("work"); node.attrs.insert( "model".to_string(), - AttrValue::String("claude-sonnet-4-5".to_string()), + AttrValue::String("claude-sonnet-4.5".to_string()), ); g.nodes.insert("work".to_string(), node); - let rule = Rule { - catalog: Catalog::builtin(), - }; + let catalog = test_catalog(); + let rule = Rule { catalog: &catalog }; let d = rule.apply(&g); assert!(d.is_empty()); } @@ -79,9 +78,8 @@ mod tests { AttrValue::String("nonexistent-model-xyz".to_string()), ); g.nodes.insert("work".to_string(), node); - let rule = Rule { - catalog: Catalog::builtin(), - }; + let catalog = test_catalog(); + let rule = Rule { catalog: &catalog }; let d = rule.apply(&g); assert_eq!(d.len(), 1); assert_eq!(d[0].severity, Severity::Warning); @@ -96,9 +94,8 @@ mod tests { node.attrs .insert("model".to_string(), AttrValue::String("opus".to_string())); g.nodes.insert("work".to_string(), node); - let rule = Rule { - catalog: Catalog::builtin(), - }; + let catalog = test_catalog(); + let rule = Rule { catalog: &catalog }; let d = rule.apply(&g); assert!(d.is_empty()); } @@ -109,25 +106,23 @@ mod tests { let mut node = Node::new("work"); node.attrs.insert( "provider".to_string(), - AttrValue::String("google".to_string()), + AttrValue::String("nonexistent-provider".to_string()), ); g.nodes.insert("work".to_string(), node); - let rule = Rule { - catalog: Catalog::builtin(), - }; + let catalog = test_catalog(); + let rule = Rule { catalog: &catalog }; let d = rule.apply(&g); assert_eq!(d.len(), 1); assert_eq!(d[0].severity, Severity::Warning); - assert!(d[0].message.contains("google")); + assert!(d[0].message.contains("nonexistent-provider")); assert_eq!(d[0].node_id.as_deref(), Some("work")); } #[test] fn node_model_known_rule_no_model_no_provider() { let g = minimal_graph(); - let rule = Rule { - catalog: Catalog::builtin(), - }; + let catalog = test_catalog(); + let rule = Rule { catalog: &catalog }; let d = rule.apply(&g); assert!(d.is_empty()); } diff --git a/lib/components/fabro-validate/src/rules/stylesheet_model_known.rs b/lib/components/fabro-validate/src/rules/stylesheet_model_known.rs index 655da0dd6..cafd16772 100644 --- a/lib/components/fabro-validate/src/rules/stylesheet_model_known.rs +++ b/lib/components/fabro-validate/src/rules/stylesheet_model_known.rs @@ -1,6 +1,6 @@ use fabro_graphviz::graph::Graph; use fabro_graphviz::stylesheet::{Selector, parse_stylesheet}; -use fabro_model::Catalog; +use fabro_llm::lithos_catalog::Catalog; use super::model_support::{check_model_known, check_provider_known}; use crate::{Diagnostic, LintRule}; @@ -77,7 +77,7 @@ impl LintRule for Rule<'_> { #[cfg(test)] mod tests { use fabro_graphviz::graph::AttrValue; - use fabro_model::Catalog; + use fabro_llm::test_support::test_catalog; use super::Rule; use crate::rules::test_support::minimal_graph; @@ -88,11 +88,10 @@ mod tests { let mut g = minimal_graph(); g.attrs.insert( "model_stylesheet".to_string(), - AttrValue::String("* { model: claude-sonnet-4-5; provider: anthropic; }".to_string()), + AttrValue::String("* { model: claude-sonnet-4.5; provider: anthropic; }".to_string()), ); - let rule = Rule { - catalog: Catalog::builtin(), - }; + let catalog = test_catalog(); + let rule = Rule { catalog: &catalog }; let d = rule.apply(&g); assert!(d.is_empty()); } @@ -104,9 +103,8 @@ mod tests { "model_stylesheet".to_string(), AttrValue::String("#opus { model: claude-opus-4-5; }".to_string()), ); - let rule = Rule { - catalog: Catalog::builtin(), - }; + let catalog = test_catalog(); + let rule = Rule { catalog: &catalog }; let d = rule.apply(&g); assert_eq!(d.len(), 1); assert_eq!(d[0].severity, Severity::Warning); @@ -119,15 +117,14 @@ mod tests { let mut g = minimal_graph(); g.attrs.insert( "model_stylesheet".to_string(), - AttrValue::String("* { provider: google; }".to_string()), + AttrValue::String("* { provider: nonexistent-provider; }".to_string()), ); - let rule = Rule { - catalog: Catalog::builtin(), - }; + let catalog = test_catalog(); + let rule = Rule { catalog: &catalog }; let d = rule.apply(&g); assert_eq!(d.len(), 1); assert_eq!(d[0].severity, Severity::Warning); - assert!(d[0].message.contains("google")); + assert!(d[0].message.contains("nonexistent-provider")); } #[test] @@ -137,9 +134,8 @@ mod tests { "model_stylesheet".to_string(), AttrValue::String("* { model: opus; }".to_string()), ); - let rule = Rule { - catalog: Catalog::builtin(), - }; + let catalog = test_catalog(); + let rule = Rule { catalog: &catalog }; let d = rule.apply(&g); assert!(d.is_empty()); } @@ -147,9 +143,8 @@ mod tests { #[test] fn stylesheet_model_known_rule_no_stylesheet() { let g = minimal_graph(); - let rule = Rule { - catalog: Catalog::builtin(), - }; + let catalog = test_catalog(); + let rule = Rule { catalog: &catalog }; let d = rule.apply(&g); assert!(d.is_empty()); } diff --git a/lib/components/fabro-workflow/Cargo.toml b/lib/components/fabro-workflow/Cargo.toml index b326fb139..388078b5b 100644 --- a/lib/components/fabro-workflow/Cargo.toml +++ b/lib/components/fabro-workflow/Cargo.toml @@ -39,7 +39,6 @@ fabro-util = { path = "../../foundation/fabro-util" } fabro-redact.workspace = true fabro-checkpoint = { path = "../fabro-checkpoint" } fabro-llm = { path = "../fabro-llm" } -fabro-model = { path = "../../foundation/fabro-model" } fabro-core = { path = "../../foundation/fabro-core" } fabro-store = { path = "../fabro-store" } fabro-static.workspace = true @@ -75,6 +74,7 @@ tempfile = "3" toml.workspace = true fabro-vault = { path = "../../foundation/fabro-vault" } [dev-dependencies] +fabro-llm = { path = "../fabro-llm", features = ["test-support"] } fabro-store = { path = "../fabro-store", features = ["test-support"] } fabro-auth = { path = "../../foundation/fabro-auth", features = ["test-support"] } fabro-github = { path = "../fabro-github", features = ["test-support"] } diff --git a/lib/components/fabro-workflow/src/billing_rollup.rs b/lib/components/fabro-workflow/src/billing_rollup.rs index 46056eace..9414a96a3 100644 --- a/lib/components/fabro-workflow/src/billing_rollup.rs +++ b/lib/components/fabro-workflow/src/billing_rollup.rs @@ -5,10 +5,9 @@ pub use fabro_types::billing_rollup::{ #[cfg(test)] mod tests { - use fabro_model::{Catalog, ModelRef, ProviderId}; use fabro_types::{ - AttrValue, BilledTokenCounts, Graph, Node, RunProjection, RunSpec, StageCompletion, - StageOutcome, first_event_seq, test_support, + AttrValue, BilledTokenCounts, Graph, ModelId, ModelRef, Node, RunProjection, RunSpec, + StageCompletion, StageOutcome, first_event_seq, provider_ids, test_support, }; use super::billing_rollup_from_projection; @@ -50,7 +49,7 @@ mod tests { timestamp: chrono::Utc::now(), }); - let rollup = billing_rollup_from_projection(&projection, None); + let rollup = billing_rollup_from_projection(&projection); assert_eq!(rollup.stages.len(), 1); assert_eq!(rollup.stages[0].node_id, "verify"); @@ -73,10 +72,10 @@ mod tests { assert_eq!(rollup.billed_visit_count, 2); assert_eq!(rollup.by_model.len(), 2); - assert_eq!(rollup.by_model[0].model.model_id, "gpt-new"); + assert_eq!(rollup.by_model[0].model.model_id.as_str(), "gpt-new"); assert_eq!(rollup.by_model[0].stages, 1); assert_eq!(rollup.by_model[0].billing.input_tokens, 200); - assert_eq!(rollup.by_model[1].model.model_id, "gpt-old"); + assert_eq!(rollup.by_model[1].model.model_id.as_str(), "gpt-old"); assert_eq!(rollup.by_model[1].stages, 1); assert_eq!(rollup.by_model[1].billing.input_tokens, 100); } @@ -93,7 +92,7 @@ mod tests { timestamp: chrono::Utc::now(), }); - let rollup = billing_rollup_from_projection(&projection, None); + let rollup = billing_rollup_from_projection(&projection); assert_eq!(rollup.stages.len(), 1); assert_eq!(rollup.stages[0].node_id, "build"); @@ -126,20 +125,16 @@ mod tests { timestamp: chrono::Utc::now(), }); - let rollup = billing_rollup_from_projection(&projection, None); + let rollup = billing_rollup_from_projection(&projection); assert_eq!(rollup.stages.len(), 0); assert_eq!(rollup.timing.wall_time_ms, 0); } #[test] - fn rollup_prices_in_flight_stage_usage_using_catalog() { + fn rollup_keeps_in_flight_stage_usage_unpriced() { let mut projection = test_projection(); - let model = ModelRef { - provider: ProviderId::openai(), - model_id: "gpt-5.4".into(), - speed: None, - }; + let model = ModelRef::new(provider_ids::openai(), ModelId::new("gpt-5.4")); let stage = projection.stage_entry("agent", 1, first_event_seq(1)); stage.started_at = Some(chrono::Utc::now()); stage.usage = BilledTokenCounts { @@ -150,22 +145,18 @@ mod tests { }; stage.model = Some(model.clone()); - let priced = billing_rollup_from_projection(&projection, Some(Catalog::builtin())); - let unpriced = billing_rollup_from_projection(&projection, None); + let rollup = billing_rollup_from_projection(&projection); - assert_eq!(priced.stages.len(), 1); - assert_eq!(priced.stages[0].node_id, "agent"); - let stage_cost = priced.stages[0].billing.total_usd_micros; - assert!( - stage_cost.is_some_and(|cost| cost > 0), - "expected priced stage cost, got {stage_cost:?}" - ); - assert_eq!(priced.totals.total_usd_micros, stage_cost); - assert_eq!(priced.by_model.len(), 1); - assert_eq!(priced.by_model[0].billing.total_usd_micros, stage_cost); - assert_eq!(unpriced.stages.len(), 1); - assert_eq!(unpriced.stages[0].billing.total_usd_micros, None); - assert_eq!(unpriced.totals.total_usd_micros, None); + // The rollup keeps the shape of what the events recorded. Costs come + // from the events themselves; an in-flight stage that has recorded no + // cost yet stays unpriced rather than being re-estimated here. + assert_eq!(rollup.stages.len(), 1); + assert_eq!(rollup.stages[0].node_id, "agent"); + assert_eq!(rollup.stages[0].billing.total_usd_micros, None); + assert_eq!(rollup.stages[0].billing.input_tokens, 500_000); + assert_eq!(rollup.totals.total_usd_micros, None); + assert_eq!(rollup.by_model.len(), 1); + assert_eq!(rollup.by_model[0].billing.input_tokens, 500_000); } fn run_spec_with_boundary_nodes() -> RunSpec { diff --git a/lib/components/fabro-workflow/src/error.rs b/lib/components/fabro-workflow/src/error.rs index 7bedbda70..d0b7eae10 100644 --- a/lib/components/fabro-workflow/src/error.rs +++ b/lib/components/fabro-workflow/src/error.rs @@ -2,8 +2,7 @@ use std::fmt; use std::sync::{Arc, LazyLock}; use fabro_graphviz::Error as GraphvizError; -use fabro_llm::{Error as LlmError, ProviderErrorKind}; -use fabro_model::ModelSelectionError; +use fabro_llm::{ErrorFacts, ErrorKind, LlmError, ModelSelectionError}; use fabro_template::TemplateError; pub use fabro_types::failure_signature::FailureSignature; pub use fabro_types::outcome::FailureCategory; @@ -19,30 +18,21 @@ use crate::outcome::{FailureDetail, Outcome, StageOutcome}; /// Classify an LLM error into a `FailureCategory` based on its structure. #[must_use] -pub fn classify_sdk_error(err: &LlmError) -> FailureCategory { - match err { - LlmError::Provider { kind, .. } => match kind { - ProviderErrorKind::RateLimit | ProviderErrorKind::Server => { - FailureCategory::TransientInfra - } - ProviderErrorKind::ContextLength | ProviderErrorKind::QuotaExceeded => { - FailureCategory::BudgetExhausted - } - ProviderErrorKind::Authentication - | ProviderErrorKind::AccessDenied - | ProviderErrorKind::NotFound - | ProviderErrorKind::InvalidRequest - | ProviderErrorKind::ContentFilter => FailureCategory::Deterministic, - }, - LlmError::RequestTimeout { .. } | LlmError::Network { .. } | LlmError::Stream { .. } => { - FailureCategory::TransientInfra - } - LlmError::Interrupt { .. } => FailureCategory::Canceled, - LlmError::InvalidToolCall { .. } - | LlmError::NoObjectGenerated { .. } - | LlmError::InvalidRequest { .. } - | LlmError::Configuration { .. } - | LlmError::UnsupportedToolChoice { .. } => FailureCategory::Deterministic, +pub fn classify_sdk_error(err: &E) -> FailureCategory { + match err.kind() { + ErrorKind::RateLimit + | ErrorKind::Server + | ErrorKind::Network + | ErrorKind::Timeout + | ErrorKind::StreamDecode => FailureCategory::TransientInfra, + ErrorKind::ContextLength | ErrorKind::QuotaExceeded => FailureCategory::BudgetExhausted, + ErrorKind::Cancelled => FailureCategory::Canceled, + // Configuration, model selection, auth, access, not-found, invalid + // request, content filter, provider, decode, resource limit, and + // middleware failures are deterministic. `ErrorKind` is + // non-exhaustive: a category added by a newer lithos never enables + // automatic retry either. + _ => FailureCategory::Deterministic, } } @@ -534,7 +524,7 @@ impl Error { Self::Stage { stage, .. } => { matches!(stage, ErrorStage::Handler | ErrorStage::Engine) } - Self::Llm(sdk_err) => sdk_err.retryable(), + Self::Llm(sdk_err) => sdk_err.is_retryable(), Self::Parse(_) | Self::Validation(_) | Self::ValidationFailed { .. } @@ -700,6 +690,12 @@ impl From for Error { } } +impl From for Error { + fn from(err: fabro_llm::Error) -> Self { + Self::Llm(LlmError::from(err)) + } +} + impl From for Error { fn from(e: GraphvizError) -> Self { match e { @@ -748,9 +744,25 @@ pub type Result = std::result::Result; #[cfg(test)] mod tests { use fabro_checkpoint::MetadataError; - use fabro_llm::{Error as SdkError, ProviderErrorDetail}; + use fabro_llm::RetryClassification; use super::*; + + /// A stored LLM error of `kind` from the `openai` provider. + fn sdk_error(kind: ErrorKind, message: &str) -> LlmError { + LlmError::from( + fabro_llm::Error::new(kind, message).with_provider(fabro_types::provider_ids::openai()), + ) + } + + /// A transient failure the provider may be asked to repeat. + fn transient_error(kind: ErrorKind, message: &str) -> LlmError { + LlmError::from( + fabro_llm::Error::new(kind, message) + .with_provider(fabro_types::provider_ids::openai()) + .with_retry(RetryClassification::Safe), + ) + } use crate::outcome::OutcomeExt; #[derive(Debug)] @@ -1144,38 +1156,23 @@ mod tests { #[test] fn llm_error_display() { - let sdk_err = SdkError::Network { - message: "connection refused".into(), - source: None, - }; + let sdk_err = transient_error(ErrorKind::Network, "connection refused"); let err = Error::Llm(sdk_err); - assert_eq!( - err.to_string(), - "LLM error: Network error: connection refused" - ); + assert_eq!(err.to_string(), "LLM error: connection refused"); } #[test] fn llm_error_retryable_delegates_to_sdk() { - let retryable = Error::Llm(SdkError::Network { - message: "timeout".into(), - source: None, - }); + let retryable = Error::Llm(transient_error(ErrorKind::Network, "timeout")); assert!(retryable.is_retryable()); - let non_retryable = Error::Llm(SdkError::Configuration { - message: "bad config".into(), - source: None, - }); + let non_retryable = Error::Llm(sdk_error(ErrorKind::Configuration, "bad config")); assert!(!non_retryable.is_retryable()); } #[test] fn llm_error_from_sdk_error() { - let sdk_err = SdkError::Stream { - message: "broken pipe".into(), - source: None, - }; + let sdk_err = transient_error(ErrorKind::StreamDecode, "broken pipe"); let err = Error::from(sdk_err); assert!(matches!(err, Error::Llm(_))); } @@ -1224,45 +1221,31 @@ mod tests { #[test] fn failure_class_llm_rate_limit() { - let err = Error::Llm(SdkError::Provider { - kind: ProviderErrorKind::RateLimit, - detail: Box::new(ProviderErrorDetail::new("too fast", "openai")), - }); + let err = Error::Llm(transient_error(ErrorKind::RateLimit, "too fast")); assert_eq!(err.failure_category(), FailureCategory::TransientInfra); } #[test] fn failure_class_llm_context_length() { - let err = Error::Llm(SdkError::Provider { - kind: ProviderErrorKind::ContextLength, - detail: Box::new(ProviderErrorDetail::new("too long", "openai")), - }); + let err = Error::Llm(sdk_error(ErrorKind::ContextLength, "too long")); assert_eq!(err.failure_category(), FailureCategory::BudgetExhausted); } #[test] fn failure_class_llm_auth() { - let err = Error::Llm(SdkError::Provider { - kind: ProviderErrorKind::Authentication, - detail: Box::new(ProviderErrorDetail::new("bad key", "openai")), - }); + let err = Error::Llm(sdk_error(ErrorKind::Authentication, "bad key")); assert_eq!(err.failure_category(), FailureCategory::Deterministic); } #[test] fn failure_class_llm_abort() { - let err = Error::Llm(SdkError::Interrupt { - message: "user cancelled".into(), - }); + let err = Error::Llm(sdk_error(ErrorKind::Cancelled, "user cancelled")); assert_eq!(err.failure_category(), FailureCategory::Canceled); } #[test] fn failure_class_llm_timeout() { - let err = Error::Llm(SdkError::RequestTimeout { - message: "timed out".into(), - source: None, - }); + let err = Error::Llm(transient_error(ErrorKind::Timeout, "timed out")); assert_eq!(err.failure_category(), FailureCategory::TransientInfra); } @@ -1270,79 +1253,55 @@ mod tests { #[test] fn classify_sdk_rate_limit() { - let err = SdkError::Provider { - kind: ProviderErrorKind::RateLimit, - detail: Box::new(ProviderErrorDetail::new("too fast", "openai")), - }; + let err = transient_error(ErrorKind::RateLimit, "too fast"); assert_eq!(classify_sdk_error(&err), FailureCategory::TransientInfra); } #[test] fn classify_sdk_server() { - let err = SdkError::Provider { - kind: ProviderErrorKind::Server, - detail: Box::new(ProviderErrorDetail::new("500", "openai")), - }; + let err = transient_error(ErrorKind::Server, "500"); assert_eq!(classify_sdk_error(&err), FailureCategory::TransientInfra); } #[test] fn classify_sdk_context_length() { - let err = SdkError::Provider { - kind: ProviderErrorKind::ContextLength, - detail: Box::new(ProviderErrorDetail::new("too long", "openai")), - }; + let err = sdk_error(ErrorKind::ContextLength, "too long"); assert_eq!(classify_sdk_error(&err), FailureCategory::BudgetExhausted); } #[test] fn classify_sdk_quota_exceeded() { - let err = SdkError::Provider { - kind: ProviderErrorKind::QuotaExceeded, - detail: Box::new(ProviderErrorDetail::new("out of quota", "openai")), - }; + let err = sdk_error(ErrorKind::QuotaExceeded, "out of quota"); assert_eq!(classify_sdk_error(&err), FailureCategory::BudgetExhausted); } #[test] fn classify_sdk_auth() { - let err = SdkError::Provider { - kind: ProviderErrorKind::Authentication, - detail: Box::new(ProviderErrorDetail::new("bad key", "openai")), - }; + let err = sdk_error(ErrorKind::Authentication, "bad key"); assert_eq!(classify_sdk_error(&err), FailureCategory::Deterministic); } #[test] fn classify_sdk_request_timeout() { - let err = SdkError::RequestTimeout { - message: "timed out".into(), - source: None, - }; + let err = transient_error(ErrorKind::Timeout, "timed out"); assert_eq!(classify_sdk_error(&err), FailureCategory::TransientInfra); } #[test] fn classify_sdk_abort() { - let err = SdkError::Interrupt { - message: "cancelled".into(), - }; + let err = sdk_error(ErrorKind::Cancelled, "cancelled"); assert_eq!(classify_sdk_error(&err), FailureCategory::Canceled); } #[test] fn classify_sdk_invalid_tool_call() { - let err = SdkError::InvalidToolCall { - message: "bad tool".into(), - }; + let err = sdk_error(ErrorKind::InvalidRequest, "bad tool"); assert_eq!(classify_sdk_error(&err), FailureCategory::Deterministic); } #[test] fn classify_sdk_invalid_request() { - let err = SdkError::InvalidRequest { - message: "unsupported reasoning effort".into(), - }; + let err = sdk_error(ErrorKind::InvalidRequest, "unsupported reasoning effort"); assert_eq!(classify_sdk_error(&err), FailureCategory::Deterministic); } @@ -1982,10 +1941,7 @@ mod tests { #[test] fn failure_signature_hint_llm_returns_some() { - let err = Error::Llm(SdkError::Provider { - kind: ProviderErrorKind::Authentication, - detail: Box::new(ProviderErrorDetail::new("bad key", "openai")), - }); + let err = Error::Llm(sdk_error(ErrorKind::Authentication, "bad key")); assert_eq!( err.failure_signature_hint(), Some(FailureSignature( @@ -2010,10 +1966,7 @@ mod tests { #[test] fn to_fail_outcome_llm_has_class_and_signature() { - let err = Error::Llm(SdkError::Provider { - kind: ProviderErrorKind::Authentication, - detail: Box::new(ProviderErrorDetail::new("bad key", "openai")), - }); + let err = Error::Llm(sdk_error(ErrorKind::Authentication, "bad key")); let outcome = err.to_fail_outcome(); assert_eq!(outcome.status, crate::outcome::StageOutcome::Failed { retry_requested: false, @@ -2040,10 +1993,7 @@ mod tests { #[test] fn to_fail_outcome_includes_error_message_as_reason() { - let err = Error::Llm(SdkError::Network { - message: "connection refused".into(), - source: None, - }); + let err = Error::Llm(transient_error(ErrorKind::Network, "connection refused")); let outcome = err.to_fail_outcome(); assert!( outcome @@ -2055,10 +2005,7 @@ mod tests { #[test] fn to_fail_outcome_no_context_updates() { - let err = Error::Llm(SdkError::Network { - message: "refused".into(), - source: None, - }); + let err = Error::Llm(transient_error(ErrorKind::Network, "refused")); let outcome = err.to_fail_outcome(); assert!(outcome.context_updates.is_empty()); } @@ -2110,10 +2057,7 @@ mod tests { Error::engine("engine err"), Error::publish("publish err"), Error::handler("handler err"), - Error::Llm(SdkError::Network { - message: "refused".into(), - source: None, - }), + Error::Llm(transient_error(ErrorKind::Network, "refused")), Error::Checkpoint("cp err".into()), Error::Stylesheet("style err".into()), Error::Io("io err".into()), @@ -2220,10 +2164,7 @@ mod tests { use crate::event::Event; // 1. Create SdkError → Error - let sdk_err = SdkError::Provider { - kind: ProviderErrorKind::RateLimit, - detail: Box::new(ProviderErrorDetail::new("too fast", "openai")), - }; + let sdk_err = transient_error(ErrorKind::RateLimit, "too fast"); let arc_err = Error::Llm(sdk_err); assert_eq!(arc_err.failure_category(), FailureCategory::TransientInfra); @@ -2295,10 +2236,7 @@ mod tests { fn e2e_serde_stability_agent_error() { use fabro_agent::Error as AgentError; - let err = AgentError::Llm(SdkError::Provider { - kind: ProviderErrorKind::RateLimit, - detail: Box::new(ProviderErrorDetail::new("too fast", "openai")), - }); + let err = AgentError::Llm(transient_error(ErrorKind::RateLimit, "too fast")); let json = serde_json::to_string(&err).unwrap(); let v: serde_json::Value = serde_json::from_str(&json).unwrap(); assert_eq!(v["type"], "llm"); diff --git a/lib/components/fabro-workflow/src/event/convert.rs b/lib/components/fabro-workflow/src/event/convert.rs index f89d53bf6..da235b8ac 100644 --- a/lib/components/fabro-workflow/src/event/convert.rs +++ b/lib/components/fabro-workflow/src/event/convert.rs @@ -1,9 +1,8 @@ use ::fabro_types::{ - EventBody, RunControlAction, RunEvent, RunId, StageOutcome, run_event as fabro_types, + EventBody, RunControlAction, RunEvent, RunId, StageOutcome, UsdMicros, run_event as fabro_types, }; use chrono::Utc; use fabro_agent::{AgentEvent, SandboxEvent, SkillActivationSource}; -use fabro_model::UsdMicros; use uuid::Uuid; use super::Event; @@ -658,19 +657,18 @@ fn event_body_from_event(event: &Event) -> EventBody { text, model, usage, - cost_usd, - cost_source, + cost, tool_call_count, context_window, reasoning, } => { - let billing = billed_token_counts_from_llm(usage) - .with_reported_cost(cost_usd.map(UsdMicros::from_usd)); + let billing = billed_token_counts_from_llm(*usage) + .with_reported_cost(cost.as_ref().map(UsdMicros::from_cost)); EventBody::AgentMessage(fabro_types::AgentMessageProps { text: text.clone(), model: model.clone(), billing, - cost_source: *cost_source, + cost_source: cost.map(|cost| cost.source), tool_call_count: *tool_call_count, visit: *visit, message: None, @@ -1463,17 +1461,16 @@ mod tests { use std::collections::BTreeMap; use ::fabro_types::{ - AutomationRef, EventBody, FailureReason, ParallelBranchId, Principal, RunNoticeCode, - RunNoticeLevel, RunProvenance, StageId, SystemActorKind, fixtures, - run_event as fabro_types, test_support, + AutomationRef, EventBody, FailureReason, ModelId, ModelRef, ParallelBranchId, Principal, + ProviderId, RunNoticeCode, RunNoticeLevel, RunProvenance, StageId, SystemActorKind, + TokenCounts as LlmTokenCounts, fixtures, provider_ids, run_event as fabro_types, + test_support, }; use chrono::Utc; use fabro_agent::{ AgentEvent, McpToolSummary, MemoryFileSummary, SandboxEvent, SkillActivationSource, SkillSummary, }; - use fabro_llm::types::TokenCounts as LlmTokenCounts; - use fabro_model::{ModelRef, ProviderId}; use super::*; use crate::error::Error; @@ -2525,14 +2522,12 @@ mod tests { visit: 1, event: AgentEvent::AssistantMessage { text: "ok".to_string(), - model: ModelRef { - provider: ProviderId::anthropic(), - model_id: "claude-sonnet".into(), - speed: None, - }, + model: ModelRef::new( + provider_ids::anthropic(), + ModelId::new("claude-sonnet"), + ), usage: LlmTokenCounts::default(), - cost_usd: None, - cost_source: None, + cost: None, tool_call_count: 0, context_window: None, reasoning: None, @@ -2556,18 +2551,16 @@ mod tests { visit: 1, event: AgentEvent::AssistantMessage { text: "ok".to_string(), - model: ModelRef { - provider: ProviderId::new("custom_proxy"), - model_id: "proxy-model".into(), - speed: None, - }, + model: ModelRef::new( + ProviderId::new("custom_proxy"), + ModelId::new("proxy-model"), + ), usage: LlmTokenCounts { - input_tokens: 12, - output_tokens: 34, + input: 12, + output: 34, ..LlmTokenCounts::default() }, - cost_usd: None, - cost_source: None, + cost: None, tool_call_count: 0, context_window: None, reasoning: None, @@ -2581,7 +2574,7 @@ mod tests { panic!("expected agent message body"); }; assert_eq!(message.model.provider, ProviderId::new("custom_proxy")); - assert_eq!(message.model.model_id, "proxy-model"); + assert_eq!(message.model.model_id.as_str(), "proxy-model"); assert_eq!(message.billing.input_tokens, 12); assert_eq!(message.billing.output_tokens, 34); assert_eq!(message.billing.total_usd_micros, None); @@ -2594,18 +2587,20 @@ mod tests { visit: 1, event: AgentEvent::AssistantMessage { text: "ok".to_string(), - model: ModelRef { - provider: ProviderId::new("openrouter"), - model_id: "openai/gpt-5.4".into(), - speed: None, - }, + model: ModelRef::new( + ProviderId::new("openrouter"), + ModelId::new("openai/gpt-5.4"), + ), usage: LlmTokenCounts { - input_tokens: 12, - output_tokens: 34, + input: 12, + output: 34, ..LlmTokenCounts::default() }, - cost_usd: Some(0.125), - cost_source: Some(fabro_model::CostSource::Authoritative), + cost: Some(::fabro_types::Cost { + usd_micros: 125_000, + + source: ::fabro_types::CostSource::Provider, + }), tool_call_count: 0, context_window: None, reasoning: None, @@ -2621,7 +2616,7 @@ mod tests { assert_eq!(message.billing.total_usd_micros, Some(125_000)); assert_eq!( message.cost_source, - Some(fabro_model::CostSource::Authoritative) + Some(::fabro_types::CostSource::Provider) ); } @@ -2649,14 +2644,9 @@ mod tests { visit: 1, event: AgentEvent::AssistantMessage { text: "ok".to_string(), - model: ModelRef { - provider: ProviderId::openai(), - model_id: "gpt-5.4".into(), - speed: None, - }, + model: ModelRef::new(provider_ids::openai(), ModelId::new("gpt-5.4")), usage: LlmTokenCounts::default(), - cost_usd: None, - cost_source: None, + cost: None, tool_call_count: 0, context_window: Some(context_window), reasoning: None, @@ -2684,14 +2674,9 @@ mod tests { visit: 1, event: AgentEvent::AssistantMessage { text: String::new(), - model: ModelRef { - provider: ProviderId::openai(), - model_id: "gpt-5.4".into(), - speed: None, - }, + model: ModelRef::new(provider_ids::openai(), ModelId::new("gpt-5.4")), usage: LlmTokenCounts::default(), - cost_usd: None, - cost_source: None, + cost: None, tool_call_count: 1, context_window: None, reasoning: Some(::fabro_types::ReasoningOutput::new( diff --git a/lib/components/fabro-workflow/src/event/events.rs b/lib/components/fabro-workflow/src/event/events.rs index f23bc5c35..f8adcb0ca 100644 --- a/lib/components/fabro-workflow/src/event/events.rs +++ b/lib/components/fabro-workflow/src/event/events.rs @@ -4,13 +4,12 @@ use ::fabro_types::{ AutomationRef, BilledTokenCounts, BlobHash, BlockedReason, CommandTermination, DiffSummary, FailureReason, ForkSourceRef, GitContext, PairId, PairMessageId, PairSystemMessageKind, PairTarget, ParallelBranchId, ParallelBranchResult, PendingReason, PermissionLevel, Principal, - PullRequestCreationId, PullRequestLink, ReviewTarget, RunFailure, RunId, RunNoticeLevel, - RunPairEndedReason, RunPairFailedReason, RunProvenance, RunRunnableSource, RunTarget, - RunTiming, SandboxProviderKind, StageId, StageOutcome, StageTiming, SuccessReason, - WorkflowVersionId, run_event as fabro_types, + PullRequestCreationId, PullRequestLink, ReasoningEffort, ReviewTarget, RunFailure, RunId, + RunNoticeLevel, RunPairEndedReason, RunPairFailedReason, RunProvenance, RunRunnableSource, + RunTarget, RunTiming, SandboxProviderKind, Speed, StageId, StageOutcome, StageTiming, + SuccessReason, WorkflowVersionId, run_event as fabro_types, }; use fabro_agent::{AgentEvent, SandboxEvent}; -use fabro_model::{ReasoningEffort, Speed}; use serde::{Deserialize, Serialize}; use crate::error::{Error, run_failure_from_error}; diff --git a/lib/components/fabro-workflow/src/event/redaction.rs b/lib/components/fabro-workflow/src/event/redaction.rs index 339f1edd3..9cfe29ee7 100644 --- a/lib/components/fabro-workflow/src/event/redaction.rs +++ b/lib/components/fabro-workflow/src/event/redaction.rs @@ -30,10 +30,11 @@ pub fn event_payload_from_redacted_json(line: &str, run_id: &RunId) -> Result RegisteredTool { let name = definition.name.to_string(); RegisteredTool { - definition: LlmToolDefinition { - name: name.clone(), - description: definition.description.to_string(), - parameters: definition.parameters.clone(), - }, + definition: LlmToolDefinition::function( + name.clone(), + definition.description.to_string(), + definition.parameters.clone(), + ), executor: Arc::new(move |args, _context: ToolContext| { let name = name.clone(); let services = services.clone(); @@ -394,40 +392,41 @@ fn control_attr<'a>(node: &'a Node, key: &str) -> Option<&'a str> { } fn parse_reasoning_effort(node: &Node, value: &str) -> Result { - value.parse::().map_err(|source| { - Error::handler_with_source( - format!( - "Invalid reasoning_effort \"{value}\" for node \"{}\"; expected one of: {}", - node.id, - expected_values(ReasoningEffort::variants()), + controls::parse_reasoning_effort(value).ok_or_else(|| { + Error::handler(format!( + "Invalid reasoning_effort \"{value}\" for node \"{}\"; expected one of: {}", + node.id, + expected_values( + controls::REASONING_EFFORTS + .iter() + .map(|effort| controls::reasoning_effort_name(*effort)) ), - source, - ) + )) }) } fn parse_speed(node: &Node, value: &str) -> Result { - value.parse::().map_err(|source| { - Error::handler_with_source( - format!( - "Invalid speed \"{value}\" for node \"{}\"; expected one of: {}", - node.id, - expected_values(Speed::variants()), + controls::parse_speed(value).ok_or_else(|| { + Error::handler(format!( + "Invalid speed \"{value}\" for node \"{}\"; expected one of: {}", + node.id, + expected_values( + controls::SPEEDS + .iter() + .map(|speed| controls::speed_name(*speed)) ), - source, - ) + )) }) } -fn expected_values(values: &[T]) -> String -where - T: ToString, -{ - values - .iter() - .map(ToString::to_string) - .collect::>() - .join(", ") +fn expected_values<'a>(values: impl Iterator) -> String { + values.collect::>().join(", ") +} + +/// Node-level `max_tokens`, as the client's `u32` output budget. +fn node_max_output_tokens(node: &Node) -> Option { + node.max_tokens() + .and_then(|tokens| u32::try_from(tokens).ok()) } /// Shared state for tracking file modifications from agent tool calls. @@ -760,7 +759,7 @@ impl LiveAgentInvocation { error: fabro_agent::Error, allow_failover: bool, emitter: &Arc, - ) -> Result { + ) -> Result { let disposition = classify_agent_error(error, allow_failover); self.abort_and_discard(emitter).await; match disposition { @@ -778,7 +777,7 @@ impl LiveAgentInvocation { async fn record_input_usage(&mut self) { self.event_forwarder.wait_for_processing_end().await; - self.total_usage += self.session.last_input_usage(); + billing::add_usage(&mut self.total_usage, self.session.last_input_usage()); UsdMicros::accumulate(&mut self.total_cost, self.session.last_input_cost()); } } @@ -792,7 +791,7 @@ impl AgentApiBackend { source: Arc, steering_hub: Arc, ) -> Self { - let catalog = Arc::new(Catalog::from_builtin().expect("default catalog should build")); + let catalog = Arc::new(fabro_llm::default_catalog()); Self::new_with_catalog( model, provider_id.into(), @@ -893,22 +892,33 @@ impl AgentApiBackend { let Some(requested_effort) = requested.reasoning_effort else { return FallbackControls::Usable(requested); }; - let Some(offering) = self - .catalog - .get_on_provider(&target.provider, target.model.as_str()) - else { + let Some(offering) = catalog::model_on_provider( + &self.catalog, + target.provider.as_str(), + target.model.as_str(), + ) else { // A catalog-unknown passthrough target has no advertised controls. // Preserve the request and let the provider validate it. return FallbackControls::Usable(requested); }; - let effective_effort = self.catalog.settings_for(offering).and_then(|settings| { - requested_effort.closest_supported(&settings.controls.reasoning_effort) + let capabilities = offering.model.capabilities(); + let effective_effort = controls::closest_supported_effort(requested_effort, |effort| { + capabilities.reasoning_effort(effort).is_supported() }); match effective_effort { Some(effort) => FallbackControls::Usable(EffectiveRequestControls { reasoning_effort: Some(effort), speed: requested.speed, }), + // No level is verified. Unless the requested one is verified + // unsupported, preserve it and let the provider validate, as for + // a passthrough target. + None if !capabilities + .reasoning_effort(requested_effort) + .is_unsupported() => + { + FallbackControls::Usable(requested) + } None => FallbackControls::NoNearbyReasoningLevel(requested_effort), } } @@ -919,7 +929,7 @@ impl AgentApiBackend { provider: &ProviderId, requested_controls: EffectiveRequestControls, ) -> (FallbackPlan, Vec) { - let primary_model = self.catalog.canonical_model_id(provider, model); + let primary_model = catalog::canonical_model_id(&self.catalog, provider, model); let original = LlmRoute { target: FallbackTarget::new(provider, &primary_model), controls: requested_controls, @@ -1021,7 +1031,7 @@ impl AgentApiBackend { route.controls, node, sandbox, - self.source.as_ref(), + Arc::clone(&self.source), Arc::clone(&self.catalog), self.tool_env.as_ref(), tool_hooks, @@ -1045,7 +1055,7 @@ impl AgentApiBackend { controls: EffectiveRequestControls, node: &Node, sandbox: &Arc, - source: &dyn CredentialSource, + source: Arc, catalog: Arc, tool_env: Option<&Arc>, tool_hooks: Option>, @@ -1053,9 +1063,7 @@ impl AgentApiBackend { tool_secrets: ToolSecrets, fabro_run_tools: Option, ) -> Result { - let client = Client::from_source(source, Arc::clone(&catalog)) - .await - .map_err(|e| Error::handler_with_source("Failed to create LLM client", e))?; + let client = build_llm_client(&catalog, source).await?; let profile_builder = AgentProfileBuilder::new( provider.profile_kind, @@ -1067,10 +1075,7 @@ impl AgentApiBackend { let profile_builder = if provider.profile_kind == AgentProfileKind::Claude5 { profile_builder.with_web_fetch_summarizer(Some(WebFetchSummarizer { client: client.clone(), - model_id: ModelHandle::ByName { - provider: provider.provider_id.clone(), - model: model.to_string(), - }, + model_id: ModelHandle::new(provider.provider_id.clone(), ModelId::new(model)), })) } else { profile_builder @@ -1078,7 +1083,7 @@ impl AgentApiBackend { let mut profile = profile_builder.build(); let config = SessionOptions { - max_tokens: node.max_tokens(), + max_tokens: node_max_output_tokens(node), reasoning_effort: controls.reasoning_effort, speed: controls.speed, tool_hooks, @@ -1196,7 +1201,7 @@ impl AgentApiBackend { async fn failover_agent_session( &self, fallback_plan: &mut FallbackPlan, - initial_error: fabro_llm::Error, + initial_error: LlmError, request: &CodergenRunRequest<'_>, input: &str, stage_scope: &StageScope, @@ -1236,7 +1241,7 @@ impl AgentApiBackend { route.controls, request.node, request.sandbox, - self.source.as_ref(), + Arc::clone(&self.source), Arc::clone(&self.catalog), self.tool_env.as_ref(), request.tool_hooks.clone(), @@ -1368,11 +1373,15 @@ impl AgentApiBackend { ); } - fn route_max_tokens(&self, node: &Node, route: &LlmRoute) -> Option { - node.max_tokens().or_else(|| { - self.catalog - .get_on_provider(&route.target.provider, route.target.model.as_str()) - .and_then(|model| model.limits.max_output) + fn route_max_tokens(&self, node: &Node, route: &LlmRoute) -> Option { + node_max_output_tokens(node).or_else(|| { + catalog::model_on_provider( + &self.catalog, + route.target.provider.as_str(), + route.target.model.as_str(), + ) + .and_then(|entry| entry.model.limits()) + .map(|limits| u32::try_from(limits.max_output_tokens).unwrap_or(u32::MAX)) }) } @@ -1383,23 +1392,27 @@ impl AgentApiBackend { route: &LlmRoute, messages: Vec, response_format: Option, - ) -> Request { - Request { - model: route.target.model.to_string(), - messages, - provider: Some(route.target.provider.to_string()), - tools: None, - tool_choice: None, - response_format, - temperature: None, - top_p: None, - max_tokens: self.route_max_tokens(node, route), - stop_sequences: None, - reasoning_effort: route.controls.reasoning_effort, - speed: route.controls.speed, - metadata: None, - provider_options: None, + ) -> Result { + let mut builder = + Request::builder().model(format!("{}/{}", route.target.provider, route.target.model)); + for message in messages { + builder = builder.message(message); } + if let Some(format) = response_format { + builder = builder.response_format(format); + } + if let Some(max_tokens) = self.route_max_tokens(node, route) { + builder = builder.max_output_tokens(max_tokens); + } + if let Some(effort) = route.controls.reasoning_effort { + builder = builder.reasoning_effort(effort); + } + if let Some(speed) = route.controls.speed { + builder = builder.speed(speed); + } + builder + .build() + .map_err(|err| Error::handler(format!("invalid LLM request: {err}"))) } async fn complete_one_shot_request( @@ -1412,35 +1425,46 @@ impl AgentApiBackend { plan: &mut FallbackPlan, ) -> Result { loop { - match client.complete(&request).await { + match client.complete(request.clone()).await { Ok(response) => { let route = plan.current(); return Ok(OneShotCompletion { response, - model: ModelRef { - provider: route.target.provider.clone(), - model_id: route.target.model.clone(), - speed: route.controls.speed, - }, + model: ModelRef::new( + route.target.provider.clone(), + route.target.model.clone(), + ) + .with_speed(route.controls.speed), }); } - Err(error) if error.failover_eligible() && plan.has_next() => { + Err(error) if failover_eligible(&error) && plan.has_next() => { let error_message = error.to_string(); plan.advance(); Self::emit_failover(node, emitter, stage_scope, plan, &error_message); request = self.route_request( node, plan.current(), - request.messages, - request.response_format, - ); + request.messages().to_vec(), + request.response_format().cloned(), + )?; } - Err(error) => return Err(Error::Llm(error)), + Err(error) => return Err(Error::Llm(LlmError::from(error))), } } } } +/// Build the LLM client a stage session dispatches through. +async fn build_llm_client( + catalog: &Arc, + source: Arc, +) -> Result { + fabro_llm::build_client(Catalog::clone(catalog), source, ClientOptions::standard()) + .await + .map(|built| built.client) + .map_err(|e| Error::handler_with_source("Failed to create LLM client", e)) +} + #[async_trait] impl CodergenBackend for AgentApiBackend { async fn shutdown(&self, emitter: &Arc) { @@ -1458,9 +1482,7 @@ impl CodergenBackend for AgentApiBackend { let emitter = request.emitter; let stage_scope = request.stage_scope; - let client = Client::from_source(self.source.as_ref(), Arc::clone(&self.catalog)) - .await - .map_err(|e| Error::handler_with_source("Failed to create LLM client", e))?; + let client = build_llm_client(&self.catalog, Arc::clone(&self.source)).await?; let model = node.model().unwrap_or(&self.model); let provider = self.resolve_provider_context(model, node.provider())?; @@ -1471,9 +1493,9 @@ impl CodergenBackend for AgentApiBackend { let mut messages = Vec::new(); if let Some(sys) = system_prompt { - messages.push(Message::system(sys)); + messages.push(Message::text(Role::System, sys)); } - messages.push(Message::user(prompt)); + messages.push(Message::text(Role::User, prompt)); let output_schema = structured_output::parse_node_output_schema(node)?; let response_format = output_schema @@ -1491,7 +1513,7 @@ impl CodergenBackend for AgentApiBackend { fallback_plan.current(), messages.clone(), response_format.clone(), - ); + )?; let inference_start = Instant::now(); let completion_result = self @@ -1506,10 +1528,10 @@ impl CodergenBackend for AgentApiBackend { .await; inference_duration = inference_duration.saturating_add(inference_start.elapsed()); let completion = completion_result?; - total_usage += completion.response.usage.clone(); + billing::add_usage(&mut total_usage, completion.response.usage); UsdMicros::accumulate( &mut total_cost, - completion.response.cost_usd.map(UsdMicros::from_usd), + completion.response.cost.as_ref().map(UsdMicros::from_cost), ); let response_text = completion.response.text(); @@ -1531,18 +1553,15 @@ impl CodergenBackend for AgentApiBackend { let repair_message = error.repair_message(schema, previous_validation_error.as_ref()); previous_validation_error = Some(error); - messages.push(Message::assistant(response_text)); - messages.push(Message::user(repair_message)); + messages.push(Message::text(Role::Assistant, response_text)); + messages.push(Message::text(Role::User, repair_message)); repair_attempts += 1; continue; } - let stage_usage = billed_model_usage_from_llm( - self.catalog.as_ref(), - &completion.model, - &total_usage, - )? - .with_reported_cost(total_cost); + let stage_usage = + billed_model_usage_from_llm(self.catalog.as_ref(), &completion.model, total_usage)? + .with_reported_cost(total_cost); return Ok(CodergenResult::Text { text: response_text, @@ -1570,8 +1589,8 @@ impl CodergenBackend for AgentApiBackend { }; // Take a cached session if reusing, otherwise create a new one. Cancel - // checks bracket `Client::from_source(...)` so cancellation arriving - // during credential refresh is not lost. + // checks bracket the client build so cancellation arriving during + // credential refresh is not lost. if request.cancel_token.is_cancelled() { return Err(Error::Cancelled); } @@ -1784,12 +1803,12 @@ impl CodergenBackend for AgentApiBackend { let stage_usage = billed_model_usage_from_llm( self.catalog.as_ref(), - &ModelRef { - provider: live.session.provider_id(), - model_id: live.session.model().into(), - speed: live.session.speed(), - }, - &live.total_usage, + &ModelRef::new( + live.session.provider_id(), + ModelId::new(live.session.model()), + ) + .with_speed(live.session.speed()), + live.total_usage, )? .with_reported_cost(live.total_cost); @@ -1883,13 +1902,15 @@ mod tests { use fabro_agent::{AgentProfile, LocalSandbox, ToolRegistry}; use fabro_api::types; use fabro_auth::{VaultCredentialSource, test_support as auth_test_support}; - use fabro_llm::provider::{ProviderAdapter, StreamEventStream}; - use fabro_llm::{Error as LlmError, ProviderErrorDetail, ProviderErrorKind}; + use fabro_llm::adapter::{ProviderAdapter, ResolvedCall}; + use fabro_llm::lithos_catalog::AdapterId; + use fabro_llm::test_support::{client_with_adapters, test_catalog, test_catalog_with_overlay}; + use fabro_llm::{ErrorKind, ResponseStream, RetryClassification}; use fabro_tool::FabroToolBackend; use fabro_types::{ - EventEnvelope, FailureReason, Run, RunId, RunLifecycle, RunLinks, RunOrigin, + ContentPart, EventEnvelope, FailureReason, Run, RunId, RunLifecycle, RunLinks, RunOrigin, RunPairStatusResponse, RunProjection, RunStatus, RunTimestamps, SuccessReason, WorkflowRef, - test_support, + provider_ids, test_support, }; use fabro_vault::{SecretType, Vault}; use futures::stream; @@ -1920,7 +1941,7 @@ mod tests { } fn provider_id(&self) -> ProviderId { - ProviderId::openai() + provider_ids::openai() } fn model(&self) -> &str { @@ -1947,129 +1968,143 @@ mod tests { } } - struct ShutdownTestProvider; + struct ShutdownTestProvider { + id: AdapterId, + } - #[async_trait] - impl ProviderAdapter for ShutdownTestProvider { - fn name(&self) -> &str { - "openai" - } - - async fn complete( - &self, - _request: &Request, - ) -> Result { - unreachable!("shutdown test never calls LLM completion") - } - - async fn stream(&self, _request: &Request) -> Result { - Ok(Box::pin(stream::empty())) + impl ShutdownTestProvider { + fn new() -> Self { + Self { + id: AdapterId::new("mock"), + } } } - struct RefusalTestProvider; + #[async_trait] + impl ProviderAdapter for ShutdownTestProvider { + fn id(&self) -> &AdapterId { + &self.id + } + + async fn complete(&self, _call: &ResolvedCall) -> Result { + unreachable!("shutdown test never calls LLM completion") + } + + async fn stream(&self, _call: &ResolvedCall) -> Result { + Ok(ResponseStream::new(stream::empty())) + } + } + + struct RefusalTestProvider { + id: AdapterId, + } + + impl RefusalTestProvider { + fn new() -> Self { + Self { + id: AdapterId::new("mock"), + } + } + } #[async_trait] impl ProviderAdapter for RefusalTestProvider { - fn name(&self) -> &str { - "anthropic" + fn id(&self) -> &AdapterId { + &self.id } - async fn complete( - &self, - _request: &Request, - ) -> Result { + async fn complete(&self, _call: &ResolvedCall) -> Result { Err(refusal_llm_error()) } - async fn stream(&self, _request: &Request) -> Result { - Ok(Box::pin(stream::empty())) + async fn stream(&self, _call: &ResolvedCall) -> Result { + Ok(ResponseStream::new(stream::empty())) } } struct TextTestProvider { - provider: &'static str, - text: &'static str, + id: AdapterId, + text: &'static str, + } + + impl TextTestProvider { + fn new(text: &'static str) -> Self { + Self { + id: AdapterId::new("mock"), + text, + } + } } #[async_trait] impl ProviderAdapter for TextTestProvider { - fn name(&self) -> &str { - self.provider + fn id(&self) -> &AdapterId { + &self.id } - async fn complete( - &self, - request: &Request, - ) -> Result { - Ok(fabro_llm::types::Response { - id: "msg_fallback".to_string(), - model: request.model.clone(), - provider: self.provider.to_string(), - message: Message::assistant(self.text), - finish_reason: fabro_llm::types::FinishReason::Stop, - usage: TokenCounts { - input_tokens: 3, - output_tokens: 2, - ..TokenCounts::default() - }, - raw: None, - warnings: vec![], - rate_limit: None, - cost_usd: None, - cost_source: None, - }) + async fn complete(&self, call: &ResolvedCall) -> Result { + let handle = call.route().handle(); + let mut response = + Response::new(handle.provider().clone(), handle.model().clone(), vec![ + ContentPart::Text { + text: self.text.to_string(), + }, + ]); + response.id = Some("msg_fallback".to_string()); + response.usage = TokenCounts { + input: 3, + output: 2, + ..TokenCounts::default() + }; + Ok(response) } - async fn stream(&self, _request: &Request) -> Result { - Ok(Box::pin(stream::empty())) + async fn stream(&self, _call: &ResolvedCall) -> Result { + Ok(ResponseStream::new(stream::empty())) } } + /// An OpenAI-compatible mock provider served by `server`, with one model. + fn mock_provider_overlay(provider: &str, model: &str, base_url: &str, env_var: &str) -> String { + format!( + r#" +[providers.{provider}] +display_name = "{provider}" +adapter = "openai-compatible" +codec = "openai-chat" +base_url = {base_url} +auth = {{ type = "bearer" }} +default_model = "{model}" + +[providers.{provider}.metadata.fabro] +agent_profile = "openai" +credentials = ["env:{env_var}"] + +[providers.{provider}.models.{model}] +display_name = "{model}" +api_model = "{model}" +limits = {{ context_tokens = 8192, max_output_tokens = 1024 }} +capabilities = {{ text = true, tools = true, response_format = {{ json_object = true, json_schema = true }} }} +"#, + base_url = toml::Value::String(base_url.to_string()), + ) + } + fn mock_llm_catalog(server: &MockServer) -> Arc { - let settings: LlmCatalogSettings = toml::from_str(&format!( - r#" -[providers.mock] -adapter = "openai_compatible" -agent_profile = "openai" -base_url = "{}" - -[providers.mock.auth] -credentials = ["env:MOCK_API_KEY"] - -[models.mock-model] -provider = "mock" -display_name = "Mock Model" -family = "mock" -default = true - -[models.mock-model.limits] -context_window = 8192 -max_output = 1024 - -[models.mock-model.features] -tools = true -vision = false -reasoning = false -"#, - server.base_url() - )) - .unwrap(); - Arc::new(Catalog::from_builtin_with_overrides(&settings).unwrap()) + Arc::new(test_catalog_with_overlay(&mock_provider_overlay( + "mock", + "mock-model", + &server.base_url(), + "MOCK_API_KEY", + ))) } + /// Modal and OpenRouter ship disabled; enable them the way an operator + /// would so their models become fallback targets. fn enabled_fallback_catalog() -> Arc { - let settings: LlmCatalogSettings = toml::from_str( - r" -[providers.modal] -enabled = true - -[providers.openrouter] -enabled = true -", - ) - .expect("fallback catalog overrides should parse"); - Arc::new(Catalog::from_builtin_with_overrides(&settings).unwrap()) + Arc::new(test_catalog_with_overlay( + "[providers.modal.metadata.fabro]\nenabled = true\n\n[providers.openrouter.metadata.fabro]\nenabled = true\n", + )) } fn mock_api_backend(server: &MockServer) -> AgentApiBackend { @@ -2082,7 +2117,7 @@ enabled = true }); AgentApiBackend::new_with_catalog( "mock-model".to_string(), - ProviderId::from("mock"), + ProviderId::new("mock"), ModelFallbackPolicy::default(), source, SteeringHub::for_tests(), @@ -2091,59 +2126,22 @@ enabled = true } fn fallback_api_backend(server: &MockServer) -> AgentApiBackend { - let settings: LlmCatalogSettings = toml::from_str(&format!( - r#" -[providers.primary] -adapter = "openai_compatible" -agent_profile = "openai" -base_url = "{}/primary" - -[providers.primary.auth] -credentials = ["env:PRIMARY_API_KEY"] - -[providers.primary.models.test-model] -display_name = "Primary Test Model" -family = "test" -default = true - -[providers.primary.models.test-model.limits] -context_window = 8192 -max_output = 1024 - -[providers.primary.models.test-model.features] -tools = true -vision = false -reasoning = false - -[providers.fallback] -adapter = "openai_compatible" -agent_profile = "openai" -base_url = "{}/fallback" - -[providers.fallback.auth] -credentials = ["env:FALLBACK_API_KEY"] - -[providers.fallback.models.test-model] -display_name = "Fallback Test Model" -family = "test" -default = true - -[providers.fallback.models.test-model.limits] -context_window = 8192 -max_output = 1024 - -[providers.fallback.models.test-model.features] -tools = true -vision = false -reasoning = false -"#, - server.base_url(), - server.base_url(), - )) - .expect("fallback catalog should parse"); - let catalog = Arc::new( - Catalog::from_builtin_with_overrides(&settings).expect("catalog should build"), + let overlay = format!( + "{}\n{}", + mock_provider_overlay( + "primary", + "test-model", + &format!("{}/primary", server.base_url()), + "PRIMARY_API_KEY", + ), + mock_provider_overlay( + "fallback", + "test-model", + &format!("{}/fallback", server.base_url()), + "FALLBACK_API_KEY", + ), ); + let catalog = Arc::new(test_catalog_with_overlay(&overlay)); let source = auth_test_support::env_credential_source(|name| match name { "PRIMARY_API_KEY" | "FALLBACK_API_KEY" => Some("sk-test".to_string()), _ => None, @@ -2192,7 +2190,7 @@ reasoning = false "delta": { "content": text }, - "finish_reason": null + "finish_reason": "stop" }] }); let usage_chunk = serde_json::json!({ @@ -2263,20 +2261,20 @@ reasoning = false fn agent_backend_stores_config() { let backend = AgentApiBackend::new( "claude-opus-4-6".to_string(), - ProviderId::openai(), + provider_ids::openai(), ModelFallbackPolicy::default(), auth_test_support::vault_only_credential_source(), SteeringHub::for_tests(), ); assert_eq!(backend.model, "claude-opus-4-6"); - assert_eq!(backend.provider_id, ProviderId::openai()); + assert_eq!(backend.provider_id, provider_ids::openai()); } #[test] fn agent_backend_initializes_empty_sessions() { let backend = AgentApiBackend::new( "claude-opus-4-6".to_string(), - ProviderId::anthropic(), + provider_ids::anthropic(), ModelFallbackPolicy::default(), auth_test_support::vault_only_credential_source(), SteeringHub::for_tests(), @@ -2311,7 +2309,10 @@ reasoning = false .get(definition.name) .expect("shared Fabro run tool should be registered"); assert_eq!(registered.definition.description, definition.description); - assert_eq!(registered.definition.parameters, definition.parameters); + assert_eq!( + fabro_agent::tool_registry::ToolDefinitionExt::parameters(®istered.definition), + &definition.parameters + ); } } @@ -2962,9 +2963,9 @@ reasoning = false fn build_profile_can_register_subagent_tools() { let mut profile = AgentProfileBuilder::new( AgentProfileKind::Anthropic, - ProviderId::anthropic(), + provider_ids::anthropic(), "claude-opus-4-6", - Arc::new(Catalog::from_builtin().unwrap()), + Arc::new(test_catalog()), ) .build(); let supervisor = SubAgentSupervisor::new(1); @@ -3012,7 +3013,7 @@ reasoning = false // Parent turn 1: spawn a subagent. let parent_spawn = server.mock(|when, then| { when.method(POST) - .path("/chat/completions") + .path("/v1/chat/completions") .body_includes("PARENT_PROMPT_MARKER") .body_excludes(r#""role":"tool""#); then.status(200) @@ -3027,7 +3028,7 @@ reasoning = false // while the child keeps running in the background. let parent_final = server.mock(|when, then| { when.method(POST) - .path("/chat/completions") + .path("/v1/chat/completions") .body_includes("call_spawn_helper"); then.status(200) .header("content-type", "text/event-stream") @@ -3036,7 +3037,7 @@ reasoning = false // Child turn 1: the child session uses a tool. let child_read = server.mock(|when, then| { when.method(POST) - .path("/chat/completions") + .path("/v1/chat/completions") .body_includes("CHILD_TASK_MARKER") .body_excludes("PARENT_PROMPT_MARKER") .body_excludes(r#""role":"tool""#); @@ -3051,7 +3052,7 @@ reasoning = false // Child turn 2: the tool result is back; the child completes. let child_final = server.mock(|when, then| { when.method(POST) - .path("/chat/completions") + .path("/v1/chat/completions") .body_includes("call_child_read"); then.status(200) .header("content-type", "text/event-stream") @@ -3111,79 +3112,44 @@ reasoning = false #[test] fn api_backend_provider_pin_wins_over_priority_selection() { - let settings: LlmCatalogSettings = toml::from_str( - r" -[providers.openrouter] -enabled = true -", - ) - .unwrap(); let backend = AgentApiBackend::new_with_catalog( "gpt-5.4".to_string(), - ProviderId::from("openrouter"), + ProviderId::new("openrouter"), ModelFallbackPolicy::default(), auth_test_support::vault_only_credential_source(), SteeringHub::for_tests(), - Arc::new(Catalog::from_builtin_with_overrides(&settings).unwrap()), + Arc::new(test_catalog_with_overlay(OPENROUTER_ENABLED)), ); let provider = backend.resolve_provider_context("gpt-5.4", None).unwrap(); - assert_eq!(provider.provider_id, ProviderId::from("openrouter")); + assert_eq!(provider.provider_id, ProviderId::new("openrouter")); } #[test] fn api_backend_node_provider_attr_overrides_backend_pin() { let backend = AgentApiBackend::new_with_catalog( "gpt-5.4".to_string(), - ProviderId::from("openrouter"), + ProviderId::new("openrouter"), ModelFallbackPolicy::default(), auth_test_support::vault_only_credential_source(), SteeringHub::for_tests(), - Arc::new(Catalog::from_builtin().unwrap()), + Arc::new(test_catalog()), ); let provider = backend .resolve_provider_context("gpt-5.4", Some("openai")) .unwrap(); - assert_eq!(provider.provider_id, ProviderId::openai()); + assert_eq!(provider.provider_id, provider_ids::openai()); } #[test] fn api_backend_resolves_custom_catalog_provider_profile() { - let settings: LlmCatalogSettings = toml::from_str( - r#" -[providers.acme] -adapter = "openai_compatible" -agent_profile = "openai" -base_url = "https://api.acme.test/v1" - -[providers.acme.auth] -credentials = ["env:ACME_API_KEY"] - -[models.acme-llama] -provider = "acme" -display_name = "Acme Llama" -family = "llama" -training = "2026-01" -default = true - -[models.acme-llama.limits] -context_window = 131072 -max_output = 8192 - -[models.acme-llama.features] -tools = true -vision = false -reasoning = false -"#, - ) - .unwrap(); - let catalog = Arc::new(Catalog::from_builtin_with_overrides(&settings).unwrap()); + let catalog = Arc::new(test_catalog_with_overlay(ACME_LLAMA_OVERLAY)); let backend = AgentApiBackend::new_with_catalog( "acme-llama".to_string(), - ProviderId::from("acme"), + ProviderId::new("acme"), ModelFallbackPolicy::default(), auth_test_support::vault_only_credential_source(), SteeringHub::for_tests(), @@ -3194,43 +3160,16 @@ reasoning = false .resolve_provider_context("acme-llama", None) .unwrap(); - assert_eq!(provider.provider_id, ProviderId::from("acme")); + assert_eq!(provider.provider_id, ProviderId::new("acme")); assert_eq!(provider.profile_kind, AgentProfileKind::OpenAi); } #[test] fn api_backend_resolves_model_agent_profile_override() { - let settings: LlmCatalogSettings = toml::from_str( - r#" -[providers.acme] -adapter = "openai_compatible" -agent_profile = "openai" -base_url = "https://api.acme.test/v1" - -[models.acme-claude] -provider = "acme" -display_name = "Acme Claude" -family = "claude" -training = "2026-01" -default = true -agent_profile = "anthropic" -aliases = ["ac"] - -[models.acme-claude.limits] -context_window = 131072 -max_output = 8192 - -[models.acme-claude.features] -tools = true -vision = false -reasoning = false -"#, - ) - .unwrap(); - let catalog = Arc::new(Catalog::from_builtin_with_overrides(&settings).unwrap()); + let catalog = Arc::new(test_catalog_with_overlay(ACME_CLAUDE_OVERLAY)); let backend = AgentApiBackend::new_with_catalog( "acme-claude".to_string(), - ProviderId::from("acme"), + ProviderId::new("acme"), ModelFallbackPolicy::default(), auth_test_support::vault_only_credential_source(), SteeringHub::for_tests(), @@ -3239,7 +3178,7 @@ reasoning = false let provider = backend.resolve_provider_context("ac", None).unwrap(); - assert_eq!(provider.provider_id, ProviderId::from("acme")); + assert_eq!(provider.provider_id, ProviderId::new("acme")); assert_eq!(provider.profile_kind, AgentProfileKind::Anthropic); } @@ -3247,34 +3186,27 @@ reasoning = false fn api_backend_selects_claude5_profile_for_sonnet5() { let backend = AgentApiBackend::new_with_catalog( "claude-sonnet-5".to_string(), - ProviderId::anthropic(), + provider_ids::anthropic(), ModelFallbackPolicy::default(), auth_test_support::vault_only_credential_source(), SteeringHub::for_tests(), - Arc::new(Catalog::from_builtin().unwrap()), + Arc::new(test_catalog()), ); let provider = backend .resolve_provider_context("claude-sonnet-5", None) .unwrap(); - assert_eq!(provider.provider_id, ProviderId::anthropic()); + assert_eq!(provider.provider_id, provider_ids::anthropic()); assert_eq!(provider.profile_kind, AgentProfileKind::Claude5); } #[test] fn api_backend_preserves_default_provider_for_legacy_model_identifier() { - let settings: LlmCatalogSettings = toml::from_str( - r" -[providers.openrouter] -enabled = true -", - ) - .unwrap(); - let catalog = Arc::new(Catalog::from_builtin_with_overrides(&settings).unwrap()); + let catalog = Arc::new(test_catalog_with_overlay(OPENROUTER_ENABLED)); let backend = AgentApiBackend::new_with_catalog( "openai/gpt-5.4".to_string(), - ProviderId::from("openrouter"), + ProviderId::new("openrouter"), ModelFallbackPolicy::default(), auth_test_support::vault_only_credential_source(), SteeringHub::for_tests(), @@ -3285,7 +3217,7 @@ enabled = true .resolve_provider_context("openai/gpt-5.4", None) .unwrap(); - assert_eq!(provider.provider_id, ProviderId::from("openrouter")); + assert_eq!(provider.provider_id, ProviderId::new("openrouter")); assert_eq!(provider.profile_kind, AgentProfileKind::OpenAi); } @@ -3293,7 +3225,7 @@ enabled = true fn run_model_controls_apply_when_node_omits_controls() { let backend = AgentApiBackend::new( "gpt-5.4".to_string(), - ProviderId::openai(), + provider_ids::openai(), ModelFallbackPolicy::default(), auth_test_support::vault_only_credential_source(), SteeringHub::for_tests(), @@ -3314,7 +3246,7 @@ enabled = true fn node_controls_override_run_model_controls() { let backend = AgentApiBackend::new( "gpt-5.4".to_string(), - ProviderId::openai(), + provider_ids::openai(), ModelFallbackPolicy::default(), auth_test_support::vault_only_credential_source(), SteeringHub::for_tests(), @@ -3330,20 +3262,20 @@ enabled = true ); node.attrs.insert( "speed".to_string(), - fabro_graphviz::graph::AttrValue::String("standard".to_string()), + fabro_graphviz::graph::AttrValue::String("balanced".to_string()), ); let controls = backend.resolve_effective_request_controls(&node).unwrap(); assert_eq!(controls.reasoning_effort, Some(ReasoningEffort::High)); - assert_eq!(controls.speed, Some(Speed::Standard)); + assert_eq!(controls.speed, Some(Speed::Balanced)); } #[test] fn omitted_reasoning_effort_stays_unset() { let backend = AgentApiBackend::new( "gpt-5.4".to_string(), - ProviderId::openai(), + provider_ids::openai(), ModelFallbackPolicy::default(), auth_test_support::vault_only_credential_source(), SteeringHub::for_tests(), @@ -3411,7 +3343,7 @@ enabled = true ])); let backend = AgentApiBackend::new_with_catalog( "claude-fable-5".to_string(), - ProviderId::anthropic(), + provider_ids::anthropic(), policy, auth_test_support::vault_only_credential_source(), SteeringHub::for_tests(), @@ -3419,7 +3351,7 @@ enabled = true ); let (mut plan, notices) = backend.fallback_plan( "claude-fable-5", - &ProviderId::anthropic(), + &provider_ids::anthropic(), EffectiveRequestControls::default(), ); @@ -3454,7 +3386,7 @@ enabled = true .unwrap(); let backend = AgentApiBackend::new( "claude-opus-4-6".to_string(), - ProviderId::anthropic(), + provider_ids::anthropic(), ModelFallbackPolicy::default(), Arc::new(VaultCredentialSource::with_env_lookup( Arc::new(AsyncRwLock::new(vault)), @@ -3463,11 +3395,14 @@ enabled = true SteeringHub::for_tests(), ); - let client = Client::from_source(backend.source.as_ref(), Arc::clone(&backend.catalog)) + let client = build_llm_client(&backend.catalog, Arc::clone(&backend.source)) .await .unwrap(); - assert_eq!(client.provider_names(), vec!["anthropic"]); + assert_eq!( + client.available_providers().iter().collect::>(), + vec![&provider_ids::anthropic()] + ); } #[tokio::test] @@ -3479,24 +3414,24 @@ enabled = true )])); let backend = AgentApiBackend::new( "claude-fable-5".to_string(), - ProviderId::anthropic(), + provider_ids::anthropic(), fallback_policy, auth_test_support::vault_only_credential_source(), SteeringHub::for_tests(), ); - let mut providers = HashMap::new(); - providers.insert( - "anthropic".to_string(), - Arc::new(RefusalTestProvider) as Arc, + let client = client_with_adapters( + vec![ + ( + "anthropic", + Arc::new(RefusalTestProvider::new()) as Arc, + ), + ( + "openai", + Arc::new(TextTestProvider::new("fallback ok")) as Arc, + ), + ], + ClientOptions::default(), ); - providers.insert( - "openai".to_string(), - Arc::new(TextTestProvider { - provider: "openai", - text: "fallback ok", - }) as Arc, - ); - let client = Client::new(providers, Some("anthropic".to_string()), Vec::new()); let node = Node::new("ask"); let context = Context::new(); let stage_scope = StageScope::for_handler(&context, &node.id); @@ -3508,25 +3443,15 @@ enabled = true *emitted_failover_for_listener.lock().unwrap() = Some(props.clone()); } }); - let request = Request { - model: "claude-fable-5".to_string(), - messages: vec![Message::user("Hello")], - provider: Some("anthropic".to_string()), - tools: None, - tool_choice: None, - response_format: None, - temperature: None, - top_p: None, - max_tokens: Some(128), - stop_sequences: None, - reasoning_effort: None, - speed: None, - metadata: None, - provider_options: None, - }; + let request = Request::builder() + .model("anthropic/claude-fable-5") + .user("Hello") + .max_output_tokens(128) + .build() + .unwrap(); let (mut fallback_plan, notices) = backend.fallback_plan( "claude-fable-5", - &ProviderId::anthropic(), + &provider_ids::anthropic(), EffectiveRequestControls::default(), ); assert!(notices.is_empty()); @@ -3544,8 +3469,8 @@ enabled = true .unwrap(); assert_eq!(completion.response.text(), "fallback ok"); - assert_eq!(completion.model.provider, ProviderId::openai()); - assert_eq!(completion.model.model_id, "gpt-5.5"); + assert_eq!(completion.model.provider, provider_ids::openai()); + assert_eq!(completion.model.model_id.as_str(), "gpt-5.5"); let failover = emitted_failover .lock() .unwrap() @@ -3567,7 +3492,7 @@ enabled = true async fn explicit_provider_one_shot_stays_on_fallback_during_output_repair() { let server = MockServer::start(); let primary_failure = server.mock(|when, then| { - when.method(POST).path("/primary/chat/completions"); + when.method(POST).path("/primary/v1/chat/completions"); then.status(401) .header("content-type", "application/json") .json_body(serde_json::json!({ @@ -3579,7 +3504,7 @@ enabled = true }); let fallback_response = server.mock(|when, then| { when.method(POST) - .path("/fallback/chat/completions") + .path("/fallback/v1/chat/completions") .body_excludes(r#""role":"assistant""#); then.status(200) .header("content-type", "application/json") @@ -3587,7 +3512,7 @@ enabled = true }); let fallback_repair = server.mock(|when, then| { when.method(POST) - .path("/fallback/chat/completions") + .path("/fallback/v1/chat/completions") .body_includes(r#""role":"assistant""#) .body_includes("not json"); then.status(200) @@ -3638,7 +3563,7 @@ enabled = true let server = MockServer::start(); let first = server.mock(|when, then| { when.method(POST) - .path("/chat/completions") + .path("/v1/chat/completions") .body_includes(r#""type":"json_schema""#) .body_excludes(r#""role":"assistant""#); then.status(200) @@ -3651,7 +3576,7 @@ enabled = true }); let repair = server.mock(|when, then| { when.method(POST) - .path("/chat/completions") + .path("/v1/chat/completions") .body_includes(r#""type":"json_schema""#) .body_includes(r#""role":"assistant""#) .body_includes("not json") @@ -3697,8 +3622,8 @@ enabled = true }; assert_eq!(text, r#"{"passed":true}"#); let usage = usage.expect("usage should be aggregated"); - assert_eq!(usage.tokens().input_tokens, 21); - assert_eq!(usage.tokens().output_tokens, 3); + assert_eq!(usage.tokens().input, 21); + assert_eq!(usage.tokens().output, 3); assert_eq!(usage.total_usd_micros, Some(100_000)); } @@ -3707,7 +3632,7 @@ enabled = true let server = MockServer::start(); let first = server.mock(|when, then| { when.method(POST) - .path("/chat/completions") + .path("/v1/chat/completions") .body_includes(r#""stream":true"#) .body_excludes(r#""role":"assistant""#); then.status(200) @@ -3716,7 +3641,7 @@ enabled = true }); let repair = server.mock(|when, then| { when.method(POST) - .path("/chat/completions") + .path("/v1/chat/completions") .body_includes(r#""stream":true"#) .body_includes(r#""role":"assistant""#) .body_includes("not json") @@ -3760,8 +3685,8 @@ enabled = true }; assert_eq!(text, r#"{"passed":true}"#); let usage = usage.expect("usage should be aggregated"); - assert_eq!(usage.tokens().input_tokens, 41); - assert_eq!(usage.tokens().output_tokens, 7); + assert_eq!(usage.tokens().input, 41); + assert_eq!(usage.tokens().output, 7); } #[tokio::test] @@ -3769,7 +3694,7 @@ enabled = true let server = MockServer::start(); let first = server.mock(|when, then| { when.method(POST) - .path("/chat/completions") + .path("/v1/chat/completions") .body_includes(r#""stream":true"#) .body_excludes(r#""role":"assistant""#); then.status(200) @@ -3778,7 +3703,7 @@ enabled = true }); let first_repair = server.mock(|when, then| { when.method(POST) - .path("/chat/completions") + .path("/v1/chat/completions") .body_includes("JSON Pointer `/findings/0/rationale`") .body_excludes("unchanged from your previous repair"); then.status(200) @@ -3787,7 +3712,7 @@ enabled = true }); let second_repair = server.mock(|when, then| { when.method(POST) - .path("/chat/completions") + .path("/v1/chat/completions") .body_includes("JSON Pointer `/findings/0/rationale`") .body_includes("unchanged from your previous repair"); then.status(200) @@ -3839,7 +3764,7 @@ enabled = true let server = MockServer::start(); let primary_response = server.mock(|when, then| { when.method(POST) - .path("/primary/chat/completions") + .path("/primary/v1/chat/completions") .body_includes(r#""stream":true"#) .body_excludes(r#""role":"assistant""#); then.status(200) @@ -3848,7 +3773,7 @@ enabled = true }); let failed_repair = server.mock(|when, then| { when.method(POST) - .path("/primary/chat/completions") + .path("/primary/v1/chat/completions") .body_includes(r#""role":"assistant""#) .body_includes("not json"); then.status(401) @@ -3862,7 +3787,7 @@ enabled = true }); let fallback_response = server.mock(|when, then| { when.method(POST) - .path("/fallback/chat/completions") + .path("/fallback/v1/chat/completions") .body_includes(r#""stream":true"#) .body_excludes(r#""role":"assistant""#); then.status(200) @@ -3910,7 +3835,7 @@ enabled = true let server = MockServer::start(); let tool_call = server.mock(|when, then| { when.method(POST) - .path("/chat/completions") + .path("/v1/chat/completions") .body_includes(r#""stream":true"#) .body_excludes(r#""role":"tool""#); then.status(200) @@ -3923,7 +3848,7 @@ enabled = true }); let completion = server.mock(|when, then| { when.method(POST) - .path("/chat/completions") + .path("/v1/chat/completions") .body_includes("call_web_search"); then.status(200) .header("content-type", "text/event-stream") @@ -3993,7 +3918,7 @@ enabled = true async fn api_backend_shutdown_closes_cached_sessions_once() { let backend = AgentApiBackend::new( "gpt-5.4".to_string(), - ProviderId::openai(), + provider_ids::openai(), ModelFallbackPolicy::default(), auth_test_support::vault_only_credential_source(), SteeringHub::for_tests(), @@ -4008,12 +3933,13 @@ enabled = true .push(event.event_name().to_string()); }); - let mut providers = HashMap::new(); - providers.insert( - "openai".to_string(), - Arc::new(ShutdownTestProvider) as Arc, + let client = client_with_adapters( + vec![( + "openai", + Arc::new(ShutdownTestProvider::new()) as Arc, + )], + ClientOptions::default(), ); - let client = Client::new(providers, Some("openai".to_string()), Vec::new()); let session = Session::new( client, Arc::new(ShutdownTestProfile::new()), @@ -4025,7 +3951,7 @@ enabled = true ); let (fallback_plan, notices) = backend.fallback_plan( "gpt-5.4", - &ProviderId::openai(), + &provider_ids::openai(), EffectiveRequestControls::default(), ); assert!(notices.is_empty()); @@ -4051,12 +3977,13 @@ enabled = true #[tokio::test] async fn session_end_barrier_preserves_child_close_ordering() { - let mut providers = HashMap::new(); - providers.insert( - "openai".to_string(), - Arc::new(ShutdownTestProvider) as Arc, + let client = client_with_adapters( + vec![( + "openai", + Arc::new(ShutdownTestProvider::new()) as Arc, + )], + ClientOptions::default(), ); - let client = Client::new(providers, Some("openai".to_string()), Vec::new()); let mut session = Session::new( client, Arc::new(ShutdownTestProfile::new()), @@ -4114,47 +4041,93 @@ enabled = true // --- Bridge guard tests --- fn failover_eligible_llm_error() -> LlmError { - LlmError::Network { - message: "boom".into(), - source: None, - } + LlmError::from( + fabro_llm::Error::new(ErrorKind::Network, "boom") + .with_provider(provider_ids::openai()) + .with_retry(RetryClassification::Safe), + ) } fn non_failover_llm_error() -> LlmError { - LlmError::Provider { - kind: ProviderErrorKind::InvalidRequest, - detail: Box::new(ProviderErrorDetail { - message: "bad key".into(), - provider: "openai".into(), - status_code: Some(401), - error_code: None, - retry_after: None, - raw: None, - }), - } + LlmError::from( + fabro_llm::Error::new(ErrorKind::InvalidRequest, "bad key") + .with_provider(provider_ids::openai()) + .with_status(401), + ) } - fn refusal_llm_error() -> LlmError { - LlmError::Provider { - kind: ProviderErrorKind::ContentFilter, - detail: Box::new(ProviderErrorDetail { - message: "claude-fable-5 refused the request".into(), - provider: "anthropic".into(), - status_code: None, - error_code: Some("refusal".into()), - retry_after: None, - raw: Some(serde_json::json!({ - "stop_reason": "refusal", - "stop_details": { - "type": "refusal", - "category": "cyber", - "explanation": "This request was declined." - } - })), - }), - } + fn refusal_llm_error() -> fabro_llm::Error { + fabro_llm::Error::new( + ErrorKind::ContentFilter, + "claude-fable-5 refused the request", + ) + .with_provider(provider_ids::anthropic()) + .with_provider_code("refusal") + .with_raw_data(serde_json::json!({ + "stop_reason": "refusal", + "stop_details": { + "type": "refusal", + "category": "cyber", + "explanation": "This request was declined." + } + })) } + const OPENROUTER_ENABLED: &str = "[providers.openrouter.metadata.fabro]\nenabled = true\n"; + + /// An operator-defined OpenAI-compatible provider whose models take the + /// provider's `openai` agent profile. + const ACME_LLAMA_OVERLAY: &str = r#" +[providers.acme] +display_name = "Acme" +adapter = "openai-compatible" +codec = "openai-chat" +base_url = "https://api.acme.test/v1" +auth = { type = "bearer" } +default_model = "acme-llama" + +[providers.acme.metadata.fabro] +agent_profile = "openai" +credentials = ["env:ACME_API_KEY"] + +[providers.acme.models.acme-llama] +display_name = "Acme Llama" +api_model = "acme-llama" +limits = { context_tokens = 131072, max_output_tokens = 8192 } +capabilities = { text = true, tools = true } + +[providers.acme.models.acme-llama.metadata.fabro] +family = "llama" +training = "2026-01" +"#; + + /// The same provider serving a Claude model that overrides the profile. + const ACME_CLAUDE_OVERLAY: &str = r#" +[providers.acme] +display_name = "Acme" +adapter = "openai-compatible" +codec = "openai-chat" +base_url = "https://api.acme.test/v1" +auth = { type = "bearer" } +default_model = "acme-claude" + +[providers.acme.metadata.fabro] +agent_profile = "openai" +credentials = ["env:ACME_API_KEY"] + +[providers.acme.models.acme-claude] +display_name = "Acme Claude" +aliases = ["ac"] +api_model = "acme-claude" +limits = { context_tokens = 131072, max_output_tokens = 8192 } +capabilities = { text = true, tools = true } + +[providers.acme.models.acme-claude.metadata.fabro] +family = "claude" +training = "2026-01" +agent_profile = "anthropic" +"#; + #[tokio::test] async fn spawn_bridge_task_sets_cancelled_and_cancels_session_token() { let run_token = CancellationToken::new(); @@ -4335,7 +4308,7 @@ enabled = true #[test] fn classify_refusal_llm_returns_failover_when_allowed() { - let err = fabro_agent::Error::Llm(refusal_llm_error()); + let err = fabro_agent::Error::Llm(LlmError::from(refusal_llm_error())); assert!(matches!( classify_agent_error(err, true), AgentApiErrorDisposition::FailoverEligible(_) @@ -4344,7 +4317,7 @@ enabled = true #[test] fn classify_refusal_llm_returns_terminal_when_not_allowed() { - let err = fabro_agent::Error::Llm(refusal_llm_error()); + let err = fabro_agent::Error::Llm(LlmError::from(refusal_llm_error())); match classify_agent_error(err, false) { AgentApiErrorDisposition::Terminal(Error::Llm(llm_err)) => { assert!(llm_err.to_string().contains("claude-fable-5 refused")); diff --git a/lib/components/fabro-workflow/src/handler/llm/preamble.rs b/lib/components/fabro-workflow/src/handler/llm/preamble.rs index dc01e65ab..5f812e533 100644 --- a/lib/components/fabro-workflow/src/handler/llm/preamble.rs +++ b/lib/components/fabro-workflow/src/handler/llm/preamble.rs @@ -589,23 +589,18 @@ fn build_summary_preamble( #[cfg(test)] mod tests { use fabro_graphviz::graph::AttrValue; - use fabro_llm::types::TokenCounts; - use fabro_model::{Catalog, ModelRef, ProviderId}; + use fabro_types::{ModelId, ModelRef, TokenCounts, provider_ids}; use super::*; use crate::outcome::{BilledModelUsage, billed_model_usage_from_llm}; - fn stage_usage(model: &str, input_tokens: i64, output_tokens: i64) -> BilledModelUsage { + fn stage_usage(model: &str, input: u64, output: u64) -> BilledModelUsage { billed_model_usage_from_llm( - Catalog::builtin(), - &ModelRef { - provider: ProviderId::anthropic(), - model_id: model.into(), - speed: None, - }, - &TokenCounts { - input_tokens, - output_tokens, + &fabro_llm::test_support::test_catalog(), + &ModelRef::new(provider_ids::anthropic(), ModelId::new(model)), + TokenCounts { + input, + output, ..TokenCounts::default() }, ) diff --git a/lib/components/fabro-workflow/src/handler/llm/router.rs b/lib/components/fabro-workflow/src/handler/llm/router.rs index 24dc97e0e..3e0580016 100644 --- a/lib/components/fabro-workflow/src/handler/llm/router.rs +++ b/lib/components/fabro-workflow/src/handler/llm/router.rs @@ -81,7 +81,7 @@ mod tests { use async_trait::async_trait; use fabro_agent::{LocalSandbox, Sandbox}; use fabro_graphviz::graph::{AttrValue, Node}; - use fabro_model::{ReasoningEffort, Speed}; + use fabro_types::{ReasoningEffort, Speed}; use tokio_util::sync::CancellationToken; use super::*; diff --git a/lib/components/fabro-workflow/src/handler/llm/routing.rs b/lib/components/fabro-workflow/src/handler/llm/routing.rs index 3ce9618db..c3db8ae5d 100644 --- a/lib/components/fabro-workflow/src/handler/llm/routing.rs +++ b/lib/components/fabro-workflow/src/handler/llm/routing.rs @@ -1,6 +1,7 @@ use fabro_graphviz::graph::{self, Node}; -use fabro_model::{AgentProfileKind, Catalog, ProviderId}; -use fabro_types::AgentBackend; +use fabro_llm::lithos_catalog::Catalog; +use fabro_llm::{ModelSelectionError, catalog, selection}; +use fabro_types::{AgentBackend, AgentProfileKind, ProviderId}; use crate::error::Error; @@ -50,38 +51,33 @@ pub(crate) fn resolve_provider_context( provider_attr: Option<&str>, ) -> Result { let provider_id = if let Some(provider) = provider_attr { - let requested = ProviderId::from(provider); - catalog - .provider(&requested) - .ok_or_else(|| { - Error::Precondition(format!("Provider \"{provider}\" is not configured")) - })? - .id - .clone() - } else if catalog - .get_on_provider(default_provider_id, model) - .is_some() - { + catalog::canonical_provider_id(catalog, provider).ok_or_else(|| { + Error::Precondition(format!("Provider \"{provider}\" is not configured")) + })? + } else if catalog::model_on_provider(catalog, default_provider_id.as_str(), model).is_some() { // The run's selected provider is a pin whenever it offers the model. default_provider_id.clone() } else { - match catalog.select(model, None, &catalog.all_provider_ids()) { - Ok(model) => model.provider.clone(), - Err(fabro_model::ModelSelectionError::UnknownSelector { .. }) => { - default_provider_id.clone() - } + match selection::select( + catalog, + model, + None, + &catalog::enabled_provider_ids(catalog), + ) { + Ok(entry) => entry.provider.id().clone(), + Err(ModelSelectionError::UnknownSelector { .. }) => default_provider_id.clone(), Err(error) => return Err(error.into()), } }; - let provider = catalog.provider(&provider_id).ok_or_else(|| { - Error::Precondition(format!("Provider \"{provider_id}\" is not configured")) - })?; - let profile_kind = catalog - .effective_agent_profile(&provider.id, Some(model)) + let provider_id = + catalog::canonical_provider_id(catalog, provider_id.as_str()).ok_or_else(|| { + Error::Precondition(format!("Provider \"{provider_id}\" is not configured")) + })?; + let profile_kind = catalog::agent_profile(catalog, provider_id.as_str(), Some(model)) .expect("validated provider should resolve an agent profile"); Ok(ProviderContext { - provider_id: provider.id.clone(), + provider_id, profile_kind, }) } diff --git a/lib/components/fabro-workflow/src/handler/prompt.rs b/lib/components/fabro-workflow/src/handler/prompt.rs index 5e2058c3c..97c110b2e 100644 --- a/lib/components/fabro-workflow/src/handler/prompt.rs +++ b/lib/components/fabro-workflow/src/handler/prompt.rs @@ -220,9 +220,8 @@ mod tests { use std::time::Duration; use fabro_graphviz::graph::AttrValue; - use fabro_model::{ReasoningEffort, Speed}; use fabro_store::{Database, RunDatabase, StageId}; - use fabro_types::{fixtures, test_support}; + use fabro_types::{ReasoningEffort, Speed, fixtures, test_support}; use object_store::memory::InMemory; use tempfile::TempDir; @@ -690,33 +689,32 @@ mod tests { tokio::fs::write(workspace.path().join("CLAUDE.md"), "anthropic memory") .await .unwrap(); - let overrides: fabro_model::catalog::LlmCatalogSettings = toml::from_str( + let catalog = Arc::new(fabro_llm::test_support::test_catalog_with_overlay( r#" -[providers.acme] -adapter = "openai_compatible" -agent_profile = "openai" -base_url = "https://api.acme.test/v1" - -[models.acme-claude] -provider = "acme" -display_name = "Acme Claude" -family = "claude" -default = true -agent_profile = "anthropic" -aliases = ["ac"] - -[models.acme-claude.limits] -context_window = 1000 - -[models.acme-claude.features] -tools = true -vision = false -reasoning = false -"#, - ) - .unwrap(); - let catalog = - Arc::new(fabro_model::Catalog::from_builtin_with_overrides(&overrides).unwrap()); + [providers.acme] + display_name = "Acme" + adapter = "openai-compatible" + codec = "openai-chat" + base_url = "https://api.acme.test/v1" + auth = { type = "bearer" } + default_model = "acme-claude" + + [providers.acme.metadata.fabro] + agent_profile = "openai" + credentials = ["env:ACME_API_KEY"] + + [providers.acme.models.acme-claude] + display_name = "Acme Claude" + aliases = ["ac"] + api_model = "acme-claude" + limits = { context_tokens = 1000, max_output_tokens = 500 } + capabilities = { text = true, tools = true } + + [providers.acme.models.acme-claude.metadata.fabro] + family = "claude" + agent_profile = "anthropic" + "#, + )); let mut services = make_services(); services.run = services .run @@ -725,7 +723,7 @@ reasoning = false ))) .with_catalog_context( Arc::clone(&catalog), - fabro_model::ProviderId::new("acme"), + fabro_types::ProviderId::new("acme"), "acme-claude".to_string(), ); @@ -766,32 +764,32 @@ reasoning = false tokio::fs::write(workspace.path().join("CLAUDE.md"), "anthropic memory") .await .unwrap(); - let overrides: fabro_model::catalog::LlmCatalogSettings = toml::from_str( + let catalog = Arc::new(fabro_llm::test_support::test_catalog_with_overlay( r#" -[providers.acme] -adapter = "openai_compatible" -agent_profile = "openai" -base_url = "https://api.acme.test/v1" - -[models.acme-claude] -provider = "acme" -display_name = "Acme Claude" -family = "claude" -default = true -agent_profile = "anthropic" - -[models.acme-claude.limits] -context_window = 1000 - -[models.acme-claude.features] -tools = true -vision = false -reasoning = false -"#, - ) - .unwrap(); - let catalog = - Arc::new(fabro_model::Catalog::from_builtin_with_overrides(&overrides).unwrap()); + [providers.acme] + display_name = "Acme" + adapter = "openai-compatible" + codec = "openai-chat" + base_url = "https://api.acme.test/v1" + auth = { type = "bearer" } + default_model = "acme-claude" + + [providers.acme.metadata.fabro] + agent_profile = "openai" + credentials = ["env:ACME_API_KEY"] + + [providers.acme.models.acme-claude] + display_name = "Acme Claude" + aliases = ["ac"] + api_model = "acme-claude" + limits = { context_tokens = 1000, max_output_tokens = 500 } + capabilities = { text = true, tools = true } + + [providers.acme.models.acme-claude.metadata.fabro] + family = "claude" + agent_profile = "anthropic" + "#, + )); let mut services = make_services(); services.run = services .run @@ -800,7 +798,7 @@ reasoning = false ))) .with_catalog_context( Arc::clone(&catalog), - fabro_model::ProviderId::new("acme"), + fabro_types::ProviderId::new("acme"), "acme-claude".to_string(), ); diff --git a/lib/components/fabro-workflow/src/handler/structured_output.rs b/lib/components/fabro-workflow/src/handler/structured_output.rs index 3403d87c4..eba502034 100644 --- a/lib/components/fabro-workflow/src/handler/structured_output.rs +++ b/lib/components/fabro-workflow/src/handler/structured_output.rs @@ -2,7 +2,7 @@ use std::fmt::Write as _; use std::sync::{Arc, LazyLock}; use fabro_graphviz::graph::Node; -use fabro_llm::types::{ResponseFormat, ResponseFormatType}; +use fabro_llm::types::ResponseFormat; use jsonschema::error::ValidationErrorKind; use jsonschema::paths::Location; use jsonschema::{ValidationError, Validator}; @@ -392,17 +392,16 @@ pub(crate) fn parse_node_output_schema(node: &Node) -> Result ResponseFormat { match schema { - OutputSchemaKind::Routing => ResponseFormat { - kind: ResponseFormatType::JsonObject, - json_schema: None, - strict: false, - }, - OutputSchemaKind::JsonSchema { schema, .. } => ResponseFormat { - kind: ResponseFormatType::JsonSchema, - json_schema: Some(schema.clone()), - strict: true, + OutputSchemaKind::Routing => ResponseFormat::JsonObject, + OutputSchemaKind::JsonSchema { schema, .. } => ResponseFormat::JsonSchema { + name: "output_schema".to_string(), + schema: schema.clone(), }, } } @@ -1119,12 +1118,10 @@ mod tests { let format = prompt_response_format(&schema); - assert_eq!(format.kind, ResponseFormatType::JsonSchema); - assert_eq!( - format.json_schema, - Some(serde_json::json!({"type": "object"})) - ); - assert!(format.strict); + assert_eq!(format, ResponseFormat::JsonSchema { + name: "output_schema".to_string(), + schema: serde_json::json!({"type": "object"}), + }); } #[test] diff --git a/lib/components/fabro-workflow/src/lifecycle/git.rs b/lib/components/fabro-workflow/src/lifecycle/git.rs index 4f1b30602..509f1a397 100644 --- a/lib/components/fabro-workflow/src/lifecycle/git.rs +++ b/lib/components/fabro-workflow/src/lifecycle/git.rs @@ -645,7 +645,6 @@ mod tests { use fabro_core::lifecycle::RunLifecycle; use fabro_core::state::ExecutionState; use fabro_graphviz::graph::types::{AttrValue, Edge, Graph, Node}; - use fabro_model::Catalog; use fabro_store::{EventEnvelope, RunDatabase, RunProjection}; use fabro_types::run_event::{MetadataSnapshotFailureKind, MetadataSnapshotPhase}; use fabro_types::{BlobHash, EventBody, RunEvent, WorkflowSettings, fixtures, test_support}; @@ -1307,10 +1306,10 @@ mod tests { None, finalize_locations, tokio_util::sync::CancellationToken::new(), - fabro_model::ProviderId::anthropic(), + fabro_types::provider_ids::anthropic(), "claude-sonnet-4-6".to_string(), auth_test_support::vault_only_credential_source(), - Arc::new(Catalog::from_builtin().expect("default catalog should build")), + Arc::new(fabro_llm::test_support::test_catalog()), Arc::new(SandboxGitRuntime::new()), Arc::clone(&lifecycle.metadata_runtime), lifecycle.metadata_writer.clone(), diff --git a/lib/components/fabro-workflow/src/model_fallback.rs b/lib/components/fabro-workflow/src/model_fallback.rs index f6de2fcd4..295d1892a 100644 --- a/lib/components/fabro-workflow/src/model_fallback.rs +++ b/lib/components/fabro-workflow/src/model_fallback.rs @@ -1,10 +1,10 @@ use std::collections::{BTreeMap, HashMap, HashSet}; -use fabro_model::{ - Catalog, FallbackTarget, Model, ModelSelectionError, ProviderId, ReasoningEffort, -}; +use fabro_llm::catalog::ModelEntry; +use fabro_llm::lithos_catalog::Catalog; +use fabro_llm::{FallbackTarget, ModelSelectionError, catalog, selection}; use fabro_types::settings::{ModelRef, ResolvedModelRef}; -use fabro_types::{RunNoticeCode, RunNoticeLevel}; +use fabro_types::{ProviderId, ReasoningEffort, RunNoticeCode, RunNoticeLevel, controls}; use crate::Error; @@ -31,7 +31,7 @@ impl ModelFallbackPolicy { provider: &ProviderId, model: &str, ) -> Option<&'a [FallbackTarget]> { - self.chain_for_canonical(&catalog.canonical_model_id(provider, model)) + self.chain_for_canonical(&catalog::canonical_model_id(catalog, provider, model)) } /// Look up a chain by an already-canonicalized requested model ID. @@ -179,9 +179,12 @@ impl ModelFallbackNotice { requested_model, target, requested_effort, - } => format!( - "Model fallback `{target}` for requested model `{requested_model}` was skipped because it has no reasoning level near `{requested_effort}`." - ), + } => { + let requested_effort = controls::reasoning_effort_name(*requested_effort); + format!( + "Model fallback `{target}` for requested model `{requested_model}` was skipped because it has no reasoning level near `{requested_effort}`." + ) + } Self::ChainEmpty { requested_model } => format!( "No usable model fallbacks remain for requested model `{requested_model}` after filtering its configured candidates." ), @@ -205,8 +208,12 @@ pub fn resolve_model_fallbacks( for (raw_key, references) in configured { require_bare_model_key(catalog, raw_key)?; - let selected = - catalog.resolve_selection_with_catalog_fallback(Some(raw_key), None, &eligible)?; + let selected = selection::resolve_selection_with_catalog_fallback( + catalog, + Some(raw_key), + None, + &eligible, + )?; let requested_model = selected.model; if let Some(previous) = @@ -218,7 +225,8 @@ pub fn resolve_model_fallbacks( } let primary = FallbackTarget::new(&selected.provider, &requested_model); - let primary_model = catalog.get_on_provider(&selected.provider, &requested_model); + let primary_model = + catalog::model_on_provider(catalog, selected.provider.as_str(), &requested_model); let mut targets = Vec::new(); for model_ref in references { @@ -226,7 +234,7 @@ pub fn resolve_model_fallbacks( catalog, &requested_model, &primary, - primary_model, + primary_model.as_ref(), &eligible, model_ref, )? { @@ -292,7 +300,7 @@ fn resolve_fallback_candidate( catalog: &Catalog, requested_model: &str, primary: &FallbackTarget, - primary_model: Option<&Model>, + primary_model: Option<&ModelEntry<'_>>, eligible: &HashSet, model_ref: &ModelRef, ) -> Result { @@ -300,7 +308,7 @@ fn resolve_fallback_candidate( Ok(match model_ref.resolve(catalog)? { ResolvedModelRef::Provider(provider_name) => { - let provider = catalog.provider_id(&provider_name)?; + let provider = selection::require_provider(catalog, &provider_name)?; if !eligible.contains(&provider) { return Ok(FallbackCandidate::Skipped( ModelFallbackNotice::ProviderUnconfigured { @@ -319,8 +327,10 @@ fn resolve_fallback_candidate( }, )); }; - match catalog.closest(&provider, primary_model) { - Some(model) => FallbackCandidate::Target(FallbackTarget::new(provider, &model.id)), + match catalog::closest_model(catalog, provider.as_str(), primary_model.model) { + Some(entry) => { + FallbackCandidate::Target(FallbackTarget::new(provider, entry.model.id())) + } None => FallbackCandidate::Skipped(ModelFallbackNotice::NoCompatibleModel { requested_model: requested_model.to_string(), reference, @@ -332,7 +342,7 @@ fn resolve_fallback_candidate( provider: Some(provider_name), selector, } => { - let provider = catalog.provider_id(&provider_name)?; + let provider = selection::require_provider(catalog, &provider_name)?; if !eligible.contains(&provider) { return Ok(FallbackCandidate::Skipped( ModelFallbackNotice::ProviderUnconfigured { @@ -342,10 +352,11 @@ fn resolve_fallback_candidate( }, )); } - match catalog.resolve_on_provider(&provider, &selector) { - Ok(info) => { - FallbackCandidate::Target(FallbackTarget::new(&info.provider, &info.id)) - } + match selection::resolve_on_provider(catalog, &provider, &selector) { + Ok(entry) => FallbackCandidate::Target(FallbackTarget::new( + entry.provider.id(), + entry.model.id(), + )), Err(ModelSelectionError::UnknownSelectorOnProvider { .. }) => { FallbackCandidate::Target(FallbackTarget::new(provider, selector)) } @@ -355,8 +366,11 @@ fn resolve_fallback_candidate( ResolvedModelRef::Model { provider: None, selector, - } => match catalog.select(&selector, None, eligible) { - Ok(info) => FallbackCandidate::Target(FallbackTarget::new(&info.provider, &info.id)), + } => match selection::select(catalog, &selector, None, eligible) { + Ok(entry) => FallbackCandidate::Target(FallbackTarget::new( + entry.provider.id(), + entry.model.id(), + )), Err(ModelSelectionError::NoEligibleOffering { providers, .. }) => { FallbackCandidate::Skipped(ModelFallbackNotice::NoConfiguredOffering { requested_model: requested_model.to_string(), @@ -376,7 +390,10 @@ fn resolve_fallback_candidate( mod tests { use std::collections::BTreeMap; - use fabro_model::{Catalog, FallbackTarget, ProviderId}; + use fabro_llm::FallbackTarget; + use fabro_llm::lithos_catalog::Catalog; + use fabro_llm::test_support::test_catalog_with_overlay; + use fabro_types::ProviderId; use super::{ModelFallbackNotice, resolve_model_fallbacks}; @@ -388,14 +405,7 @@ mod tests { } fn openrouter_catalog() -> Catalog { - let overrides = toml::from_str( - r" -[providers.openrouter] -enabled = true -", - ) - .expect("catalog override should parse"); - Catalog::from_builtin_with_overrides(&overrides).expect("catalog should build") + test_catalog_with_overlay("[providers.openrouter.metadata.fabro]\nenabled = true\n") } #[test] @@ -497,19 +507,9 @@ enabled = true #[test] fn resolves_the_requested_production_policy_as_independent_chains() { - let catalog = { - let overrides = toml::from_str( - r" -[providers.modal] -enabled = true - -[providers.openrouter] -enabled = true -", - ) - .expect("catalog override should parse"); - Catalog::from_builtin_with_overrides(&overrides).expect("catalog should build") - }; + let catalog = test_catalog_with_overlay( + "[providers.modal.metadata.fabro]\nenabled = true\n\n[providers.openrouter.metadata.fabro]\nenabled = true\n", + ); let eligible = [ ProviderId::new("modal"), ProviderId::new("moonshot"), diff --git a/lib/components/fabro-workflow/src/operations/create.rs b/lib/components/fabro-workflow/src/operations/create.rs index bcb3087d5..188154f1b 100644 --- a/lib/components/fabro-workflow/src/operations/create.rs +++ b/lib/components/fabro-workflow/src/operations/create.rs @@ -12,12 +12,12 @@ use std::sync::Arc; use fabro_config::Storage; use fabro_graphviz::graph::{AttrValue, Graph}; -use fabro_model::{Catalog, ProviderId}; +use fabro_llm::lithos_catalog::Catalog; use fabro_store::{BlobStore, Database}; use fabro_template::TemplateContext; use fabro_types::{ - AutomationRef, BlobHash, ForkSourceRef, GitContext, ManifestPath, RunId, RunProvenance, - RunTarget, WorkflowSettings, WorkflowVersionId, + AutomationRef, BlobHash, ForkSourceRef, GitContext, ManifestPath, ProviderId, RunId, + RunProvenance, RunTarget, WorkflowSettings, WorkflowVersionId, }; use fabro_util::json::normalize_json_value; use tokio::task::spawn_blocking; @@ -688,7 +688,7 @@ mod tests { use fabro_store::Database; use fabro_types::settings::InterpString; use fabro_types::settings::run::RunMode; - use fabro_types::{EventBody, WorkflowSettings, fixtures, test_support}; + use fabro_types::{EventBody, WorkflowSettings, fixtures, provider_ids, test_support}; use fabro_util::error::collect_chain; use fabro_validate::Severity; use object_store::local::LocalFileSystem; @@ -731,60 +731,32 @@ mod tests { } fn test_catalog() -> Arc { - Arc::new(Catalog::from_builtin().unwrap()) + Arc::new(fabro_llm::test_support::test_catalog()) } + /// OpenAI and OpenRouter both offering GPT-5.6 Sol as their default, so a + /// portable selector resolves to whichever provider is ready. fn portable_model_catalog() -> Arc { - let settings: fabro_model::catalog::LlmCatalogSettings = toml::from_str( + Arc::new(fabro_llm::test_support::test_catalog_with_overlay( r#" -[providers.openai] -display_name = "OpenAI" -adapter = "openai" -agent_profile = "openai" -priority = 90 - -[providers.openai.models."gpt-5.6-sol"] -display_name = "GPT-5.6 Sol" -family = "gpt-5" -aliases = ["gpt-56-sol"] -default = true - -[providers.openai.models."gpt-5.6-sol".limits] -context_window = 1000 - -[providers.openai.models."gpt-5.6-sol".features] -tools = true -vision = false -reasoning = false - -[providers.openrouter] -display_name = "OpenRouter" -adapter = "openai_compatible" -agent_profile = "openai" -priority = 25 - -[providers.openrouter.models."gpt-5.6-sol"] -api_id = "openai/gpt-5.6-sol" -display_name = "GPT-5.6 Sol (via OpenRouter)" -family = "gpt-5" -aliases = ["gpt-56-sol"] -default = true - -[providers.openrouter.models."gpt-5.6-sol".limits] -context_window = 1000 - -[providers.openrouter.models."gpt-5.6-sol".features] -tools = true -vision = false -reasoning = false -"#, - ) - .unwrap(); - Arc::new(Catalog::from_settings(&settings).unwrap()) + [providers.openai] + priority = 90 + default_model = "gpt-5.6-sol" + + [providers.openrouter] + priority = 25 + default_model = "gpt-5.6-sol" + + [providers.openrouter.metadata.fabro] + enabled = true + "#, + )) } fn test_provider_ids() -> Vec { - Catalog::builtin().all_provider_ids().into_iter().collect() + fabro_llm::catalog::enabled_provider_ids(&fabro_llm::test_support::test_catalog()) + .into_iter() + .collect() } fn compile_input(request: &CreateRunInput) -> CreateRunCompileInput { @@ -2056,25 +2028,25 @@ reasoning = false }"#; let catalog = portable_model_catalog(); let cases = [ - (vec![ProviderId::openai()], None, ProviderId::openai()), + (vec![provider_ids::openai()], None, provider_ids::openai()), ( vec![ProviderId::new("openrouter")], None, ProviderId::new("openrouter"), ), ( - vec![ProviderId::openai(), ProviderId::new("openrouter")], + vec![provider_ids::openai(), ProviderId::new("openrouter")], None, - ProviderId::openai(), + provider_ids::openai(), ), ( - vec![ProviderId::openai(), ProviderId::new("openrouter")], + vec![provider_ids::openai(), ProviderId::new("openrouter")], Some("openrouter"), ProviderId::new("openrouter"), ), ]; - for selector in ["gpt-56-sol", "openai/gpt-5.6-sol"] { + for selector in ["gpt-56-sol", "gpt-5.6"] { for (ready, explicit_provider, expected_provider) in &cases { let dir = tempfile::tempdir().unwrap(); let mut settings = test_default_settings(); diff --git a/lib/components/fabro-workflow/src/operations/start.rs b/lib/components/fabro-workflow/src/operations/start.rs index 520e09a18..d55e531e3 100644 --- a/lib/components/fabro-workflow/src/operations/start.rs +++ b/lib/components/fabro-workflow/src/operations/start.rs @@ -6,9 +6,8 @@ use std::time::{Duration, Instant}; use fabro_auth::{CredentialSource, VaultCredentialSource}; use fabro_interview::{AutoApproveInterviewer, Interviewer}; -use fabro_llm::client::Client as LlmClient; +use fabro_llm::lithos_catalog::Catalog; use fabro_mcp::config::McpServerSettings; -use fabro_model::{Catalog, ProviderId}; use fabro_sandbox::daytona::DaytonaConfig; use fabro_sandbox::from_environment::{ daytona_config_from_environment, docker_config_from_environment_with_secrets, @@ -24,7 +23,7 @@ use fabro_types::settings::run::{ RunPrepareSettings as ResolvedRunPrepareSettings, }; use fabro_types::{ - ManifestPath, RunId, RunRunnableSource, RunSpec, RunTarget, SandboxProviderKind, + ManifestPath, ProviderId, RunId, RunRunnableSource, RunSpec, RunTarget, SandboxProviderKind, TargetValidationError, }; use fabro_util::error::collect_chain; @@ -751,15 +750,7 @@ async fn configured_providers_for_start( Arc::clone(vault), process_env_var, )); - match LlmClient::from_source_report(source.as_ref(), catalog).await { - Ok(report) => report - .client - .provider_names() - .into_iter() - .map(ProviderId::new) - .collect(), - Err(_) => Vec::new(), - } + source.resolve_all(catalog.as_ref()).await.ready } fn git_checkpoint_options_from_start( @@ -1321,7 +1312,7 @@ mod tests { }; use fabro_types::{ BilledModelUsage, GitContext, ManifestPath, RunTarget, StageTiming, WorkflowSettings, - fixtures, test_support, + fixtures, provider_ids, test_support, }; use fabro_vault::SecretType; use object_store::memory::InMemory; @@ -1442,73 +1433,32 @@ mod tests { } fn test_catalog() -> Arc { - Arc::new(Catalog::from_builtin().expect("default catalog should build")) + Arc::new(fabro_llm::test_support::test_catalog()) } fn test_provider_ids() -> Vec { - Catalog::builtin().all_provider_ids().into_iter().collect() + fabro_llm::catalog::enabled_provider_ids(&fabro_llm::test_support::test_catalog()) + .into_iter() + .collect() } + /// OpenAI and OpenRouter both offering GPT-5.6 Sol as their default, so a + /// portable selector resolves to whichever provider is ready. fn portable_model_catalog() -> Catalog { - let settings: fabro_model::catalog::LlmCatalogSettings = toml::from_str( + fabro_llm::test_support::test_catalog_with_overlay( r#" -[providers.openai] -display_name = "OpenAI" -adapter = "openai" -agent_profile = "openai" -priority = 90 - -[providers.openai.models."gpt-5.6-sol"] -display_name = "GPT-5.6 Sol" -family = "gpt-5" -aliases = ["gpt-56-sol"] -default = true - -[providers.openai.models."gpt-5.6-sol".limits] -context_window = 1000 - -[providers.openai.models."gpt-5.6-sol".features] -tools = true -vision = false -reasoning = false - -[providers.openai.models."gpt-5.4-mini"] -display_name = "GPT-5.4 Mini" -family = "gpt-5" -aliases = ["mini"] - -[providers.openai.models."gpt-5.4-mini".limits] -context_window = 1000 - -[providers.openai.models."gpt-5.4-mini".features] -tools = true -vision = false -reasoning = false - -[providers.openrouter] -display_name = "OpenRouter" -adapter = "openai_compatible" -agent_profile = "openai" -priority = 25 - -[providers.openrouter.models."gpt-5.6-sol"] -api_id = "openai/gpt-5.6-sol" -display_name = "GPT-5.6 Sol (via OpenRouter)" -family = "gpt-5" -aliases = ["gpt-56-sol"] -default = true - -[providers.openrouter.models."gpt-5.6-sol".limits] -context_window = 1000 - -[providers.openrouter.models."gpt-5.6-sol".features] -tools = true -vision = false -reasoning = false -"#, + [providers.openai] + priority = 90 + default_model = "gpt-5.6-sol" + + [providers.openrouter] + priority = 25 + default_model = "gpt-5.6-sol" + + [providers.openrouter.metadata.fabro] + enabled = true + "#, ) - .unwrap(); - Catalog::from_settings(&settings).unwrap() } #[test] @@ -1525,40 +1475,40 @@ reasoning = false assert!(matches!( error, - Error::ModelSelection(fabro_model::ModelSelectionError::ProviderUnavailable { + Error::ModelSelection(fabro_llm::ModelSelectionError::ProviderUnavailable { provider - }) if provider == ProviderId::openai() + }) if provider == provider_ids::openai() )); } #[test] fn resolve_start_llm_infers_provider_from_model_alias() { - let overrides: fabro_model::catalog::LlmCatalogSettings = toml::from_str( + let catalog = fabro_llm::test_support::test_catalog_with_overlay( r#" -[providers.acme] -adapter = "openai_compatible" -agent_profile = "openai" -base_url = "https://api.acme.test/v1" - -[models.acme-claude] -provider = "acme" -display_name = "Acme Claude" -family = "claude" -default = true -agent_profile = "anthropic" -aliases = ["ac"] - -[models.acme-claude.limits] -context_window = 1000 - -[models.acme-claude.features] -tools = true -vision = false -reasoning = false -"#, - ) - .unwrap(); - let catalog = Catalog::from_builtin_with_overrides(&overrides).unwrap(); + [providers.acme] + display_name = "Acme" + adapter = "openai-compatible" + codec = "openai-chat" + base_url = "https://api.acme.test/v1" + auth = { type = "bearer" } + default_model = "acme-claude" + + [providers.acme.metadata.fabro] + agent_profile = "openai" + credentials = ["env:ACME_API_KEY"] + + [providers.acme.models.acme-claude] + display_name = "Acme Claude" + aliases = ["ac"] + api_model = "acme-claude" + limits = { context_tokens = 1000, max_output_tokens = 500 } + capabilities = { text = true, tools = true } + + [providers.acme.models.acme-claude.metadata.fabro] + family = "claude" + agent_profile = "anthropic" + "#, + ); let mut settings = ResolvedRunSettings::default(); settings.model.name = Some("ac".to_string()); diff --git a/lib/components/fabro-workflow/src/operations/validate.rs b/lib/components/fabro-workflow/src/operations/validate.rs index dfcb38777..3a8db0c5b 100644 --- a/lib/components/fabro-workflow/src/operations/validate.rs +++ b/lib/components/fabro-workflow/src/operations/validate.rs @@ -2,8 +2,8 @@ use std::collections::HashMap; use std::path::PathBuf; use std::sync::Arc; -use fabro_model::{Catalog, ProviderId}; -use fabro_types::WorkflowSettings; +use fabro_llm::lithos_catalog::Catalog; +use fabro_types::{ProviderId, WorkflowSettings}; use super::create::{configured_default_provider, preprocess_and_validate, template_context}; use super::source::{ResolveWorkflowInput, WorkflowInput, resolve_workflow}; diff --git a/lib/components/fabro-workflow/src/outcome.rs b/lib/components/fabro-workflow/src/outcome.rs index 330969de1..e41717ca5 100644 --- a/lib/components/fabro-workflow/src/outcome.rs +++ b/lib/components/fabro-workflow/src/outcome.rs @@ -1,56 +1,37 @@ pub use fabro_core::outcome::{ FailureCategory, FailureDetail, OutcomeMeta, StageOutcome, StageState, }; -use fabro_llm::types::TokenCounts as LlmTokenCounts; -use fabro_model::{ - BilledTokenCounts, Catalog, ModelBillingInput, ModelRef, ModelUsage, TokenCounts, -}; +use fabro_llm::catalog; +use fabro_llm::lithos_catalog::Catalog; pub use fabro_types::BilledModelUsage; +use fabro_types::{BilledTokenCounts, ModelRef, TokenCounts}; use crate::error::{Error, FailureSignature, classify_failure_reason}; pub type Outcome = fabro_core::Outcome>; +/// Bills `usage` on `model` from catalog pricing. +/// +/// The provider must be one the catalog knows; a passthrough model on a known +/// provider is billed with no cost, since the catalog has no rates for it. pub fn billed_model_usage_from_llm( catalog: &Catalog, model: &ModelRef, - usage: &LlmTokenCounts, + usage: TokenCounts, ) -> Result { - let tokens = token_counts_from_llm_usage(usage); - let facts = catalog.billing_facts_for(model, &tokens).ok_or_else(|| { - Error::Precondition(format!("Provider \"{}\" is not configured", model.provider)) - })?; - let input = ModelBillingInput { - usage: ModelUsage { - model: model.clone(), - tokens, - }, - facts, - }; - - let total_usd_micros = catalog - .pricing_for(model) - .and_then(|pricing| pricing.bill(&input)) - .map(|amount| amount.0); - - Ok(BilledModelUsage { - input, - total_usd_micros, - }) + if catalog::provider(catalog, model.provider.as_str()).is_none() { + return Err(Error::Precondition(format!( + "Provider \"{}\" is not configured", + model.provider + ))); + } + let cost = catalog::estimate_cost(catalog, model, usage); + Ok(BilledModelUsage::new(model.clone(), usage, cost)) } #[must_use] -pub fn billed_token_counts_from_llm(usage: &LlmTokenCounts) -> BilledTokenCounts { - let tokens = token_counts_from_llm_usage(usage); - BilledTokenCounts { - input_tokens: tokens.input_tokens, - output_tokens: tokens.output_tokens, - total_tokens: tokens.total_tokens(), - reasoning_tokens: tokens.reasoning_tokens, - cache_read_tokens: tokens.cache_read_tokens, - cache_write_tokens: tokens.cache_write_tokens, - total_usd_micros: None, - } +pub fn billed_token_counts_from_llm(usage: TokenCounts) -> BilledTokenCounts { + BilledTokenCounts::from_token_counts(usage, None) } pub trait OutcomeExt: Sized { @@ -141,58 +122,56 @@ pub fn format_cost(cost: f64) -> String { format!("${cost:.2}") } -fn token_counts_from_llm_usage(usage: &LlmTokenCounts) -> TokenCounts { - usage.clone() -} - #[cfg(test)] mod tests { - use fabro_llm::types::TokenCounts; - use fabro_model::catalog::LlmCatalogSettings; - use fabro_model::{Catalog, ModelRef, ProviderId, Speed, UsdMicros}; + use fabro_llm::lithos_catalog::Catalog; + use fabro_llm::test_support::{test_catalog, test_catalog_with_overlay}; + use fabro_types::{ModelId, ModelRef, ProviderId, Speed, TokenCounts, UsdMicros, provider_ids}; use super::{OutcomeExt, billed_model_usage_from_llm}; fn model_ref(provider: ProviderId, model_id: &str, speed: Option) -> ModelRef { - ModelRef { - provider, - model_id: model_id.into(), - speed, - } + ModelRef::new(provider, ModelId::new(model_id)).with_speed(speed) + } + + fn catalog() -> Catalog { + test_catalog() } #[test] fn billed_model_usage_from_llm_bills_openai_cached_input_and_reasoning_output() { + // Stay under the 272k long-context tier so the standard rates apply. let usage = TokenCounts { - input_tokens: 500_000, - output_tokens: 125_000, - reasoning_tokens: 25_000, - cache_read_tokens: 250_000, + input: 100_000, + output: 25_000, + reasoning: 5_000, + cache_read: 50_000, ..TokenCounts::default() }; let billed = billed_model_usage_from_llm( - Catalog::builtin(), - &model_ref(ProviderId::openai(), "gpt-5.4", None), - &usage, + &catalog(), + &model_ref(provider_ids::openai(), "gpt-5.4", None), + usage, ) .unwrap(); - assert_eq!(billed.total_usd_micros, Some(3_562_500)); - assert_eq!(billed.tokens().output_tokens, 125_000); - assert_eq!(billed.tokens().reasoning_tokens, 25_000); + // 100k input at $2.50/M + 50k cached at $0.25/M + 30k output at $15/M. + assert_eq!(billed.total_usd_micros, Some(712_500)); + assert_eq!(billed.tokens().output, 25_000); + assert_eq!(billed.tokens().reasoning, 5_000); } #[test] fn response_cost_overrides_catalog_estimate() { let usage = TokenCounts { - input_tokens: 11, - output_tokens: 7, + input: 11, + output: 7, ..TokenCounts::default() }; let billed = billed_model_usage_from_llm( - Catalog::builtin(), - &model_ref(ProviderId::openai(), "gpt-5.4", None), - &usage, + &catalog(), + &model_ref(provider_ids::openai(), "gpt-5.4", None), + usage, ) .unwrap() .with_reported_cost(Some(UsdMicros(125_000))); @@ -213,142 +192,88 @@ mod tests { #[test] fn billed_model_usage_from_llm_bills_anthropic_fast_mode_cache_write_pricing() { let usage = TokenCounts { - input_tokens: 100_000, - output_tokens: 10_000, - reasoning_tokens: 5_000, - cache_read_tokens: 20_000, - cache_write_tokens: 30_000, + input: 100_000, + output: 10_000, + reasoning: 5_000, + cache_read: 20_000, + cache_write: 30_000, }; let billed = billed_model_usage_from_llm( - Catalog::builtin(), + &catalog(), &model_ref( - ProviderId::anthropic(), - "claude-opus-4-6", + provider_ids::anthropic(), + "claude-opus-5", Some(Speed::Fast), ), - &usage, + usage, ) .unwrap(); - assert_eq!(billed.total_usd_micros, Some(6_435_000)); + // Fast rates: $10/M input, $50/M output (incl. reasoning), $1/M cache + // read, $12.50/M cache write. + assert_eq!(billed.total_usd_micros, Some(2_145_000)); } #[test] fn billed_model_usage_from_llm_uses_injected_custom_catalog() { - let settings: LlmCatalogSettings = toml::from_str( + let catalog = test_catalog_with_overlay( r#" [providers.proxy] display_name = "Proxy" -adapter = "openai_compatible" -agent_profile = "openai" -billing_policy = "openai" +adapter = "openai-compatible" +codec = "openai-chat" base_url = "https://proxy.example/v1" +auth = { type = "bearer" } +default_model = "canonical-model" -[models.canonical-model] -provider = "proxy" -api_id = "wire-model" +[providers.proxy.models.canonical-model] display_name = "Canonical Model" -family = "proxy" -default = true - -[models.canonical-model.limits] -context_window = 1000 - -[models.canonical-model.features] -tools = true -vision = false -reasoning = false - -[models.canonical-model.costs] -input_cost_per_mtok = 1.0 -output_cost_per_mtok = 2.0 +api_model = "wire-model" +limits = { context_tokens = 1000, max_output_tokens = 500 } +capabilities = { text = true, tools = true } +pricing = { input_usd_micros_per_million = 1000000, output_usd_micros_per_million = 2000000 } "#, - ) - .unwrap(); - let catalog = Catalog::from_settings(&settings).unwrap(); + ); let usage = TokenCounts { - input_tokens: 500_000, - output_tokens: 250_000, + input: 1_000_000, + output: 500_000, ..TokenCounts::default() }; - let billed = billed_model_usage_from_llm( &catalog, &model_ref(ProviderId::new("proxy"), "canonical-model", None), - &usage, + usage, ) .unwrap(); - assert_eq!(&billed.model().provider, &ProviderId::new("proxy")); + assert_eq!(billed.total_usd_micros, Some(2_000_000)); assert_eq!(billed.model_id(), "canonical-model"); - assert_eq!(billed.total_usd_micros, Some(1_000_000)); } #[test] - fn billed_model_usage_from_llm_does_not_bill_provider_api_id() { - let settings: LlmCatalogSettings = toml::from_str( - r#" -[providers.proxy] -display_name = "Proxy" -adapter = "openai_compatible" -agent_profile = "openai" -billing_policy = "openai" -base_url = "https://proxy.example/v1" - -[models.canonical-model] -provider = "proxy" -api_id = "wire-model" -display_name = "Canonical Model" -family = "proxy" -default = true - -[models.canonical-model.limits] -context_window = 1000 - -[models.canonical-model.features] -tools = true -vision = false -reasoning = false - -[models.canonical-model.costs] -input_cost_per_mtok = 1.0 -output_cost_per_mtok = 2.0 -"#, - ) - .unwrap(); - let catalog = Catalog::from_settings(&settings).unwrap(); - + fn passthrough_model_on_known_provider_has_no_cost() { let billed = billed_model_usage_from_llm( - &catalog, - &model_ref(ProviderId::new("proxy"), "wire-model", None), - &TokenCounts { - input_tokens: 500_000, - output_tokens: 250_000, + &catalog(), + &model_ref(provider_ids::openai(), "brand-new-model", None), + TokenCounts { + input: 10, + output: 5, ..TokenCounts::default() }, ) .unwrap(); - - assert_eq!(billed.model_id(), "wire-model"); assert_eq!(billed.total_usd_micros, None); + assert_eq!(billed.tokens().input, 10); } #[test] - fn billed_model_usage_round_trips_dense_token_counts() { - let usage = TokenCounts { - input_tokens: 100, - output_tokens: 40, - reasoning_tokens: 5, - cache_read_tokens: 20, - cache_write_tokens: 10, - }; - let billed = billed_model_usage_from_llm( - Catalog::builtin(), - &model_ref(ProviderId::anthropic(), "claude-opus-4-6", None), - &usage, + fn unknown_provider_is_a_precondition_failure() { + let error = billed_model_usage_from_llm( + &catalog(), + &model_ref(ProviderId::new("nowhere"), "model", None), + TokenCounts::default(), ) - .unwrap(); - - assert_eq!(billed.tokens().clone(), usage); + .unwrap_err(); + assert!(error.to_string().contains("not configured"), "{error}"); } } diff --git a/lib/components/fabro-workflow/src/pipeline/execute/tests.rs b/lib/components/fabro-workflow/src/pipeline/execute/tests.rs index 7e8647f6c..ec254b3d2 100644 --- a/lib/components/fabro-workflow/src/pipeline/execute/tests.rs +++ b/lib/components/fabro-workflow/src/pipeline/execute/tests.rs @@ -87,13 +87,8 @@ fn test_run_id(label: &str) -> RunId { } } -fn test_catalog() -> Arc { - Arc::new( - fabro_model::Catalog::from_builtin_with_overrides( - &fabro_model::catalog::LlmCatalogSettings::default(), - ) - .expect("default catalog should build"), - ) +fn test_catalog() -> Arc { + Arc::new(fabro_llm::test_support::test_catalog()) } fn test_emitter(label: &str) -> Emitter { @@ -270,7 +265,7 @@ async fn execute_test_run_with_options( }, llm: LlmSpec { model: "test-model".to_string(), - provider_id: fabro_model::ProviderId::anthropic(), + provider_id: fabro_types::provider_ids::anthropic(), fallbacks: ModelFallbackPolicy::default(), mcp_servers: Vec::new(), model_controls: RunModelControls::default(), @@ -330,7 +325,7 @@ async fn execute_runs_start_to_exit_and_returns_final_context() { }, llm: LlmSpec { model: "test-model".to_string(), - provider_id: fabro_model::ProviderId::anthropic(), + provider_id: fabro_types::provider_ids::anthropic(), fallbacks: ModelFallbackPolicy::default(), mcp_servers: Vec::new(), model_controls: RunModelControls::default(), @@ -471,7 +466,7 @@ async fn resumed_in_flight_node_starts_a_new_stage_execution() { }, llm: LlmSpec { model: "test-model".to_string(), - provider_id: fabro_model::ProviderId::anthropic(), + provider_id: fabro_types::provider_ids::anthropic(), fallbacks: ModelFallbackPolicy::default(), mcp_servers: Vec::new(), model_controls: RunModelControls::default(), @@ -585,7 +580,7 @@ async fn run_with_lifecycle( }, llm: LlmSpec { model: "test-model".to_string(), - provider_id: fabro_model::ProviderId::anthropic(), + provider_id: fabro_types::provider_ids::anthropic(), fallbacks: ModelFallbackPolicy::default(), mcp_servers: Vec::new(), model_controls: RunModelControls::default(), diff --git a/lib/components/fabro-workflow/src/pipeline/finalize.rs b/lib/components/fabro-workflow/src/pipeline/finalize.rs index a91ebbd6e..bc1ec1860 100644 --- a/lib/components/fabro-workflow/src/pipeline/finalize.rs +++ b/lib/components/fabro-workflow/src/pipeline/finalize.rs @@ -81,7 +81,7 @@ fn build_conclusion_from_projection( final_git_commit_sha: Option, ) -> Conclusion { let billing = projection - .map(|projection| billing_rollup::billing_rollup_from_projection(projection, None)) + .map(billing_rollup::billing_rollup_from_projection) .unwrap_or_default(); let (stages, total_retries) = projection .map(|projection| billing.conclusion_stages(projection)) @@ -343,7 +343,7 @@ async fn compute_final_patch( #[cfg(any(test, feature = "test-support"))] pub(crate) fn billing_from_projection(projection: &RunProjection) -> Option { - billing_rollup::billing_rollup_from_projection(projection, None).billing_if_present() + billing_rollup::billing_rollup_from_projection(projection).billing_if_present() } pub(crate) fn build_terminal_event( @@ -580,7 +580,6 @@ mod tests { use bytes::Bytes; use fabro_auth::test_support as auth_test_support; use fabro_graphviz::graph::Graph; - use fabro_model::Catalog; use fabro_sandbox::test_support::MockSandbox; use fabro_store::{Database, EventEnvelope, RunDatabase, RunProjection}; use fabro_types::run_event::{MetadataSnapshotFailureKind, MetadataSnapshotPhase}; @@ -997,10 +996,10 @@ mod tests { None, locations, tokio_util::sync::CancellationToken::new(), - fabro_model::ProviderId::anthropic(), + fabro_types::provider_ids::anthropic(), "claude-sonnet-4-6".to_string(), auth_test_support::vault_only_credential_source(), - Arc::new(Catalog::from_builtin().expect("default catalog should build")), + Arc::new(fabro_llm::test_support::test_catalog()), Arc::new(SandboxGitRuntime::new()), metadata_runtime, metadata_writer, @@ -1030,10 +1029,10 @@ mod tests { None, locations, tokio_util::sync::CancellationToken::new(), - fabro_model::ProviderId::anthropic(), + fabro_types::provider_ids::anthropic(), "claude-sonnet-4-6".to_string(), auth_test_support::vault_only_credential_source(), - Arc::new(Catalog::from_builtin().expect("default catalog should build")), + Arc::new(fabro_llm::test_support::test_catalog()), Arc::new(SandboxGitRuntime::new()), Arc::new(RunMetadataRuntime::new()), None, diff --git a/lib/components/fabro-workflow/src/pipeline/initialize.rs b/lib/components/fabro-workflow/src/pipeline/initialize.rs index d05d9acd9..38fc31e51 100644 --- a/lib/components/fabro-workflow/src/pipeline/initialize.rs +++ b/lib/components/fabro-workflow/src/pipeline/initialize.rs @@ -4,13 +4,11 @@ use std::sync::Arc; use std::time::Instant; use fabro_agent::{Sandbox, ToolSecrets}; -use fabro_auth::{ - CredentialSource, ExtraHeadersCredentialSource, VaultCredentialSource, auth_issue_message, -}; +use fabro_auth::{CredentialSource, ExtraHeadersCredentialSource, VaultCredentialSource}; use fabro_github::token_source::InstallationTokenSource; use fabro_graphviz::graph; use fabro_hooks::{HookContext, HookDecision, HookEvent, HookExecutionContext, HookRunner}; -use fabro_model::Catalog; +use fabro_llm::lithos_catalog::Catalog; use fabro_sandbox::{ GitSetupIntent, SandboxEventCallback, SandboxSpec, reconnect_for_run_with_callback, shell_quote, }; @@ -278,37 +276,22 @@ async fn build_registry( return Ok((build_llm_registry(), false)); } - match llm_source.resolve(catalog.as_ref()).await { - Ok(result) if result.credentials.is_empty() => { - if graph_needs_llm { - let detail = (!result.auth_issues.is_empty()).then(|| { - result - .auth_issues - .iter() - .map(|(provider, issue)| auth_issue_message(provider, issue)) - .collect::>() - .join("; ") - }); - let prefix = detail.map_or_else( - || "No LLM providers configured".to_string(), - |detail| format!("No usable LLM providers configured: {detail}"), - ); - return Err(Error::Precondition(format!( - "{prefix}. Set ANTHROPIC_API_KEY or OPENAI_API_KEY, or pass --dry-run to simulate." - ))); - } - Ok((build_no_backend(), false)) - } - Ok(_result) => Ok((build_llm_registry(), false)), - Err(e) => { - if graph_needs_llm { - return Err(Error::Precondition(format!( - "Failed to initialize LLM client: {e}. Set ANTHROPIC_API_KEY or OPENAI_API_KEY, or pass --dry-run to simulate.", - ))); - } - Ok((build_no_backend(), false)) + let result = llm_source.resolve_all(catalog.as_ref()).await; + if result.ready.is_empty() { + if graph_needs_llm { + let detail = + (!result.auth_issues.is_empty()).then(|| result.issue_messages().join("; ")); + let prefix = detail.map_or_else( + || "No LLM providers configured".to_string(), + |detail| format!("No usable LLM providers configured: {detail}"), + ); + return Err(Error::Precondition(format!( + "{prefix}. Set ANTHROPIC_API_KEY or OPENAI_API_KEY, or pass --dry-run to simulate." + ))); } + return Ok((build_no_backend(), false)); } + Ok((build_llm_registry(), false)) } async fn tool_secrets_from_configured_sources(vault: &Arc>) -> ToolSecrets { @@ -781,7 +764,7 @@ mod tests { } fn test_catalog() -> Arc { - Arc::new(Catalog::from_builtin().expect("default catalog should build")) + Arc::new(fabro_llm::test_support::test_catalog()) } fn memory_store() -> Arc { @@ -911,7 +894,7 @@ mod tests { sandbox: SandboxSpec::Local { working_directory }, llm: LlmSpec { model: "test-model".to_string(), - provider_id: fabro_model::ProviderId::anthropic(), + provider_id: fabro_types::provider_ids::anthropic(), fallbacks: ModelFallbackPolicy::default(), mcp_servers: Vec::new(), model_controls: RunModelControls::default(), @@ -1101,17 +1084,16 @@ mod tests { assert_eq!(initialized.model, "test-model"); assert_eq!( initialized.engine.run.provider_id, - fabro_model::ProviderId::anthropic() + fabro_types::provider_ids::anthropic() ); assert!( initialized .engine .run .llm_source - .resolve(&initialized.engine.run.catalog) + .resolve_all(&initialized.engine.run.catalog) .await - .unwrap() - .credentials + .ready .is_empty() ); } @@ -1229,7 +1211,7 @@ mod tests { let (_registry, effective_dry_run) = build_registry( &LlmSpec { model: "claude-opus-4-6".to_string(), - provider_id: fabro_model::ProviderId::anthropic(), + provider_id: fabro_types::provider_ids::anthropic(), fallbacks: ModelFallbackPolicy::default(), mcp_servers: Vec::new(), model_controls: RunModelControls::default(), @@ -1261,17 +1243,22 @@ mod tests { let expected_session_id = run_id.to_string(); let source = build_llm_source(vault, run_id); - let resolved = source.resolve(test_catalog().as_ref()).await.unwrap(); + let catalog = test_catalog(); + let resolved = source.resolve_all(catalog.as_ref()).await; - assert!(!resolved.credentials.is_empty()); - for credential in &resolved.credentials { - assert_eq!( - credential - .extra_headers - .get(SESSION_ID_HEADER) - .map(String::as_str), - Some(expected_session_id.as_str()) - ); + assert!(!resolved.ready.is_empty()); + for provider in &resolved.ready { + let provider = catalog.provider(provider.as_str()).unwrap(); + let credentials = source.credentials(provider).await.unwrap(); + let fabro_llm::credentials::Credentials::Http(http) = credentials else { + panic!("vault credentials should be HTTP credentials"); + }; + let session_header = http + .extra_headers + .iter() + .find(|header| header.name == SESSION_ID_HEADER) + .map(|header| header.value.expose_secret()); + assert_eq!(session_header, Some(expected_session_id.as_str())); } } @@ -1351,7 +1338,7 @@ mod tests { }, llm: LlmSpec { model: "fake-acp".to_string(), - provider_id: fabro_model::ProviderId::openai(), + provider_id: fabro_types::provider_ids::openai(), fallbacks: ModelFallbackPolicy::default(), mcp_servers: Vec::new(), model_controls: RunModelControls::default(), @@ -1454,7 +1441,7 @@ mod tests { }, llm: LlmSpec { model: "test-model".to_string(), - provider_id: fabro_model::ProviderId::anthropic(), + provider_id: fabro_types::provider_ids::anthropic(), fallbacks: ModelFallbackPolicy::default(), mcp_servers: Vec::new(), model_controls: RunModelControls::default(), @@ -1596,7 +1583,7 @@ mod tests { }, llm: LlmSpec { model: "test-model".to_string(), - provider_id: fabro_model::ProviderId::anthropic(), + provider_id: fabro_types::provider_ids::anthropic(), fallbacks: ModelFallbackPolicy::default(), mcp_servers: Vec::new(), model_controls: RunModelControls::default(), diff --git a/lib/components/fabro-workflow/src/pipeline/publish.rs b/lib/components/fabro-workflow/src/pipeline/publish.rs index 42010e133..6455ec3f1 100644 --- a/lib/components/fabro-workflow/src/pipeline/publish.rs +++ b/lib/components/fabro-workflow/src/pipeline/publish.rs @@ -179,7 +179,7 @@ impl Concluded { merge_strategy: pr_config.merge_strategy, }), run_store: &self.services.run_store, - llm_source: self.services.llm_source.as_ref(), + llm_source: Arc::clone(&self.services.llm_source), catalog: Arc::clone(&self.services.catalog), conclusion: Some(&self.conclusion), run_state: None, diff --git a/lib/components/fabro-workflow/src/pipeline/pull_request.rs b/lib/components/fabro-workflow/src/pipeline/pull_request.rs index 10f324dd3..74c5cfaa5 100644 --- a/lib/components/fabro-workflow/src/pipeline/pull_request.rs +++ b/lib/components/fabro-workflow/src/pipeline/pull_request.rs @@ -5,12 +5,11 @@ use std::time::Duration; use fabro_auth::CredentialSource; use fabro_github::{self as github_app, ssh_url_to_https}; use fabro_graphviz::parser; -use fabro_llm::client::Client; -use fabro_llm::generate::{GenerateParams, generate_object}; -use fabro_model::{Catalog, ProviderId}; +use fabro_llm::lithos_catalog::Catalog; +use fabro_llm::{Client, ClientOptions, Request, selection, structured}; use fabro_store::RunProjection; -use fabro_types::PullRequestLink; use fabro_types::settings::run::MergeStrategy; +use fabro_types::{ProviderId, PullRequestLink, Role}; use fabro_util::text::strip_goal_decoration; use tokio::time::sleep; use tracing::{debug, info, warn}; @@ -72,10 +71,10 @@ fn truncation_caps( eligible: &HashSet, catalog: &Catalog, ) -> TruncationCaps { - let ctx = catalog - .select(model, None, eligible) + let ctx = selection::select(catalog, model, None, eligible) .ok() - .and_then(|m| usize::try_from(m.context_window()).ok()) + .and_then(|entry| entry.model.limits()) + .and_then(|limits| usize::try_from(limits.context_tokens).ok()) .unwrap_or(UNKNOWN_MODEL_CTX); truncation_caps_for_context_window(ctx) @@ -334,14 +333,19 @@ pub async fn build_pr_content( goal: &str, model: &str, run_store: &RunStoreHandle, - llm_source: &dyn CredentialSource, + llm_source: Arc, catalog: Arc, conclusion: Option<&Conclusion>, run_state: Option<&RunProjection>, ) -> Result { - let client = Client::from_source(llm_source, Arc::clone(&catalog)) - .await - .map_err(|e| format!("Failed to create LLM client: {e}"))?; + let client = fabro_llm::build_client( + Catalog::clone(&catalog), + llm_source, + ClientOptions::standard(), + ) + .await + .map_err(|e| format!("Failed to create LLM client: {e}"))? + .client; build_pr_content_with_client( diff, @@ -385,7 +389,7 @@ async fn build_pr_content_with_client( let run_spec = run_state.map(|state| state.spec.clone()); let dot_source = run_state.and_then(|state| state.spec.graph_source.clone()); - let eligible = client.provider_ids(); + let eligible = client.available_providers().iter().cloned().collect(); let caps = truncation_caps(model, &eligible, catalog); let truncated_diff = truncate_chars(diff, caps.diff); @@ -398,18 +402,18 @@ async fn build_pr_content_with_client( format!("Goal: {goal}\n\nDiff:\n```\n{truncated_diff}\n```") }; - let params = GenerateParams::new(model, client) + let request = Request::builder() + .model(model) .system(PR_BODY_SYSTEM_PROMPT) - .prompt(prompt); + .message(fabro_types::Message::text(Role::User, prompt)) + .build() + .map_err(|e| format!("invalid PR content request: {e}"))?; + let completion = + structured::complete_object(&client, request, "pr_content", PR_CONTENT_SCHEMA.clone()) + .await + .map_err(|e| format!("LLM generation failed: {e}"))?; - let result = generate_object(params, PR_CONTENT_SCHEMA.clone()) - .await - .map_err(|e| format!("LLM generation failed: {e}"))?; - - let output = result - .output - .ok_or_else(|| "LLM generation returned no structured output".to_string())?; - let generated: PrContent = serde_json::from_value(output) + let generated: PrContent = serde_json::from_value(completion.object) .map_err(|e| format!("Failed to deserialize PR content: {e}"))?; let title = if generated.title.trim().is_empty() { @@ -458,7 +462,7 @@ pub struct OpenPullRequestRequest<'a> { pub draft: bool, pub auto_merge: Option, pub run_store: &'a RunStoreHandle, - pub llm_source: &'a dyn CredentialSource, + pub llm_source: Arc, pub catalog: Arc, pub conclusion: Option<&'a Conclusion>, pub run_state: Option<&'a RunProjection>, @@ -612,7 +616,7 @@ pub async fn open_pull_request( req.goal, req.model, req.run_store, - req.llm_source, + Arc::clone(&req.llm_source), Arc::clone(&req.catalog), req.conclusion, req.run_state, @@ -684,18 +688,15 @@ mod tests { use chrono::Utc; use fabro_auth::{CredentialSource, VaultCredentialSource}; use fabro_graphviz::graph::Graph; - use fabro_llm::Error as LlmError; - use fabro_llm::client::Client; - use fabro_llm::provider::{ProviderAdapter, StreamEventStream}; - use fabro_llm::types::{FinishReason, Message, Request, Response, StreamEvent, TokenCounts}; - use fabro_model::catalog::{LlmCatalogSettings, ProviderCatalogSettings}; + use fabro_llm::adapter::{ProviderAdapter, ResolvedCall}; + use fabro_llm::lithos_catalog::AdapterId; + use fabro_llm::{Response, ResponseStream}; use fabro_store::Database; use fabro_types::{ - BilledTokenCounts, RunProjection, RunSpec, SuccessReason, WorkflowSettings, - first_event_seq, fixtures, test_support, + BilledTokenCounts, ContentPart, RunProjection, RunSpec, SuccessReason, TokenCounts, + WorkflowSettings, first_event_seq, fixtures, test_support, }; use fabro_vault::{SecretType, Vault}; - use futures::stream; use httpmock::Method::{GET, POST}; use httpmock::MockServer; use object_store::memory::InMemory; @@ -705,77 +706,53 @@ mod tests { use crate::event::{Event, append_event}; use crate::records::StageSummary; + /// Answers every completion with one fixed text, attributed to the route + /// that was asked. struct MockProvider { - name: String, + id: AdapterId, response_text: String, } impl MockProvider { - fn new(name: &str, text: &str) -> Self { + fn new(text: &str) -> Self { Self { - name: name.to_string(), + id: AdapterId::new("mock"), response_text: text.to_string(), } } + + fn response(&self, call: &ResolvedCall) -> Response { + let handle = call.route().handle(); + let mut response = + Response::new(handle.provider().clone(), handle.model().clone(), vec![ + ContentPart::Text { + text: self.response_text.clone(), + }, + ]); + response.id = Some("resp_1".to_string()); + response.usage = TokenCounts { + input: 10, + output: 20, + ..TokenCounts::default() + }; + response + } } #[async_trait::async_trait] impl ProviderAdapter for MockProvider { - fn name(&self) -> &str { - &self.name + fn id(&self) -> &AdapterId { + &self.id } - async fn complete(&self, _request: &Request) -> Result { - Ok(Response { - id: "resp_1".into(), - model: "mock-model".into(), - provider: "mock".into(), - message: Message::assistant(&self.response_text), - finish_reason: FinishReason::Stop, - usage: TokenCounts { - input_tokens: 10, - output_tokens: 20, - ..Default::default() - }, - raw: None, - warnings: vec![], - rate_limit: None, - cost_usd: None, - cost_source: None, - }) + async fn complete(&self, call: &ResolvedCall) -> Result { + Ok(self.response(call)) } - async fn stream(&self, _request: &Request) -> Result { - let text = self.response_text.clone(); - let events = vec![ - Ok(StreamEvent::text_delta(&text, Some("t1".into()))), - Ok(StreamEvent::finish( - FinishReason::Stop, - TokenCounts { - input_tokens: 10, - output_tokens: 20, - ..Default::default() - }, - Response { - id: "resp_1".into(), - model: "mock-model".into(), - provider: "mock".into(), - message: Message::assistant(&text), - finish_reason: FinishReason::Stop, - usage: TokenCounts { - input_tokens: 10, - output_tokens: 20, - ..Default::default() - }, - raw: None, - warnings: vec![], - rate_limit: None, - cost_usd: None, - cost_source: None, - }, - )), - ]; - Ok(Box::pin(stream::iter(events))) + async fn stream(&self, call: &ResolvedCall) -> Result { + Ok(fabro_llm::test_support::response_to_stream( + self.response(call), + )) } } @@ -789,30 +766,48 @@ mod tests { } fn test_catalog_with_provider_base_url(provider: &str, base_url: &str) -> Arc { - let mut settings = LlmCatalogSettings::default(); - settings - .providers - .insert(provider.to_string(), ProviderCatalogSettings { - base_url: Some(base_url.to_string()), - ..ProviderCatalogSettings::default() - }); - Arc::new( - Catalog::from_builtin_with_overrides(&settings) - .expect("catalog with custom base_url should build"), + Arc::new(fabro_llm::test_support::test_catalog_with_provider_base_url(provider, base_url)) + } + + /// The catalog every mock-backed test resolves against: the built-ins plus + /// a `mock` provider that passes any model name through. + fn mock_catalog() -> Catalog { + fabro_llm::test_support::test_catalog_with_overlay( + r#" +[providers.mock] +display_name = "Mock" +adapter = "openai-compatible" +codec = "openai-chat" +base_url = "http://mock.invalid/v1" +auth = { type = "bearer" } +allow_passthrough = true + +[providers.mock.metadata.fabro] +agent_profile = "openai" +credentials = ["env:MOCK_API_KEY"] + +[providers.mock.models.mock-model] +display_name = "Mock Model" +api_model = "mock-model" +limits = { context_tokens = 8192, max_output_tokens = 1024 } +capabilities = { text = true, tools = true, response_format = { json_object = true, json_schema = true } } +"#, ) } + /// A client over [`mock_catalog`] whose `provider_name` answers with + /// `text`. fn explicit_client(provider_name: &str, text: &str) -> Arc { - let mut providers: HashMap> = HashMap::new(); - providers.insert( - provider_name.to_string(), - Arc::new(MockProvider::new(provider_name, text)), - ); - Arc::new(Client::new( - providers, - Some(provider_name.to_string()), - vec![], - )) + let adapter: Arc = Arc::new(MockProvider::new(text)); + let mut options = fabro_llm::ClientOptions::default(); + options + .adapters + .push((fabro_types::ProviderId::new(provider_name), adapter)); + Arc::new( + fabro_llm::build_offline_client(mock_catalog(), options) + .expect("mock client should build") + .client, + ) } fn test_projection() -> RunProjection { @@ -1074,7 +1069,7 @@ mod tests { "Implement feature", "mock-model", &run_store.clone().into(), - Catalog::builtin(), + &mock_catalog(), Some(&make_test_conclusion()), None, explicit_client( @@ -1148,7 +1143,7 @@ mod tests { "Implement feature", "mock-model", &run_store.clone().into(), - Catalog::builtin(), + &mock_catalog(), Some(&make_test_conclusion()), None, explicit_client( @@ -1246,7 +1241,7 @@ mod tests { "Implement feature", "mock-model", &run_store.clone().into(), - Catalog::builtin(), + &mock_catalog(), Some(&make_test_conclusion()), None, explicit_client( @@ -1271,7 +1266,7 @@ mod tests { "Implement feature", "gpt-5.4", &run_store.clone().into(), - Catalog::builtin(), + &mock_catalog(), Some(&make_test_conclusion()), None, explicit_client( @@ -1329,7 +1324,7 @@ mod tests { "Implement feature", "gpt-5.4", &run_store_handle, - llm_source.as_ref(), + llm_source, catalog, Some(&make_test_conclusion()), None, @@ -1490,8 +1485,8 @@ mod tests { assert_eq!( truncation_caps( "unknown-model", - &Catalog::builtin().all_provider_ids(), - Catalog::builtin(), + &fabro_llm::catalog::enabled_provider_ids(&mock_catalog()), + &mock_catalog(), ), TruncationCaps { diff: 80_000, @@ -1517,7 +1512,7 @@ mod tests { draft: false, auto_merge: None, run_store: &harness.run_store, - llm_source: harness.llm_source.as_ref(), + llm_source: Arc::clone(&harness.llm_source), catalog: harness.catalog.clone(), conclusion: None, run_state: None, @@ -1556,7 +1551,7 @@ mod tests { "Implement feature", "mock-model", &run_store.clone().into(), - Catalog::builtin(), + &mock_catalog(), Some(&make_test_conclusion()), None, explicit_client("mock", &payload), @@ -1579,7 +1574,7 @@ mod tests { "## Plan:", "mock-model", &run_store.clone().into(), - Catalog::builtin(), + &mock_catalog(), Some(&make_test_conclusion()), None, explicit_client("mock", &payload), @@ -1669,7 +1664,7 @@ mod tests { "Implement feature", "mock-model", &run_store.clone().into(), - Catalog::builtin(), + &mock_catalog(), Some(&make_test_conclusion()), None, explicit_client("mock", &payload), @@ -1945,7 +1940,7 @@ mod tests { draft: false, auto_merge: None, run_store: &harness.run_store, - llm_source: harness.llm_source.as_ref(), + llm_source: Arc::clone(&harness.llm_source), catalog: harness.catalog.clone(), conclusion: None, run_state: None, @@ -1993,7 +1988,7 @@ mod tests { draft: false, auto_merge: None, run_store: &harness.run_store, - llm_source: harness.llm_source.as_ref(), + llm_source: Arc::clone(&harness.llm_source), catalog: harness.catalog.clone(), conclusion: None, run_state: None, @@ -2030,7 +2025,7 @@ mod tests { draft: false, auto_merge: None, run_store: &harness.run_store, - llm_source: harness.llm_source.as_ref(), + llm_source: Arc::clone(&harness.llm_source), catalog: harness.catalog.clone(), conclusion: None, run_state: None, diff --git a/lib/components/fabro-workflow/src/pipeline/transform.rs b/lib/components/fabro-workflow/src/pipeline/transform.rs index 542d52c43..2768c2496 100644 --- a/lib/components/fabro-workflow/src/pipeline/transform.rs +++ b/lib/components/fabro-workflow/src/pipeline/transform.rs @@ -113,7 +113,7 @@ mod tests { use std::sync::Arc; use fabro_graphviz::graph::AttrValue; - use fabro_model::Catalog; + use fabro_llm::lithos_catalog::Catalog; use super::*; use crate::file_resolver::FilesystemFileResolver; @@ -129,7 +129,7 @@ mod tests { } fn test_catalog() -> Arc { - Arc::new(Catalog::from_builtin().unwrap()) + Arc::new(fabro_llm::test_support::test_catalog()) } fn transform_options() -> TransformOptions { diff --git a/lib/components/fabro-workflow/src/pipeline/types.rs b/lib/components/fabro-workflow/src/pipeline/types.rs index c65f1fdf6..669db75e3 100644 --- a/lib/components/fabro-workflow/src/pipeline/types.rs +++ b/lib/components/fabro-workflow/src/pipeline/types.rs @@ -4,14 +4,14 @@ use std::sync::Arc; use fabro_graphviz::graph::Graph; use fabro_interview::Interviewer; +use fabro_llm::lithos_catalog::Catalog; use fabro_mcp::config::McpServerSettings; -use fabro_model::{Catalog, ProviderId}; use fabro_sandbox::SandboxSpec; use fabro_template::TemplateContext; use fabro_types::settings::run::{ PullRequestSettings, ResolvedGithubIntegration, RunModelControls, }; -use fabro_types::{ManifestPath, RunId, RunProjection}; +use fabro_types::{ManifestPath, ProviderId, RunId, RunProjection}; use fabro_validate::{Diagnostic, Severity}; use fabro_vault::Vault; use tokio::sync::RwLock as AsyncRwLock; diff --git a/lib/components/fabro-workflow/src/pipeline/validate.rs b/lib/components/fabro-workflow/src/pipeline/validate.rs index 3c550f5df..200a66de8 100644 --- a/lib/components/fabro-workflow/src/pipeline/validate.rs +++ b/lib/components/fabro-workflow/src/pipeline/validate.rs @@ -1,4 +1,4 @@ -use fabro_model::Catalog; +use fabro_llm::lithos_catalog::Catalog; use fabro_validate::LintRule; use super::types::{Transformed, Validated}; diff --git a/lib/components/fabro-workflow/src/run_materialization.rs b/lib/components/fabro-workflow/src/run_materialization.rs index d1118b076..714e184e1 100644 --- a/lib/components/fabro-workflow/src/run_materialization.rs +++ b/lib/components/fabro-workflow/src/run_materialization.rs @@ -1,10 +1,11 @@ use std::collections::HashSet; use fabro_graphviz::graph::Graph; -use fabro_model::{Catalog, ModelSelectionError, ProviderId}; -use fabro_types::WorkflowSettings; +use fabro_llm::lithos_catalog::Catalog; +use fabro_llm::{ModelSelectionError, selection}; use fabro_types::settings::InterpString; use fabro_types::settings::run::RunGoal; +use fabro_types::{ProviderId, WorkflowSettings}; use crate::error::Error; @@ -61,7 +62,7 @@ fn materialize_run_with_eligible_providers( )?; settings.run.model.name = Some(resolved_model); - settings.run.model.provider = Some(resolved_provider.into_inner()); + settings.run.model.provider = Some(resolved_provider.into_string()); let goal = graph.goal().to_string(); settings.run.goal = if goal.is_empty() { @@ -93,9 +94,14 @@ pub(crate) fn resolve_run_model( .filter(|provider| !provider.is_empty()) .map(ProviderId::new); let selected = if catalog_fallback { - catalog.resolve_selection_with_catalog_fallback(model, provider.as_ref(), eligible)? + selection::resolve_selection_with_catalog_fallback( + catalog, + model, + provider.as_ref(), + eligible, + )? } else { - catalog.resolve_selection(model, provider.as_ref(), eligible)? + selection::resolve_selection(catalog, model, provider.as_ref(), eligible)? }; Ok((selected.model, selected.provider)) } diff --git a/lib/components/fabro-workflow/src/services.rs b/lib/components/fabro-workflow/src/services.rs index 9fa4c21e1..b1662de01 100644 --- a/lib/components/fabro-workflow/src/services.rs +++ b/lib/components/fabro-workflow/src/services.rs @@ -6,13 +6,11 @@ use std::time::Duration; use fabro_agent::{Sandbox, ToolEnvProvider}; use fabro_auth::CredentialSource; -#[cfg(test)] -use fabro_auth::ResolvedCredentials; use fabro_github::token_source::InstallationTokenSource; use fabro_hooks::{HookContext, HookDecision, HookExecutionContext, HookRunner}; use fabro_interview::Interviewer; -use fabro_model::{Catalog, ProviderId}; -use fabro_types::{ManifestPath, RunId}; +use fabro_llm::lithos_catalog::Catalog; +use fabro_types::{ManifestPath, ProviderId, RunId}; use tokio_util::sync::CancellationToken; use crate::event::Emitter; @@ -271,12 +269,13 @@ impl EngineServices { #[async_trait::async_trait] impl CredentialSource for StubCredentialSource { - async fn resolve(&self, catalog: &Catalog) -> anyhow::Result { - let _ = catalog; - Ok(ResolvedCredentials { - credentials: Vec::new(), - auth_issues: Vec::new(), - }) + async fn credentials( + &self, + provider: &fabro_llm::lithos_catalog::CatalogProvider, + ) -> Result { + Err(fabro_auth::ResolveError::NotConfigured( + provider.id().clone(), + )) } async fn configured_providers(&self, catalog: &Catalog) -> Vec { @@ -318,10 +317,10 @@ impl EngineServices { None, locations, CancellationToken::new(), - ProviderId::anthropic(), - "claude-sonnet-4-6".to_string(), + fabro_types::provider_ids::anthropic(), + "claude-sonnet-4.6".to_string(), Arc::new(StubCredentialSource), - Arc::new(Catalog::from_builtin().expect("default catalog should build")), + Arc::new(fabro_llm::default_catalog()), Arc::new(SandboxGitRuntime::new()), Arc::new(RunMetadataRuntime::new()), None, diff --git a/lib/components/fabro-workflow/src/test_support.rs b/lib/components/fabro-workflow/src/test_support.rs index 039cb91cf..9efbeae84 100644 --- a/lib/components/fabro-workflow/src/test_support.rs +++ b/lib/components/fabro-workflow/src/test_support.rs @@ -8,10 +8,13 @@ use fabro_agent::Sandbox; use fabro_auth::{CredentialSource, test_support as auth_test_support}; use fabro_graphviz::graph::Graph as GvGraph; use fabro_interview::AutoApproveInterviewer; -use fabro_model::Catalog; -#[cfg(feature = "test-support")] -use fabro_model::ProviderId; +use fabro_llm::catalog; +use fabro_llm::lithos_catalog::Catalog; +use fabro_llm::test_support::test_catalog; use fabro_store::{ArtifactStore, RunProjection, test_support as store_test_support}; +#[cfg(feature = "test-support")] +use fabro_types::ProviderId; +use fabro_types::{ModelId, ModelRef, provider_ids}; use object_store::local::LocalFileSystem; use crate::artifact_upload::ArtifactSink; @@ -36,7 +39,7 @@ pub(crate) fn test_configured_provider_ids( assume_ready: bool, ) -> Vec { if assume_ready { - catalog.all_provider_ids().into_iter().collect() + catalog::enabled_provider_ids(catalog).into_iter().collect() } else { configured_provider_ids } @@ -82,26 +85,20 @@ async fn execute_and_emit_terminal(initialized: InitializedState) -> Executed { #[must_use] pub fn test_usage( model_id: &str, - input_tokens: i64, - output_tokens: i64, + input_tokens: u64, + output_tokens: u64, ) -> fabro_types::BilledModelUsage { - serde_json::from_value(serde_json::json!({ - "input": { - "usage": { - "model": { - "provider": "openai", - "model_id": model_id - }, - "tokens": { - "input_tokens": input_tokens, - "output_tokens": output_tokens - } - }, - "facts": { "algorithm": "openai" } + let mut usage = fabro_types::BilledModelUsage::new( + ModelRef::new(provider_ids::openai(), ModelId::new(model_id)), + fabro_types::TokenCounts { + input: input_tokens, + output: output_tokens, + ..fabro_types::TokenCounts::default() }, - "total_usd_micros": input_tokens + output_tokens - })) - .expect("test_usage JSON must deserialise") + None, + ); + usage.total_usd_micros = Some(i64::try_from(input_tokens + output_tokens).unwrap_or(i64::MAX)); + usage } /// Append the `RunStartRequested → RunRunnable → RunStarting → RunRunning` @@ -272,12 +269,12 @@ async fn initialized( options.hook_runner, locations, run_options.cancel_token.clone(), - fabro_model::ProviderId::anthropic(), + provider_ids::anthropic(), "claude-sonnet-4-6".to_string(), options .llm_source .unwrap_or_else(auth_test_support::vault_only_credential_source), - Arc::new(Catalog::from_builtin().expect("default catalog should build")), + Arc::new(test_catalog()), Arc::new(SandboxGitRuntime::new()), Arc::new(RunMetadataRuntime::new()), None, diff --git a/lib/components/fabro-workflow/src/transforms/model_resolution.rs b/lib/components/fabro-workflow/src/transforms/model_resolution.rs index 7fb61b9d9..f7450f712 100644 --- a/lib/components/fabro-workflow/src/transforms/model_resolution.rs +++ b/lib/components/fabro-workflow/src/transforms/model_resolution.rs @@ -2,7 +2,9 @@ use std::collections::HashSet; use std::sync::Arc; use fabro_graphviz::graph::{AttrValue, Graph}; -use fabro_model::{Catalog, ProviderId}; +use fabro_llm::lithos_catalog::Catalog; +use fabro_llm::{catalog, selection}; +use fabro_types::ProviderId; use super::Transform; use crate::error::Error; @@ -19,7 +21,7 @@ pub struct ModelResolutionTransform { impl ModelResolutionTransform { #[must_use] pub fn new(catalog: Arc) -> Self { - let eligible_providers = catalog.all_provider_ids(); + let eligible_providers = catalog::enabled_provider_ids(&catalog); Self { catalog, default_provider: None, @@ -65,14 +67,19 @@ impl ModelResolutionTransform { explicit_provider: Option<&ProviderId>, ) -> Result<(String, ProviderId), Error> { let selected = if self.catalog_fallback { - self.catalog.resolve_selection_with_catalog_fallback( + selection::resolve_selection_with_catalog_fallback( + &self.catalog, Some(model), explicit_provider, &self.eligible_providers, ) } else { - self.catalog - .resolve_selection(Some(model), explicit_provider, &self.eligible_providers) + selection::resolve_selection( + &self.catalog, + Some(model), + explicit_provider, + &self.eligible_providers, + ) }?; Ok((selected.model, selected.provider)) } @@ -148,45 +155,41 @@ mod tests { use std::sync::Arc; use fabro_graphviz::graph::{AttrValue, Graph, Node}; - use fabro_model::catalog::LlmCatalogSettings; + use fabro_llm::test_support::{test_catalog, test_catalog_with_overlay}; use super::*; + /// An operator-defined provider with one aliased model, the shape an + /// `[llm]` overlay produces. It joins the built-ins rather than replacing + /// them: lithos catalogs are layered, never standalone. fn custom_catalog() -> Arc { - let settings: LlmCatalogSettings = toml::from_str( + Arc::new(test_catalog_with_overlay( r#" -[providers.venice] -display_name = "Venice" -adapter = "openai_compatible" -agent_profile = "openai" +[providers.acme-venice] +display_name = "Acme Venice" +adapter = "openai-compatible" +codec = "openai-chat" base_url = "https://api.venice.ai/api/v1" +auth = { type = "bearer" } +priority = 200 +default_model = "venice-large" -[providers.venice.auth] +[providers.acme-venice.metadata.fabro] +agent_profile = "openai" credentials = ["env:VENICE_API_KEY"] -[models."venice-large"] -provider = "venice" +[providers.acme-venice.models.venice-large] display_name = "Venice Large" -family = "venice" -default = true aliases = ["vl"] - -[models."venice-large".limits] -context_window = 128000 - -[models."venice-large".features] -tools = true -vision = false -reasoning = false +api_model = "venice-large" +limits = { context_tokens = 128000, max_output_tokens = 8192 } +capabilities = { text = true, tools = true } "#, - ) - .unwrap(); - Arc::new(Catalog::from_settings(&settings).unwrap()) + )) } fn builtin_transform() -> ModelResolutionTransform { - let catalog = Catalog::from_builtin().unwrap(); - ModelResolutionTransform::new(Arc::new(catalog)) + ModelResolutionTransform::new(Arc::new(test_catalog())) } #[test] @@ -195,7 +198,7 @@ reasoning = false let mut node = Node::new("a"); node.attrs.insert( "model".to_string(), - AttrValue::String("claude-sonnet-4-5".to_string()), + AttrValue::String("claude-sonnet-4.5".to_string()), ); graph.nodes.insert("a".to_string(), node); @@ -216,7 +219,7 @@ reasoning = false let mut node = Node::new("a"); node.attrs.insert( "model".to_string(), - AttrValue::String("claude-sonnet-4-5".to_string()), + AttrValue::String("claude-sonnet-4.5".to_string()), ); node.attrs.insert( "provider".to_string(), @@ -238,7 +241,7 @@ reasoning = false .attrs .get("model") .and_then(AttrValue::as_str), - Some("claude-sonnet-4-5") + Some("claude-sonnet-4.5") ); } @@ -345,20 +348,15 @@ reasoning = false .attrs .get("provider") .and_then(AttrValue::as_str), - Some("venice") + Some("acme-venice") ); } #[test] fn fallback_resolution_keeps_ready_preference_for_unpinned_nodes() { - let overrides: LlmCatalogSettings = toml::from_str( - r" -[providers.openrouter] -enabled = true -", - ) - .unwrap(); - let catalog = Arc::new(Catalog::from_builtin_with_overrides(&overrides).unwrap()); + let catalog = Arc::new(test_catalog_with_overlay( + "[providers.openrouter.metadata.fabro]\nenabled = true\n", + )); let mut graph = Graph::new("test"); let mut portable = Node::new("portable"); portable.attrs.insert( @@ -414,7 +412,7 @@ enabled = true .attrs .get("default_provider") .and_then(AttrValue::as_str), - Some("venice") + Some("acme-venice") ); } } diff --git a/lib/components/fabro-workflow/tests/it/integration.rs b/lib/components/fabro-workflow/tests/it/integration.rs index 399f07f37..94571848b 100644 --- a/lib/components/fabro-workflow/tests/it/integration.rs +++ b/lib/components/fabro-workflow/tests/it/integration.rs @@ -30,10 +30,11 @@ use fabro_interview::{ Answer, AnswerValue, AutoApproveInterviewer, CallbackInterviewer, Interviewer, QueueInterviewer, RecordingInterviewer, }; -use fabro_model::catalog::{LlmCatalogSettings, ProviderCatalogSettings}; -use fabro_model::{Catalog, ProviderId}; +use fabro_llm::lithos_catalog::Catalog; use fabro_store::{ArtifactKey, ArtifactStore}; -use fabro_types::{EventBody, RunEvent, RunId, StageId, WorkflowSettings, parse_blob_ref}; +use fabro_types::{ + EventBody, ProviderId, RunEvent, RunId, StageId, WorkflowSettings, parse_blob_ref, +}; use fabro_validate::{Severity, validate, validate_or_raise}; use fabro_workflow::artifact; use fabro_workflow::context::Context; @@ -66,21 +67,11 @@ use tokio_util::sync::CancellationToken; use ulid::Ulid; fn default_catalog() -> Arc { - Arc::new(Catalog::from_builtin().expect("default catalog should build")) + Arc::new(fabro_llm::test_support::test_catalog()) } fn catalog_with_provider_base_url(provider: &str, base_url: &str) -> Arc { - let mut settings = LlmCatalogSettings::default(); - settings - .providers - .insert(provider.to_string(), ProviderCatalogSettings { - base_url: Some(base_url.to_string()), - ..ProviderCatalogSettings::default() - }); - Arc::new( - Catalog::from_builtin_with_overrides(&settings) - .expect("catalog with custom base_url should build"), - ) + Arc::new(fabro_llm::test_support::test_catalog_with_provider_base_url(provider, base_url)) } fn local_env() -> Arc { @@ -2583,7 +2574,7 @@ async fn shared_thread_compaction_before_routing_audit_succeeds() { "model": "compact-model", "choices": [{ "delta": {"content": text}, - "finish_reason": null + "finish_reason": "stop" }] }); let usage_chunk = serde_json::json!({ @@ -2629,7 +2620,7 @@ async fn shared_thread_compaction_before_routing_audit_succeeds() { server .mock_async(move |when, then| { when.method(POST) - .path("/chat/completions") + .path("/v1/chat/completions") .body_includes(r#""stream":true"#) .body_includes(prompt) .body_excludes(next_prompt); @@ -2648,7 +2639,7 @@ async fn shared_thread_compaction_before_routing_audit_succeeds() { let audit_mock = server .mock_async(|when, then| { when.method(POST) - .path("/chat/completions") + .path("/v1/chat/completions") .body_includes(r#""stream":true"#) .body_includes("Audit shared-thread work"); then.status(200) @@ -2660,7 +2651,7 @@ async fn shared_thread_compaction_before_routing_audit_succeeds() { let compaction_mock = server .mock_async(|when, then| { when.method(POST) - .path("/chat/completions") + .path("/v1/chat/completions") .body_excludes(r#""stream":true"#); then.status(200) .header("content-type", "application/json") @@ -2670,41 +2661,36 @@ async fn shared_thread_compaction_before_routing_audit_succeeds() { }) .await; - let settings: LlmCatalogSettings = toml::from_str(&format!( - r#" + let catalog = Arc::new(fabro_llm::test_support::test_catalog_with_overlay( + &format!( + r#" [providers.compact] -adapter = "openai_compatible" -agent_profile = "openai" -base_url = "{}" +display_name = "Compact" +adapter = "openai-compatible" +codec = "openai-chat" +base_url = {base_url} +auth = {{ type = "bearer" }} +default_model = "compact-model" -[providers.compact.auth] +[providers.compact.metadata.fabro] +agent_profile = "openai" credentials = ["env:COMPACT_API_KEY"] -[models.compact-model] -provider = "compact" +[providers.compact.models.compact-model] display_name = "Compact Model" -family = "mock" -default = true - -[models.compact-model.limits] -context_window = 100000 -max_output = 1024 - -[models.compact-model.features] -tools = true -vision = false -reasoning = false +api_model = "compact-model" +limits = {{ context_tokens = 100000, max_output_tokens = 1024 }} +capabilities = {{ text = true, tools = true, response_format = {{ json_object = true, json_schema = true }} }} "#, - server.base_url() - )) - .expect("test catalog should parse"); - let catalog = Arc::new(Catalog::from_builtin_with_overrides(&settings).unwrap()); + base_url = toml::Value::String(server.base_url()), + ), + )); let source = auth_test_support::env_credential_source(|name| { (name == "COMPACT_API_KEY").then(|| "sk-test".to_string()) }); let backend = AgentApiBackend::new_with_catalog( "compact-model".to_string(), - ProviderId::from("compact"), + ProviderId::new("compact"), ModelFallbackPolicy::default(), source, Arc::new(SteeringHub::new(Arc::new(Emitter::default()))), @@ -2817,7 +2803,7 @@ async fn workflow_persists_authoritative_openrouter_cost_for_agent_stage() { "model": "openai/gpt-5.4", "choices": [{ "delta": {"content": "done"}, - "finish_reason": null + "finish_reason": "stop" }] }); let usage_chunk = serde_json::json!({ @@ -2835,7 +2821,7 @@ async fn workflow_persists_authoritative_openrouter_cost_for_agent_stage() { let completion_mock = server .mock_async(|when, then| { when.method(POST) - .path("/chat/completions") + .path("/v1/chat/completions") .body_includes(r#""stream":true"#) .body_includes("Report completion"); then.status(200) @@ -2844,22 +2830,23 @@ async fn workflow_persists_authoritative_openrouter_cost_for_agent_stage() { }) .await; - let settings: LlmCatalogSettings = toml::from_str(&format!( - r#" -[providers.openrouter] + let catalog = Arc::new(fabro_llm::test_support::test_catalog_with_overlay( + &format!( + "[providers.openrouter] +base_url = {} + +[providers.openrouter.metadata.fabro] enabled = true -base_url = "{}" -"#, - server.base_url() - )) - .expect("test catalog should parse"); - let catalog = Arc::new(Catalog::from_builtin_with_overrides(&settings).unwrap()); +", + toml::Value::String(server.base_url()), + ), + )); let source = auth_test_support::env_credential_source(|name| { (name == "OPENROUTER_API_KEY").then(|| "sk-test".to_string()) }); let backend = AgentApiBackend::new_with_catalog( "openai/gpt-5.4".to_string(), - ProviderId::from("openrouter"), + ProviderId::new("openrouter"), ModelFallbackPolicy::default(), source, Arc::new(SteeringHub::new(Arc::new(Emitter::default()))), @@ -5263,12 +5250,7 @@ async fn import_e2e_through_engine() { use fabro_workflow::transforms::ModelResolutionTransform; let dir = tempfile::tempdir().unwrap(); - let catalog = std::sync::Arc::new( - fabro_model::Catalog::from_builtin_with_overrides( - &fabro_model::catalog::LlmCatalogSettings::default(), - ) - .unwrap(), - ); + let catalog = std::sync::Arc::new(fabro_llm::test_support::test_catalog()); std::fs::write( dir.path().join("val.fabro"), r#"digraph validate { @@ -7323,9 +7305,7 @@ mod real_llm { use async_trait::async_trait; use fabro_auth::EnvCredentialSource; use fabro_graphviz::graph::Node; - use fabro_llm::client::Client; - use fabro_llm::providers::OpenAiAdapter; - use fabro_llm::types::{Message, Request}; + use fabro_llm::{Client, ClientOptions, Request}; use fabro_types::WorkflowSettings; use fabro_workflow::error::Error; use fabro_workflow::handler::agent::{ @@ -7352,25 +7332,16 @@ mod real_llm { impl LlmCodergenBackend { async fn complete(&self, prompt: &str) -> Result { - let request = Request { - model: self.model.clone(), - messages: vec![Message::user(prompt)], - provider: Some(self.provider.clone()), - tools: None, - tool_choice: None, - response_format: None, - temperature: Some(0.0), - top_p: None, - max_tokens: Some(200), - stop_sequences: None, - reasoning_effort: None, - speed: None, - metadata: None, - provider_options: None, - }; + let request = Request::builder() + .model(format!("{}/{}", self.provider, self.model)) + .user(prompt) + .temperature(0.0) + .max_output_tokens(200) + .build() + .map_err(|e| Error::handler(e.to_string()))?; let response = self .client - .complete(&request) + .complete(request) .await .map_err(|e| Error::handler(e.to_string()))?; Ok(CodergenResult::Text { @@ -7399,27 +7370,44 @@ mod real_llm { } } + /// A client whose `openai` provider is the twin at `base_url`, + /// authenticated with `api_key`. + async fn twin_openai_client(base_url: String, api_key: String) -> Arc { + let catalog = fabro_llm::build_catalog(&fabro_config::LlmLayer::default(), &move |name| { + (name == fabro_static::EnvVars::OPENAI_BASE_URL).then(|| base_url.clone()) + }) + .expect("twin catalog should build"); + Arc::new( + fabro_llm::test_support::client_from_env( + catalog, + move |name| { + (name == fabro_static::EnvVars::OPENAI_API_KEY).then(|| api_key.clone()) + }, + ClientOptions::standard(), + ) + .await, + ) + } + async fn make_llm_client() -> Option> { + use fabro_llm::lithos_catalog::Catalog; + if fabro_test::TestMode::from_env().is_twin() { let (base_url, api_key) = fabro_test::e2e_openai!(); - let adapter: Arc = - Arc::new(OpenAiAdapter::new(api_key).with_base_url(base_url)); - let mut providers: HashMap> = - HashMap::new(); - providers.insert("openai".to_string(), adapter); - return Some(Arc::new(Client::new( - providers, - Some("openai".to_string()), - Vec::new(), - ))); + return Some(twin_openai_client(base_url, api_key).await); } fabro_test::require_env("ANTHROPIC_API_KEY")?; - let source = EnvCredentialSource::new(); + let source: Arc = Arc::new(EnvCredentialSource::new()); Some(Arc::new( - Client::from_source(&source, super::default_catalog()) - .await - .expect("unified-llm client should initialize from env source"), + fabro_llm::build_client( + Catalog::clone(&super::default_catalog()), + source, + ClientOptions::standard(), + ) + .await + .expect("LLM client should initialize from env source") + .client, )) } @@ -7583,14 +7571,7 @@ mod real_llm { .load(twin) .await; - let adapter: Arc = - Arc::new(OpenAiAdapter::new(namespace.clone()).with_base_url(twin.base_url.clone())); - let providers = HashMap::from([("openai".to_string(), adapter)]); - let client = Arc::new(Client::new( - providers, - Some("openai".to_string()), - Vec::new(), - )); + let client = twin_openai_client(twin.base_url.clone(), namespace.clone()).await; let mut graph = Graph::new("ForEachSecurityReview"); graph.attrs.insert( @@ -8207,8 +8188,8 @@ async fn workflow_run_with_vault_only_openai_codex_builds_pr_body() { "Implement feature", "gpt-5.4", &run_store_handle, - llm_source.as_ref(), - catalog, + Arc::clone(&llm_source), + Arc::clone(&catalog), Some(&Conclusion { timestamp: Utc::now(), status: StageOutcome::Succeeded, diff --git a/lib/components/fabro-workflow/tests/materialize_run.rs b/lib/components/fabro-workflow/tests/materialize_run.rs index 830a73a3b..be8a28522 100644 --- a/lib/components/fabro-workflow/tests/materialize_run.rs +++ b/lib/components/fabro-workflow/tests/materialize_run.rs @@ -1,9 +1,9 @@ use fabro_graphviz::graph::Graph; use fabro_graphviz::parser; -use fabro_model::{Catalog, ProviderId}; -use fabro_types::WorkflowSettings; +use fabro_llm::test_support::test_catalog; use fabro_types::settings::InterpString; use fabro_types::settings::run::{PullRequestSettings, RunGoal, RunModelSettings, RunNamespace}; +use fabro_types::{WorkflowSettings, provider_ids}; use fabro_workflow::run_materialization::materialize_run; fn graph(source: &str) -> Graph { @@ -34,8 +34,8 @@ fn materialize_run_applies_graph_and_catalog_defaults() { ..WorkflowSettings::default() }; - let materialized = materialize_run(settings, &graph(source), Catalog::builtin(), &[ - ProviderId::anthropic(), + let materialized = materialize_run(settings, &graph(source), &test_catalog(), &[ + provider_ids::anthropic(), ]) .unwrap(); let resolved = &materialized.run; @@ -61,8 +61,8 @@ fn materialize_run_uses_configured_provider_defaults() { let materialized = materialize_run( WorkflowSettings::default(), &graph(source), - Catalog::builtin(), - &[ProviderId::openai()], + &test_catalog(), + &[provider_ids::openai()], ) .unwrap(); let resolved = &materialized.run;