scriptoriumadmin/fabro
1
0
Fork 0
mirror of https://github.com/fabro-sh/fabro.git synced 2026-09-10 22:43:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Port fabro-workflow, hooks, validate, and store to the lithos types

Browse source 
Workflow LLM handlers build lithos requests, bill from lithos usage and
cost, and classify failures from lithos `ErrorKind`. Model resolution and
fallback use the fabro-llm selection and catalog helpers. Validation
rules read the lithos catalog, and store fixtures use the new
`BilledModelUsage` shape.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Bryan Helmkamp 2026-09-09 17:26:57 -06:00
parent e557774b0b
commit 82bcafcfca
No known key found for this signature in database
49 changed files with 1455 additions and 1798 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
lib/components/fabro-hooks/Cargo.toml
View file

@ -16,7 +16,6 @@ workspace = true
fabro-agent = { path = "../fabro-agent" }
fabro-auth = { path = "../../foundation/fabro-auth" }
fabro-llm = { path = "../fabro-llm" }
fabro-model = { path = "../../foundation/fabro-model" }
fabro-redact.workspace = true
fabro-types = { path = "../../foundation/fabro-types" }
fabro-util = { path = "../../foundation/fabro-util" }

4
lib/components/fabro-hooks/src/bridge.rs
View file

@ -78,7 +78,7 @@ mod tests {
use std::path::PathBuf;
use std::sync::Mutex;
use fabro_model::Catalog;
use fabro_llm::lithos_catalog::Catalog;
use fabro_types::fixtures;
use super::*;
@ -100,7 +100,7 @@ mod tests {
context: &HookContext,
_sandbox: Arc<dyn Sandbox>,
execution_context: &HookExecutionContext,
_llm_source: &dyn fabro_auth::CredentialSource,
_llm_source: Arc<dyn fabro_auth::CredentialSource>,
_catalog: Arc<Catalog>,
) -> HookResult {
self.captured_contexts.lock().unwrap().push(context.clone());

173
lib/components/fabro-hooks/src/executor.rs
View file

@ -7,12 +7,11 @@ use async_trait::async_trait;
use fabro_agent::Sandbox;
use fabro_agent::tool_registry::ToolContext;
use fabro_auth::CredentialSource;
use fabro_llm::client::Client as LlmClient;
use fabro_llm::generate::{GenerateParams, generate_object};
use fabro_llm::types::{Message, Request, ToolResult};
use fabro_model::Catalog;
use fabro_llm::lithos_catalog::Catalog;
use fabro_llm::{Client, ClientOptions, Request, structured};
use fabro_redact::redacted_url_for_log;
use fabro_types::settings::{InterpString, ResolveCtx, ResolveError};
use fabro_types::{Message, Role, ToolCall, tool_call_arguments, tool_result_from_json};
use tokio::process::Command as TokioCommand;
use tokio::time::timeout as tokio_timeout;
use tokio_util::sync::CancellationToken;
@ -49,7 +48,7 @@ pub trait HookExecutor: Send + Sync {
context: &HookContext,
sandbox: Arc<dyn Sandbox>,
execution_context: &HookExecutionContext,
llm_source: &dyn CredentialSource,
llm_source: Arc<dyn CredentialSource>,
catalog: Arc<Catalog>,
) -> HookResult;
}
@ -282,7 +281,7 @@ impl HookExecutorImpl {
prompt: &InterpString,
model: Option<&InterpString>,
context: &HookContext,
llm_source: &dyn CredentialSource,
llm_source: Arc<dyn CredentialSource>,
catalog: Arc<Catalog>,
) -> HookDecision {
let (prompt, model) = match Self::resolve_prompt_and_model(prompt, model) {
@ -299,33 +298,48 @@ impl HookExecutorImpl {
let user_msg = Self::build_hook_user_message(&prompt, context);
Self::execute_llm_with_timeout(definition.timeout(), "prompt", || async move {
let client = match LlmClient::from_source(llm_source, catalog).await {
Ok(client) => Arc::new(client),
let client = match Self::build_client(catalog, llm_source).await {
Ok(client) => client,
Err(e) => {
tracing::warn!(error = %e, "prompt hook client creation failed, proceeding");
return HookDecision::Proceed;
}
};
let params = GenerateParams::new(&resolved_model, client)
let request = Request::builder()
.model(&resolved_model)
.system(HOOK_EVALUATOR_SYSTEM_PROMPT)
.prompt(user_msg)
.max_tokens(1024);
.user(user_msg)
.max_output_tokens(1024)
.build();
let request = match request {
Ok(request) => request,
Err(e) => {
tracing::warn!(error = %e, "prompt hook request invalid, proceeding");
return HookDecision::Proceed;
}
};
match generate_object(params, HOOK_RESPONSE_SCHEMA.clone()).await {
Ok(result) => if let Some(obj) = result.output { match serde_json::from_value::<PromptHookResponse>(obj) {
Ok(resp) if resp.ok => HookDecision::Proceed,
Ok(resp) => HookDecision::Block {
reason: resp.reason,
},
Err(e) => {
tracing::warn!(error = %e, "prompt hook response deserialize failed, proceeding");
HookDecision::Proceed
match structured::complete_object(
&client,
request,
"hook_response",
HOOK_RESPONSE_SCHEMA.clone(),
)
.await
{
Ok(completion) => {
match serde_json::from_value::<PromptHookResponse>(completion.object) {
Ok(resp) if resp.ok => HookDecision::Proceed,
Ok(resp) => HookDecision::Block {
reason: resp.reason,
},
Err(e) => {
tracing::warn!(error = %e, "prompt hook response deserialize failed, proceeding");
HookDecision::Proceed
}
}
} } else {
tracing::warn!("prompt hook returned no structured output, proceeding");
HookDecision::Proceed
},
}
Err(e) => {
tracing::warn!(error = %e, "prompt hook LLM call failed, proceeding");
HookDecision::Proceed
@ -347,7 +361,7 @@ impl HookExecutorImpl {
max_tool_rounds: Option<u32>,
context: &HookContext,
sandbox: Arc<dyn Sandbox>,
llm_source: &dyn CredentialSource,
llm_source: Arc<dyn CredentialSource>,
catalog: Arc<Catalog>,
) -> HookDecision {
let (prompt, model) = match Self::resolve_prompt_and_model(prompt, model) {
@ -364,7 +378,7 @@ impl HookExecutorImpl {
let user_msg = Self::build_hook_user_message(&prompt, context);
Self::execute_llm_with_timeout(definition.timeout(), "agent", || async move {
let client = match LlmClient::from_source(llm_source, catalog).await {
let client = match Self::build_client(catalog, llm_source).await {
Ok(c) => c,
Err(e) => {
tracing::warn!(error = %e, "agent hook client creation failed, proceeding");
@ -378,32 +392,30 @@ impl HookExecutorImpl {
let tool_defs = registry.definitions();
let mut messages = vec![
Message::system(HOOK_EVALUATOR_SYSTEM_PROMPT),
Message::user(user_msg),
Message::text(Role::System, HOOK_EVALUATOR_SYSTEM_PROMPT),
Message::text(Role::User, user_msg),
];
let rounds = max_tool_rounds.unwrap_or(50);
let cancel = CancellationToken::new();
for _ in 0..rounds {
let request = Request {
model: resolved_model.clone(),
messages: messages.clone(),
provider: None,
tools: Some(tool_defs.clone()),
tool_choice: None,
response_format: None,
temperature: None,
top_p: None,
max_tokens: None,
stop_sequences: None,
reasoning_effort: None,
speed: None,
metadata: None,
provider_options: None,
let mut builder = Request::builder().model(&resolved_model);
for message in &messages {
builder = builder.message(message.clone());
}
for tool in &tool_defs {
builder = builder.tool(tool.clone());
}
let request = match builder.build() {
Ok(request) => request,
Err(e) => {
tracing::warn!(error = %e, "agent hook request invalid, proceeding");
return HookDecision::Proceed;
}
};
let response = match client.complete(&request).await {
let response = match client.complete(request).await {
Ok(r) => r,
Err(e) => {
tracing::warn!(error = %e, "agent hook LLM call failed, proceeding");
@ -411,13 +423,14 @@ impl HookExecutorImpl {
}
};
let tool_calls = response.tool_calls();
let tool_calls: Vec<ToolCall> = response.tool_calls().cloned().collect();
if tool_calls.is_empty() {
return Self::parse_prompt_response(&response.text());
}
messages.push(response.message.clone());
messages.push(response.into_message());
let mut results = Vec::with_capacity(tool_calls.len());
for tc in &tool_calls {
let tool = registry.get(&tc.name).cloned();
let ctx = ToolContext {
@ -430,22 +443,27 @@ impl HookExecutorImpl {
agent_event_emitter: None,
};
let result = match tool {
Some(t) => match (t.executor)(tc.arguments.clone(), ctx).await {
Ok(output) => {
ToolResult::success(tc.id.clone(), serde_json::json!(output))
}
Err(err) => ToolResult::error(tc.id.clone(), err),
Some(t) => match (t.executor)(tool_call_arguments(tc), ctx).await {
Ok(output) => tool_result_from_json(
tc.id.clone(),
serde_json::Value::String(output),
false,
),
Err(err) => tool_result_from_json(
tc.id.clone(),
serde_json::Value::String(err),
true,
),
},
None => {
ToolResult::error(tc.id.clone(), format!("Unknown tool: {}", tc.name))
}
None => tool_result_from_json(
tc.id.clone(),
serde_json::Value::String(format!("Unknown tool: {}", tc.name)),
true,
),
};
messages.push(Message::tool_result(
result.tool_call_id,
result.content,
result.is_error,
));
results.push(fabro_types::ContentPart::ToolResult(result));
}
messages.push(Message::new(Role::Tool, results));
}
tracing::warn!("agent hook exhausted max tool rounds, proceeding");
@ -454,6 +472,21 @@ impl HookExecutorImpl {
.await
}
/// The LLM client hooks dispatch through: every provider the source can
/// serve, with standard retries.
async fn build_client(
catalog: Arc<Catalog>,
llm_source: Arc<dyn CredentialSource>,
) -> Result<Client, fabro_llm::LlmSetupError> {
fabro_llm::build_client(
Catalog::clone(&catalog),
llm_source,
ClientOptions::standard(),
)
.await
.map(|built| built.client)
}
/// Build an HTTP client for the given TLS mode.
fn build_http_client(tls: TlsMode) -> fabro_http::HttpClient {
let accept_invalid = matches!(tls, TlsMode::NoVerify | TlsMode::Off);
@ -629,7 +662,7 @@ impl HookExecutor for HookExecutorImpl {
context: &HookContext,
sandbox: Arc<dyn Sandbox>,
execution_context: &HookExecutionContext,
llm_source: &dyn CredentialSource,
llm_source: Arc<dyn CredentialSource>,
catalog: Arc<Catalog>,
) -> HookResult {
use std::sync::OnceLock;
@ -751,7 +784,7 @@ mod tests {
}
fn test_catalog() -> Arc<Catalog> {
Arc::new(Catalog::from_builtin().unwrap())
Arc::new(fabro_llm::default_catalog())
}
fn test_http_client() -> fabro_http::HttpClient {
@ -841,7 +874,7 @@ mod tests {
&ctx,
sandbox,
&HookExecutionContext::default(),
source.as_ref(),
Arc::clone(&source),
test_catalog(),
)
.await;
@ -862,7 +895,7 @@ mod tests {
&ctx,
sandbox,
&HookExecutionContext::default(),
source.as_ref(),
Arc::clone(&source),
test_catalog(),
)
.await;
@ -882,7 +915,7 @@ mod tests {
&ctx,
sandbox,
&HookExecutionContext::default(),
source.as_ref(),
Arc::clone(&source),
test_catalog(),
)
.await;
@ -902,7 +935,7 @@ mod tests {
&ctx,
sandbox,
&HookExecutionContext::default(),
source.as_ref(),
Arc::clone(&source),
test_catalog(),
)
.await;
@ -926,7 +959,7 @@ mod tests {
&ctx,
sandbox,
&HookExecutionContext::default(),
source.as_ref(),
Arc::clone(&source),
test_catalog(),
)
.await;
@ -955,7 +988,7 @@ mod tests {
&ctx,
sandbox,
&HookExecutionContext::default(),
source.as_ref(),
Arc::clone(&source),
test_catalog(),
)
.await;
@ -1441,7 +1474,7 @@ mod tests {
&ctx,
sandbox,
&HookExecutionContext::default(),
source.as_ref(),
Arc::clone(&source),
test_catalog(),
)
.await;
@ -1475,7 +1508,7 @@ mod tests {
&interp("{{ env.MISSING_HOOK_VALUE }}"),
None,
&make_context(),
test_llm_source().as_ref(),
test_llm_source(),
test_catalog(),
)
.await;
@ -1503,7 +1536,7 @@ mod tests {
Some(1),
&make_context(),
make_sandbox(),
test_llm_source().as_ref(),
test_llm_source(),
test_catalog(),
)
.await;

12
lib/components/fabro-hooks/src/runner.rs
View file

@ -5,7 +5,7 @@ use fabro_agent::Sandbox;
use fabro_auth::CredentialSource;
#[cfg(test)]
use fabro_auth::test_support;
use fabro_model::Catalog;
use fabro_llm::lithos_catalog::Catalog;
use crate::config::{HookDefinition, HookSettings};
use crate::executor::{HookExecutor, HookExecutorImpl};
@ -47,7 +47,7 @@ impl HookRunner {
config,
executor,
llm_source: test_support::vault_only_credential_source(),
catalog: Arc::new(Catalog::from_builtin().expect("default catalog should build")),
catalog: Arc::new(fabro_llm::default_catalog()),
compiled_matchers,
}
}
@ -158,7 +158,7 @@ impl HookRunner {
context,
sandbox.clone(),
execution_context,
self.llm_source.as_ref(),
Arc::clone(&self.llm_source),
Arc::clone(&self.catalog),
)
.await;
@ -213,7 +213,7 @@ impl HookRunner {
context,
sandbox.clone(),
execution_context,
self.llm_source.as_ref(),
Arc::clone(&self.llm_source),
Arc::clone(&self.catalog),
)
.await;
@ -256,7 +256,7 @@ mod tests {
_context: &HookContext,
_sandbox: Arc<dyn Sandbox>,
_execution_context: &HookExecutionContext,
_llm_source: &dyn CredentialSource,
_llm_source: Arc<dyn CredentialSource>,
_catalog: Arc<Catalog>,
) -> HookResult {
HookResult {
@ -282,7 +282,7 @@ mod tests {
}
fn test_catalog() -> Arc<Catalog> {
Arc::new(Catalog::from_builtin().expect("default catalog should build"))
Arc::new(fabro_llm::default_catalog())
}
fn make_hook(event: HookEvent, name: &str) -> HookDefinition {

4
lib/components/fabro-hooks/tests/host_command_hooks.rs
View file

@ -7,7 +7,7 @@ use fabro_hooks::{
HookContext, HookDecision, HookDefinition, HookEvent, HookExecutionContext, HookRunner,
HookSettings, InterpString,
};
use fabro_model::Catalog;
use fabro_llm::lithos_catalog::Catalog;
use fabro_types::RunId;
use tokio::fs;
@ -16,7 +16,7 @@ fn test_llm_source() -> Arc<dyn CredentialSource> {
}
fn test_catalog() -> Arc<Catalog> {
Arc::new(Catalog::from_builtin().expect("default catalog should build"))
Arc::new(fabro_llm::default_catalog())
}
fn local_sandbox() -> Arc<dyn Sandbox> {

4
lib/components/fabro-manifest/src/lib.rs
View file

@ -557,10 +557,6 @@ fn remotely_available_sha(
/// Resolve a workflow reference and reject it when neither its config nor
/// its graph exists on disk.
/// A missing workflow surfaces as `fabro_config::Error::WorkflowNotFound`.
#[expect(
clippy::result_large_err,
reason = "callers match on the concrete config error to classify missing workflows"
)]
fn resolve_existing_workflow_location(
workflow: &Path,
cwd: &Path,

70
lib/components/fabro-store/src/run_state.rs
View file

@ -1564,8 +1564,8 @@ fn conclusion_from_completed(
props: &RunCompletedProps,
timestamp: DateTime<Utc>,
) -> Result<Conclusion> {
let (stages, total_retries) = billing_rollup::billing_rollup_from_projection(projection, None)
.conclusion_stages(projection);
let (stages, total_retries) =
billing_rollup::billing_rollup_from_projection(projection).conclusion_stages(projection);
Ok(Conclusion {
timestamp,
status: StageOutcome::from_str(&props.status)
@ -1588,8 +1588,8 @@ fn conclusion_from_failed(
props: &RunFailedProps,
timestamp: DateTime<Utc>,
) -> Conclusion {
let (stages, total_retries) = billing_rollup::billing_rollup_from_projection(projection, None)
.conclusion_stages(projection);
let (stages, total_retries) =
billing_rollup::billing_rollup_from_projection(projection).conclusion_stages(projection);
Conclusion {
timestamp,
status: StageOutcome::Failed {
@ -1759,8 +1759,8 @@ mod tests {
AgentBackend, AgentControlState, AttrValue, AutomationRef, BilledModelUsage,
BilledTokenCounts, BlobHash, BlockedReason, Checkpoint, CheckpointRecord,
CommandTermination, EventBody, FailureCategory, FailureDetail, FailureReason, Graph,
McpServerStatus, Node, Outcome, ParallelBranchId, PendingReason, PermissionLevel,
PullRequestCreationStatus, PullRequestLink, QuestionType, ReasoningEffort,
McpServerStatus, ModelId, Node, Outcome, ParallelBranchId, PendingReason, PermissionLevel,
ProviderId, PullRequestCreationStatus, PullRequestLink, QuestionType, ReasoningEffort,
RunApprovalState, RunBillingSummary, RunControlAction, RunDiff, RunEvent, RunSize, RunSpec,
RunStatus, Speed, StageContextWindowBreakdownItem, StageContextWindowCategory,
StageContextWindowCountMethod, StageContextWindowProjection, StageContextWindowStaleness,
@ -1812,11 +1812,11 @@ mod tests {
fn llm_started() -> EventBody {
EventBody::AgentLlmStarted(AgentLlmStartedProps {
requested_model: ModelRef {
provider: "anthropic".parse().unwrap(),
model_id: "claude-fable-5".into(),
speed: Some(Speed::Fast),
},
requested_model: ModelRef::new(
ProviderId::new("anthropic"),
ModelId::new("claude-fable-5"),
)
.with_speed(Some(Speed::Fast)),
visit: 1,
})
}
@ -2324,18 +2324,10 @@ mod tests {
fn test_usage(model_id: &str, input_tokens: i64, output_tokens: i64) -> BilledModelUsage {
serde_json::from_value(json!({
"input": {
"usage": {
"model": {
"provider": "openai",
"model_id": model_id
},
"tokens": {
"input_tokens": input_tokens,
"output_tokens": output_tokens
}
},
"facts": { "algorithm": "openai" }
"model": { "provider": "openai", "model_id": model_id },
"tokens": {
"input": input_tokens,
"output": output_tokens
},
"total_usd_micros": input_tokens + output_tokens
}))
@ -5388,21 +5380,13 @@ mod tests {
fn billed_usage() -> BilledModelUsage {
serde_json::from_value(json!({
"input": {
"usage": {
"model": {
"provider": "openai",
"model_id": "gpt-test"
},
"tokens": {
"input_tokens": 10,
"output_tokens": 5,
"reasoning_tokens": 2,
"cache_read_tokens": 3,
"cache_write_tokens": 4
}
},
"facts": { "algorithm": "openai" }
"model": { "provider": "openai", "model_id": "gpt-test" },
"tokens": {
"input": 10,
"output": 5,
"reasoning": 2,
"cache_read": 3,
"cache_write": 4
},
"total_usd_micros": 123
}))
@ -7622,11 +7606,11 @@ mod tests {
fn started() -> EventBody {
EventBody::AgentLlmStarted(AgentLlmStartedProps {
requested_model: ModelRef {
provider: "anthropic".parse().unwrap(),
model_id: "claude-fable-5".into(),
speed: Some(Speed::Fast),
},
requested_model: ModelRef::new(
ProviderId::new("anthropic"),
ModelId::new("claude-fable-5"),
)
.with_speed(Some(Speed::Fast)),
visit: 1,
})
}

16
lib/components/fabro-store/tests/serializable_projection.rs
View file

@ -49,18 +49,10 @@ fn sample_checkpoint() -> Checkpoint {
fn sample_usage() -> BilledModelUsage {
serde_json::from_value(json!({
"input": {
"usage": {
"model": {
"provider": "openai",
"model_id": "gpt-5.2"
},
"tokens": {
"input_tokens": 123,
"output_tokens": 45
}
},
"facts": { "algorithm": "openai" }
"model": { "provider": "openai", "model_id": "gpt-5.2" },
"tokens": {
"input": 123,
"output": 45
},
"total_usd_micros": 168
}))

3
lib/components/fabro-validate/Cargo.toml
View file

@ -15,10 +15,11 @@ workspace = true
[dependencies]
fabro-acp = { path = "../fabro-acp", default-features = false }
fabro-graphviz = { path = "../fabro-graphviz" }
fabro-model = { path = "../../foundation/fabro-model" }
fabro-llm = { path = "../fabro-llm" }
fabro-types = { path = "../../foundation/fabro-types" }
serde = { workspace = true }
thiserror = { workspace = true }
[dev-dependencies]
fabro-llm = { path = "../fabro-llm", features = ["test-support"] }
toml = { workspace = true }

45
lib/components/fabro-validate/src/lib.rs
View file

@ -1,7 +1,7 @@
pub mod rules;
use fabro_graphviz::graph::Graph;
use fabro_model::Catalog;
use fabro_llm::lithos_catalog::Catalog;
use serde::{Deserialize, Serialize};
/// Severity level for validation diagnostics.
@ -153,8 +153,8 @@ pub fn validate_with_catalog_or_raise(
mod tests {
use fabro_graphviz::graph::{AttrValue, Edge, Graph, Node};
use fabro_graphviz::parser;
use fabro_model::catalog::LlmCatalogSettings;
use fabro_model::{Catalog, ProviderId};
use fabro_llm::lithos_catalog::Catalog;
use fabro_llm::test_support::test_catalog_with_overlay;
use super::*;
@ -208,35 +208,30 @@ mod tests {
g
}
/// An operator-defined provider layered over the built-ins, the shape an
/// `[llm]` overlay produces.
fn custom_catalog() -> Catalog {
let settings: LlmCatalogSettings = toml::from_str(
test_catalog_with_overlay(
r#"
[providers.venice]
display_name = "Venice"
adapter = "openai_compatible"
agent_profile = "openai"
[providers.acme-venice]
display_name = "Acme Venice"
adapter = "openai-compatible"
codec = "openai-chat"
base_url = "https://api.venice.ai/api/v1"
auth = { type = "bearer" }
default_model = "venice-large"
[providers.venice.auth]
[providers.acme-venice.metadata.fabro]
agent_profile = "openai"
credentials = ["env:VENICE_API_KEY"]
[models."venice-large"]
provider = "venice"
[providers.acme-venice.models.venice-large]
display_name = "Venice Large"
family = "venice"
default = true
[models."venice-large".limits]
context_window = 128000
[models."venice-large".features]
tools = true
vision = false
reasoning = false
api_model = "venice-large"
limits = { context_tokens = 128000, max_output_tokens = 8192 }
capabilities = { text = true, tools = true }
"#,
)
.unwrap();
Catalog::from_settings(&settings).unwrap()
}
#[test]
@ -377,7 +372,7 @@ reasoning = false
#[test]
fn validate_with_catalog_accepts_custom_catalog_entries() {
let g = graph_with_model_and_provider("venice-large", "venice");
let g = graph_with_model_and_provider("venice-large", "acme-venice");
let catalog = custom_catalog();
let diagnostics = validate_with_catalog(&g, &catalog, &[]);
@ -406,7 +401,7 @@ reasoning = false
assert!(
diagnostics.iter().any(|d| d.rule == "node_model_known"
&& d.message.contains("missing-provider")
&& d.message.contains(ProviderId::new("venice").as_str())),
&& d.message.contains("acme-venice")),
"missing provider diagnostic not found: {diagnostics:?}"
);
}

3
lib/components/fabro-validate/src/rules/mod.rs
View file

@ -1,3 +1,4 @@
use fabro_llm::lithos_catalog::Catalog;
mod all_conditional_edges;
mod auto_status_deprecated;
mod backend_valid;
@ -82,7 +83,7 @@ pub fn built_in_rules() -> Vec<Box<dyn LintRule>> {
/// Returns lint rules that require the caller's resolved model catalog.
#[must_use]
pub fn catalog_rules(catalog: &fabro_model::Catalog) -> Vec<Box<dyn LintRule + '_>> {
pub fn catalog_rules(catalog: &Catalog) -> Vec<Box<dyn LintRule + '_>> {
vec![
stylesheet_model_known::rule(catalog),
node_model_known::rule(catalog),

19
lib/components/fabro-validate/src/rules/model_support.rs
View file

@ -1,13 +1,16 @@
use fabro_llm::catalog;
use fabro_llm::lithos_catalog::Catalog;
use crate::{Diagnostic, Severity};
pub(super) fn check_model_known(
rule_name: &str,
catalog: &fabro_model::Catalog,
catalog: &Catalog,
model: &str,
context: &str,
node_id: Option<String>,
) -> Option<Diagnostic> {
if catalog.is_model_selector(model) {
if catalog::is_model_selector(catalog, model) {
return None;
}
Some(Diagnostic {
@ -26,21 +29,17 @@ pub(super) fn check_model_known(
pub(super) fn check_provider_known(
rule_name: &str,
catalog: &fabro_model::Catalog,
catalog: &Catalog,
provider: &str,
context: &str,
node_id: Option<String>,
) -> Option<Diagnostic> {
if catalog
.provider(&fabro_model::ProviderId::new(provider))
.is_some()
{
if catalog::is_provider_selector(catalog, provider) {
return None;
}
let valid: Vec<&str> = catalog
.providers()
let valid: Vec<String> = catalog::listed_providers(catalog)
.iter()
.map(|provider| provider.id.as_str())
.map(|entry| entry.provider.id().to_string())
.collect();
let valid_str = valid.join(", ");
Some(Diagnostic {

35
lib/components/fabro-validate/src/rules/node_model_known.rs
View file

@ -1,5 +1,5 @@
use fabro_graphviz::graph::Graph;
use fabro_model::Catalog;
use fabro_llm::lithos_catalog::Catalog;
use super::model_support::{check_model_known, check_provider_known};
use crate::{Diagnostic, LintRule};
@ -48,7 +48,7 @@ impl LintRule for Rule<'_> {
#[cfg(test)]
mod tests {
use fabro_graphviz::graph::{AttrValue, Node};
use fabro_model::Catalog;
use fabro_llm::test_support::test_catalog;
use super::Rule;
use crate::rules::test_support::minimal_graph;
@ -60,12 +60,11 @@ mod tests {
let mut node = Node::new("work");
node.attrs.insert(
"model".to_string(),
AttrValue::String("claude-sonnet-4-5".to_string()),
AttrValue::String("claude-sonnet-4.5".to_string()),
);
g.nodes.insert("work".to_string(), node);
let rule = Rule {
catalog: Catalog::builtin(),
};
let catalog = test_catalog();
let rule = Rule { catalog: &catalog };
let d = rule.apply(&g);
assert!(d.is_empty());
}
@ -79,9 +78,8 @@ mod tests {
AttrValue::String("nonexistent-model-xyz".to_string()),
);
g.nodes.insert("work".to_string(), node);
let rule = Rule {
catalog: Catalog::builtin(),
};
let catalog = test_catalog();
let rule = Rule { catalog: &catalog };
let d = rule.apply(&g);
assert_eq!(d.len(), 1);
assert_eq!(d[0].severity, Severity::Warning);
@ -96,9 +94,8 @@ mod tests {
node.attrs
.insert("model".to_string(), AttrValue::String("opus".to_string()));
g.nodes.insert("work".to_string(), node);
let rule = Rule {
catalog: Catalog::builtin(),
};
let catalog = test_catalog();
let rule = Rule { catalog: &catalog };
let d = rule.apply(&g);
assert!(d.is_empty());
}
@ -109,25 +106,23 @@ mod tests {
let mut node = Node::new("work");
node.attrs.insert(
"provider".to_string(),
AttrValue::String("google".to_string()),
AttrValue::String("nonexistent-provider".to_string()),
);
g.nodes.insert("work".to_string(), node);
let rule = Rule {
catalog: Catalog::builtin(),
};
let catalog = test_catalog();
let rule = Rule { catalog: &catalog };
let d = rule.apply(&g);
assert_eq!(d.len(), 1);
assert_eq!(d[0].severity, Severity::Warning);
assert!(d[0].message.contains("google"));
assert!(d[0].message.contains("nonexistent-provider"));
assert_eq!(d[0].node_id.as_deref(), Some("work"));
}
#[test]
fn node_model_known_rule_no_model_no_provider() {
let g = minimal_graph();
let rule = Rule {
catalog: Catalog::builtin(),
};
let catalog = test_catalog();
let rule = Rule { catalog: &catalog };
let d = rule.apply(&g);
assert!(d.is_empty());
}

35
lib/components/fabro-validate/src/rules/stylesheet_model_known.rs
View file

@ -1,6 +1,6 @@
use fabro_graphviz::graph::Graph;
use fabro_graphviz::stylesheet::{Selector, parse_stylesheet};
use fabro_model::Catalog;
use fabro_llm::lithos_catalog::Catalog;
use super::model_support::{check_model_known, check_provider_known};
use crate::{Diagnostic, LintRule};
@ -77,7 +77,7 @@ impl LintRule for Rule<'_> {
#[cfg(test)]
mod tests {
use fabro_graphviz::graph::AttrValue;
use fabro_model::Catalog;
use fabro_llm::test_support::test_catalog;
use super::Rule;
use crate::rules::test_support::minimal_graph;
@ -88,11 +88,10 @@ mod tests {
let mut g = minimal_graph();
g.attrs.insert(
"model_stylesheet".to_string(),
AttrValue::String("* { model: claude-sonnet-4-5; provider: anthropic; }".to_string()),
AttrValue::String("* { model: claude-sonnet-4.5; provider: anthropic; }".to_string()),
);
let rule = Rule {
catalog: Catalog::builtin(),
};
let catalog = test_catalog();
let rule = Rule { catalog: &catalog };
let d = rule.apply(&g);
assert!(d.is_empty());
}
@ -104,9 +103,8 @@ mod tests {
"model_stylesheet".to_string(),
AttrValue::String("#opus { model: claude-opus-4-5; }".to_string()),
);
let rule = Rule {
catalog: Catalog::builtin(),
};
let catalog = test_catalog();
let rule = Rule { catalog: &catalog };
let d = rule.apply(&g);
assert_eq!(d.len(), 1);
assert_eq!(d[0].severity, Severity::Warning);
@ -119,15 +117,14 @@ mod tests {
let mut g = minimal_graph();
g.attrs.insert(
"model_stylesheet".to_string(),
AttrValue::String("* { provider: google; }".to_string()),
AttrValue::String("* { provider: nonexistent-provider; }".to_string()),
);
let rule = Rule {
catalog: Catalog::builtin(),
};
let catalog = test_catalog();
let rule = Rule { catalog: &catalog };
let d = rule.apply(&g);
assert_eq!(d.len(), 1);
assert_eq!(d[0].severity, Severity::Warning);
assert!(d[0].message.contains("google"));
assert!(d[0].message.contains("nonexistent-provider"));
}
#[test]
@ -137,9 +134,8 @@ mod tests {
"model_stylesheet".to_string(),
AttrValue::String("* { model: opus; }".to_string()),
);
let rule = Rule {
catalog: Catalog::builtin(),
};
let catalog = test_catalog();
let rule = Rule { catalog: &catalog };
let d = rule.apply(&g);
assert!(d.is_empty());
}
@ -147,9 +143,8 @@ mod tests {
#[test]
fn stylesheet_model_known_rule_no_stylesheet() {
let g = minimal_graph();
let rule = Rule {
catalog: Catalog::builtin(),
};
let catalog = test_catalog();
let rule = Rule { catalog: &catalog };
let d = rule.apply(&g);
assert!(d.is_empty());
}

2
lib/components/fabro-workflow/Cargo.toml
View file

@ -39,7 +39,6 @@ fabro-util = { path = "../../foundation/fabro-util" }
fabro-redact.workspace = true
fabro-checkpoint = { path = "../fabro-checkpoint" }
fabro-llm = { path = "../fabro-llm" }
fabro-model = { path = "../../foundation/fabro-model" }
fabro-core = { path = "../../foundation/fabro-core" }
fabro-store = { path = "../fabro-store" }
fabro-static.workspace = true
@ -75,6 +74,7 @@ tempfile = "3"
toml.workspace = true
fabro-vault = { path = "../../foundation/fabro-vault" }
[dev-dependencies]
fabro-llm = { path = "../fabro-llm", features = ["test-support"] }
fabro-store = { path = "../fabro-store", features = ["test-support"] }
fabro-auth = { path = "../../foundation/fabro-auth", features = ["test-support"] }
fabro-github = { path = "../fabro-github", features = ["test-support"] }

49
lib/components/fabro-workflow/src/billing_rollup.rs
View file

@ -5,10 +5,9 @@ pub use fabro_types::billing_rollup::{
#[cfg(test)]
mod tests {
use fabro_model::{Catalog, ModelRef, ProviderId};
use fabro_types::{
AttrValue, BilledTokenCounts, Graph, Node, RunProjection, RunSpec, StageCompletion,
StageOutcome, first_event_seq, test_support,
AttrValue, BilledTokenCounts, Graph, ModelId, ModelRef, Node, RunProjection, RunSpec,
StageCompletion, StageOutcome, first_event_seq, provider_ids, test_support,
};
use super::billing_rollup_from_projection;
@ -50,7 +49,7 @@ mod tests {
timestamp: chrono::Utc::now(),
});
let rollup = billing_rollup_from_projection(&projection, None);
let rollup = billing_rollup_from_projection(&projection);
assert_eq!(rollup.stages.len(), 1);
assert_eq!(rollup.stages[0].node_id, "verify");
@ -73,10 +72,10 @@ mod tests {
assert_eq!(rollup.billed_visit_count, 2);
assert_eq!(rollup.by_model.len(), 2);
assert_eq!(rollup.by_model[0].model.model_id, "gpt-new");
assert_eq!(rollup.by_model[0].model.model_id.as_str(), "gpt-new");
assert_eq!(rollup.by_model[0].stages, 1);
assert_eq!(rollup.by_model[0].billing.input_tokens, 200);
assert_eq!(rollup.by_model[1].model.model_id, "gpt-old");
assert_eq!(rollup.by_model[1].model.model_id.as_str(), "gpt-old");
assert_eq!(rollup.by_model[1].stages, 1);
assert_eq!(rollup.by_model[1].billing.input_tokens, 100);
}
@ -93,7 +92,7 @@ mod tests {
timestamp: chrono::Utc::now(),
});
let rollup = billing_rollup_from_projection(&projection, None);
let rollup = billing_rollup_from_projection(&projection);
assert_eq!(rollup.stages.len(), 1);
assert_eq!(rollup.stages[0].node_id, "build");
@ -126,20 +125,16 @@ mod tests {
timestamp: chrono::Utc::now(),
});
let rollup = billing_rollup_from_projection(&projection, None);
let rollup = billing_rollup_from_projection(&projection);
assert_eq!(rollup.stages.len(), 0);
assert_eq!(rollup.timing.wall_time_ms, 0);
}
#[test]
fn rollup_prices_in_flight_stage_usage_using_catalog() {
fn rollup_keeps_in_flight_stage_usage_unpriced() {
let mut projection = test_projection();
let model = ModelRef {
provider: ProviderId::openai(),
model_id: "gpt-5.4".into(),
speed: None,
};
let model = ModelRef::new(provider_ids::openai(), ModelId::new("gpt-5.4"));
let stage = projection.stage_entry("agent", 1, first_event_seq(1));
stage.started_at = Some(chrono::Utc::now());
stage.usage = BilledTokenCounts {
@ -150,22 +145,18 @@ mod tests {
};
stage.model = Some(model.clone());
let priced = billing_rollup_from_projection(&projection, Some(Catalog::builtin()));
let unpriced = billing_rollup_from_projection(&projection, None);
let rollup = billing_rollup_from_projection(&projection);
assert_eq!(priced.stages.len(), 1);
assert_eq!(priced.stages[0].node_id, "agent");
let stage_cost = priced.stages[0].billing.total_usd_micros;
assert!(
stage_cost.is_some_and(|cost| cost > 0),
"expected priced stage cost, got {stage_cost:?}"
);
assert_eq!(priced.totals.total_usd_micros, stage_cost);
assert_eq!(priced.by_model.len(), 1);
assert_eq!(priced.by_model[0].billing.total_usd_micros, stage_cost);
assert_eq!(unpriced.stages.len(), 1);
assert_eq!(unpriced.stages[0].billing.total_usd_micros, None);
assert_eq!(unpriced.totals.total_usd_micros, None);
// The rollup keeps the shape of what the events recorded. Costs come
// from the events themselves; an in-flight stage that has recorded no
// cost yet stays unpriced rather than being re-estimated here.
assert_eq!(rollup.stages.len(), 1);
assert_eq!(rollup.stages[0].node_id, "agent");
assert_eq!(rollup.stages[0].billing.total_usd_micros, None);
assert_eq!(rollup.stages[0].billing.input_tokens, 500_000);
assert_eq!(rollup.totals.total_usd_micros, None);
assert_eq!(rollup.by_model.len(), 1);
assert_eq!(rollup.by_model[0].billing.input_tokens, 500_000);
}
fn run_spec_with_boundary_nodes() -> RunSpec {

194
lib/components/fabro-workflow/src/error.rs
View file

@ -2,8 +2,7 @@ use std::fmt;
use std::sync::{Arc, LazyLock};
use fabro_graphviz::Error as GraphvizError;
use fabro_llm::{Error as LlmError, ProviderErrorKind};
use fabro_model::ModelSelectionError;
use fabro_llm::{ErrorFacts, ErrorKind, LlmError, ModelSelectionError};
use fabro_template::TemplateError;
pub use fabro_types::failure_signature::FailureSignature;
pub use fabro_types::outcome::FailureCategory;
@ -19,30 +18,21 @@ use crate::outcome::{FailureDetail, Outcome, StageOutcome};
/// Classify an LLM error into a `FailureCategory` based on its structure.
#[must_use]
pub fn classify_sdk_error(err: &LlmError) -> FailureCategory {
match err {
LlmError::Provider { kind, .. } => match kind {
ProviderErrorKind::RateLimit | ProviderErrorKind::Server => {
FailureCategory::TransientInfra
}
ProviderErrorKind::ContextLength | ProviderErrorKind::QuotaExceeded => {
FailureCategory::BudgetExhausted
}
ProviderErrorKind::Authentication
| ProviderErrorKind::AccessDenied
| ProviderErrorKind::NotFound
| ProviderErrorKind::InvalidRequest
| ProviderErrorKind::ContentFilter => FailureCategory::Deterministic,
},
LlmError::RequestTimeout { .. } | LlmError::Network { .. } | LlmError::Stream { .. } => {
FailureCategory::TransientInfra
}
LlmError::Interrupt { .. } => FailureCategory::Canceled,
LlmError::InvalidToolCall { .. }
| LlmError::NoObjectGenerated { .. }
| LlmError::InvalidRequest { .. }
| LlmError::Configuration { .. }
| LlmError::UnsupportedToolChoice { .. } => FailureCategory::Deterministic,
pub fn classify_sdk_error<E: ErrorFacts + ?Sized>(err: &E) -> FailureCategory {
match err.kind() {
ErrorKind::RateLimit
| ErrorKind::Server
| ErrorKind::Network
| ErrorKind::Timeout
| ErrorKind::StreamDecode => FailureCategory::TransientInfra,
ErrorKind::ContextLength | ErrorKind::QuotaExceeded => FailureCategory::BudgetExhausted,
ErrorKind::Cancelled => FailureCategory::Canceled,
// Configuration, model selection, auth, access, not-found, invalid
// request, content filter, provider, decode, resource limit, and
// middleware failures are deterministic. `ErrorKind` is
// non-exhaustive: a category added by a newer lithos never enables
// automatic retry either.
_ => FailureCategory::Deterministic,
}
}
@ -534,7 +524,7 @@ impl Error {
Self::Stage { stage, .. } => {
matches!(stage, ErrorStage::Handler | ErrorStage::Engine)
}
Self::Llm(sdk_err) => sdk_err.retryable(),
Self::Llm(sdk_err) => sdk_err.is_retryable(),
Self::Parse(_)
| Self::Validation(_)
| Self::ValidationFailed { .. }
@ -700,6 +690,12 @@ impl From<LlmError> for Error {
}
}
impl From<fabro_llm::Error> for Error {
fn from(err: fabro_llm::Error) -> Self {
Self::Llm(LlmError::from(err))
}
}
impl From<GraphvizError> for Error {
fn from(e: GraphvizError) -> Self {
match e {
@ -748,9 +744,25 @@ pub type Result<T> = std::result::Result<T, Error>;
#[cfg(test)]
mod tests {
use fabro_checkpoint::MetadataError;
use fabro_llm::{Error as SdkError, ProviderErrorDetail};
use fabro_llm::RetryClassification;
use super::*;
/// A stored LLM error of `kind` from the `openai` provider.
fn sdk_error(kind: ErrorKind, message: &str) -> LlmError {
LlmError::from(
fabro_llm::Error::new(kind, message).with_provider(fabro_types::provider_ids::openai()),
)
}
/// A transient failure the provider may be asked to repeat.
fn transient_error(kind: ErrorKind, message: &str) -> LlmError {
LlmError::from(
fabro_llm::Error::new(kind, message)
.with_provider(fabro_types::provider_ids::openai())
.with_retry(RetryClassification::Safe),
)
}
use crate::outcome::OutcomeExt;
#[derive(Debug)]
@ -1144,38 +1156,23 @@ mod tests {
#[test]
fn llm_error_display() {
let sdk_err = SdkError::Network {
message: "connection refused".into(),
source: None,
};
let sdk_err = transient_error(ErrorKind::Network, "connection refused");
let err = Error::Llm(sdk_err);
assert_eq!(
err.to_string(),
"LLM error: Network error: connection refused"
);
assert_eq!(err.to_string(), "LLM error: connection refused");
}
#[test]
fn llm_error_retryable_delegates_to_sdk() {
let retryable = Error::Llm(SdkError::Network {
message: "timeout".into(),
source: None,
});
let retryable = Error::Llm(transient_error(ErrorKind::Network, "timeout"));
assert!(retryable.is_retryable());
let non_retryable = Error::Llm(SdkError::Configuration {
message: "bad config".into(),
source: None,
});
let non_retryable = Error::Llm(sdk_error(ErrorKind::Configuration, "bad config"));
assert!(!non_retryable.is_retryable());
}
#[test]
fn llm_error_from_sdk_error() {
let sdk_err = SdkError::Stream {
message: "broken pipe".into(),
source: None,
};
let sdk_err = transient_error(ErrorKind::StreamDecode, "broken pipe");
let err = Error::from(sdk_err);
assert!(matches!(err, Error::Llm(_)));
}
@ -1224,45 +1221,31 @@ mod tests {
#[test]
fn failure_class_llm_rate_limit() {
let err = Error::Llm(SdkError::Provider {
kind: ProviderErrorKind::RateLimit,
detail: Box::new(ProviderErrorDetail::new("too fast", "openai")),
});
let err = Error::Llm(transient_error(ErrorKind::RateLimit, "too fast"));
assert_eq!(err.failure_category(), FailureCategory::TransientInfra);
}
#[test]
fn failure_class_llm_context_length() {
let err = Error::Llm(SdkError::Provider {
kind: ProviderErrorKind::ContextLength,
detail: Box::new(ProviderErrorDetail::new("too long", "openai")),
});
let err = Error::Llm(sdk_error(ErrorKind::ContextLength, "too long"));
assert_eq!(err.failure_category(), FailureCategory::BudgetExhausted);
}
#[test]
fn failure_class_llm_auth() {
let err = Error::Llm(SdkError::Provider {
kind: ProviderErrorKind::Authentication,
detail: Box::new(ProviderErrorDetail::new("bad key", "openai")),
});
let err = Error::Llm(sdk_error(ErrorKind::Authentication, "bad key"));
assert_eq!(err.failure_category(), FailureCategory::Deterministic);
}
#[test]
fn failure_class_llm_abort() {
let err = Error::Llm(SdkError::Interrupt {
message: "user cancelled".into(),
});
let err = Error::Llm(sdk_error(ErrorKind::Cancelled, "user cancelled"));
assert_eq!(err.failure_category(), FailureCategory::Canceled);
}
#[test]
fn failure_class_llm_timeout() {
let err = Error::Llm(SdkError::RequestTimeout {
message: "timed out".into(),
source: None,
});
let err = Error::Llm(transient_error(ErrorKind::Timeout, "timed out"));
assert_eq!(err.failure_category(), FailureCategory::TransientInfra);
}
@ -1270,79 +1253,55 @@ mod tests {
#[test]
fn classify_sdk_rate_limit() {
let err = SdkError::Provider {
kind: ProviderErrorKind::RateLimit,
detail: Box::new(ProviderErrorDetail::new("too fast", "openai")),
};
let err = transient_error(ErrorKind::RateLimit, "too fast");
assert_eq!(classify_sdk_error(&err), FailureCategory::TransientInfra);
}
#[test]
fn classify_sdk_server() {
let err = SdkError::Provider {
kind: ProviderErrorKind::Server,
detail: Box::new(ProviderErrorDetail::new("500", "openai")),
};
let err = transient_error(ErrorKind::Server, "500");
assert_eq!(classify_sdk_error(&err), FailureCategory::TransientInfra);
}
#[test]
fn classify_sdk_context_length() {
let err = SdkError::Provider {
kind: ProviderErrorKind::ContextLength,
detail: Box::new(ProviderErrorDetail::new("too long", "openai")),
};
let err = sdk_error(ErrorKind::ContextLength, "too long");
assert_eq!(classify_sdk_error(&err), FailureCategory::BudgetExhausted);
}
#[test]
fn classify_sdk_quota_exceeded() {
let err = SdkError::Provider {
kind: ProviderErrorKind::QuotaExceeded,
detail: Box::new(ProviderErrorDetail::new("out of quota", "openai")),
};
let err = sdk_error(ErrorKind::QuotaExceeded, "out of quota");
assert_eq!(classify_sdk_error(&err), FailureCategory::BudgetExhausted);
}
#[test]
fn classify_sdk_auth() {
let err = SdkError::Provider {
kind: ProviderErrorKind::Authentication,
detail: Box::new(ProviderErrorDetail::new("bad key", "openai")),
};
let err = sdk_error(ErrorKind::Authentication, "bad key");
assert_eq!(classify_sdk_error(&err), FailureCategory::Deterministic);
}
#[test]
fn classify_sdk_request_timeout() {
let err = SdkError::RequestTimeout {
message: "timed out".into(),
source: None,
};
let err = transient_error(ErrorKind::Timeout, "timed out");
assert_eq!(classify_sdk_error(&err), FailureCategory::TransientInfra);
}
#[test]
fn classify_sdk_abort() {
let err = SdkError::Interrupt {
message: "cancelled".into(),
};
let err = sdk_error(ErrorKind::Cancelled, "cancelled");
assert_eq!(classify_sdk_error(&err), FailureCategory::Canceled);
}
#[test]
fn classify_sdk_invalid_tool_call() {
let err = SdkError::InvalidToolCall {
message: "bad tool".into(),
};
let err = sdk_error(ErrorKind::InvalidRequest, "bad tool");
assert_eq!(classify_sdk_error(&err), FailureCategory::Deterministic);
}
#[test]
fn classify_sdk_invalid_request() {
let err = SdkError::InvalidRequest {
message: "unsupported reasoning effort".into(),
};
let err = sdk_error(ErrorKind::InvalidRequest, "unsupported reasoning effort");
assert_eq!(classify_sdk_error(&err), FailureCategory::Deterministic);
}
@ -1982,10 +1941,7 @@ mod tests {
#[test]
fn failure_signature_hint_llm_returns_some() {
let err = Error::Llm(SdkError::Provider {
kind: ProviderErrorKind::Authentication,
detail: Box::new(ProviderErrorDetail::new("bad key", "openai")),
});
let err = Error::Llm(sdk_error(ErrorKind::Authentication, "bad key"));
assert_eq!(
err.failure_signature_hint(),
Some(FailureSignature(
@ -2010,10 +1966,7 @@ mod tests {
#[test]
fn to_fail_outcome_llm_has_class_and_signature() {
let err = Error::Llm(SdkError::Provider {
kind: ProviderErrorKind::Authentication,
detail: Box::new(ProviderErrorDetail::new("bad key", "openai")),
});
let err = Error::Llm(sdk_error(ErrorKind::Authentication, "bad key"));
let outcome = err.to_fail_outcome();
assert_eq!(outcome.status, crate::outcome::StageOutcome::Failed {
retry_requested: false,
@ -2040,10 +1993,7 @@ mod tests {
#[test]
fn to_fail_outcome_includes_error_message_as_reason() {
let err = Error::Llm(SdkError::Network {
message: "connection refused".into(),
source: None,
});
let err = Error::Llm(transient_error(ErrorKind::Network, "connection refused"));
let outcome = err.to_fail_outcome();
assert!(
outcome
@ -2055,10 +2005,7 @@ mod tests {
#[test]
fn to_fail_outcome_no_context_updates() {
let err = Error::Llm(SdkError::Network {
message: "refused".into(),
source: None,
});
let err = Error::Llm(transient_error(ErrorKind::Network, "refused"));
let outcome = err.to_fail_outcome();
assert!(outcome.context_updates.is_empty());
}
@ -2110,10 +2057,7 @@ mod tests {
Error::engine("engine err"),
Error::publish("publish err"),
Error::handler("handler err"),
Error::Llm(SdkError::Network {
message: "refused".into(),
source: None,
}),
Error::Llm(transient_error(ErrorKind::Network, "refused")),
Error::Checkpoint("cp err".into()),
Error::Stylesheet("style err".into()),
Error::Io("io err".into()),
@ -2220,10 +2164,7 @@ mod tests {
use crate::event::Event;
// 1. Create SdkError → Error
let sdk_err = SdkError::Provider {
kind: ProviderErrorKind::RateLimit,
detail: Box::new(ProviderErrorDetail::new("too fast", "openai")),
};
let sdk_err = transient_error(ErrorKind::RateLimit, "too fast");
let arc_err = Error::Llm(sdk_err);
assert_eq!(arc_err.failure_category(), FailureCategory::TransientInfra);
@ -2295,10 +2236,7 @@ mod tests {
fn e2e_serde_stability_agent_error() {
use fabro_agent::Error as AgentError;
let err = AgentError::Llm(SdkError::Provider {
kind: ProviderErrorKind::RateLimit,
detail: Box::new(ProviderErrorDetail::new("too fast", "openai")),
});
let err = AgentError::Llm(transient_error(ErrorKind::RateLimit, "too fast"));
let json = serde_json::to_string(&err).unwrap();
let v: serde_json::Value = serde_json::from_str(&json).unwrap();
assert_eq!(v["type"], "llm");

91
lib/components/fabro-workflow/src/event/convert.rs
View file

@ -1,9 +1,8 @@
use ::fabro_types::{
EventBody, RunControlAction, RunEvent, RunId, StageOutcome, run_event as fabro_types,
EventBody, RunControlAction, RunEvent, RunId, StageOutcome, UsdMicros, run_event as fabro_types,
};
use chrono::Utc;
use fabro_agent::{AgentEvent, SandboxEvent, SkillActivationSource};
use fabro_model::UsdMicros;
use uuid::Uuid;
use super::Event;
@ -658,19 +657,18 @@ fn event_body_from_event(event: &Event) -> EventBody {
text,
model,
usage,
cost_usd,
cost_source,
cost,
tool_call_count,
context_window,
reasoning,
} => {
let billing = billed_token_counts_from_llm(usage)
.with_reported_cost(cost_usd.map(UsdMicros::from_usd));
let billing = billed_token_counts_from_llm(*usage)
.with_reported_cost(cost.as_ref().map(UsdMicros::from_cost));
EventBody::AgentMessage(fabro_types::AgentMessageProps {
text: text.clone(),
model: model.clone(),
billing,
cost_source: *cost_source,
cost_source: cost.map(|cost| cost.source),
tool_call_count: *tool_call_count,
visit: *visit,
message: None,
@ -1463,17 +1461,16 @@ mod tests {
use std::collections::BTreeMap;
use ::fabro_types::{
AutomationRef, EventBody, FailureReason, ParallelBranchId, Principal, RunNoticeCode,
RunNoticeLevel, RunProvenance, StageId, SystemActorKind, fixtures,
run_event as fabro_types, test_support,
AutomationRef, EventBody, FailureReason, ModelId, ModelRef, ParallelBranchId, Principal,
ProviderId, RunNoticeCode, RunNoticeLevel, RunProvenance, StageId, SystemActorKind,
TokenCounts as LlmTokenCounts, fixtures, provider_ids, run_event as fabro_types,
test_support,
};
use chrono::Utc;
use fabro_agent::{
AgentEvent, McpToolSummary, MemoryFileSummary, SandboxEvent, SkillActivationSource,
SkillSummary,
};
use fabro_llm::types::TokenCounts as LlmTokenCounts;
use fabro_model::{ModelRef, ProviderId};
use super::*;
use crate::error::Error;
@ -2525,14 +2522,12 @@ mod tests {
visit: 1,
event: AgentEvent::AssistantMessage {
text: "ok".to_string(),
model: ModelRef {
provider: ProviderId::anthropic(),
model_id: "claude-sonnet".into(),
speed: None,
},
model: ModelRef::new(
provider_ids::anthropic(),
ModelId::new("claude-sonnet"),
),
usage: LlmTokenCounts::default(),
cost_usd: None,
cost_source: None,
cost: None,
tool_call_count: 0,
context_window: None,
reasoning: None,
@ -2556,18 +2551,16 @@ mod tests {
visit: 1,
event: AgentEvent::AssistantMessage {
text: "ok".to_string(),
model: ModelRef {
provider: ProviderId::new("custom_proxy"),
model_id: "proxy-model".into(),
speed: None,
},
model: ModelRef::new(
ProviderId::new("custom_proxy"),
ModelId::new("proxy-model"),
),
usage: LlmTokenCounts {
input_tokens: 12,
output_tokens: 34,
input: 12,
output: 34,
..LlmTokenCounts::default()
},
cost_usd: None,
cost_source: None,
cost: None,
tool_call_count: 0,
context_window: None,
reasoning: None,
@ -2581,7 +2574,7 @@ mod tests {
panic!("expected agent message body");
};
assert_eq!(message.model.provider, ProviderId::new("custom_proxy"));
assert_eq!(message.model.model_id, "proxy-model");
assert_eq!(message.model.model_id.as_str(), "proxy-model");
assert_eq!(message.billing.input_tokens, 12);
assert_eq!(message.billing.output_tokens, 34);
assert_eq!(message.billing.total_usd_micros, None);
@ -2594,18 +2587,20 @@ mod tests {
visit: 1,
event: AgentEvent::AssistantMessage {
text: "ok".to_string(),
model: ModelRef {
provider: ProviderId::new("openrouter"),
model_id: "openai/gpt-5.4".into(),
speed: None,
},
model: ModelRef::new(
ProviderId::new("openrouter"),
ModelId::new("openai/gpt-5.4"),
),
usage: LlmTokenCounts {
input_tokens: 12,
output_tokens: 34,
input: 12,
output: 34,
..LlmTokenCounts::default()
},
cost_usd: Some(0.125),
cost_source: Some(fabro_model::CostSource::Authoritative),
cost: Some(::fabro_types::Cost {
usd_micros: 125_000,
source: ::fabro_types::CostSource::Provider,
}),
tool_call_count: 0,
context_window: None,
reasoning: None,
@ -2621,7 +2616,7 @@ mod tests {
assert_eq!(message.billing.total_usd_micros, Some(125_000));
assert_eq!(
message.cost_source,
Some(fabro_model::CostSource::Authoritative)
Some(::fabro_types::CostSource::Provider)
);
}
@ -2649,14 +2644,9 @@ mod tests {
visit: 1,
event: AgentEvent::AssistantMessage {
text: "ok".to_string(),
model: ModelRef {
provider: ProviderId::openai(),
model_id: "gpt-5.4".into(),
speed: None,
},
model: ModelRef::new(provider_ids::openai(), ModelId::new("gpt-5.4")),
usage: LlmTokenCounts::default(),
cost_usd: None,
cost_source: None,
cost: None,
tool_call_count: 0,
context_window: Some(context_window),
reasoning: None,
@ -2684,14 +2674,9 @@ mod tests {
visit: 1,
event: AgentEvent::AssistantMessage {
text: String::new(),
model: ModelRef {
provider: ProviderId::openai(),
model_id: "gpt-5.4".into(),
speed: None,
},
model: ModelRef::new(provider_ids::openai(), ModelId::new("gpt-5.4")),
usage: LlmTokenCounts::default(),
cost_usd: None,
cost_source: None,
cost: None,
tool_call_count: 1,
context_window: None,
reasoning: Some(::fabro_types::ReasoningOutput::new(

9
lib/components/fabro-workflow/src/event/events.rs
View file

@ -4,13 +4,12 @@ use ::fabro_types::{
AutomationRef, BilledTokenCounts, BlobHash, BlockedReason, CommandTermination, DiffSummary,
FailureReason, ForkSourceRef, GitContext, PairId, PairMessageId, PairSystemMessageKind,
PairTarget, ParallelBranchId, ParallelBranchResult, PendingReason, PermissionLevel, Principal,
PullRequestCreationId, PullRequestLink, ReviewTarget, RunFailure, RunId, RunNoticeLevel,
RunPairEndedReason, RunPairFailedReason, RunProvenance, RunRunnableSource, RunTarget,
RunTiming, SandboxProviderKind, StageId, StageOutcome, StageTiming, SuccessReason,
WorkflowVersionId, run_event as fabro_types,
PullRequestCreationId, PullRequestLink, ReasoningEffort, ReviewTarget, RunFailure, RunId,
RunNoticeLevel, RunPairEndedReason, RunPairFailedReason, RunProvenance, RunRunnableSource,
RunTarget, RunTiming, SandboxProviderKind, Speed, StageId, StageOutcome, StageTiming,
SuccessReason, WorkflowVersionId, run_event as fabro_types,
};
use fabro_agent::{AgentEvent, SandboxEvent};
use fabro_model::{ReasoningEffort, Speed};
use serde::{Deserialize, Serialize};
use crate::error::{Error, run_failure_from_error};

16
lib/components/fabro-workflow/src/event/redaction.rs
View file

@ -30,10 +30,11 @@ pub fn event_payload_from_redacted_json(line: &str, run_id: &RunId) -> Result<Ev
#[cfg(test)]
mod tests {
use ::fabro_types::{ReasoningOutput, fixtures, run_event as fabro_types};
use ::fabro_types::{
ModelId, ModelRef, ReasoningOutput, TokenCounts as LlmTokenCounts, fixtures, provider_ids,
run_event as fabro_types,
};
use fabro_agent::AgentEvent;
use fabro_llm::types::TokenCounts as LlmTokenCounts;
use fabro_model::{ModelRef, ProviderId};
use super::*;
use crate::event::{Event, to_run_event};
@ -124,14 +125,9 @@ mod tests {
visit: 1,
event: AgentEvent::AssistantMessage {
text: "done".to_string(),
model: ModelRef {
provider: ProviderId::openai(),
model_id: "gpt-5.4".into(),
speed: None,
},
model: ModelRef::new(provider_ids::openai(), ModelId::new("gpt-5.4")),
usage: LlmTokenCounts::default(),
cost_usd: None,
cost_source: None,
cost: None,
tool_call_count: 0,
context_window: None,
reasoning: Some(ReasoningOutput::new(

16
lib/components/fabro-workflow/src/event/sink.rs
View file

@ -296,7 +296,7 @@ mod tests {
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use ::fabro_types::{Graph, RunNoticeLevel, WorkflowSettings, fixtures};
use ::fabro_types::{Graph, ModelId, RunNoticeLevel, WorkflowSettings, fixtures};
use fabro_types::test_support;
use tokio::sync::Mutex as AsyncMutex;
@ -389,14 +389,12 @@ mod tests {
visit: 1,
event: fabro_agent::AgentEvent::AssistantMessage {
text: String::new(),
model: fabro_model::ModelRef {
provider: fabro_model::ProviderId::openai(),
model_id: "gpt-5.4".into(),
speed: None,
},
usage: fabro_llm::types::TokenCounts::default(),
cost_usd: None,
cost_source: None,
model: ::fabro_types::ModelRef::new(
::fabro_types::provider_ids::openai(),
ModelId::new("gpt-5.4"),
),
usage: ::fabro_types::TokenCounts::default(),
cost: None,
tool_call_count: 1,
context_window: None,
reasoning: Some(::fabro_types::ReasoningOutput::new(

3
lib/components/fabro-workflow/src/handler/agent.rs
View file

@ -445,9 +445,8 @@ mod tests {
use std::time::Duration;
use fabro_graphviz::graph::AttrValue;
use fabro_model::{ReasoningEffort, Speed};
use fabro_store::{Database, RunDatabase, StageId};
use fabro_types::{fixtures, test_support};
use fabro_types::{ReasoningEffort, Speed, fixtures, test_support};
use object_store::memory::InMemory;
use tempfile::TempDir;

3
lib/components/fabro-workflow/src/handler/llm/activation_lease.rs
View file

@ -1,8 +1,7 @@
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use fabro_model::{ReasoningEffort, Speed};
use fabro_types::{PermissionLevel, SessionCapability, StageId};
use fabro_types::{PermissionLevel, ReasoningEffort, SessionCapability, Speed, StageId};
use crate::error::Error;
use crate::event::{Emitter, Event};

927
lib/components/fabro-workflow/src/handler/llm/api.rs
View file

File diff suppressed because it is too large Load diff

19
lib/components/fabro-workflow/src/handler/llm/preamble.rs
View file

@ -589,23 +589,18 @@ fn build_summary_preamble(
#[cfg(test)]
mod tests {
use fabro_graphviz::graph::AttrValue;
use fabro_llm::types::TokenCounts;
use fabro_model::{Catalog, ModelRef, ProviderId};
use fabro_types::{ModelId, ModelRef, TokenCounts, provider_ids};
use super::*;
use crate::outcome::{BilledModelUsage, billed_model_usage_from_llm};
fn stage_usage(model: &str, input_tokens: i64, output_tokens: i64) -> BilledModelUsage {
fn stage_usage(model: &str, input: u64, output: u64) -> BilledModelUsage {
billed_model_usage_from_llm(
Catalog::builtin(),
&ModelRef {
provider: ProviderId::anthropic(),
model_id: model.into(),
speed: None,
},
&TokenCounts {
input_tokens,
output_tokens,
&fabro_llm::test_support::test_catalog(),
&ModelRef::new(provider_ids::anthropic(), ModelId::new(model)),
TokenCounts {
input,
output,
..TokenCounts::default()
},
)

2
lib/components/fabro-workflow/src/handler/llm/router.rs
View file

@ -81,7 +81,7 @@ mod tests {
use async_trait::async_trait;
use fabro_agent::{LocalSandbox, Sandbox};
use fabro_graphviz::graph::{AttrValue, Node};
use fabro_model::{ReasoningEffort, Speed};
use fabro_types::{ReasoningEffort, Speed};
use tokio_util::sync::CancellationToken;
use super::*;

46
lib/components/fabro-workflow/src/handler/llm/routing.rs
View file

@ -1,6 +1,7 @@
use fabro_graphviz::graph::{self, Node};
use fabro_model::{AgentProfileKind, Catalog, ProviderId};
use fabro_types::AgentBackend;
use fabro_llm::lithos_catalog::Catalog;
use fabro_llm::{ModelSelectionError, catalog, selection};
use fabro_types::{AgentBackend, AgentProfileKind, ProviderId};
use crate::error::Error;
@ -50,38 +51,33 @@ pub(crate) fn resolve_provider_context(
provider_attr: Option<&str>,
) -> Result<ProviderContext, Error> {
let provider_id = if let Some(provider) = provider_attr {
let requested = ProviderId::from(provider);
catalog
.provider(&requested)
.ok_or_else(|| {
Error::Precondition(format!("Provider \"{provider}\" is not configured"))
})?
.id
.clone()
} else if catalog
.get_on_provider(default_provider_id, model)
.is_some()
{
catalog::canonical_provider_id(catalog, provider).ok_or_else(|| {
Error::Precondition(format!("Provider \"{provider}\" is not configured"))
})?
} else if catalog::model_on_provider(catalog, default_provider_id.as_str(), model).is_some() {
// The run's selected provider is a pin whenever it offers the model.
default_provider_id.clone()
} else {
match catalog.select(model, None, &catalog.all_provider_ids()) {
Ok(model) => model.provider.clone(),
Err(fabro_model::ModelSelectionError::UnknownSelector { .. }) => {
default_provider_id.clone()
}
match selection::select(
catalog,
model,
None,
&catalog::enabled_provider_ids(catalog),
) {
Ok(entry) => entry.provider.id().clone(),
Err(ModelSelectionError::UnknownSelector { .. }) => default_provider_id.clone(),
Err(error) => return Err(error.into()),
}
};
let provider = catalog.provider(&provider_id).ok_or_else(|| {
Error::Precondition(format!("Provider \"{provider_id}\" is not configured"))
})?;
let profile_kind = catalog
.effective_agent_profile(&provider.id, Some(model))
let provider_id =
catalog::canonical_provider_id(catalog, provider_id.as_str()).ok_or_else(|| {
Error::Precondition(format!("Provider \"{provider_id}\" is not configured"))
})?;
let profile_kind = catalog::agent_profile(catalog, provider_id.as_str(), Some(model))
.expect("validated provider should resolve an agent profile");
Ok(ProviderContext {
provider_id: provider.id.clone(),
provider_id,
profile_kind,
})
}

108
lib/components/fabro-workflow/src/handler/prompt.rs
View file

@ -220,9 +220,8 @@ mod tests {
use std::time::Duration;
use fabro_graphviz::graph::AttrValue;
use fabro_model::{ReasoningEffort, Speed};
use fabro_store::{Database, RunDatabase, StageId};
use fabro_types::{fixtures, test_support};
use fabro_types::{ReasoningEffort, Speed, fixtures, test_support};
use object_store::memory::InMemory;
use tempfile::TempDir;
@ -690,33 +689,32 @@ mod tests {
tokio::fs::write(workspace.path().join("CLAUDE.md"), "anthropic memory")
.await
.unwrap();
let overrides: fabro_model::catalog::LlmCatalogSettings = toml::from_str(
let catalog = Arc::new(fabro_llm::test_support::test_catalog_with_overlay(
r#"
[providers.acme]
adapter = "openai_compatible"
agent_profile = "openai"
base_url = "https://api.acme.test/v1"
[models.acme-claude]
provider = "acme"
display_name = "Acme Claude"
family = "claude"
default = true
agent_profile = "anthropic"
aliases = ["ac"]
[models.acme-claude.limits]
context_window = 1000
[models.acme-claude.features]
tools = true
vision = false
reasoning = false
"#,
)
.unwrap();
let catalog =
Arc::new(fabro_model::Catalog::from_builtin_with_overrides(&overrides).unwrap());
[providers.acme]
display_name = "Acme"
adapter = "openai-compatible"
codec = "openai-chat"
base_url = "https://api.acme.test/v1"
auth = { type = "bearer" }
default_model = "acme-claude"
[providers.acme.metadata.fabro]
agent_profile = "openai"
credentials = ["env:ACME_API_KEY"]
[providers.acme.models.acme-claude]
display_name = "Acme Claude"
aliases = ["ac"]
api_model = "acme-claude"
limits = { context_tokens = 1000, max_output_tokens = 500 }
capabilities = { text = true, tools = true }
[providers.acme.models.acme-claude.metadata.fabro]
family = "claude"
agent_profile = "anthropic"
"#,
));
let mut services = make_services();
services.run = services
.run
@ -725,7 +723,7 @@ reasoning = false
)))
.with_catalog_context(
Arc::clone(&catalog),
fabro_model::ProviderId::new("acme"),
fabro_types::ProviderId::new("acme"),
"acme-claude".to_string(),
);
@ -766,32 +764,32 @@ reasoning = false
tokio::fs::write(workspace.path().join("CLAUDE.md"), "anthropic memory")
.await
.unwrap();
let overrides: fabro_model::catalog::LlmCatalogSettings = toml::from_str(
let catalog = Arc::new(fabro_llm::test_support::test_catalog_with_overlay(
r#"
[providers.acme]
adapter = "openai_compatible"
agent_profile = "openai"
base_url = "https://api.acme.test/v1"
[models.acme-claude]
provider = "acme"
display_name = "Acme Claude"
family = "claude"
default = true
agent_profile = "anthropic"
[models.acme-claude.limits]
context_window = 1000
[models.acme-claude.features]
tools = true
vision = false
reasoning = false
"#,
)
.unwrap();
let catalog =
Arc::new(fabro_model::Catalog::from_builtin_with_overrides(&overrides).unwrap());
[providers.acme]
display_name = "Acme"
adapter = "openai-compatible"
codec = "openai-chat"
base_url = "https://api.acme.test/v1"
auth = { type = "bearer" }
default_model = "acme-claude"
[providers.acme.metadata.fabro]
agent_profile = "openai"
credentials = ["env:ACME_API_KEY"]
[providers.acme.models.acme-claude]
display_name = "Acme Claude"
aliases = ["ac"]
api_model = "acme-claude"
limits = { context_tokens = 1000, max_output_tokens = 500 }
capabilities = { text = true, tools = true }
[providers.acme.models.acme-claude.metadata.fabro]
family = "claude"
agent_profile = "anthropic"
"#,
));
let mut services = make_services();
services.run = services
.run
@ -800,7 +798,7 @@ reasoning = false
)))
.with_catalog_context(
Arc::clone(&catalog),
fabro_model::ProviderId::new("acme"),
fabro_types::ProviderId::new("acme"),
"acme-claude".to_string(),
);

29
lib/components/fabro-workflow/src/handler/structured_output.rs
View file

@ -2,7 +2,7 @@ use std::fmt::Write as _;
use std::sync::{Arc, LazyLock};
use fabro_graphviz::graph::Node;
use fabro_llm::types::{ResponseFormat, ResponseFormatType};
use fabro_llm::types::ResponseFormat;
use jsonschema::error::ValidationErrorKind;
use jsonschema::paths::Location;
use jsonschema::{ValidationError, Validator};
@ -392,17 +392,16 @@ pub(crate) fn parse_node_output_schema(node: &Node) -> Result<Option<OutputSchem
}
#[must_use]
/// The provider response format for a node's output schema.
///
/// Providers with native structured output enforce the JSON schema; every
/// provider still gets validated locally afterwards.
pub(crate) fn prompt_response_format(schema: &OutputSchemaKind) -> ResponseFormat {
match schema {
OutputSchemaKind::Routing => ResponseFormat {
kind: ResponseFormatType::JsonObject,
json_schema: None,
strict: false,
},
OutputSchemaKind::JsonSchema { schema, .. } => ResponseFormat {
kind: ResponseFormatType::JsonSchema,
json_schema: Some(schema.clone()),
strict: true,
OutputSchemaKind::Routing => ResponseFormat::JsonObject,
OutputSchemaKind::JsonSchema { schema, .. } => ResponseFormat::JsonSchema {
name: "output_schema".to_string(),
schema: schema.clone(),
},
}
}
@ -1119,12 +1118,10 @@ mod tests {
let format = prompt_response_format(&schema);
assert_eq!(format.kind, ResponseFormatType::JsonSchema);
assert_eq!(
format.json_schema,
Some(serde_json::json!({"type": "object"}))
);
assert!(format.strict);
assert_eq!(format, ResponseFormat::JsonSchema {
name: "output_schema".to_string(),
schema: serde_json::json!({"type": "object"}),
});
}
#[test]

5
lib/components/fabro-workflow/src/lifecycle/git.rs
View file

@ -645,7 +645,6 @@ mod tests {
use fabro_core::lifecycle::RunLifecycle;
use fabro_core::state::ExecutionState;
use fabro_graphviz::graph::types::{AttrValue, Edge, Graph, Node};
use fabro_model::Catalog;
use fabro_store::{EventEnvelope, RunDatabase, RunProjection};
use fabro_types::run_event::{MetadataSnapshotFailureKind, MetadataSnapshotPhase};
use fabro_types::{BlobHash, EventBody, RunEvent, WorkflowSettings, fixtures, test_support};
@ -1307,10 +1306,10 @@ mod tests {
None,
finalize_locations,
tokio_util::sync::CancellationToken::new(),
fabro_model::ProviderId::anthropic(),
fabro_types::provider_ids::anthropic(),
"claude-sonnet-4-6".to_string(),
auth_test_support::vault_only_credential_source(),
Arc::new(Catalog::from_builtin().expect("default catalog should build")),
Arc::new(fabro_llm::test_support::test_catalog()),
Arc::new(SandboxGitRuntime::new()),
Arc::clone(&lifecycle.metadata_runtime),
lifecycle.metadata_writer.clone(),

90
lib/components/fabro-workflow/src/model_fallback.rs
View file

@ -1,10 +1,10 @@
use std::collections::{BTreeMap, HashMap, HashSet};
use fabro_model::{
Catalog, FallbackTarget, Model, ModelSelectionError, ProviderId, ReasoningEffort,
};
use fabro_llm::catalog::ModelEntry;
use fabro_llm::lithos_catalog::Catalog;
use fabro_llm::{FallbackTarget, ModelSelectionError, catalog, selection};
use fabro_types::settings::{ModelRef, ResolvedModelRef};
use fabro_types::{RunNoticeCode, RunNoticeLevel};
use fabro_types::{ProviderId, ReasoningEffort, RunNoticeCode, RunNoticeLevel, controls};
use crate::Error;
@ -31,7 +31,7 @@ impl ModelFallbackPolicy {
provider: &ProviderId,
model: &str,
) -> Option<&'a [FallbackTarget]> {
self.chain_for_canonical(&catalog.canonical_model_id(provider, model))
self.chain_for_canonical(&catalog::canonical_model_id(catalog, provider, model))
}
/// Look up a chain by an already-canonicalized requested model ID.
@ -179,9 +179,12 @@ impl ModelFallbackNotice {
requested_model,
target,
requested_effort,
} => format!(
"Model fallback `{target}` for requested model `{requested_model}` was skipped because it has no reasoning level near `{requested_effort}`."
),
} => {
let requested_effort = controls::reasoning_effort_name(*requested_effort);
format!(
"Model fallback `{target}` for requested model `{requested_model}` was skipped because it has no reasoning level near `{requested_effort}`."
)
}
Self::ChainEmpty { requested_model } => format!(
"No usable model fallbacks remain for requested model `{requested_model}` after filtering its configured candidates."
),
@ -205,8 +208,12 @@ pub fn resolve_model_fallbacks(
for (raw_key, references) in configured {
require_bare_model_key(catalog, raw_key)?;
let selected =
catalog.resolve_selection_with_catalog_fallback(Some(raw_key), None, &eligible)?;
let selected = selection::resolve_selection_with_catalog_fallback(
catalog,
Some(raw_key),
None,
&eligible,
)?;
let requested_model = selected.model;
if let Some(previous) =
@ -218,7 +225,8 @@ pub fn resolve_model_fallbacks(
}
let primary = FallbackTarget::new(&selected.provider, &requested_model);
let primary_model = catalog.get_on_provider(&selected.provider, &requested_model);
let primary_model =
catalog::model_on_provider(catalog, selected.provider.as_str(), &requested_model);
let mut targets = Vec::new();
for model_ref in references {
@ -226,7 +234,7 @@ pub fn resolve_model_fallbacks(
catalog,
&requested_model,
&primary,
primary_model,
primary_model.as_ref(),
&eligible,
model_ref,
)? {
@ -292,7 +300,7 @@ fn resolve_fallback_candidate(
catalog: &Catalog,
requested_model: &str,
primary: &FallbackTarget,
primary_model: Option<&Model>,
primary_model: Option<&ModelEntry<'_>>,
eligible: &HashSet<ProviderId>,
model_ref: &ModelRef,
) -> Result<FallbackCandidate, Error> {
@ -300,7 +308,7 @@ fn resolve_fallback_candidate(
Ok(match model_ref.resolve(catalog)? {
ResolvedModelRef::Provider(provider_name) => {
let provider = catalog.provider_id(&provider_name)?;
let provider = selection::require_provider(catalog, &provider_name)?;
if !eligible.contains(&provider) {
return Ok(FallbackCandidate::Skipped(
ModelFallbackNotice::ProviderUnconfigured {
@ -319,8 +327,10 @@ fn resolve_fallback_candidate(
},
));
};
match catalog.closest(&provider, primary_model) {
Some(model) => FallbackCandidate::Target(FallbackTarget::new(provider, &model.id)),
match catalog::closest_model(catalog, provider.as_str(), primary_model.model) {
Some(entry) => {
FallbackCandidate::Target(FallbackTarget::new(provider, entry.model.id()))
}
None => FallbackCandidate::Skipped(ModelFallbackNotice::NoCompatibleModel {
requested_model: requested_model.to_string(),
reference,
@ -332,7 +342,7 @@ fn resolve_fallback_candidate(
provider: Some(provider_name),
selector,
} => {
let provider = catalog.provider_id(&provider_name)?;
let provider = selection::require_provider(catalog, &provider_name)?;
if !eligible.contains(&provider) {
return Ok(FallbackCandidate::Skipped(
ModelFallbackNotice::ProviderUnconfigured {
@ -342,10 +352,11 @@ fn resolve_fallback_candidate(
},
));
}
match catalog.resolve_on_provider(&provider, &selector) {
Ok(info) => {
FallbackCandidate::Target(FallbackTarget::new(&info.provider, &info.id))
}
match selection::resolve_on_provider(catalog, &provider, &selector) {
Ok(entry) => FallbackCandidate::Target(FallbackTarget::new(
entry.provider.id(),
entry.model.id(),
)),
Err(ModelSelectionError::UnknownSelectorOnProvider { .. }) => {
FallbackCandidate::Target(FallbackTarget::new(provider, selector))
}
@ -355,8 +366,11 @@ fn resolve_fallback_candidate(
ResolvedModelRef::Model {
provider: None,
selector,
} => match catalog.select(&selector, None, eligible) {
Ok(info) => FallbackCandidate::Target(FallbackTarget::new(&info.provider, &info.id)),
} => match selection::select(catalog, &selector, None, eligible) {
Ok(entry) => FallbackCandidate::Target(FallbackTarget::new(
entry.provider.id(),
entry.model.id(),
)),
Err(ModelSelectionError::NoEligibleOffering { providers, .. }) => {
FallbackCandidate::Skipped(ModelFallbackNotice::NoConfiguredOffering {
requested_model: requested_model.to_string(),
@ -376,7 +390,10 @@ fn resolve_fallback_candidate(
mod tests {
use std::collections::BTreeMap;
use fabro_model::{Catalog, FallbackTarget, ProviderId};
use fabro_llm::FallbackTarget;
use fabro_llm::lithos_catalog::Catalog;
use fabro_llm::test_support::test_catalog_with_overlay;
use fabro_types::ProviderId;
use super::{ModelFallbackNotice, resolve_model_fallbacks};
@ -388,14 +405,7 @@ mod tests {
}
fn openrouter_catalog() -> Catalog {
let overrides = toml::from_str(
r"
[providers.openrouter]
enabled = true
",
)
.expect("catalog override should parse");
Catalog::from_builtin_with_overrides(&overrides).expect("catalog should build")
test_catalog_with_overlay("[providers.openrouter.metadata.fabro]\nenabled = true\n")
}
#[test]
@ -497,19 +507,9 @@ enabled = true
#[test]
fn resolves_the_requested_production_policy_as_independent_chains() {
let catalog = {
let overrides = toml::from_str(
r"
[providers.modal]
enabled = true
[providers.openrouter]
enabled = true
",
)
.expect("catalog override should parse");
Catalog::from_builtin_with_overrides(&overrides).expect("catalog should build")
};
let catalog = test_catalog_with_overlay(
"[providers.modal.metadata.fabro]\nenabled = true\n\n[providers.openrouter.metadata.fabro]\nenabled = true\n",
);
let eligible = [
ProviderId::new("modal"),
ProviderId::new("moonshot"),

84
lib/components/fabro-workflow/src/operations/create.rs
View file

@ -12,12 +12,12 @@ use std::sync::Arc;
use fabro_config::Storage;
use fabro_graphviz::graph::{AttrValue, Graph};
use fabro_model::{Catalog, ProviderId};
use fabro_llm::lithos_catalog::Catalog;
use fabro_store::{BlobStore, Database};
use fabro_template::TemplateContext;
use fabro_types::{
AutomationRef, BlobHash, ForkSourceRef, GitContext, ManifestPath, RunId, RunProvenance,
RunTarget, WorkflowSettings, WorkflowVersionId,
AutomationRef, BlobHash, ForkSourceRef, GitContext, ManifestPath, ProviderId, RunId,
RunProvenance, RunTarget, WorkflowSettings, WorkflowVersionId,
};
use fabro_util::json::normalize_json_value;
use tokio::task::spawn_blocking;
@ -688,7 +688,7 @@ mod tests {
use fabro_store::Database;
use fabro_types::settings::InterpString;
use fabro_types::settings::run::RunMode;
use fabro_types::{EventBody, WorkflowSettings, fixtures, test_support};
use fabro_types::{EventBody, WorkflowSettings, fixtures, provider_ids, test_support};
use fabro_util::error::collect_chain;
use fabro_validate::Severity;
use object_store::local::LocalFileSystem;
@ -731,60 +731,32 @@ mod tests {
}
fn test_catalog() -> Arc<Catalog> {
Arc::new(Catalog::from_builtin().unwrap())
Arc::new(fabro_llm::test_support::test_catalog())
}
/// OpenAI and OpenRouter both offering GPT-5.6 Sol as their default, so a
/// portable selector resolves to whichever provider is ready.
fn portable_model_catalog() -> Arc<Catalog> {
let settings: fabro_model::catalog::LlmCatalogSettings = toml::from_str(
Arc::new(fabro_llm::test_support::test_catalog_with_overlay(
r#"
[providers.openai]
display_name = "OpenAI"
adapter = "openai"
agent_profile = "openai"
priority = 90
[providers.openai.models."gpt-5.6-sol"]
display_name = "GPT-5.6 Sol"
family = "gpt-5"
aliases = ["gpt-56-sol"]
default = true
[providers.openai.models."gpt-5.6-sol".limits]
context_window = 1000
[providers.openai.models."gpt-5.6-sol".features]
tools = true
vision = false
reasoning = false
[providers.openrouter]
display_name = "OpenRouter"
adapter = "openai_compatible"
agent_profile = "openai"
priority = 25
[providers.openrouter.models."gpt-5.6-sol"]
api_id = "openai/gpt-5.6-sol"
display_name = "GPT-5.6 Sol (via OpenRouter)"
family = "gpt-5"
aliases = ["gpt-56-sol"]
default = true
[providers.openrouter.models."gpt-5.6-sol".limits]
context_window = 1000
[providers.openrouter.models."gpt-5.6-sol".features]
tools = true
vision = false
reasoning = false
"#,
)
.unwrap();
Arc::new(Catalog::from_settings(&settings).unwrap())
[providers.openai]
priority = 90
default_model = "gpt-5.6-sol"
[providers.openrouter]
priority = 25
default_model = "gpt-5.6-sol"
[providers.openrouter.metadata.fabro]
enabled = true
"#,
))
}
fn test_provider_ids() -> Vec<ProviderId> {
Catalog::builtin().all_provider_ids().into_iter().collect()
fabro_llm::catalog::enabled_provider_ids(&fabro_llm::test_support::test_catalog())
.into_iter()
.collect()
}
fn compile_input(request: &CreateRunInput) -> CreateRunCompileInput {
@ -2056,25 +2028,25 @@ reasoning = false
}"#;
let catalog = portable_model_catalog();
let cases = [
(vec![ProviderId::openai()], None, ProviderId::openai()),
(vec![provider_ids::openai()], None, provider_ids::openai()),
(
vec![ProviderId::new("openrouter")],
None,
ProviderId::new("openrouter"),
),
(
vec![ProviderId::openai(), ProviderId::new("openrouter")],
vec![provider_ids::openai(), ProviderId::new("openrouter")],
None,
ProviderId::openai(),
provider_ids::openai(),
),
(
vec![ProviderId::openai(), ProviderId::new("openrouter")],
vec![provider_ids::openai(), ProviderId::new("openrouter")],
Some("openrouter"),
ProviderId::new("openrouter"),
),
];
for selector in ["gpt-56-sol", "openai/gpt-5.6-sol"] {
for selector in ["gpt-56-sol", "gpt-5.6"] {
for (ready, explicit_provider, expected_provider) in &cases {
let dir = tempfile::tempdir().unwrap();
let mut settings = test_default_settings();

148
lib/components/fabro-workflow/src/operations/start.rs
View file

@ -6,9 +6,8 @@ use std::time::{Duration, Instant};
use fabro_auth::{CredentialSource, VaultCredentialSource};
use fabro_interview::{AutoApproveInterviewer, Interviewer};
use fabro_llm::client::Client as LlmClient;
use fabro_llm::lithos_catalog::Catalog;
use fabro_mcp::config::McpServerSettings;
use fabro_model::{Catalog, ProviderId};
use fabro_sandbox::daytona::DaytonaConfig;
use fabro_sandbox::from_environment::{
daytona_config_from_environment, docker_config_from_environment_with_secrets,
@ -24,7 +23,7 @@ use fabro_types::settings::run::{
RunPrepareSettings as ResolvedRunPrepareSettings,
};
use fabro_types::{
ManifestPath, RunId, RunRunnableSource, RunSpec, RunTarget, SandboxProviderKind,
ManifestPath, ProviderId, RunId, RunRunnableSource, RunSpec, RunTarget, SandboxProviderKind,
TargetValidationError,
};
use fabro_util::error::collect_chain;
@ -751,15 +750,7 @@ async fn configured_providers_for_start(
Arc::clone(vault),
process_env_var,
));
match LlmClient::from_source_report(source.as_ref(), catalog).await {
Ok(report) => report
.client
.provider_names()
.into_iter()
.map(ProviderId::new)
.collect(),
Err(_) => Vec::new(),
}
source.resolve_all(catalog.as_ref()).await.ready
}
fn git_checkpoint_options_from_start(
@ -1321,7 +1312,7 @@ mod tests {
};
use fabro_types::{
BilledModelUsage, GitContext, ManifestPath, RunTarget, StageTiming, WorkflowSettings,
fixtures, test_support,
fixtures, provider_ids, test_support,
};
use fabro_vault::SecretType;
use object_store::memory::InMemory;
@ -1442,73 +1433,32 @@ mod tests {
}
fn test_catalog() -> Arc<Catalog> {
Arc::new(Catalog::from_builtin().expect("default catalog should build"))
Arc::new(fabro_llm::test_support::test_catalog())
}
fn test_provider_ids() -> Vec<ProviderId> {
Catalog::builtin().all_provider_ids().into_iter().collect()
fabro_llm::catalog::enabled_provider_ids(&fabro_llm::test_support::test_catalog())
.into_iter()
.collect()
}
/// OpenAI and OpenRouter both offering GPT-5.6 Sol as their default, so a
/// portable selector resolves to whichever provider is ready.
fn portable_model_catalog() -> Catalog {
let settings: fabro_model::catalog::LlmCatalogSettings = toml::from_str(
fabro_llm::test_support::test_catalog_with_overlay(
r#"
[providers.openai]
display_name = "OpenAI"
adapter = "openai"
agent_profile = "openai"
priority = 90
[providers.openai.models."gpt-5.6-sol"]
display_name = "GPT-5.6 Sol"
family = "gpt-5"
aliases = ["gpt-56-sol"]
default = true
[providers.openai.models."gpt-5.6-sol".limits]
context_window = 1000
[providers.openai.models."gpt-5.6-sol".features]
tools = true
vision = false
reasoning = false
[providers.openai.models."gpt-5.4-mini"]
display_name = "GPT-5.4 Mini"
family = "gpt-5"
aliases = ["mini"]
[providers.openai.models."gpt-5.4-mini".limits]
context_window = 1000
[providers.openai.models."gpt-5.4-mini".features]
tools = true
vision = false
reasoning = false
[providers.openrouter]
display_name = "OpenRouter"
adapter = "openai_compatible"
agent_profile = "openai"
priority = 25
[providers.openrouter.models."gpt-5.6-sol"]
api_id = "openai/gpt-5.6-sol"
display_name = "GPT-5.6 Sol (via OpenRouter)"
family = "gpt-5"
aliases = ["gpt-56-sol"]
default = true
[providers.openrouter.models."gpt-5.6-sol".limits]
context_window = 1000
[providers.openrouter.models."gpt-5.6-sol".features]
tools = true
vision = false
reasoning = false
"#,
[providers.openai]
priority = 90
default_model = "gpt-5.6-sol"
[providers.openrouter]
priority = 25
default_model = "gpt-5.6-sol"
[providers.openrouter.metadata.fabro]
enabled = true
"#,
)
.unwrap();
Catalog::from_settings(&settings).unwrap()
}
#[test]
@ -1525,40 +1475,40 @@ reasoning = false
assert!(matches!(
error,
Error::ModelSelection(fabro_model::ModelSelectionError::ProviderUnavailable {
Error::ModelSelection(fabro_llm::ModelSelectionError::ProviderUnavailable {
provider
}) if provider == ProviderId::openai()
}) if provider == provider_ids::openai()
));
}
#[test]
fn resolve_start_llm_infers_provider_from_model_alias() {
let overrides: fabro_model::catalog::LlmCatalogSettings = toml::from_str(
let catalog = fabro_llm::test_support::test_catalog_with_overlay(
r#"
[providers.acme]
adapter = "openai_compatible"
agent_profile = "openai"
base_url = "https://api.acme.test/v1"
[models.acme-claude]
provider = "acme"
display_name = "Acme Claude"
family = "claude"
default = true
agent_profile = "anthropic"
aliases = ["ac"]
[models.acme-claude.limits]
context_window = 1000
[models.acme-claude.features]
tools = true
vision = false
reasoning = false
"#,
)
.unwrap();
let catalog = Catalog::from_builtin_with_overrides(&overrides).unwrap();
[providers.acme]
display_name = "Acme"
adapter = "openai-compatible"
codec = "openai-chat"
base_url = "https://api.acme.test/v1"
auth = { type = "bearer" }
default_model = "acme-claude"
[providers.acme.metadata.fabro]
agent_profile = "openai"
credentials = ["env:ACME_API_KEY"]
[providers.acme.models.acme-claude]
display_name = "Acme Claude"
aliases = ["ac"]
api_model = "acme-claude"
limits = { context_tokens = 1000, max_output_tokens = 500 }
capabilities = { text = true, tools = true }
[providers.acme.models.acme-claude.metadata.fabro]
family = "claude"
agent_profile = "anthropic"
"#,
);
let mut settings = ResolvedRunSettings::default();
settings.model.name = Some("ac".to_string());

4
lib/components/fabro-workflow/src/operations/validate.rs
View file

@ -2,8 +2,8 @@ use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::Arc;
use fabro_model::{Catalog, ProviderId};
use fabro_types::WorkflowSettings;
use fabro_llm::lithos_catalog::Catalog;
use fabro_types::{ProviderId, WorkflowSettings};
use super::create::{configured_default_provider, preprocess_and_validate, template_context};
use super::source::{ResolveWorkflowInput, WorkflowInput, resolve_workflow};

243
lib/components/fabro-workflow/src/outcome.rs
View file

@ -1,56 +1,37 @@
pub use fabro_core::outcome::{
FailureCategory, FailureDetail, OutcomeMeta, StageOutcome, StageState,
};
use fabro_llm::types::TokenCounts as LlmTokenCounts;
use fabro_model::{
BilledTokenCounts, Catalog, ModelBillingInput, ModelRef, ModelUsage, TokenCounts,
};
use fabro_llm::catalog;
use fabro_llm::lithos_catalog::Catalog;
pub use fabro_types::BilledModelUsage;
use fabro_types::{BilledTokenCounts, ModelRef, TokenCounts};
use crate::error::{Error, FailureSignature, classify_failure_reason};
pub type Outcome = fabro_core::Outcome<Option<BilledModelUsage>>;
/// Bills `usage` on `model` from catalog pricing.
///
/// The provider must be one the catalog knows; a passthrough model on a known
/// provider is billed with no cost, since the catalog has no rates for it.
pub fn billed_model_usage_from_llm(
catalog: &Catalog,
model: &ModelRef,
usage: &LlmTokenCounts,
usage: TokenCounts,
) -> Result<BilledModelUsage, Error> {
let tokens = token_counts_from_llm_usage(usage);
let facts = catalog.billing_facts_for(model, &tokens).ok_or_else(|| {
Error::Precondition(format!("Provider \"{}\" is not configured", model.provider))
})?;
let input = ModelBillingInput {
usage: ModelUsage {
model: model.clone(),
tokens,
},
facts,
};
let total_usd_micros = catalog
.pricing_for(model)
.and_then(|pricing| pricing.bill(&input))
.map(|amount| amount.0);
Ok(BilledModelUsage {
input,
total_usd_micros,
})
if catalog::provider(catalog, model.provider.as_str()).is_none() {
return Err(Error::Precondition(format!(
"Provider \"{}\" is not configured",
model.provider
)));
}
let cost = catalog::estimate_cost(catalog, model, usage);
Ok(BilledModelUsage::new(model.clone(), usage, cost))
}
#[must_use]
pub fn billed_token_counts_from_llm(usage: &LlmTokenCounts) -> BilledTokenCounts {
let tokens = token_counts_from_llm_usage(usage);
BilledTokenCounts {
input_tokens: tokens.input_tokens,
output_tokens: tokens.output_tokens,
total_tokens: tokens.total_tokens(),
reasoning_tokens: tokens.reasoning_tokens,
cache_read_tokens: tokens.cache_read_tokens,
cache_write_tokens: tokens.cache_write_tokens,
total_usd_micros: None,
}
pub fn billed_token_counts_from_llm(usage: TokenCounts) -> BilledTokenCounts {
BilledTokenCounts::from_token_counts(usage, None)
}
pub trait OutcomeExt: Sized {
@ -141,58 +122,56 @@ pub fn format_cost(cost: f64) -> String {
format!("${cost:.2}")
}
fn token_counts_from_llm_usage(usage: &LlmTokenCounts) -> TokenCounts {
usage.clone()
}
#[cfg(test)]
mod tests {
use fabro_llm::types::TokenCounts;
use fabro_model::catalog::LlmCatalogSettings;
use fabro_model::{Catalog, ModelRef, ProviderId, Speed, UsdMicros};
use fabro_llm::lithos_catalog::Catalog;
use fabro_llm::test_support::{test_catalog, test_catalog_with_overlay};
use fabro_types::{ModelId, ModelRef, ProviderId, Speed, TokenCounts, UsdMicros, provider_ids};
use super::{OutcomeExt, billed_model_usage_from_llm};
fn model_ref(provider: ProviderId, model_id: &str, speed: Option<Speed>) -> ModelRef {
ModelRef {
provider,
model_id: model_id.into(),
speed,
}
ModelRef::new(provider, ModelId::new(model_id)).with_speed(speed)
}
fn catalog() -> Catalog {
test_catalog()
}
#[test]
fn billed_model_usage_from_llm_bills_openai_cached_input_and_reasoning_output() {
// Stay under the 272k long-context tier so the standard rates apply.
let usage = TokenCounts {
input_tokens: 500_000,
output_tokens: 125_000,
reasoning_tokens: 25_000,
cache_read_tokens: 250_000,
input: 100_000,
output: 25_000,
reasoning: 5_000,
cache_read: 50_000,
..TokenCounts::default()
};
let billed = billed_model_usage_from_llm(
Catalog::builtin(),
&model_ref(ProviderId::openai(), "gpt-5.4", None),
&usage,
&catalog(),
&model_ref(provider_ids::openai(), "gpt-5.4", None),
usage,
)
.unwrap();
assert_eq!(billed.total_usd_micros, Some(3_562_500));
assert_eq!(billed.tokens().output_tokens, 125_000);
assert_eq!(billed.tokens().reasoning_tokens, 25_000);
// 100k input at $2.50/M + 50k cached at $0.25/M + 30k output at $15/M.
assert_eq!(billed.total_usd_micros, Some(712_500));
assert_eq!(billed.tokens().output, 25_000);
assert_eq!(billed.tokens().reasoning, 5_000);
}
#[test]
fn response_cost_overrides_catalog_estimate() {
let usage = TokenCounts {
input_tokens: 11,
output_tokens: 7,
input: 11,
output: 7,
..TokenCounts::default()
};
let billed = billed_model_usage_from_llm(
Catalog::builtin(),
&model_ref(ProviderId::openai(), "gpt-5.4", None),
&usage,
&catalog(),
&model_ref(provider_ids::openai(), "gpt-5.4", None),
usage,
)
.unwrap()
.with_reported_cost(Some(UsdMicros(125_000)));
@ -213,142 +192,88 @@ mod tests {
#[test]
fn billed_model_usage_from_llm_bills_anthropic_fast_mode_cache_write_pricing() {
let usage = TokenCounts {
input_tokens: 100_000,
output_tokens: 10_000,
reasoning_tokens: 5_000,
cache_read_tokens: 20_000,
cache_write_tokens: 30_000,
input: 100_000,
output: 10_000,
reasoning: 5_000,
cache_read: 20_000,
cache_write: 30_000,
};
let billed = billed_model_usage_from_llm(
Catalog::builtin(),
&catalog(),
&model_ref(
ProviderId::anthropic(),
"claude-opus-4-6",
provider_ids::anthropic(),
"claude-opus-5",
Some(Speed::Fast),
),
&usage,
usage,
)
.unwrap();
assert_eq!(billed.total_usd_micros, Some(6_435_000));
// Fast rates: $10/M input, $50/M output (incl. reasoning), $1/M cache
// read, $12.50/M cache write.
assert_eq!(billed.total_usd_micros, Some(2_145_000));
}
#[test]
fn billed_model_usage_from_llm_uses_injected_custom_catalog() {
let settings: LlmCatalogSettings = toml::from_str(
let catalog = test_catalog_with_overlay(
r#"
[providers.proxy]
display_name = "Proxy"
adapter = "openai_compatible"
agent_profile = "openai"
billing_policy = "openai"
adapter = "openai-compatible"
codec = "openai-chat"
base_url = "https://proxy.example/v1"
auth = { type = "bearer" }
default_model = "canonical-model"
[models.canonical-model]
provider = "proxy"
api_id = "wire-model"
[providers.proxy.models.canonical-model]
display_name = "Canonical Model"
family = "proxy"
default = true
[models.canonical-model.limits]
context_window = 1000
[models.canonical-model.features]
tools = true
vision = false
reasoning = false
[models.canonical-model.costs]
input_cost_per_mtok = 1.0
output_cost_per_mtok = 2.0
api_model = "wire-model"
limits = { context_tokens = 1000, max_output_tokens = 500 }
capabilities = { text = true, tools = true }
pricing = { input_usd_micros_per_million = 1000000, output_usd_micros_per_million = 2000000 }
"#,
)
.unwrap();
let catalog = Catalog::from_settings(&settings).unwrap();
);
let usage = TokenCounts {
input_tokens: 500_000,
output_tokens: 250_000,
input: 1_000_000,
output: 500_000,
..TokenCounts::default()
};
let billed = billed_model_usage_from_llm(
&catalog,
&model_ref(ProviderId::new("proxy"), "canonical-model", None),
&usage,
usage,
)
.unwrap();
assert_eq!(&billed.model().provider, &ProviderId::new("proxy"));
assert_eq!(billed.total_usd_micros, Some(2_000_000));
assert_eq!(billed.model_id(), "canonical-model");
assert_eq!(billed.total_usd_micros, Some(1_000_000));
}
#[test]
fn billed_model_usage_from_llm_does_not_bill_provider_api_id() {
let settings: LlmCatalogSettings = toml::from_str(
r#"
[providers.proxy]
display_name = "Proxy"
adapter = "openai_compatible"
agent_profile = "openai"
billing_policy = "openai"
base_url = "https://proxy.example/v1"
[models.canonical-model]
provider = "proxy"
api_id = "wire-model"
display_name = "Canonical Model"
family = "proxy"
default = true
[models.canonical-model.limits]
context_window = 1000
[models.canonical-model.features]
tools = true
vision = false
reasoning = false
[models.canonical-model.costs]
input_cost_per_mtok = 1.0
output_cost_per_mtok = 2.0
"#,
)
.unwrap();
let catalog = Catalog::from_settings(&settings).unwrap();
fn passthrough_model_on_known_provider_has_no_cost() {
let billed = billed_model_usage_from_llm(
&catalog,
&model_ref(ProviderId::new("proxy"), "wire-model", None),
&TokenCounts {
input_tokens: 500_000,
output_tokens: 250_000,
&catalog(),
&model_ref(provider_ids::openai(), "brand-new-model", None),
TokenCounts {
input: 10,
output: 5,
..TokenCounts::default()
},
)
.unwrap();
assert_eq!(billed.model_id(), "wire-model");
assert_eq!(billed.total_usd_micros, None);
assert_eq!(billed.tokens().input, 10);
}
#[test]
fn billed_model_usage_round_trips_dense_token_counts() {
let usage = TokenCounts {
input_tokens: 100,
output_tokens: 40,
reasoning_tokens: 5,
cache_read_tokens: 20,
cache_write_tokens: 10,
};
let billed = billed_model_usage_from_llm(
Catalog::builtin(),
&model_ref(ProviderId::anthropic(), "claude-opus-4-6", None),
&usage,
fn unknown_provider_is_a_precondition_failure() {
let error = billed_model_usage_from_llm(
&catalog(),
&model_ref(ProviderId::new("nowhere"), "model", None),
TokenCounts::default(),
)
.unwrap();
assert_eq!(billed.tokens().clone(), usage);
.unwrap_err();
assert!(error.to_string().contains("not configured"), "{error}");
}
}

17
lib/components/fabro-workflow/src/pipeline/execute/tests.rs
View file

@ -87,13 +87,8 @@ fn test_run_id(label: &str) -> RunId {
}
}
fn test_catalog() -> Arc<fabro_model::Catalog> {
Arc::new(
fabro_model::Catalog::from_builtin_with_overrides(
&fabro_model::catalog::LlmCatalogSettings::default(),
)
.expect("default catalog should build"),
)
fn test_catalog() -> Arc<fabro_llm::lithos_catalog::Catalog> {
Arc::new(fabro_llm::test_support::test_catalog())
}
fn test_emitter(label: &str) -> Emitter {
@ -270,7 +265,7 @@ async fn execute_test_run_with_options(
},
llm: LlmSpec {
model: "test-model".to_string(),
provider_id: fabro_model::ProviderId::anthropic(),
provider_id: fabro_types::provider_ids::anthropic(),
fallbacks: ModelFallbackPolicy::default(),
mcp_servers: Vec::new(),
model_controls: RunModelControls::default(),
@ -330,7 +325,7 @@ async fn execute_runs_start_to_exit_and_returns_final_context() {
},
llm: LlmSpec {
model: "test-model".to_string(),
provider_id: fabro_model::ProviderId::anthropic(),
provider_id: fabro_types::provider_ids::anthropic(),
fallbacks: ModelFallbackPolicy::default(),
mcp_servers: Vec::new(),
model_controls: RunModelControls::default(),
@ -471,7 +466,7 @@ async fn resumed_in_flight_node_starts_a_new_stage_execution() {
},
llm: LlmSpec {
model: "test-model".to_string(),
provider_id: fabro_model::ProviderId::anthropic(),
provider_id: fabro_types::provider_ids::anthropic(),
fallbacks: ModelFallbackPolicy::default(),
mcp_servers: Vec::new(),
model_controls: RunModelControls::default(),
@ -585,7 +580,7 @@ async fn run_with_lifecycle(
},
llm: LlmSpec {
model: "test-model".to_string(),
provider_id: fabro_model::ProviderId::anthropic(),
provider_id: fabro_types::provider_ids::anthropic(),
fallbacks: ModelFallbackPolicy::default(),
mcp_servers: Vec::new(),
model_controls: RunModelControls::default(),

13
lib/components/fabro-workflow/src/pipeline/finalize.rs
View file

@ -81,7 +81,7 @@ fn build_conclusion_from_projection(
final_git_commit_sha: Option<String>,
) -> Conclusion {
let billing = projection
.map(|projection| billing_rollup::billing_rollup_from_projection(projection, None))
.map(billing_rollup::billing_rollup_from_projection)
.unwrap_or_default();
let (stages, total_retries) = projection
.map(|projection| billing.conclusion_stages(projection))
@ -343,7 +343,7 @@ async fn compute_final_patch(
#[cfg(any(test, feature = "test-support"))]
pub(crate) fn billing_from_projection(projection: &RunProjection) -> Option<BilledTokenCounts> {
billing_rollup::billing_rollup_from_projection(projection, None).billing_if_present()
billing_rollup::billing_rollup_from_projection(projection).billing_if_present()
}
pub(crate) fn build_terminal_event(
@ -580,7 +580,6 @@ mod tests {
use bytes::Bytes;
use fabro_auth::test_support as auth_test_support;
use fabro_graphviz::graph::Graph;
use fabro_model::Catalog;
use fabro_sandbox::test_support::MockSandbox;
use fabro_store::{Database, EventEnvelope, RunDatabase, RunProjection};
use fabro_types::run_event::{MetadataSnapshotFailureKind, MetadataSnapshotPhase};
@ -997,10 +996,10 @@ mod tests {
None,
locations,
tokio_util::sync::CancellationToken::new(),
fabro_model::ProviderId::anthropic(),
fabro_types::provider_ids::anthropic(),
"claude-sonnet-4-6".to_string(),
auth_test_support::vault_only_credential_source(),
Arc::new(Catalog::from_builtin().expect("default catalog should build")),
Arc::new(fabro_llm::test_support::test_catalog()),
Arc::new(SandboxGitRuntime::new()),
metadata_runtime,
metadata_writer,
@ -1030,10 +1029,10 @@ mod tests {
None,
locations,
tokio_util::sync::CancellationToken::new(),
fabro_model::ProviderId::anthropic(),
fabro_types::provider_ids::anthropic(),
"claude-sonnet-4-6".to_string(),
auth_test_support::vault_only_credential_source(),
Arc::new(Catalog::from_builtin().expect("default catalog should build")),
Arc::new(fabro_llm::test_support::test_catalog()),
Arc::new(SandboxGitRuntime::new()),
Arc::new(RunMetadataRuntime::new()),
None,

93
lib/components/fabro-workflow/src/pipeline/initialize.rs
View file

@ -4,13 +4,11 @@ use std::sync::Arc;
use std::time::Instant;
use fabro_agent::{Sandbox, ToolSecrets};
use fabro_auth::{
CredentialSource, ExtraHeadersCredentialSource, VaultCredentialSource, auth_issue_message,
};
use fabro_auth::{CredentialSource, ExtraHeadersCredentialSource, VaultCredentialSource};
use fabro_github::token_source::InstallationTokenSource;
use fabro_graphviz::graph;
use fabro_hooks::{HookContext, HookDecision, HookEvent, HookExecutionContext, HookRunner};
use fabro_model::Catalog;
use fabro_llm::lithos_catalog::Catalog;
use fabro_sandbox::{
GitSetupIntent, SandboxEventCallback, SandboxSpec, reconnect_for_run_with_callback, shell_quote,
};
@ -278,37 +276,22 @@ async fn build_registry(
return Ok((build_llm_registry(), false));
}
match llm_source.resolve(catalog.as_ref()).await {
Ok(result) if result.credentials.is_empty() => {
if graph_needs_llm {
let detail = (!result.auth_issues.is_empty()).then(|| {
result
.auth_issues
.iter()
.map(|(provider, issue)| auth_issue_message(provider, issue))
.collect::<Vec<_>>()
.join("; ")
});
let prefix = detail.map_or_else(
|| "No LLM providers configured".to_string(),
|detail| format!("No usable LLM providers configured: {detail}"),
);
return Err(Error::Precondition(format!(
"{prefix}. Set ANTHROPIC_API_KEY or OPENAI_API_KEY, or pass --dry-run to simulate."
)));
}
Ok((build_no_backend(), false))
}
Ok(_result) => Ok((build_llm_registry(), false)),
Err(e) => {
if graph_needs_llm {
return Err(Error::Precondition(format!(
"Failed to initialize LLM client: {e}. Set ANTHROPIC_API_KEY or OPENAI_API_KEY, or pass --dry-run to simulate.",
)));
}
Ok((build_no_backend(), false))
let result = llm_source.resolve_all(catalog.as_ref()).await;
if result.ready.is_empty() {
if graph_needs_llm {
let detail =
(!result.auth_issues.is_empty()).then(|| result.issue_messages().join("; "));
let prefix = detail.map_or_else(
|| "No LLM providers configured".to_string(),
|detail| format!("No usable LLM providers configured: {detail}"),
);
return Err(Error::Precondition(format!(
"{prefix}. Set ANTHROPIC_API_KEY or OPENAI_API_KEY, or pass --dry-run to simulate."
)));
}
return Ok((build_no_backend(), false));
}
Ok((build_llm_registry(), false))
}
async fn tool_secrets_from_configured_sources(vault: &Arc<AsyncRwLock<Vault>>) -> ToolSecrets {
@ -781,7 +764,7 @@ mod tests {
}
fn test_catalog() -> Arc<Catalog> {
Arc::new(Catalog::from_builtin().expect("default catalog should build"))
Arc::new(fabro_llm::test_support::test_catalog())
}
fn memory_store() -> Arc<Database> {
@ -911,7 +894,7 @@ mod tests {
sandbox: SandboxSpec::Local { working_directory },
llm: LlmSpec {
model: "test-model".to_string(),
provider_id: fabro_model::ProviderId::anthropic(),
provider_id: fabro_types::provider_ids::anthropic(),
fallbacks: ModelFallbackPolicy::default(),
mcp_servers: Vec::new(),
model_controls: RunModelControls::default(),
@ -1101,17 +1084,16 @@ mod tests {
assert_eq!(initialized.model, "test-model");
assert_eq!(
initialized.engine.run.provider_id,
fabro_model::ProviderId::anthropic()
fabro_types::provider_ids::anthropic()
);
assert!(
initialized
.engine
.run
.llm_source
.resolve(&initialized.engine.run.catalog)
.resolve_all(&initialized.engine.run.catalog)
.await
.unwrap()
.credentials
.ready
.is_empty()
);
}
@ -1229,7 +1211,7 @@ mod tests {
let (_registry, effective_dry_run) = build_registry(
&LlmSpec {
model: "claude-opus-4-6".to_string(),
provider_id: fabro_model::ProviderId::anthropic(),
provider_id: fabro_types::provider_ids::anthropic(),
fallbacks: ModelFallbackPolicy::default(),
mcp_servers: Vec::new(),
model_controls: RunModelControls::default(),
@ -1261,17 +1243,22 @@ mod tests {
let expected_session_id = run_id.to_string();
let source = build_llm_source(vault, run_id);
let resolved = source.resolve(test_catalog().as_ref()).await.unwrap();
let catalog = test_catalog();
let resolved = source.resolve_all(catalog.as_ref()).await;
assert!(!resolved.credentials.is_empty());
for credential in &resolved.credentials {
assert_eq!(
credential
.extra_headers
.get(SESSION_ID_HEADER)
.map(String::as_str),
Some(expected_session_id.as_str())
);
assert!(!resolved.ready.is_empty());
for provider in &resolved.ready {
let provider = catalog.provider(provider.as_str()).unwrap();
let credentials = source.credentials(provider).await.unwrap();
let fabro_llm::credentials::Credentials::Http(http) = credentials else {
panic!("vault credentials should be HTTP credentials");
};
let session_header = http
.extra_headers
.iter()
.find(|header| header.name == SESSION_ID_HEADER)
.map(|header| header.value.expose_secret());
assert_eq!(session_header, Some(expected_session_id.as_str()));
}
}
@ -1351,7 +1338,7 @@ mod tests {
},
llm: LlmSpec {
model: "fake-acp".to_string(),
provider_id: fabro_model::ProviderId::openai(),
provider_id: fabro_types::provider_ids::openai(),
fallbacks: ModelFallbackPolicy::default(),
mcp_servers: Vec::new(),
model_controls: RunModelControls::default(),
@ -1454,7 +1441,7 @@ mod tests {
},
llm: LlmSpec {
model: "test-model".to_string(),
provider_id: fabro_model::ProviderId::anthropic(),
provider_id: fabro_types::provider_ids::anthropic(),
fallbacks: ModelFallbackPolicy::default(),
mcp_servers: Vec::new(),
model_controls: RunModelControls::default(),
@ -1596,7 +1583,7 @@ mod tests {
},
llm: LlmSpec {
model: "test-model".to_string(),
provider_id: fabro_model::ProviderId::anthropic(),
provider_id: fabro_types::provider_ids::anthropic(),
fallbacks: ModelFallbackPolicy::default(),
mcp_servers: Vec::new(),
model_controls: RunModelControls::default(),

2
lib/components/fabro-workflow/src/pipeline/publish.rs
View file

@ -179,7 +179,7 @@ impl Concluded {
merge_strategy: pr_config.merge_strategy,
}),
run_store: &self.services.run_store,
llm_source: self.services.llm_source.as_ref(),
llm_source: Arc::clone(&self.services.llm_source),
catalog: Arc::clone(&self.services.catalog),
conclusion: Some(&self.conclusion),
run_state: None,

235
lib/components/fabro-workflow/src/pipeline/pull_request.rs
View file

@ -5,12 +5,11 @@ use std::time::Duration;
use fabro_auth::CredentialSource;
use fabro_github::{self as github_app, ssh_url_to_https};
use fabro_graphviz::parser;
use fabro_llm::client::Client;
use fabro_llm::generate::{GenerateParams, generate_object};
use fabro_model::{Catalog, ProviderId};
use fabro_llm::lithos_catalog::Catalog;
use fabro_llm::{Client, ClientOptions, Request, selection, structured};
use fabro_store::RunProjection;
use fabro_types::PullRequestLink;
use fabro_types::settings::run::MergeStrategy;
use fabro_types::{ProviderId, PullRequestLink, Role};
use fabro_util::text::strip_goal_decoration;
use tokio::time::sleep;
use tracing::{debug, info, warn};
@ -72,10 +71,10 @@ fn truncation_caps(
eligible: &HashSet<ProviderId>,
catalog: &Catalog,
) -> TruncationCaps {
let ctx = catalog
.select(model, None, eligible)
let ctx = selection::select(catalog, model, None, eligible)
.ok()
.and_then(|m| usize::try_from(m.context_window()).ok())
.and_then(|entry| entry.model.limits())
.and_then(|limits| usize::try_from(limits.context_tokens).ok())
.unwrap_or(UNKNOWN_MODEL_CTX);
truncation_caps_for_context_window(ctx)
@ -334,14 +333,19 @@ pub async fn build_pr_content(
goal: &str,
model: &str,
run_store: &RunStoreHandle,
llm_source: &dyn CredentialSource,
llm_source: Arc<dyn CredentialSource>,
catalog: Arc<Catalog>,
conclusion: Option<&Conclusion>,
run_state: Option<&RunProjection>,
) -> Result<PrContent, String> {
let client = Client::from_source(llm_source, Arc::clone(&catalog))
.await
.map_err(|e| format!("Failed to create LLM client: {e}"))?;
let client = fabro_llm::build_client(
Catalog::clone(&catalog),
llm_source,
ClientOptions::standard(),
)
.await
.map_err(|e| format!("Failed to create LLM client: {e}"))?
.client;
build_pr_content_with_client(
diff,
@ -385,7 +389,7 @@ async fn build_pr_content_with_client(
let run_spec = run_state.map(|state| state.spec.clone());
let dot_source = run_state.and_then(|state| state.spec.graph_source.clone());
let eligible = client.provider_ids();
let eligible = client.available_providers().iter().cloned().collect();
let caps = truncation_caps(model, &eligible, catalog);
let truncated_diff = truncate_chars(diff, caps.diff);
@ -398,18 +402,18 @@ async fn build_pr_content_with_client(
format!("Goal: {goal}\n\nDiff:\n```\n{truncated_diff}\n```")
};
let params = GenerateParams::new(model, client)
let request = Request::builder()
.model(model)
.system(PR_BODY_SYSTEM_PROMPT)
.prompt(prompt);
.message(fabro_types::Message::text(Role::User, prompt))
.build()
.map_err(|e| format!("invalid PR content request: {e}"))?;
let completion =
structured::complete_object(&client, request, "pr_content", PR_CONTENT_SCHEMA.clone())
.await
.map_err(|e| format!("LLM generation failed: {e}"))?;
let result = generate_object(params, PR_CONTENT_SCHEMA.clone())
.await
.map_err(|e| format!("LLM generation failed: {e}"))?;
let output = result
.output
.ok_or_else(|| "LLM generation returned no structured output".to_string())?;
let generated: PrContent = serde_json::from_value(output)
let generated: PrContent = serde_json::from_value(completion.object)
.map_err(|e| format!("Failed to deserialize PR content: {e}"))?;
let title = if generated.title.trim().is_empty() {
@ -458,7 +462,7 @@ pub struct OpenPullRequestRequest<'a> {
pub draft: bool,
pub auto_merge: Option<AutoMergeOptions>,
pub run_store: &'a RunStoreHandle,
pub llm_source: &'a dyn CredentialSource,
pub llm_source: Arc<dyn CredentialSource>,
pub catalog: Arc<Catalog>,
pub conclusion: Option<&'a Conclusion>,
pub run_state: Option<&'a RunProjection>,
@ -612,7 +616,7 @@ pub async fn open_pull_request(
req.goal,
req.model,
req.run_store,
req.llm_source,
Arc::clone(&req.llm_source),
Arc::clone(&req.catalog),
req.conclusion,
req.run_state,
@ -684,18 +688,15 @@ mod tests {
use chrono::Utc;
use fabro_auth::{CredentialSource, VaultCredentialSource};
use fabro_graphviz::graph::Graph;
use fabro_llm::Error as LlmError;
use fabro_llm::client::Client;
use fabro_llm::provider::{ProviderAdapter, StreamEventStream};
use fabro_llm::types::{FinishReason, Message, Request, Response, StreamEvent, TokenCounts};
use fabro_model::catalog::{LlmCatalogSettings, ProviderCatalogSettings};
use fabro_llm::adapter::{ProviderAdapter, ResolvedCall};
use fabro_llm::lithos_catalog::AdapterId;
use fabro_llm::{Response, ResponseStream};
use fabro_store::Database;
use fabro_types::{
BilledTokenCounts, RunProjection, RunSpec, SuccessReason, WorkflowSettings,
first_event_seq, fixtures, test_support,
BilledTokenCounts, ContentPart, RunProjection, RunSpec, SuccessReason, TokenCounts,
WorkflowSettings, first_event_seq, fixtures, test_support,
};
use fabro_vault::{SecretType, Vault};
use futures::stream;
use httpmock::Method::{GET, POST};
use httpmock::MockServer;
use object_store::memory::InMemory;
@ -705,77 +706,53 @@ mod tests {
use crate::event::{Event, append_event};
use crate::records::StageSummary;
/// Answers every completion with one fixed text, attributed to the route
/// that was asked.
struct MockProvider {
name: String,
id: AdapterId,
response_text: String,
}
impl MockProvider {
fn new(name: &str, text: &str) -> Self {
fn new(text: &str) -> Self {
Self {
name: name.to_string(),
id: AdapterId::new("mock"),
response_text: text.to_string(),
}
}
fn response(&self, call: &ResolvedCall) -> Response {
let handle = call.route().handle();
let mut response =
Response::new(handle.provider().clone(), handle.model().clone(), vec![
ContentPart::Text {
text: self.response_text.clone(),
},
]);
response.id = Some("resp_1".to_string());
response.usage = TokenCounts {
input: 10,
output: 20,
..TokenCounts::default()
};
response
}
}
#[async_trait::async_trait]
impl ProviderAdapter for MockProvider {
fn name(&self) -> &str {
&self.name
fn id(&self) -> &AdapterId {
&self.id
}
async fn complete(&self, _request: &Request) -> Result<Response, LlmError> {
Ok(Response {
id: "resp_1".into(),
model: "mock-model".into(),
provider: "mock".into(),
message: Message::assistant(&self.response_text),
finish_reason: FinishReason::Stop,
usage: TokenCounts {
input_tokens: 10,
output_tokens: 20,
..Default::default()
},
raw: None,
warnings: vec![],
rate_limit: None,
cost_usd: None,
cost_source: None,
})
async fn complete(&self, call: &ResolvedCall) -> Result<Response, fabro_llm::Error> {
Ok(self.response(call))
}
async fn stream(&self, _request: &Request) -> Result<StreamEventStream, LlmError> {
let text = self.response_text.clone();
let events = vec![
Ok(StreamEvent::text_delta(&text, Some("t1".into()))),
Ok(StreamEvent::finish(
FinishReason::Stop,
TokenCounts {
input_tokens: 10,
output_tokens: 20,
..Default::default()
},
Response {
id: "resp_1".into(),
model: "mock-model".into(),
provider: "mock".into(),
message: Message::assistant(&text),
finish_reason: FinishReason::Stop,
usage: TokenCounts {
input_tokens: 10,
output_tokens: 20,
..Default::default()
},
raw: None,
warnings: vec![],
rate_limit: None,
cost_usd: None,
cost_source: None,
},
)),
];
Ok(Box::pin(stream::iter(events)))
async fn stream(&self, call: &ResolvedCall) -> Result<ResponseStream, fabro_llm::Error> {
Ok(fabro_llm::test_support::response_to_stream(
self.response(call),
))
}
}
@ -789,30 +766,48 @@ mod tests {
}
fn test_catalog_with_provider_base_url(provider: &str, base_url: &str) -> Arc<Catalog> {
let mut settings = LlmCatalogSettings::default();
settings
.providers
.insert(provider.to_string(), ProviderCatalogSettings {
base_url: Some(base_url.to_string()),
..ProviderCatalogSettings::default()
});
Arc::new(
Catalog::from_builtin_with_overrides(&settings)
.expect("catalog with custom base_url should build"),
Arc::new(fabro_llm::test_support::test_catalog_with_provider_base_url(provider, base_url))
}
/// The catalog every mock-backed test resolves against: the built-ins plus
/// a `mock` provider that passes any model name through.
fn mock_catalog() -> Catalog {
fabro_llm::test_support::test_catalog_with_overlay(
r#"
[providers.mock]
display_name = "Mock"
adapter = "openai-compatible"
codec = "openai-chat"
base_url = "http://mock.invalid/v1"
auth = { type = "bearer" }
allow_passthrough = true
[providers.mock.metadata.fabro]
agent_profile = "openai"
credentials = ["env:MOCK_API_KEY"]
[providers.mock.models.mock-model]
display_name = "Mock Model"
api_model = "mock-model"
limits = { context_tokens = 8192, max_output_tokens = 1024 }
capabilities = { text = true, tools = true, response_format = { json_object = true, json_schema = true } }
"#,
)
}
/// A client over [`mock_catalog`] whose `provider_name` answers with
/// `text`.
fn explicit_client(provider_name: &str, text: &str) -> Arc<Client> {
let mut providers: HashMap<String, Arc<dyn ProviderAdapter>> = HashMap::new();
providers.insert(
provider_name.to_string(),
Arc::new(MockProvider::new(provider_name, text)),
);
Arc::new(Client::new(
providers,
Some(provider_name.to_string()),
vec![],
))
let adapter: Arc<dyn ProviderAdapter> = Arc::new(MockProvider::new(text));
let mut options = fabro_llm::ClientOptions::default();
options
.adapters
.push((fabro_types::ProviderId::new(provider_name), adapter));
Arc::new(
fabro_llm::build_offline_client(mock_catalog(), options)
.expect("mock client should build")
.client,
)
}
fn test_projection() -> RunProjection {
@ -1074,7 +1069,7 @@ mod tests {
"Implement feature",
"mock-model",
&run_store.clone().into(),
Catalog::builtin(),
&mock_catalog(),
Some(&make_test_conclusion()),
None,
explicit_client(
@ -1148,7 +1143,7 @@ mod tests {
"Implement feature",
"mock-model",
&run_store.clone().into(),
Catalog::builtin(),
&mock_catalog(),
Some(&make_test_conclusion()),
None,
explicit_client(
@ -1246,7 +1241,7 @@ mod tests {
"Implement feature",
"mock-model",
&run_store.clone().into(),
Catalog::builtin(),
&mock_catalog(),
Some(&make_test_conclusion()),
None,
explicit_client(
@ -1271,7 +1266,7 @@ mod tests {
"Implement feature",
"gpt-5.4",
&run_store.clone().into(),
Catalog::builtin(),
&mock_catalog(),
Some(&make_test_conclusion()),
None,
explicit_client(
@ -1329,7 +1324,7 @@ mod tests {
"Implement feature",
"gpt-5.4",
&run_store_handle,
llm_source.as_ref(),
llm_source,
catalog,
Some(&make_test_conclusion()),
None,
@ -1490,8 +1485,8 @@ mod tests {
assert_eq!(
truncation_caps(
"unknown-model",
&Catalog::builtin().all_provider_ids(),
Catalog::builtin(),
&fabro_llm::catalog::enabled_provider_ids(&mock_catalog()),
&mock_catalog(),
),
TruncationCaps {
diff: 80_000,
@ -1517,7 +1512,7 @@ mod tests {
draft: false,
auto_merge: None,
run_store: &harness.run_store,
llm_source: harness.llm_source.as_ref(),
llm_source: Arc::clone(&harness.llm_source),
catalog: harness.catalog.clone(),
conclusion: None,
run_state: None,
@ -1556,7 +1551,7 @@ mod tests {
"Implement feature",
"mock-model",
&run_store.clone().into(),
Catalog::builtin(),
&mock_catalog(),
Some(&make_test_conclusion()),
None,
explicit_client("mock", &payload),
@ -1579,7 +1574,7 @@ mod tests {
"## Plan:",
"mock-model",
&run_store.clone().into(),
Catalog::builtin(),
&mock_catalog(),
Some(&make_test_conclusion()),
None,
explicit_client("mock", &payload),
@ -1669,7 +1664,7 @@ mod tests {
"Implement feature",
"mock-model",
&run_store.clone().into(),
Catalog::builtin(),
&mock_catalog(),
Some(&make_test_conclusion()),
None,
explicit_client("mock", &payload),
@ -1945,7 +1940,7 @@ mod tests {
draft: false,
auto_merge: None,
run_store: &harness.run_store,
llm_source: harness.llm_source.as_ref(),
llm_source: Arc::clone(&harness.llm_source),
catalog: harness.catalog.clone(),
conclusion: None,
run_state: None,
@ -1993,7 +1988,7 @@ mod tests {
draft: false,
auto_merge: None,
run_store: &harness.run_store,
llm_source: harness.llm_source.as_ref(),
llm_source: Arc::clone(&harness.llm_source),
catalog: harness.catalog.clone(),
conclusion: None,
run_state: None,
@ -2030,7 +2025,7 @@ mod tests {
draft: false,
auto_merge: None,
run_store: &harness.run_store,
llm_source: harness.llm_source.as_ref(),
llm_source: Arc::clone(&harness.llm_source),
catalog: harness.catalog.clone(),
conclusion: None,
run_state: None,

4
lib/components/fabro-workflow/src/pipeline/transform.rs
View file

@ -113,7 +113,7 @@ mod tests {
use std::sync::Arc;
use fabro_graphviz::graph::AttrValue;
use fabro_model::Catalog;
use fabro_llm::lithos_catalog::Catalog;
use super::*;
use crate::file_resolver::FilesystemFileResolver;
@ -129,7 +129,7 @@ mod tests {
}
fn test_catalog() -> Arc<Catalog> {
Arc::new(Catalog::from_builtin().unwrap())
Arc::new(fabro_llm::test_support::test_catalog())
}
fn transform_options() -> TransformOptions {

4
lib/components/fabro-workflow/src/pipeline/types.rs
View file

@ -4,14 +4,14 @@ use std::sync::Arc;
use fabro_graphviz::graph::Graph;
use fabro_interview::Interviewer;
use fabro_llm::lithos_catalog::Catalog;
use fabro_mcp::config::McpServerSettings;
use fabro_model::{Catalog, ProviderId};
use fabro_sandbox::SandboxSpec;
use fabro_template::TemplateContext;
use fabro_types::settings::run::{
PullRequestSettings, ResolvedGithubIntegration, RunModelControls,
};
use fabro_types::{ManifestPath, RunId, RunProjection};
use fabro_types::{ManifestPath, ProviderId, RunId, RunProjection};
use fabro_validate::{Diagnostic, Severity};
use fabro_vault::Vault;
use tokio::sync::RwLock as AsyncRwLock;

2
lib/components/fabro-workflow/src/pipeline/validate.rs
View file

@ -1,4 +1,4 @@
use fabro_model::Catalog;
use fabro_llm::lithos_catalog::Catalog;
use fabro_validate::LintRule;
use super::types::{Transformed, Validated};

16
lib/components/fabro-workflow/src/run_materialization.rs
View file

@ -1,10 +1,11 @@
use std::collections::HashSet;
use fabro_graphviz::graph::Graph;
use fabro_model::{Catalog, ModelSelectionError, ProviderId};
use fabro_types::WorkflowSettings;
use fabro_llm::lithos_catalog::Catalog;
use fabro_llm::{ModelSelectionError, selection};
use fabro_types::settings::InterpString;
use fabro_types::settings::run::RunGoal;
use fabro_types::{ProviderId, WorkflowSettings};
use crate::error::Error;
@ -61,7 +62,7 @@ fn materialize_run_with_eligible_providers(
)?;
settings.run.model.name = Some(resolved_model);
settings.run.model.provider = Some(resolved_provider.into_inner());
settings.run.model.provider = Some(resolved_provider.into_string());
let goal = graph.goal().to_string();
settings.run.goal = if goal.is_empty() {
@ -93,9 +94,14 @@ pub(crate) fn resolve_run_model(
.filter(|provider| !provider.is_empty())
.map(ProviderId::new);
let selected = if catalog_fallback {
catalog.resolve_selection_with_catalog_fallback(model, provider.as_ref(), eligible)?
selection::resolve_selection_with_catalog_fallback(
catalog,
model,
provider.as_ref(),
eligible,
)?
} else {
catalog.resolve_selection(model, provider.as_ref(), eligible)?
selection::resolve_selection(catalog, model, provider.as_ref(), eligible)?
};
Ok((selected.model, selected.provider))
}

25
lib/components/fabro-workflow/src/services.rs
View file

@ -6,13 +6,11 @@ use std::time::Duration;
use fabro_agent::{Sandbox, ToolEnvProvider};
use fabro_auth::CredentialSource;
#[cfg(test)]
use fabro_auth::ResolvedCredentials;
use fabro_github::token_source::InstallationTokenSource;
use fabro_hooks::{HookContext, HookDecision, HookExecutionContext, HookRunner};
use fabro_interview::Interviewer;
use fabro_model::{Catalog, ProviderId};
use fabro_types::{ManifestPath, RunId};
use fabro_llm::lithos_catalog::Catalog;
use fabro_types::{ManifestPath, ProviderId, RunId};
use tokio_util::sync::CancellationToken;
use crate::event::Emitter;
@ -271,12 +269,13 @@ impl EngineServices {
#[async_trait::async_trait]
impl CredentialSource for StubCredentialSource {
async fn resolve(&self, catalog: &Catalog) -> anyhow::Result<ResolvedCredentials> {
let _ = catalog;
Ok(ResolvedCredentials {
credentials: Vec::new(),
auth_issues: Vec::new(),
})
async fn credentials(
&self,
provider: &fabro_llm::lithos_catalog::CatalogProvider,
) -> Result<fabro_llm::credentials::Credentials, fabro_auth::ResolveError> {
Err(fabro_auth::ResolveError::NotConfigured(
provider.id().clone(),
))
}
async fn configured_providers(&self, catalog: &Catalog) -> Vec<ProviderId> {
@ -318,10 +317,10 @@ impl EngineServices {
None,
locations,
CancellationToken::new(),
ProviderId::anthropic(),
"claude-sonnet-4-6".to_string(),
fabro_types::provider_ids::anthropic(),
"claude-sonnet-4.6".to_string(),
Arc::new(StubCredentialSource),
Arc::new(Catalog::from_builtin().expect("default catalog should build")),
Arc::new(fabro_llm::default_catalog()),
Arc::new(SandboxGitRuntime::new()),
Arc::new(RunMetadataRuntime::new()),
None,

45
lib/components/fabro-workflow/src/test_support.rs
View file

@ -8,10 +8,13 @@ use fabro_agent::Sandbox;
use fabro_auth::{CredentialSource, test_support as auth_test_support};
use fabro_graphviz::graph::Graph as GvGraph;
use fabro_interview::AutoApproveInterviewer;
use fabro_model::Catalog;
#[cfg(feature = "test-support")]
use fabro_model::ProviderId;
use fabro_llm::catalog;
use fabro_llm::lithos_catalog::Catalog;
use fabro_llm::test_support::test_catalog;
use fabro_store::{ArtifactStore, RunProjection, test_support as store_test_support};
#[cfg(feature = "test-support")]
use fabro_types::ProviderId;
use fabro_types::{ModelId, ModelRef, provider_ids};
use object_store::local::LocalFileSystem;
use crate::artifact_upload::ArtifactSink;
@ -36,7 +39,7 @@ pub(crate) fn test_configured_provider_ids(
assume_ready: bool,
) -> Vec<ProviderId> {
if assume_ready {
catalog.all_provider_ids().into_iter().collect()
catalog::enabled_provider_ids(catalog).into_iter().collect()
} else {
configured_provider_ids
}
@ -82,26 +85,20 @@ async fn execute_and_emit_terminal(initialized: InitializedState) -> Executed {
#[must_use]
pub fn test_usage(
model_id: &str,
input_tokens: i64,
output_tokens: i64,
input_tokens: u64,
output_tokens: u64,
) -> fabro_types::BilledModelUsage {
serde_json::from_value(serde_json::json!({
"input": {
"usage": {
"model": {
"provider": "openai",
"model_id": model_id
},
"tokens": {
"input_tokens": input_tokens,
"output_tokens": output_tokens
}
},
"facts": { "algorithm": "openai" }
let mut usage = fabro_types::BilledModelUsage::new(
ModelRef::new(provider_ids::openai(), ModelId::new(model_id)),
fabro_types::TokenCounts {
input: input_tokens,
output: output_tokens,
..fabro_types::TokenCounts::default()
},
"total_usd_micros": input_tokens + output_tokens
}))
.expect("test_usage JSON must deserialise")
None,
);
usage.total_usd_micros = Some(i64::try_from(input_tokens + output_tokens).unwrap_or(i64::MAX));
usage
}
/// Append the `RunStartRequested → RunRunnable → RunStarting → RunRunning`
@ -272,12 +269,12 @@ async fn initialized(
options.hook_runner,
locations,
run_options.cancel_token.clone(),
fabro_model::ProviderId::anthropic(),
provider_ids::anthropic(),
"claude-sonnet-4-6".to_string(),
options
.llm_source
.unwrap_or_else(auth_test_support::vault_only_credential_source),
Arc::new(Catalog::from_builtin().expect("default catalog should build")),
Arc::new(test_catalog()),
Arc::new(SandboxGitRuntime::new()),
Arc::new(RunMetadataRuntime::new()),
None,

82
lib/components/fabro-workflow/src/transforms/model_resolution.rs
View file

@ -2,7 +2,9 @@ use std::collections::HashSet;
use std::sync::Arc;
use fabro_graphviz::graph::{AttrValue, Graph};
use fabro_model::{Catalog, ProviderId};
use fabro_llm::lithos_catalog::Catalog;
use fabro_llm::{catalog, selection};
use fabro_types::ProviderId;
use super::Transform;
use crate::error::Error;
@ -19,7 +21,7 @@ pub struct ModelResolutionTransform {
impl ModelResolutionTransform {
#[must_use]
pub fn new(catalog: Arc<Catalog>) -> Self {
let eligible_providers = catalog.all_provider_ids();
let eligible_providers = catalog::enabled_provider_ids(&catalog);
Self {
catalog,
default_provider: None,
@ -65,14 +67,19 @@ impl ModelResolutionTransform {
explicit_provider: Option<&ProviderId>,
) -> Result<(String, ProviderId), Error> {
let selected = if self.catalog_fallback {
self.catalog.resolve_selection_with_catalog_fallback(
selection::resolve_selection_with_catalog_fallback(
&self.catalog,
Some(model),
explicit_provider,
&self.eligible_providers,
)
} else {
self.catalog
.resolve_selection(Some(model), explicit_provider, &self.eligible_providers)
selection::resolve_selection(
&self.catalog,
Some(model),
explicit_provider,
&self.eligible_providers,
)
}?;
Ok((selected.model, selected.provider))
}
@ -148,45 +155,41 @@ mod tests {
use std::sync::Arc;
use fabro_graphviz::graph::{AttrValue, Graph, Node};
use fabro_model::catalog::LlmCatalogSettings;
use fabro_llm::test_support::{test_catalog, test_catalog_with_overlay};
use super::*;
/// An operator-defined provider with one aliased model, the shape an
/// `[llm]` overlay produces. It joins the built-ins rather than replacing
/// them: lithos catalogs are layered, never standalone.
fn custom_catalog() -> Arc<Catalog> {
let settings: LlmCatalogSettings = toml::from_str(
Arc::new(test_catalog_with_overlay(
r#"
[providers.venice]
display_name = "Venice"
adapter = "openai_compatible"
agent_profile = "openai"
[providers.acme-venice]
display_name = "Acme Venice"
adapter = "openai-compatible"
codec = "openai-chat"
base_url = "https://api.venice.ai/api/v1"
auth = { type = "bearer" }
priority = 200
default_model = "venice-large"
[providers.venice.auth]
[providers.acme-venice.metadata.fabro]
agent_profile = "openai"
credentials = ["env:VENICE_API_KEY"]
[models."venice-large"]
provider = "venice"
[providers.acme-venice.models.venice-large]
display_name = "Venice Large"
family = "venice"
default = true
aliases = ["vl"]
[models."venice-large".limits]
context_window = 128000
[models."venice-large".features]
tools = true
vision = false
reasoning = false
api_model = "venice-large"
limits = { context_tokens = 128000, max_output_tokens = 8192 }
capabilities = { text = true, tools = true }
"#,
)
.unwrap();
Arc::new(Catalog::from_settings(&settings).unwrap())
))
}
fn builtin_transform() -> ModelResolutionTransform {
let catalog = Catalog::from_builtin().unwrap();
ModelResolutionTransform::new(Arc::new(catalog))
ModelResolutionTransform::new(Arc::new(test_catalog()))
}
#[test]
@ -195,7 +198,7 @@ reasoning = false
let mut node = Node::new("a");
node.attrs.insert(
"model".to_string(),
AttrValue::String("claude-sonnet-4-5".to_string()),
AttrValue::String("claude-sonnet-4.5".to_string()),
);
graph.nodes.insert("a".to_string(), node);
@ -216,7 +219,7 @@ reasoning = false
let mut node = Node::new("a");
node.attrs.insert(
"model".to_string(),
AttrValue::String("claude-sonnet-4-5".to_string()),
AttrValue::String("claude-sonnet-4.5".to_string()),
);
node.attrs.insert(
"provider".to_string(),
@ -238,7 +241,7 @@ reasoning = false
.attrs
.get("model")
.and_then(AttrValue::as_str),
Some("claude-sonnet-4-5")
Some("claude-sonnet-4.5")
);
}
@ -345,20 +348,15 @@ reasoning = false
.attrs
.get("provider")
.and_then(AttrValue::as_str),
Some("venice")
Some("acme-venice")
);
}
#[test]
fn fallback_resolution_keeps_ready_preference_for_unpinned_nodes() {
let overrides: LlmCatalogSettings = toml::from_str(
r"
[providers.openrouter]
enabled = true
",
)
.unwrap();
let catalog = Arc::new(Catalog::from_builtin_with_overrides(&overrides).unwrap());
let catalog = Arc::new(test_catalog_with_overlay(
"[providers.openrouter.metadata.fabro]\nenabled = true\n",
));
let mut graph = Graph::new("test");
let mut portable = Node::new("portable");
portable.attrs.insert(
@ -414,7 +412,7 @@ enabled = true
.attrs
.get("default_provider")
.and_then(AttrValue::as_str),
Some("venice")
Some("acme-venice")
);
}
}

191
lib/components/fabro-workflow/tests/it/integration.rs
View file

@ -30,10 +30,11 @@ use fabro_interview::{
Answer, AnswerValue, AutoApproveInterviewer, CallbackInterviewer, Interviewer,
QueueInterviewer, RecordingInterviewer,
};
use fabro_model::catalog::{LlmCatalogSettings, ProviderCatalogSettings};
use fabro_model::{Catalog, ProviderId};
use fabro_llm::lithos_catalog::Catalog;
use fabro_store::{ArtifactKey, ArtifactStore};
use fabro_types::{EventBody, RunEvent, RunId, StageId, WorkflowSettings, parse_blob_ref};
use fabro_types::{
EventBody, ProviderId, RunEvent, RunId, StageId, WorkflowSettings, parse_blob_ref,
};
use fabro_validate::{Severity, validate, validate_or_raise};
use fabro_workflow::artifact;
use fabro_workflow::context::Context;
@ -66,21 +67,11 @@ use tokio_util::sync::CancellationToken;
use ulid::Ulid;
fn default_catalog() -> Arc<Catalog> {
Arc::new(Catalog::from_builtin().expect("default catalog should build"))
Arc::new(fabro_llm::test_support::test_catalog())
}
fn catalog_with_provider_base_url(provider: &str, base_url: &str) -> Arc<Catalog> {
let mut settings = LlmCatalogSettings::default();
settings
.providers
.insert(provider.to_string(), ProviderCatalogSettings {
base_url: Some(base_url.to_string()),
..ProviderCatalogSettings::default()
});
Arc::new(
Catalog::from_builtin_with_overrides(&settings)
.expect("catalog with custom base_url should build"),
)
Arc::new(fabro_llm::test_support::test_catalog_with_provider_base_url(provider, base_url))
}
fn local_env() -> Arc<dyn fabro_agent::Sandbox> {
@ -2583,7 +2574,7 @@ async fn shared_thread_compaction_before_routing_audit_succeeds() {
"model": "compact-model",
"choices": [{
"delta": {"content": text},
"finish_reason": null
"finish_reason": "stop"
}]
});
let usage_chunk = serde_json::json!({
@ -2629,7 +2620,7 @@ async fn shared_thread_compaction_before_routing_audit_succeeds() {
server
.mock_async(move |when, then| {
when.method(POST)
.path("/chat/completions")
.path("/v1/chat/completions")
.body_includes(r#""stream":true"#)
.body_includes(prompt)
.body_excludes(next_prompt);
@ -2648,7 +2639,7 @@ async fn shared_thread_compaction_before_routing_audit_succeeds() {
let audit_mock = server
.mock_async(|when, then| {
when.method(POST)
.path("/chat/completions")
.path("/v1/chat/completions")
.body_includes(r#""stream":true"#)
.body_includes("Audit shared-thread work");
then.status(200)
@ -2660,7 +2651,7 @@ async fn shared_thread_compaction_before_routing_audit_succeeds() {
let compaction_mock = server
.mock_async(|when, then| {
when.method(POST)
.path("/chat/completions")
.path("/v1/chat/completions")
.body_excludes(r#""stream":true"#);
then.status(200)
.header("content-type", "application/json")
@ -2670,41 +2661,36 @@ async fn shared_thread_compaction_before_routing_audit_succeeds() {
})
.await;
let settings: LlmCatalogSettings = toml::from_str(&format!(
r#"
let catalog = Arc::new(fabro_llm::test_support::test_catalog_with_overlay(
&format!(
r#"
[providers.compact]
adapter = "openai_compatible"
agent_profile = "openai"
base_url = "{}"
display_name = "Compact"
adapter = "openai-compatible"
codec = "openai-chat"
base_url = {base_url}
auth = {{ type = "bearer" }}
default_model = "compact-model"
[providers.compact.auth]
[providers.compact.metadata.fabro]
agent_profile = "openai"
credentials = ["env:COMPACT_API_KEY"]
[models.compact-model]
provider = "compact"
[providers.compact.models.compact-model]
display_name = "Compact Model"
family = "mock"
default = true
[models.compact-model.limits]
context_window = 100000
max_output = 1024
[models.compact-model.features]
tools = true
vision = false
reasoning = false
api_model = "compact-model"
limits = {{ context_tokens = 100000, max_output_tokens = 1024 }}
capabilities = {{ text = true, tools = true, response_format = {{ json_object = true, json_schema = true }} }}
"#,
server.base_url()
))
.expect("test catalog should parse");
let catalog = Arc::new(Catalog::from_builtin_with_overrides(&settings).unwrap());
base_url = toml::Value::String(server.base_url()),
),
));
let source = auth_test_support::env_credential_source(|name| {
(name == "COMPACT_API_KEY").then(|| "sk-test".to_string())
});
let backend = AgentApiBackend::new_with_catalog(
"compact-model".to_string(),
ProviderId::from("compact"),
ProviderId::new("compact"),
ModelFallbackPolicy::default(),
source,
Arc::new(SteeringHub::new(Arc::new(Emitter::default()))),
@ -2817,7 +2803,7 @@ async fn workflow_persists_authoritative_openrouter_cost_for_agent_stage() {
"model": "openai/gpt-5.4",
"choices": [{
"delta": {"content": "done"},
"finish_reason": null
"finish_reason": "stop"
}]
});
let usage_chunk = serde_json::json!({
@ -2835,7 +2821,7 @@ async fn workflow_persists_authoritative_openrouter_cost_for_agent_stage() {
let completion_mock = server
.mock_async(|when, then| {
when.method(POST)
.path("/chat/completions")
.path("/v1/chat/completions")
.body_includes(r#""stream":true"#)
.body_includes("Report completion");
then.status(200)
@ -2844,22 +2830,23 @@ async fn workflow_persists_authoritative_openrouter_cost_for_agent_stage() {
})
.await;
let settings: LlmCatalogSettings = toml::from_str(&format!(
r#"
[providers.openrouter]
let catalog = Arc::new(fabro_llm::test_support::test_catalog_with_overlay(
&format!(
"[providers.openrouter]
base_url = {}
[providers.openrouter.metadata.fabro]
enabled = true
base_url = "{}"
"#,
server.base_url()
))
.expect("test catalog should parse");
let catalog = Arc::new(Catalog::from_builtin_with_overrides(&settings).unwrap());
",
toml::Value::String(server.base_url()),
),
));
let source = auth_test_support::env_credential_source(|name| {
(name == "OPENROUTER_API_KEY").then(|| "sk-test".to_string())
});
let backend = AgentApiBackend::new_with_catalog(
"openai/gpt-5.4".to_string(),
ProviderId::from("openrouter"),
ProviderId::new("openrouter"),
ModelFallbackPolicy::default(),
source,
Arc::new(SteeringHub::new(Arc::new(Emitter::default()))),
@ -5263,12 +5250,7 @@ async fn import_e2e_through_engine() {
use fabro_workflow::transforms::ModelResolutionTransform;
let dir = tempfile::tempdir().unwrap();
let catalog = std::sync::Arc::new(
fabro_model::Catalog::from_builtin_with_overrides(
&fabro_model::catalog::LlmCatalogSettings::default(),
)
.unwrap(),
);
let catalog = std::sync::Arc::new(fabro_llm::test_support::test_catalog());
std::fs::write(
dir.path().join("val.fabro"),
r#"digraph validate {
@ -7323,9 +7305,7 @@ mod real_llm {
use async_trait::async_trait;
use fabro_auth::EnvCredentialSource;
use fabro_graphviz::graph::Node;
use fabro_llm::client::Client;
use fabro_llm::providers::OpenAiAdapter;
use fabro_llm::types::{Message, Request};
use fabro_llm::{Client, ClientOptions, Request};
use fabro_types::WorkflowSettings;
use fabro_workflow::error::Error;
use fabro_workflow::handler::agent::{
@ -7352,25 +7332,16 @@ mod real_llm {
impl LlmCodergenBackend {
async fn complete(&self, prompt: &str) -> Result<CodergenResult, Error> {
let request = Request {
model: self.model.clone(),
messages: vec![Message::user(prompt)],
provider: Some(self.provider.clone()),
tools: None,
tool_choice: None,
response_format: None,
temperature: Some(0.0),
top_p: None,
max_tokens: Some(200),
stop_sequences: None,
reasoning_effort: None,
speed: None,
metadata: None,
provider_options: None,
};
let request = Request::builder()
.model(format!("{}/{}", self.provider, self.model))
.user(prompt)
.temperature(0.0)
.max_output_tokens(200)
.build()
.map_err(|e| Error::handler(e.to_string()))?;
let response = self
.client
.complete(&request)
.complete(request)
.await
.map_err(|e| Error::handler(e.to_string()))?;
Ok(CodergenResult::Text {
@ -7399,27 +7370,44 @@ mod real_llm {
}
}
/// A client whose `openai` provider is the twin at `base_url`,
/// authenticated with `api_key`.
async fn twin_openai_client(base_url: String, api_key: String) -> Arc<Client> {
let catalog = fabro_llm::build_catalog(&fabro_config::LlmLayer::default(), &move |name| {
(name == fabro_static::EnvVars::OPENAI_BASE_URL).then(|| base_url.clone())
})
.expect("twin catalog should build");
Arc::new(
fabro_llm::test_support::client_from_env(
catalog,
move |name| {
(name == fabro_static::EnvVars::OPENAI_API_KEY).then(|| api_key.clone())
},
ClientOptions::standard(),
)
.await,
)
}
async fn make_llm_client() -> Option<Arc<Client>> {
use fabro_llm::lithos_catalog::Catalog;
if fabro_test::TestMode::from_env().is_twin() {
let (base_url, api_key) = fabro_test::e2e_openai!();
let adapter: Arc<dyn fabro_llm::provider::ProviderAdapter> =
Arc::new(OpenAiAdapter::new(api_key).with_base_url(base_url));
let mut providers: HashMap<String, Arc<dyn fabro_llm::provider::ProviderAdapter>> =
HashMap::new();
providers.insert("openai".to_string(), adapter);
return Some(Arc::new(Client::new(
providers,
Some("openai".to_string()),
Vec::new(),
)));
return Some(twin_openai_client(base_url, api_key).await);
}
fabro_test::require_env("ANTHROPIC_API_KEY")?;
let source = EnvCredentialSource::new();
let source: Arc<dyn fabro_auth::CredentialSource> = Arc::new(EnvCredentialSource::new());
Some(Arc::new(
Client::from_source(&source, super::default_catalog())
.await
.expect("unified-llm client should initialize from env source"),
fabro_llm::build_client(
Catalog::clone(&super::default_catalog()),
source,
ClientOptions::standard(),
)
.await
.expect("LLM client should initialize from env source")
.client,
))
}
@ -7583,14 +7571,7 @@ mod real_llm {
.load(twin)
.await;
let adapter: Arc<dyn fabro_llm::provider::ProviderAdapter> =
Arc::new(OpenAiAdapter::new(namespace.clone()).with_base_url(twin.base_url.clone()));
let providers = HashMap::from([("openai".to_string(), adapter)]);
let client = Arc::new(Client::new(
providers,
Some("openai".to_string()),
Vec::new(),
));
let client = twin_openai_client(twin.base_url.clone(), namespace.clone()).await;
let mut graph = Graph::new("ForEachSecurityReview");
graph.attrs.insert(
@ -8207,8 +8188,8 @@ async fn workflow_run_with_vault_only_openai_codex_builds_pr_body() {
"Implement feature",
"gpt-5.4",
&run_store_handle,
llm_source.as_ref(),
catalog,
Arc::clone(&llm_source),
Arc::clone(&catalog),
Some(&Conclusion {
timestamp: Utc::now(),
status: StageOutcome::Succeeded,

12
lib/components/fabro-workflow/tests/materialize_run.rs
View file

@ -1,9 +1,9 @@
use fabro_graphviz::graph::Graph;
use fabro_graphviz::parser;
use fabro_model::{Catalog, ProviderId};
use fabro_types::WorkflowSettings;
use fabro_llm::test_support::test_catalog;
use fabro_types::settings::InterpString;
use fabro_types::settings::run::{PullRequestSettings, RunGoal, RunModelSettings, RunNamespace};
use fabro_types::{WorkflowSettings, provider_ids};
use fabro_workflow::run_materialization::materialize_run;
fn graph(source: &str) -> Graph {
@ -34,8 +34,8 @@ fn materialize_run_applies_graph_and_catalog_defaults() {
..WorkflowSettings::default()
};
let materialized = materialize_run(settings, &graph(source), Catalog::builtin(), &[
ProviderId::anthropic(),
let materialized = materialize_run(settings, &graph(source), &test_catalog(), &[
provider_ids::anthropic(),
])
.unwrap();
let resolved = &materialized.run;
@ -61,8 +61,8 @@ fn materialize_run_uses_configured_provider_defaults() {
let materialized = materialize_run(
WorkflowSettings::default(),
&graph(source),
Catalog::builtin(),
&[ProviderId::openai()],
&test_catalog(),
&[provider_ids::openai()],
)
.unwrap();
let resolved = &materialized.run;