Merge branch 'main' into fabro/run/01KM8C0SVZW77C5W018CYEVE4Y

This commit is contained in:
Bryan Helmkamp 2026-03-21 12:35:48 -04:00
commit 756915b1ea
11 changed files with 300 additions and 82 deletions

1
Cargo.lock generated
View file

@ -1534,6 +1534,7 @@ dependencies = [
"futures",
"http",
"httpmock",
"indicatif",
"insta",
"rand 0.8.5",
"reqwest 0.12.28",

View file

@ -13,13 +13,14 @@ pub async fn create_run(
args: &RunArgs,
run_defaults: RunDefaults,
styles: &Styles,
quiet: bool,
) -> anyhow::Result<(String, PathBuf)> {
let workflow_path = args
.workflow
.as_ref()
.ok_or_else(|| anyhow::anyhow!("--workflow is required"))?;
let prep = prepare_workflow(args, run_defaults, styles)?;
let prep = prepare_workflow(args, run_defaults, styles, quiet)?;
let goal = prep.graph.goal();

View file

@ -474,6 +474,7 @@ pub(crate) fn prepare_workflow(
args: &RunArgs,
mut run_defaults: RunDefaults,
styles: &Styles,
quiet: bool,
) -> anyhow::Result<PreparedWorkflow> {
let workflow_path = args
.workflow
@ -539,30 +540,32 @@ pub(crate) fn prepare_workflow(
}
}
eprintln!(
"{} {} {}",
styles.bold.apply_to("Workflow:"),
graph.name,
styles.dim.apply_to(format!(
"({} nodes, {} edges)",
graph.nodes.len(),
graph.edges.len()
)),
);
eprintln!(
"{} {}",
styles.dim.apply_to("Graph:"),
styles.dim.apply_to(relative_path(&dot_path)),
);
if !quiet {
eprintln!(
"{} {} {}",
styles.bold.apply_to("Workflow:"),
graph.name,
styles.dim.apply_to(format!(
"({} nodes, {} edges)",
graph.nodes.len(),
graph.edges.len()
)),
);
eprintln!(
"{} {}",
styles.dim.apply_to("Graph:"),
styles.dim.apply_to(relative_path(&dot_path)),
);
let goal = graph.goal();
if !goal.is_empty() {
let stripped = fabro_util::text::strip_goal_decoration(goal);
eprintln!("{} {stripped}\n", styles.bold.apply_to("Goal:"));
let goal = graph.goal();
if !goal.is_empty() {
let stripped = fabro_util::text::strip_goal_decoration(goal);
eprintln!("{} {stripped}\n", styles.bold.apply_to("Goal:"));
}
print_diagnostics(&diagnostics, styles);
}
print_diagnostics(&diagnostics, styles);
if diagnostics.iter().any(|d| d.severity == Severity::Error) {
bail!("Validation failed");
}
@ -618,7 +621,7 @@ pub async fn run_command(
model,
provider,
run_defaults,
} = prepare_workflow(&args, run_defaults, styles)?;
} = prepare_workflow(&args, run_defaults, styles, false)?;
// Extract workflow slug from the workflow path argument.
// If bare name (no extension, e.g. "smoke"), use it directly.

View file

@ -656,7 +656,7 @@ async fn main_inner() -> (String, Result<()>) {
if args.detach {
// Detach mode: create + start + print run ID
let (run_id, run_dir) =
commands::create::create_run(&args, cli_config.run_defaults, styles)
commands::create::create_run(&args, cli_config.run_defaults, styles, true)
.await?;
commands::start::start_run(&run_dir)?;
println!("{run_id}");
@ -686,7 +686,8 @@ async fn main_inner() -> (String, Result<()>) {
Box::leak(Box::new(fabro_util::terminal::Styles::detect_stderr()));
let cli_config = cli_config::load_cli_config(None)?;
let (run_id, _run_dir) =
commands::create::create_run(&args, cli_config.run_defaults, styles).await?;
commands::create::create_run(&args, cli_config.run_defaults, styles, true)
.await?;
println!("{run_id}");
}
Command::Start { run } => {

View file

@ -28,6 +28,7 @@ base64.workspace = true
bytes.workspace = true
tokio-util.workspace = true
cli-table.workspace = true
indicatif.workspace = true
clap.workspace = true
dialoguer.workspace = true
tracing.workspace = true

View file

@ -16,7 +16,8 @@ use fabro_util::terminal::Styles;
use fabro_model as catalog;
use crate::generate::{self, GenerateParams};
use crate::types::Message;
use crate::tools::Tool;
use crate::types::{ContentPart, Message};
use fabro_model::ModelInfo;
pub struct ServerConnection {
@ -76,6 +77,10 @@ pub enum ModelsCommand {
/// Test a specific model
#[arg(short, long)]
model: Option<String>,
/// Run a multi-turn tool-use test (catches reasoning round-trip bugs)
#[arg(long)]
deep: bool,
},
}
@ -829,12 +834,97 @@ async fn test_model_via_server(
.context("Failed to parse model test response from server")
}
fn build_deep_test_params(info: &ModelInfo) -> Option<GenerateParams> {
if !info.features.tools {
return None;
}
let add_tool = Tool::active(
"add",
"Add two integers and return the sum",
serde_json::json!({
"type": "object",
"properties": {
"a": { "type": "integer", "description": "First number" },
"b": { "type": "integer", "description": "Second number" }
},
"required": ["a", "b"]
}),
|args, _ctx| async move {
let a = args.get("a").and_then(|v| v.as_i64()).unwrap_or(0);
let b = args.get("b").and_then(|v| v.as_i64()).unwrap_or(0);
Ok(serde_json::json!(a + b))
},
);
let mut params = GenerateParams::new(&info.id)
.provider(&info.provider)
.prompt(
"I have three numbers: 15, 27, and 42. \
First use the add tool to compute 15 + 27, \
then use the add tool to add that result to 42. \
Finally, tell me whether the grand total is even or odd and why.",
)
.tools(vec![add_tool])
.max_tool_rounds(5)
.max_tokens(1024);
if info.features.reasoning {
params = params.reasoning_effort("high");
}
Some(params)
}
fn validate_deep_result(
result: &crate::types::GenerateResult,
info: &ModelInfo,
) -> (cli_table::Color, String) {
// Check tool use: need at least 2 steps (tool call + follow-up)
if result.steps.len() < 2 {
return (
Color::Red,
"deep: fail (model did not call tool)".to_string(),
);
}
// Check that step 0 had tool results (tool was executed)
if result.steps[0].tool_results.is_empty() {
return (Color::Red, "deep: fail (tool not executed)".to_string());
}
// Check correctness: 15+27=42, 42+42=84 — final response should contain "84"
let final_text = result.response.text();
if !final_text.contains("84") {
return (Color::Red, "deep: fail (wrong answer)".to_string());
}
// Check reasoning if the model supports it
if info.features.reasoning {
let has_reasoning = result.steps.iter().any(|step| {
step.response.message.content.iter().any(|part| {
matches!(part, ContentPart::Thinking(_))
|| matches!(part, ContentPart::Other { kind, .. } if kind == ContentPart::OPENAI_REASONING)
})
});
if !has_reasoning {
return (Color::Yellow, "deep: ok (no reasoning)".to_string());
}
}
(Color::Green, "deep: ok".to_string())
}
async fn test_models_via_server(
server: &ServerConnection,
provider: Option<&str>,
model: Option<&str>,
deep: bool,
s: &Styles,
) -> Result<()> {
if deep {
eprintln!("Warning: --deep is not supported in server mode");
}
let models_to_test = if let Some(model_id) = model {
let all = fetch_models_from_server(&server.client, &server.base_url, None).await?;
let found: Vec<_> = all.into_iter().filter(|m| m.id == model_id).collect();
@ -932,12 +1022,17 @@ pub async fn run_models(
print_models_table(&models, &styles);
}
ModelsCommand::Test { provider, model } => match &server {
ModelsCommand::Test {
provider,
model,
deep,
} => match &server {
Some(s) => {
test_models_via_server(s, provider.as_deref(), model.as_deref(), &styles).await?;
test_models_via_server(s, provider.as_deref(), model.as_deref(), deep, &styles)
.await?;
}
None => {
test_models(provider.as_deref(), model.as_deref(), &styles).await?;
test_models(provider.as_deref(), model.as_deref(), deep, &styles).await?;
}
},
}
@ -945,7 +1040,44 @@ pub async fn run_models(
Ok(())
}
async fn test_models(provider: Option<&str>, model: Option<&str>, s: &Styles) -> Result<()> {
async fn test_one_model(info: &ModelInfo, deep: bool) -> (Color, String) {
if deep {
match build_deep_test_params(info) {
None => (Color::Yellow, "deep: skipped (no tool support)".to_string()),
Some(params) => {
let result =
tokio::time::timeout(Duration::from_secs(90), generate::generate(params)).await;
match result {
Ok(Ok(ref gen_result)) => validate_deep_result(gen_result, info),
Ok(Err(e)) => (Color::Red, format!("deep: error: {e}")),
Err(_) => (Color::Red, "deep: error: timeout (90s)".to_string()),
}
}
}
} else {
let params = GenerateParams::new(&info.id)
.provider(&info.provider)
.prompt("Say OK")
.max_tokens(16);
let result =
tokio::time::timeout(Duration::from_secs(30), generate::generate(params)).await;
match result {
Ok(Ok(_)) => (Color::Green, "ok".to_string()),
Ok(Err(e)) => (Color::Red, format!("error: {e}")),
Err(_) => (Color::Red, "error: timeout (30s)".to_string()),
}
}
}
async fn test_models(
provider: Option<&str>,
model: Option<&str>,
deep: bool,
s: &Styles,
) -> Result<()> {
use rand::seq::SliceRandom;
let models_to_test = if let Some(model_id) = model {
match catalog::get_model_info(model_id) {
Some(info) => vec![info],
@ -959,40 +1091,55 @@ async fn test_models(provider: Option<&str>, model: Option<&str>, s: &Styles) ->
bail!("No models found");
}
let test_kind = if deep { "Deep testing" } else { "Testing" };
let pb = indicatif::ProgressBar::new(models_to_test.len() as u64);
pb.set_style(
indicatif::ProgressStyle::with_template(&format!(
"{{spinner:.green}} {test_kind} {{pos}}/{{len}} models {{wide_bar}} {{eta}}"
))
.unwrap(),
);
pb.enable_steady_tick(Duration::from_millis(100));
// Build (original_index, model_info) pairs, then shuffle for provider spread
let mut indexed: Vec<(usize, &ModelInfo)> = models_to_test.iter().enumerate().collect();
indexed.shuffle(&mut rand::thread_rng());
// Run tests concurrently, 6 at a time
let results: Vec<(usize, Color, String)> = futures::stream::iter(indexed)
.map(|(idx, info)| {
let pb = pb.clone();
async move {
let (color, status) = test_one_model(info, deep).await;
pb.inc(1);
(idx, color, status)
}
})
.buffer_unordered(6)
.collect()
.await;
pb.finish_and_clear();
// Sort back to original catalog order
let mut sorted_results = results;
sorted_results.sort_by_key(|(idx, _, _)| *idx);
let use_color = s.use_color;
let mut title = models_title();
title.push("RESULT".cell().bold(true));
let mut rows: Vec<Vec<CellStruct>> = Vec::new();
let mut failures = 0u32;
for info in &models_to_test {
eprint!("Testing {}...", info.id);
let params = GenerateParams::new(&info.id)
.provider(&info.provider)
.prompt("Say OK")
.max_tokens(16);
let result =
tokio::time::timeout(Duration::from_secs(30), generate::generate(params)).await;
eprintln!(" done");
let (result_color, status) = match result {
Ok(Ok(_)) => (Color::Green, "ok".to_string()),
Ok(Err(e)) => {
failures += 1;
(Color::Red, format!("error: {e}"))
}
Err(_) => {
failures += 1;
(Color::Red, "error: timeout (30s)".to_string())
}
};
let mut row = model_row(info, use_color);
for (idx, result_color, status) in &sorted_results {
if *result_color == Color::Red {
failures += 1;
}
let mut row = model_row(&models_to_test[*idx], use_color);
row.push(
status
.cell()
.foreground_color(color_if(use_color, result_color)),
.foreground_color(color_if(use_color, *result_color)),
);
rows.push(row);
}

View file

@ -523,6 +523,18 @@ fn extract_thinking_config(
.cloned()
}
/// Map a reasoning effort level to a thinking `budget_tokens` value for models
/// that don't support the `output_config.effort` parameter (e.g. claude-sonnet-4-5).
fn effort_to_budget_tokens(effort: &str, max_tokens: i64) -> i64 {
let budget = match effort {
"low" => max_tokens / 4,
"high" => max_tokens * 3 / 4,
_ => max_tokens / 2, // "medium" or unknown
};
// Anthropic requires budget_tokens >= 1024
budget.max(1024)
}
fn is_auto_cache_enabled(provider_options: Option<&serde_json::Value>) -> bool {
provider_options
.and_then(|opts| opts.get("anthropic"))
@ -1072,22 +1084,48 @@ fn build_api_request(
apply_cache_control_to_conversation_prefix(&mut api_messages);
}
let thinking = extract_thinking_config(request.provider_options.as_ref());
let explicit_thinking = extract_thinking_config(request.provider_options.as_ref());
let output_config = request
.reasoning_effort
.as_ref()
.map(|effort| serde_json::json!({"effort": effort}));
// Check whether this model supports the `output_config.effort` parameter.
// Older reasoning models (e.g. claude-sonnet-4-5) need `thinking` with
// `budget_tokens` instead.
let model_info = fabro_model::get_model_info(&request.model);
let supports_effort = model_info.as_ref().is_none_or(|m| m.features.effort);
let mut resolved_max_tokens = request
.max_tokens
.or_else(|| model_info.as_ref().and_then(|m| m.limits.max_output))
.unwrap_or(65536);
let (thinking, output_config) = if let Some(effort) = &request.reasoning_effort {
if supports_effort {
(
explicit_thinking,
Some(serde_json::json!({"effort": effort})),
)
} else if explicit_thinking.is_none() {
// Convert effort level to a thinking budget for models that don't
// support the effort parameter (e.g. claude-sonnet-4-5).
let budget = effort_to_budget_tokens(effort, resolved_max_tokens);
if resolved_max_tokens <= budget {
resolved_max_tokens = budget + 1024;
}
(
Some(serde_json::json!({"type": "enabled", "budget_tokens": budget})),
None,
)
} else {
// thinking already configured via provider_options; skip output_config
(explicit_thinking, None)
}
} else {
(explicit_thinking, None)
};
let api_request = ApiRequest {
model: request.model.clone(),
messages: api_messages,
max_tokens: request
.max_tokens
.or_else(|| {
fabro_model::get_model_info(&request.model).and_then(|m| m.limits.max_output)
})
.unwrap_or(65536),
max_tokens: resolved_max_tokens,
system: system_value,
temperature: request.temperature,
top_p: request.top_p,

View file

@ -138,6 +138,8 @@ struct ApiRequest {
#[serde(skip_serializing_if = "Option::is_none")]
metadata: Option<std::collections::HashMap<String, String>>,
store: bool,
#[serde(skip_serializing_if = "Vec::is_empty")]
include: Vec<String>,
#[serde(skip_serializing_if = "std::ops::Not::not")]
stream: bool,
}
@ -391,6 +393,12 @@ fn build_api_request(request: &Request, stream: bool, codex_mode: bool) -> ApiRe
.as_ref()
.and_then(translate_response_format);
let include = if reasoning.is_some() {
vec!["reasoning.encrypted_content".to_string()]
} else {
Vec::new()
};
let instructions = if codex_mode {
Some(instructions.unwrap_or_default())
} else {
@ -414,7 +422,12 @@ fn build_api_request(request: &Request, stream: bool, codex_mode: bool) -> ApiRe
text,
stop: request.stop_sequences.clone(),
metadata: request.metadata.clone(),
store: !codex_mode,
// store: false is required for non-Azure OpenAI endpoints. Reasoning
// items still round-trip correctly because we request encrypted_content
// via the `include` field, which embeds them in the response payload
// rather than relying on server-side storage.
store: false,
include,
stream,
}
}

View file

@ -6,7 +6,7 @@
"display_name": "Claude Opus 4.6",
"limits": { "context_window": 1000000, "max_output": 128000 },
"training": "2025-08-01",
"features": { "tools": true, "vision": true, "reasoning": true },
"features": { "tools": true, "vision": true, "reasoning": true, "effort": true },
"costs": {
"input_cost_per_mtok": 15.0,
"output_cost_per_mtok": 75.0,
@ -38,7 +38,7 @@
"display_name": "Claude Sonnet 4.6",
"limits": { "context_window": 200000, "max_output": 64000 },
"training": "2025-08-01",
"features": { "tools": true, "vision": true, "reasoning": true },
"features": { "tools": true, "vision": true, "reasoning": true, "effort": true },
"costs": {
"input_cost_per_mtok": 3.0,
"output_cost_per_mtok": 15.0,
@ -71,7 +71,7 @@
"display_name": "GPT-5.2",
"limits": { "context_window": 1047576, "max_output": 128000 },
"training": "2025-08-31",
"features": { "tools": true, "vision": true, "reasoning": true },
"features": { "tools": true, "vision": true, "reasoning": true, "effort": true },
"costs": {
"input_cost_per_mtok": 1.75,
"output_cost_per_mtok": 14.0,
@ -87,7 +87,7 @@
"display_name": "GPT-5 Mini",
"limits": { "context_window": 1047576, "max_output": 128000 },
"training": "2025-08-31",
"features": { "tools": true, "vision": true, "reasoning": true },
"features": { "tools": true, "vision": true, "reasoning": true, "effort": true },
"costs": {
"input_cost_per_mtok": 0.25,
"output_cost_per_mtok": 2.0,
@ -103,7 +103,7 @@
"display_name": "GPT-5.2 Codex",
"limits": { "context_window": 1047576, "max_output": 128000 },
"training": "2025-08-31",
"features": { "tools": true, "vision": true, "reasoning": true },
"features": { "tools": true, "vision": true, "reasoning": true, "effort": true },
"costs": {
"input_cost_per_mtok": 1.75,
"output_cost_per_mtok": 14.0,
@ -119,7 +119,7 @@
"display_name": "GPT-5.3 Codex",
"limits": { "context_window": 1047576, "max_output": 128000 },
"training": "2025-08-31",
"features": { "tools": true, "vision": true, "reasoning": true },
"features": { "tools": true, "vision": true, "reasoning": true, "effort": true },
"costs": {
"input_cost_per_mtok": 1.75,
"output_cost_per_mtok": 14.0,
@ -135,7 +135,7 @@
"display_name": "GPT-5.3 Codex Spark",
"limits": { "context_window": 131072, "max_output": 128000 },
"training": "2025-08-31",
"features": { "tools": true, "vision": false, "reasoning": true },
"features": { "tools": true, "vision": false, "reasoning": true, "effort": true },
"costs": {
"input_cost_per_mtok": null,
"output_cost_per_mtok": null,
@ -151,7 +151,7 @@
"display_name": "GPT-5.4",
"limits": { "context_window": 1047576, "max_output": 128000 },
"training": "2025-08-31",
"features": { "tools": true, "vision": true, "reasoning": true },
"features": { "tools": true, "vision": true, "reasoning": true, "effort": true },
"costs": {
"input_cost_per_mtok": 2.5,
"output_cost_per_mtok": 15.0,
@ -168,7 +168,7 @@
"display_name": "GPT-5.4 Pro",
"limits": { "context_window": 1047576, "max_output": 128000 },
"training": "2025-08-31",
"features": { "tools": true, "vision": true, "reasoning": true },
"features": { "tools": true, "vision": true, "reasoning": true, "effort": true },
"costs": {
"input_cost_per_mtok": 30.0,
"output_cost_per_mtok": 180.0,
@ -184,7 +184,7 @@
"display_name": "GPT-5.4 Mini",
"limits": { "context_window": 400000, "max_output": 128000 },
"training": "2025-08-31",
"features": { "tools": true, "vision": true, "reasoning": true },
"features": { "tools": true, "vision": true, "reasoning": true, "effort": true },
"costs": {
"input_cost_per_mtok": 0.75,
"output_cost_per_mtok": 4.50,
@ -200,7 +200,7 @@
"display_name": "Gemini 3.1 Pro (Preview)",
"limits": { "context_window": 1048576, "max_output": 65536 },
"training": "2025-01-01",
"features": { "tools": true, "vision": true, "reasoning": true },
"features": { "tools": true, "vision": true, "reasoning": true, "effort": true },
"costs": {
"input_cost_per_mtok": 2.0,
"output_cost_per_mtok": 12.0,
@ -217,7 +217,7 @@
"display_name": "Gemini 3.1 Pro Custom Tools (Preview)",
"limits": { "context_window": 1048576, "max_output": 65536 },
"training": "2025-01-01",
"features": { "tools": true, "vision": true, "reasoning": true },
"features": { "tools": true, "vision": true, "reasoning": true, "effort": true },
"costs": {
"input_cost_per_mtok": 2.0,
"output_cost_per_mtok": 12.0,
@ -233,7 +233,7 @@
"display_name": "Gemini 3 Flash (Preview)",
"limits": { "context_window": 1048576, "max_output": 65536 },
"training": "2025-01-01",
"features": { "tools": true, "vision": true, "reasoning": true },
"features": { "tools": true, "vision": true, "reasoning": true, "effort": true },
"costs": {
"input_cost_per_mtok": 0.5,
"output_cost_per_mtok": 3.0,
@ -249,7 +249,7 @@
"display_name": "Gemini 3.1 Flash Lite (Preview)",
"limits": { "context_window": 1048576, "max_output": 65536 },
"training": "2025-01-01",
"features": { "tools": true, "vision": true, "reasoning": true },
"features": { "tools": true, "vision": true, "reasoning": true, "effort": true },
"costs": {
"input_cost_per_mtok": 0.25,
"output_cost_per_mtok": 1.5,
@ -316,7 +316,7 @@
"display_name": "Mercury 2",
"limits": { "context_window": 131072, "max_output": 50000 },
"training": null,
"features": { "tools": true, "vision": false, "reasoning": true },
"features": { "tools": true, "vision": false, "reasoning": true, "effort": true },
"costs": {
"input_cost_per_mtok": 0.25,
"output_cost_per_mtok": 0.75,

View file

@ -270,6 +270,7 @@ mod tests {
tools: true,
vision: true,
reasoning: true,
effort: true,
},
costs: ModelCosts {
input_cost_per_mtok: Some(
@ -349,6 +350,7 @@ mod tests {
tools: true,
vision: true,
reasoning: true,
effort: true,
},
costs: ModelCosts {
input_cost_per_mtok: Some(
@ -402,6 +404,7 @@ mod tests {
tools: true,
vision: true,
reasoning: false,
effort: false,
},
costs: ModelCosts {
input_cost_per_mtok: Some(
@ -460,6 +463,7 @@ mod tests {
tools: true,
vision: false,
reasoning: true,
effort: true,
},
costs: ModelCosts {
input_cost_per_mtok: Some(
@ -508,6 +512,7 @@ mod tests {
tools: true,
vision: true,
reasoning: true,
effort: true,
},
costs: ModelCosts {
input_cost_per_mtok: Some(
@ -554,6 +559,7 @@ mod tests {
tools: true,
vision: true,
reasoning: true,
effort: true,
},
costs: ModelCosts {
input_cost_per_mtok: Some(
@ -620,6 +626,7 @@ mod tests {
tools: true,
vision: false,
reasoning: true,
effort: true,
},
costs: ModelCosts {
input_cost_per_mtok: None,

View file

@ -13,6 +13,12 @@ pub struct ModelFeatures {
pub tools: bool,
pub vision: bool,
pub reasoning: bool,
/// Whether the model supports the `reasoning_effort` / `effort` parameter
/// directly (e.g. Anthropic `output_config.effort`, OpenAI `reasoning.effort`).
/// Models with `reasoning=true` but `effort=false` (e.g. claude-sonnet-4-5)
/// need the older `thinking` API with `budget_tokens` instead.
#[serde(default)]
pub effort: bool,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]