From e5f0cc5dd066d5d82092e371383755651740acf1 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Sat, 21 Mar 2026 11:27:25 -0400 Subject: [PATCH] Add --deep flag to `fabro model test` for multi-turn tool-use validation Exercises a 2+ turn tool-call round-trip with reasoning_effort("high") to catch bugs like store: false that only manifest when reasoning items from turn 1 are sent back in turn 2. Co-Authored-By: Claude Opus 4.6 (1M context) --- lib/crates/fabro-llm/src/cli.rs | 150 ++++++++++++++++++++++++++++++-- 1 file changed, 145 insertions(+), 5 deletions(-) diff --git a/lib/crates/fabro-llm/src/cli.rs b/lib/crates/fabro-llm/src/cli.rs index 1f74e897e..8a9c76c01 100644 --- a/lib/crates/fabro-llm/src/cli.rs +++ b/lib/crates/fabro-llm/src/cli.rs @@ -16,7 +16,8 @@ use fabro_util::terminal::Styles; use fabro_model as catalog; use crate::generate::{self, GenerateParams}; -use crate::types::Message; +use crate::tools::Tool; +use crate::types::{ContentPart, Message}; use fabro_model::ModelInfo; pub struct ServerConnection { @@ -76,6 +77,10 @@ pub enum ModelsCommand { /// Test a specific model #[arg(short, long)] model: Option, + + /// Run a multi-turn tool-use test (catches reasoning round-trip bugs) + #[arg(long)] + deep: bool, }, } @@ -829,12 +834,94 @@ async fn test_model_via_server( .context("Failed to parse model test response from server") } +fn build_deep_test_params(info: &ModelInfo) -> Option { + if !info.features.tools { + return None; + } + + let add_tool = Tool::active( + "add", + "Add two integers and return the sum", + serde_json::json!({ + "type": "object", + "properties": { + "a": { "type": "integer", "description": "First number" }, + "b": { "type": "integer", "description": "Second number" } + }, + "required": ["a", "b"] + }), + |args, _ctx| async move { + let a = args.get("a").and_then(|v| v.as_i64()).unwrap_or(0); + let b = args.get("b").and_then(|v| v.as_i64()).unwrap_or(0); + Ok(serde_json::json!(a + b)) + }, + ); + + let mut params = GenerateParams::new(&info.id) + .provider(&info.provider) + .prompt( + "I have three numbers: 15, 27, and 42. \ + First use the add tool to compute 15 + 27, \ + then use the add tool to add that result to 42. \ + Finally, tell me whether the grand total is even or odd and why.", + ) + .tools(vec![add_tool]) + .max_tool_rounds(5) + .max_tokens(1024); + + if info.features.reasoning { + params = params.reasoning_effort("high"); + } + + Some(params) +} + +fn validate_deep_result( + result: &crate::types::GenerateResult, + info: &ModelInfo, +) -> (cli_table::Color, String) { + // Check tool use: need at least 2 steps (tool call + follow-up) + if result.steps.len() < 2 { + return (Color::Red, "deep: fail (model did not call tool)".to_string()); + } + + // Check that step 0 had tool results (tool was executed) + if result.steps[0].tool_results.is_empty() { + return (Color::Red, "deep: fail (tool not executed)".to_string()); + } + + // Check correctness: 15+27=42, 42+42=84 — final response should contain "84" + let final_text = result.response.text(); + if !final_text.contains("84") { + return (Color::Red, "deep: fail (wrong answer)".to_string()); + } + + // Check reasoning if the model supports it + if info.features.reasoning { + let has_reasoning = result.steps.iter().any(|step| { + step.response.message.content.iter().any(|part| { + matches!(part, ContentPart::Thinking(_)) + || matches!(part, ContentPart::Other { kind, .. } if kind == ContentPart::OPENAI_REASONING) + }) + }); + if !has_reasoning { + return (Color::Yellow, "deep: ok (no reasoning)".to_string()); + } + } + + (Color::Green, "deep: ok".to_string()) +} + async fn test_models_via_server( server: &ServerConnection, provider: Option<&str>, model: Option<&str>, + deep: bool, s: &Styles, ) -> Result<()> { + if deep { + eprintln!("Warning: --deep is not supported in server mode"); + } let models_to_test = if let Some(model_id) = model { let all = fetch_models_from_server(&server.client, &server.base_url, None).await?; let found: Vec<_> = all.into_iter().filter(|m| m.id == model_id).collect(); @@ -932,12 +1019,17 @@ pub async fn run_models( print_models_table(&models, &styles); } - ModelsCommand::Test { provider, model } => match &server { + ModelsCommand::Test { + provider, + model, + deep, + } => match &server { Some(s) => { - test_models_via_server(s, provider.as_deref(), model.as_deref(), &styles).await?; + test_models_via_server(s, provider.as_deref(), model.as_deref(), deep, &styles) + .await?; } None => { - test_models(provider.as_deref(), model.as_deref(), &styles).await?; + test_models(provider.as_deref(), model.as_deref(), deep, &styles).await?; } }, } @@ -945,7 +1037,12 @@ pub async fn run_models( Ok(()) } -async fn test_models(provider: Option<&str>, model: Option<&str>, s: &Styles) -> Result<()> { +async fn test_models( + provider: Option<&str>, + model: Option<&str>, + deep: bool, + s: &Styles, +) -> Result<()> { let models_to_test = if let Some(model_id) = model { match catalog::get_model_info(model_id) { Some(info) => vec![info], @@ -995,6 +1092,49 @@ async fn test_models(provider: Option<&str>, model: Option<&str>, s: &Styles) -> .foreground_color(color_if(use_color, result_color)), ); rows.push(row); + + if deep { + match build_deep_test_params(info) { + None => { + let mut deep_row = model_row(info, use_color); + deep_row.push( + "deep: skipped (no tool support)" + .cell() + .foreground_color(color_if(use_color, Color::Yellow)), + ); + rows.push(deep_row); + } + Some(params) => { + eprint!("Deep testing {}...", info.id); + let deep_result = tokio::time::timeout( + Duration::from_secs(90), + generate::generate(params), + ) + .await; + eprintln!(" done"); + + let (deep_color, deep_status) = match deep_result { + Ok(Ok(ref gen_result)) => validate_deep_result(gen_result, info), + Ok(Err(e)) => { + failures += 1; + (Color::Red, format!("deep: error: {e}")) + } + Err(_) => { + failures += 1; + (Color::Red, "deep: error: timeout (90s)".to_string()) + } + }; + + let mut deep_row = model_row(info, use_color); + deep_row.push( + deep_status + .cell() + .foreground_color(color_if(use_color, deep_color)), + ); + rows.push(deep_row); + } + } + } } let table = rows