From 5ead0145b9126fdf2e9cc11a9a5b616aa1c63f91 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Tue, 25 Aug 2026 13:36:51 -0400 Subject: [PATCH] feat(model): separate tool and reasoning tests --- docs/public/api-reference/fabro-api.yaml | 12 +- docs/public/integrations/deepseek.mdx | 2 +- docs/public/integrations/poolside.mdx | 4 +- docs/public/integrations/venice.mdx | 2 +- docs/public/reference/cli.mdx | 3 +- lib/apps/fabro-cli/src/args.rs | 9 +- lib/apps/fabro-cli/src/commands/model.rs | 53 ++++++-- lib/apps/fabro-cli/tests/it/cmd/model_test.rs | 110 +++++++++++++-- .../fabro-server/src/server/handler/models.rs | 23 +++- lib/apps/fabro-server/src/server/tests.rs | 111 +++++++++++++++ lib/components/fabro-llm/src/model_test.rs | 127 +++++++++++++----- lib/components/fabro-llm/tests/integration.rs | 2 +- lib/foundation/fabro-client/src/client.rs | 6 +- .../fabro-api-client/src/api/models-api.ts | 24 +++- 14 files changed, 406 insertions(+), 82 deletions(-) diff --git a/docs/public/api-reference/fabro-api.yaml b/docs/public/api-reference/fabro-api.yaml index 59ea3cac0..a0d9bf69d 100644 --- a/docs/public/api-reference/fabro-api.yaml +++ b/docs/public/api-reference/fabro-api.yaml @@ -5692,6 +5692,7 @@ paths: description: The canonical model ID or an alias. - $ref: "#/components/parameters/ModelTestProviderParam" - $ref: "#/components/parameters/ModelTestModeParam" + - $ref: "#/components/parameters/ModelTestReasoningEffortParam" responses: "200": description: Test result @@ -5700,7 +5701,7 @@ paths: schema: $ref: "#/components/schemas/ModelTestResult" "400": - description: Invalid test mode + description: Invalid test mode or reasoning effort headers: x-request-id: $ref: "#/components/headers/XRequestId" @@ -6234,6 +6235,15 @@ components: $ref: "#/components/schemas/ProviderId" example: openrouter + ModelTestReasoningEffortParam: + name: reasoning_effort + in: query + required: false + description: Optional native reasoning-effort level for the model test. + schema: + $ref: "#/components/schemas/ReasoningEffort" + example: high + headers: XRequestId: description: > diff --git a/docs/public/integrations/deepseek.mdx b/docs/public/integrations/deepseek.mdx index 23fcdb361..fad6524cf 100644 --- a/docs/public/integrations/deepseek.mdx +++ b/docs/public/integrations/deepseek.mdx @@ -53,7 +53,7 @@ Fabro does not use the `gpt56` profile for DeepSeek. That profile has a smaller ```bash fabro model list --provider deepseek -fabro model test --provider deepseek --model deepseek-v4-flash --deep +fabro model test --provider deepseek --model deepseek-v4-flash --tools fabro run workflow.fabro --provider deepseek --model deepseek ``` diff --git a/docs/public/integrations/poolside.mdx b/docs/public/integrations/poolside.mdx index a1923dc83..7a2fd0627 100644 --- a/docs/public/integrations/poolside.mdx +++ b/docs/public/integrations/poolside.mdx @@ -48,7 +48,7 @@ Both models support text input, tool calling, native reasoning, streaming, and a ```bash fabro model list --provider poolside -fabro model test --model laguna-xs-2.1 --deep +fabro model test --model laguna-xs-2.1 --tools fabro run workflow.fabro --model laguna-s-2.1 ``` @@ -104,7 +104,7 @@ enabled = true The OpenRouter routes use vendor-namespaced model IDs so they can coexist with direct Poolside routes: ```bash -fabro model test --model poolside/laguna-xs-2.1 --deep +fabro model test --model poolside/laguna-xs-2.1 --tools fabro run workflow.fabro --model poolside/laguna-s-2.1 ``` diff --git a/docs/public/integrations/venice.mdx b/docs/public/integrations/venice.mdx index f172ccb34..5788aa32e 100644 --- a/docs/public/integrations/venice.mdx +++ b/docs/public/integrations/venice.mdx @@ -57,7 +57,7 @@ Pin Venice when the run must use Venice: ```bash fabro model list --provider venice -fabro model test --provider venice --model deepseek-v4-flash --deep +fabro model test --provider venice --model deepseek-v4-flash --tools fabro run workflow.fabro --provider venice --model deepseek-v4-flash ``` diff --git a/docs/public/reference/cli.mdx b/docs/public/reference/cli.mdx index 530107c6a..8df61ae5b 100644 --- a/docs/public/reference/cli.mdx +++ b/docs/public/reference/cli.mdx @@ -693,11 +693,12 @@ fabro model test [OPTIONS] | Option | Description | | --- | --- | -| `--deep` | Run a multi-turn tool-use test (catches reasoning round-trip bugs) | | `-j, --jobs ` | Number of model tests to run concurrently in bulk mode
Default: `4` | | `-m, --model ` | Test a specific model | | `-p, --provider ` | Filter by provider | +| `--reasoning-effort ` | Request a reasoning-effort level (`low`, `medium`, `high`, `xhigh`, or `max`) | | `--server ` | Fabro server target: http(s) URL or absolute Unix socket path | +| `--tools` | Run a multi-turn tool-use test | ### `fabro parent` diff --git a/lib/apps/fabro-cli/src/args.rs b/lib/apps/fabro-cli/src/args.rs index f6130530a..b94293295 100644 --- a/lib/apps/fabro-cli/src/args.rs +++ b/lib/apps/fabro-cli/src/args.rs @@ -5,6 +5,7 @@ use anyhow::{Context, Result, bail}; use clap::{Args, Parser, Subcommand, ValueEnum}; use fabro_agent::cli::AgentArgs; use fabro_config::{CliLayer, CliLoggingLayer, CliOutputLayer, CliUpdatesLayer}; +use fabro_model::ReasoningEffort; use fabro_server::serve::DEFAULT_TCP_PORT; use fabro_static::EnvVars; use fabro_types::settings::cli::{OutputFormat, OutputVerbosity}; @@ -1091,9 +1092,13 @@ pub(crate) struct ModelTestArgs { )] pub(crate) jobs: usize, - /// Run a multi-turn tool-use test (catches reasoning round-trip bugs) + /// Run a multi-turn tool-use test + #[arg(long, alias = "deep")] + pub(crate) tools: bool, + + /// Request a reasoning-effort level (low, medium, high, xhigh, or max) #[arg(long)] - pub(crate) deep: bool, + pub(crate) reasoning_effort: Option, } #[derive(Args)] diff --git a/lib/apps/fabro-cli/src/commands/model.rs b/lib/apps/fabro-cli/src/commands/model.rs index dcff19c81..2f90e1f46 100644 --- a/lib/apps/fabro-cli/src/commands/model.rs +++ b/lib/apps/fabro-cli/src/commands/model.rs @@ -2,7 +2,7 @@ use anyhow::{Context, Result, bail}; use cli_table::format::{Border, Justify, Separator}; use cli_table::{Cell, CellStruct, Color, Style, Table}; use fabro_api::types as api_types; -use fabro_model::{Model, ModelTestMode, ProviderId}; +use fabro_model::{Model, ModelTestMode, ProviderId, ReasoningEffort}; use fabro_util::terminal::Styles; use futures::{StreamExt, stream}; use serde::Serialize; @@ -256,12 +256,13 @@ async fn test_models_via_server( client: &server_client::Client, provider: Option<&str>, model: Option<&str>, - deep: bool, + tools: bool, + reasoning_effort: Option, jobs: usize, styles: &Styles, json_output: bool, ) -> Result<()> { - let request_mode = deep.then_some(ModelTestMode::Deep); + let request_mode = tools.then_some(ModelTestMode::Deep); let use_color = styles.use_color; let mut title = models_title(use_color); @@ -288,7 +289,12 @@ async fn test_models_via_server( } else { Some( client - .test_model(model_id, requested_provider.as_ref(), request_mode) + .test_model( + model_id, + requested_provider.as_ref(), + request_mode, + reasoning_effort, + ) .await, ) }; @@ -375,7 +381,12 @@ async fn test_models_via_server( let client = client.clone(); async move { let result = client - .test_model(info.id.as_str(), Some(&info.provider), request_mode) + .test_model( + info.id.as_str(), + Some(&info.provider), + request_mode, + reasoning_effort, + ) .await; if !json_output { eprintln!("Testing {}... done", info.id); @@ -489,7 +500,8 @@ async fn run_models( ModelsCommand::Test(ModelTestArgs { provider, model, - deep, + tools, + reasoning_effort, jobs, .. }) => { @@ -497,7 +509,8 @@ async fn run_models( client, provider.as_deref(), model.as_deref(), - deep, + tools, + reasoning_effort, jobs, &styles, json_output, @@ -674,20 +687,24 @@ mod tests { .await; let client = test_client(&server.url("")); - let response = client.test_model("test-model", None, None).await.unwrap(); + let response = client + .test_model("test-model", None, None, None) + .await + .unwrap(); assert_eq!(response.status, api_types::ModelTestResultStatus::Ok); assert!(response.error_message.is_none()); } #[tokio::test] - async fn test_model_via_server_passes_mode_and_parses_error() { + async fn test_model_via_server_passes_mode_and_reasoning_effort() { let server = httpmock::MockServer::start_async().await; server .mock_async(|when, then| { when.method("POST") .path("/api/v1/models/test-model/test") - .query_param("mode", "deep"); + .query_param("mode", "deep") + .query_param("reasoning_effort", "high"); then.status(200) .header("Content-Type", "application/json") .body( @@ -704,7 +721,12 @@ mod tests { let client = test_client(&server.url("")); let response = client - .test_model("test-model", None, Some(ModelTestMode::Deep)) + .test_model( + "test-model", + None, + Some(ModelTestMode::Deep), + Some(ReasoningEffort::High), + ) .await .unwrap(); @@ -732,7 +754,10 @@ mod tests { .await; let client = test_client(&server.url("")); - let response = client.test_model("kimi-k2.5", None, None).await.unwrap(); + let response = client + .test_model("kimi-k2.5", None, None, None) + .await + .unwrap(); assert_eq!(response.status, api_types::ModelTestResultStatus::Skip); assert!(response.error_message.is_none()); @@ -756,7 +781,7 @@ mod tests { .await; let client = test_client(&server.url("")); - let result = client.test_model("bad-model", None, None).await; + let result = client.test_model("bad-model", None, None, None).await; assert!(result.is_err()); assert!(result.unwrap_err().to_string().contains("Model not found")); } @@ -805,6 +830,7 @@ mod tests { None, Some("venice-large"), false, + None, 1, &Styles::new(false), true, @@ -876,6 +902,7 @@ mod tests { None, None, false, + None, 2, &Styles::new(false), true, diff --git a/lib/apps/fabro-cli/tests/it/cmd/model_test.rs b/lib/apps/fabro-cli/tests/it/cmd/model_test.rs index 53fd86fb2..1cec2e7b4 100644 --- a/lib/apps/fabro-cli/tests/it/cmd/model_test.rs +++ b/lib/apps/fabro-cli/tests/it/cmd/model_test.rs @@ -86,21 +86,109 @@ fn help() { Usage: fabro model test [OPTIONS] Options: - --json Output as JSON [env: FABRO_JSON=] - --server Fabro server target: http(s) URL or absolute Unix socket path [env: FABRO_SERVER=] - --debug Enable DEBUG-level logging (default is INFO) [env: FABRO_DEBUG=] - -p, --provider Filter by provider - -m, --model Test a specific model - --no-upgrade-check Disable automatic upgrade check [env: FABRO_NO_UPGRADE_CHECK=true] - -j, --jobs Number of model tests to run concurrently in bulk mode [default: 4] - --quiet Suppress non-essential output [env: FABRO_QUIET=] - --deep Run a multi-turn tool-use test (catches reasoning round-trip bugs) - --verbose Enable verbose output [env: FABRO_VERBOSE=] - -h, --help Print help + --json + Output as JSON [env: FABRO_JSON=] + --server + Fabro server target: http(s) URL or absolute Unix socket path [env: FABRO_SERVER=] + --debug + Enable DEBUG-level logging (default is INFO) [env: FABRO_DEBUG=] + -p, --provider + Filter by provider + -m, --model + Test a specific model + --no-upgrade-check + Disable automatic upgrade check [env: FABRO_NO_UPGRADE_CHECK=true] + -j, --jobs + Number of model tests to run concurrently in bulk mode [default: 4] + --quiet + Suppress non-essential output [env: FABRO_QUIET=] + --tools + Run a multi-turn tool-use test + --verbose + Enable verbose output [env: FABRO_VERBOSE=] + --reasoning-effort + Request a reasoning-effort level (low, medium, high, xhigh, or max) + -h, --help + Print help ----- stderr ----- "); } +#[test] +fn model_test_tools_and_reasoning_effort_are_forwarded() { + let context = test_context!(); + let server = MockServer::start(); + context.set_http_target(&server.base_url()); + let list = mock_model_list(&server, [model_json("test-model", "anthropic", true)]); + let test = server.mock(|when, then| { + when.method("POST") + .path("/api/v1/models/test-model/test") + .query_param("mode", "deep") + .query_param("reasoning_effort", "low"); + then.status(200) + .header("Content-Type", "application/json") + .json_body(serde_json::json!({ + "model_id": "test-model", + "provider": "anthropic", + "status": "ok" + })); + }); + + let mut cmd = context.command(); + cmd.args([ + "model", + "test", + "--model", + "test-model", + "--tools", + "--reasoning-effort", + "low", + ]); + let output = cmd.output().expect("command should execute"); + + assert!( + output.status.success(), + "model test should succeed:\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + list.assert(); + test.assert(); +} + +#[test] +fn model_test_deep_remains_an_alias_for_tools() { + let context = test_context!(); + let server = MockServer::start(); + context.set_http_target(&server.base_url()); + let list = mock_model_list(&server, [model_json("test-model", "anthropic", true)]); + let test = server.mock(|when, then| { + when.method("POST") + .path("/api/v1/models/test-model/test") + .query_param("mode", "deep"); + then.status(200) + .header("Content-Type", "application/json") + .json_body(serde_json::json!({ + "model_id": "test-model", + "provider": "anthropic", + "status": "ok" + })); + }); + + let mut cmd = context.command(); + cmd.args(["model", "test", "--model", "test-model", "--deep"]); + let output = cmd.output().expect("command should execute"); + + assert!( + output.status.success(), + "model test should succeed:\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + list.assert(); + test.assert(); +} + #[test] fn model_test_unknown_model_errors() { let context = test_context!(); diff --git a/lib/apps/fabro-server/src/server/handler/models.rs b/lib/apps/fabro-server/src/server/handler/models.rs index 06af1d3de..6a5934274 100644 --- a/lib/apps/fabro-server/src/server/handler/models.rs +++ b/lib/apps/fabro-server/src/server/handler/models.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use fabro_auth::ApiCredential; use fabro_llm::client::Client as LlmClient; use fabro_llm::model_test::{ModelTestStatus, run_basic_model_probe}; -use fabro_model::ModelSelectionError; +use fabro_model::{ModelSelectionError, ReasoningEffort}; use fabro_redact::redact_string; use super::super::{ @@ -41,9 +41,11 @@ struct ModelListParams { #[derive(serde::Deserialize)] struct ModelTestParams { #[serde(default)] - mode: Option, + mode: Option, #[serde(default)] - provider: Option, + provider: Option, + #[serde(default)] + reasoning_effort: Option, } async fn list_models( @@ -207,6 +209,19 @@ async fn test_model( }, None => ModelTestMode::Basic, }; + let reasoning_effort = match params.reasoning_effort.as_deref() { + Some(value) => match ReasoningEffort::from_str(value) { + Ok(reasoning_effort) => Some(reasoning_effort), + Err(_) => { + return ApiError::new( + StatusCode::BAD_REQUEST, + format!("invalid reasoning effort: {value}"), + ) + .into_response(); + } + }, + None => None, + }; let llm_result = match state.resolve_llm_client().await { Ok(result) => result, Err(err) => { @@ -253,7 +268,7 @@ async fn test_model( } let client = Arc::new(llm_result.client); - let outcome = run_model_test(info, mode, client).await; + let outcome = run_model_test(info, mode, reasoning_effort, client).await; Json(serde_json::json!({ "model_id": info.id, "provider": info.provider, diff --git a/lib/apps/fabro-server/src/server/tests.rs b/lib/apps/fabro-server/src/server/tests.rs index 2bed9431f..7ac1cddc8 100644 --- a/lib/apps/fabro-server/src/server/tests.rs +++ b/lib/apps/fabro-server/src/server/tests.rs @@ -7973,6 +7973,117 @@ async fn test_model_invalid_mode_returns_400() { assert_status!(response, StatusCode::BAD_REQUEST).await; } +#[tokio::test] +async fn test_model_invalid_reasoning_effort_returns_400() { + let state = test_app_state_with_env_lookup( + default_test_server_settings(), + RunLayer::default(), + 5, + |_| None, + ); + let app = crate::test_support::build_test_router(state); + + let req = Request::builder() + .method("POST") + .uri(api("/models/claude-opus-4-6/test?reasoning_effort=bogus")) + .header("content-type", "application/json") + .body(Body::empty()) + .unwrap(); + + let response = app.oneshot(req).await.unwrap(); + assert_status!(response, StatusCode::BAD_REQUEST).await; +} + +#[tokio::test] +async fn test_model_forwards_and_validates_reasoning_effort() { + let upstream = MockServer::start(); + let completion = upstream.mock(|when, then| { + when.method(POST) + .path("/chat/completions") + .json_body_includes(r#"{"model":"acme-reasoner","reasoning_effort":"low"}"#); + then.status(200) + .header("content-type", "application/json") + .json_body(json!({ + "id": "chatcmpl-test", + "model": "acme-reasoner", + "choices": [{ + "message": {"role": "assistant", "content": "OK"}, + "finish_reason": "stop" + }], + "usage": { + "prompt_tokens": 1, + "completion_tokens": 1, + "total_tokens": 2 + } + })); + }); + let settings: LlmCatalogSettings = toml::from_str(&format!( + r#" +[providers.acme] +display_name = "Acme" +adapter = "openai_compatible" +agent_profile = "openai" +base_url = "{}" +priority = 120 + +[providers.acme.auth] +credentials = ["vault:ACME_API_KEY"] + +[providers.acme.models.acme-reasoner] +display_name = "Acme Reasoner" +family = "acme" +default = true + +[providers.acme.models.acme-reasoner.limits] +context_window = 128000 + +[providers.acme.models.acme-reasoner.features] +tools = true +vision = false +reasoning = true +reasoning_effort = "levels" + +[providers.acme.models.acme-reasoner.controls] +reasoning_effort = ["low", "high"] +"#, + upstream.base_url() + )) + .expect("catalog fixture should parse"); + let state = TestAppStateBuilder::new() + .llm_catalog_settings(settings) + .vault_entries([("ACME_API_KEY", "acme-test-key")]) + .build(); + let app = crate::test_support::build_test_router(state); + + let req = Request::builder() + .method("POST") + .uri(api( + "/models/acme-reasoner/test?provider=acme&reasoning_effort=low", + )) + .body(Body::empty()) + .unwrap(); + + let response = app.clone().oneshot(req).await.unwrap(); + let body = response_json!(response, StatusCode::OK).await; + assert_eq!(body["status"], "ok"); + + let unsupported = Request::builder() + .method("POST") + .uri(api( + "/models/acme-reasoner/test?provider=acme&reasoning_effort=medium", + )) + .body(Body::empty()) + .unwrap(); + let response = app.oneshot(unsupported).await.unwrap(); + let body = response_json!(response, StatusCode::OK).await; + assert_eq!(body["status"], "error"); + assert_eq!( + body["error_message"], + "Invalid request: model 'acme-reasoner' does not support reasoning_effort 'medium'; allowed values: low, high" + ); + completion.assert_calls(1); +} + #[tokio::test] async fn test_provider_credentials_uses_app_state_catalog() { let upstream = MockServer::start(); diff --git a/lib/components/fabro-llm/src/model_test.rs b/lib/components/fabro-llm/src/model_test.rs index de09f6fde..c0e1208e0 100644 --- a/lib/components/fabro-llm/src/model_test.rs +++ b/lib/components/fabro-llm/src/model_test.rs @@ -46,16 +46,31 @@ impl ModelTestOutcome { pub async fn run_model_test( info: &Model, mode: ModelTestMode, + reasoning_effort: Option, client: Arc, ) -> ModelTestOutcome { match mode { - ModelTestMode::Basic => run_basic_test(info, client).await, - ModelTestMode::Deep => run_deep_test(info, client).await, + ModelTestMode::Basic => run_basic_test(info, reasoning_effort, client).await, + ModelTestMode::Deep => run_tools_test(info, reasoning_effort, client).await, } } -async fn run_basic_test(info: &Model, client: Arc) -> ModelTestOutcome { - run_basic_model_probe(info.id.as_str(), &info.provider, client).await +async fn run_basic_test( + info: &Model, + reasoning_effort: Option, + client: Arc, +) -> ModelTestOutcome { + let params = build_basic_test_params( + info.id.as_str(), + info.provider.to_string(), + reasoning_effort, + client, + ); + basic_model_probe_outcome( + generate::generate(params), + Duration::from_secs(ModelTestMode::Basic.timeout_secs()), + ) + .await } /// Run the cheap single-prompt model availability probe without requiring a @@ -80,14 +95,30 @@ pub async fn run_basic_model_probe_with_timeout( client: Arc, probe_timeout: Duration, ) -> ModelTestOutcome { - let params = GenerateParams::new(model_id, client) - .provider(provider.to_string()) - .prompt("Say OK") - .max_tokens(16); + let params = build_basic_test_params(model_id, provider.to_string(), None, client); basic_model_probe_outcome(generate::generate(params), probe_timeout).await } +fn build_basic_test_params( + model_id: &str, + provider: String, + reasoning_effort: Option, + client: Arc, +) -> GenerateParams { + let max_tokens = if reasoning_effort.is_some() { 1024 } else { 16 }; + let mut params = GenerateParams::new(model_id, client) + .provider(provider) + .prompt("Say OK") + .max_tokens(max_tokens); + + if let Some(reasoning_effort) = reasoning_effort { + params = params.reasoning_effort(reasoning_effort); + } + + params +} + async fn basic_model_probe_outcome(probe: F, probe_timeout: Duration) -> ModelTestOutcome where F: Future>, @@ -99,8 +130,12 @@ where } } -async fn run_deep_test(info: &Model, client: Arc) -> ModelTestOutcome { - let Some(params) = build_deep_test_params(info, client) else { +async fn run_tools_test( + info: &Model, + reasoning_effort: Option, + client: Arc, +) -> ModelTestOutcome { + let Some(params) = build_tools_test_params(info, reasoning_effort, client) else { return ModelTestOutcome::error("model does not support tools"); }; @@ -111,7 +146,7 @@ async fn run_deep_test(info: &Model, client: Arc) -> ModelTestOutcome { .await; match result { - Ok(Ok(gen_result)) => match validate_deep_result(&gen_result) { + Ok(Ok(gen_result)) => match validate_tools_result(&gen_result) { Ok(()) => ModelTestOutcome::ok(), Err(message) => ModelTestOutcome::error(message), }, @@ -120,7 +155,11 @@ async fn run_deep_test(info: &Model, client: Arc) -> ModelTestOutcome { } } -fn build_deep_test_params(info: &Model, client: Arc) -> Option { +fn build_tools_test_params( + info: &Model, + reasoning_effort: Option, + client: Arc, +) -> Option { if !info.features.tools { return None; } @@ -159,14 +198,14 @@ fn build_deep_test_params(info: &Model, client: Arc) -> Option Result<(), String> { +fn validate_tools_result(result: &GenerateResult) -> Result<(), String> { if result.steps.len() < 2 { return Err("model did not call tool".to_string()); } @@ -241,7 +280,7 @@ mod tests { } #[tokio::test] - async fn run_model_test_deep_errors_when_model_lacks_tools() { + async fn run_model_test_tools_errors_when_model_lacks_tools() { let info = test_model_with(ModelFeatures { tools: false, vision: false, @@ -252,7 +291,7 @@ mod tests { sampling_params: true, }); - let outcome = run_model_test(&info, ModelTestMode::Deep, empty_test_client()).await; + let outcome = run_model_test(&info, ModelTestMode::Deep, None, empty_test_client()).await; assert_eq!(outcome.status, ModelTestStatus::Error); assert_eq!( @@ -274,25 +313,20 @@ mod tests { } #[test] - fn deep_test_omits_effort_for_reasoning_without_effort_controls() { - let info = test_model_with(ModelFeatures { - tools: true, - vision: false, - reasoning: true, - reasoning_effort: ReasoningEffortFeature::None, - prompt_cache: true, - cache_control_breakpoints: false, - sampling_params: true, - }); + fn basic_test_expands_output_budget_for_reasoning() { + let params = build_basic_test_params( + "test-model", + "anthropic".to_string(), + Some(ReasoningEffort::Max), + empty_test_client(), + ); - let params = build_deep_test_params(&info, empty_test_client()) - .expect("tool-capable model should produce deep-test params"); - - assert_eq!(params.reasoning_effort, None); + assert_eq!(params.reasoning_effort, Some(ReasoningEffort::Max)); + assert_eq!(params.max_tokens, Some(1024)); } #[test] - fn deep_test_uses_high_effort_when_supported() { + fn tools_test_omits_effort_when_not_requested() { let info = test_model_with(ModelFeatures { tools: true, vision: false, @@ -303,14 +337,33 @@ mod tests { sampling_params: true, }); - let params = build_deep_test_params(&info, empty_test_client()) - .expect("tool-capable model should produce deep-test params"); + let params = build_tools_test_params(&info, None, empty_test_client()) + .expect("tool-capable model should produce tools-test params"); - assert_eq!(params.reasoning_effort, Some(ReasoningEffort::High)); + assert_eq!(params.reasoning_effort, None); } #[test] - fn validate_deep_result_does_not_fail_only_for_missing_reasoning() { + fn tools_test_uses_requested_effort() { + let info = test_model_with(ModelFeatures { + tools: true, + vision: false, + reasoning: true, + reasoning_effort: ReasoningEffortFeature::Levels, + prompt_cache: true, + cache_control_breakpoints: false, + sampling_params: true, + }); + + let params = + build_tools_test_params(&info, Some(ReasoningEffort::Low), empty_test_client()) + .expect("tool-capable model should produce tools-test params"); + + assert_eq!(params.reasoning_effort, Some(ReasoningEffort::Low)); + } + + #[test] + fn validate_tools_result_does_not_fail_only_for_missing_reasoning() { let tool_results = vec![ToolResult::success("call_1", serde_json::json!(42))]; let first_step = StepResult { response: response_with_text("tool step"), @@ -328,6 +381,6 @@ mod tests { output: None, }; - assert_eq!(validate_deep_result(&result), Ok(())); + assert_eq!(validate_tools_result(&result), Ok(())); } } diff --git a/lib/components/fabro-llm/tests/integration.rs b/lib/components/fabro-llm/tests/integration.rs index a9187e709..c91cad136 100644 --- a/lib/components/fabro-llm/tests/integration.rs +++ b/lib/components/fabro-llm/tests/integration.rs @@ -73,7 +73,7 @@ async fn assert_deep_tool_round_trip( .get_on_provider(provider, model_id) .unwrap_or_else(|| panic!("{provider} {model_id} should be present")); - let outcome = run_model_test(model, ModelTestMode::Deep, client).await; + let outcome = run_model_test(model, ModelTestMode::Deep, None, client).await; assert_eq!( outcome.status, ModelTestStatus::Ok, diff --git a/lib/foundation/fabro-client/src/client.rs b/lib/foundation/fabro-client/src/client.rs index 56c4e0546..a2734d451 100644 --- a/lib/foundation/fabro-client/src/client.rs +++ b/lib/foundation/fabro-client/src/client.rs @@ -10,7 +10,7 @@ use bytes::Bytes; use fabro_api::types; use fabro_http::header::{ACCEPT, AUTHORIZATION, CONTENT_LENGTH, CONTENT_TYPE}; use fabro_http::multipart::{Form, Part}; -use fabro_model::{Model, ModelTestMode, ProviderId}; +use fabro_model::{Model, ModelTestMode, ProviderId, ReasoningEffort}; use fabro_types::settings::run::MergeStrategy; use fabro_types::{ ArtifactUpload, BlobHash, EventEnvelope, PairId, PairMessageRecord, PairMessageRequest, @@ -865,6 +865,7 @@ impl Client { id: &str, provider: Option<&ProviderId>, mode: Option, + reasoning_effort: Option, ) -> Result { let response = self .send_api(|client| async move { @@ -875,6 +876,9 @@ impl Client { if let Some(mode) = mode { request = request.mode(mode); } + if let Some(reasoning_effort) = reasoning_effort { + request = request.reasoning_effort(reasoning_effort); + } request.send().await }) .await?; diff --git a/lib/packages/fabro-api-client/src/api/models-api.ts b/lib/packages/fabro-api-client/src/api/models-api.ts index f2662fe2b..7e883d91d 100644 --- a/lib/packages/fabro-api-client/src/api/models-api.ts +++ b/lib/packages/fabro-api-client/src/api/models-api.ts @@ -37,6 +37,8 @@ import type { ProviderCredentialTestResponse } from '../models'; import type { ProviderList } from '../models'; // @ts-ignore import type { ProviderTestList } from '../models'; +// @ts-ignore +import type { ReasoningEffort } from '../models'; /** * ModelsApi - axios parameter creator */ @@ -140,10 +142,11 @@ export const ModelsApiAxiosParamCreator = function (configuration?: Configuratio * @param {string} id The canonical model ID or an alias. * @param {string} [provider] Pin the test to this provider\'s offering. When omitted, the server selects among ready providers by catalog priority. * @param {ModelTestMode} [mode] Test mode for the single-model test endpoint. Defaults to `basic`. + * @param {ReasoningEffort} [reasoningEffort] Optional native reasoning-effort level for the model test. * @param {*} [options] Override http request option. * @throws {RequiredError} */ - testModel: async (id: string, provider?: string, mode?: ModelTestMode, options: RawAxiosRequestConfig = {}): Promise => { + testModel: async (id: string, provider?: string, mode?: ModelTestMode, reasoningEffort?: ReasoningEffort, options: RawAxiosRequestConfig = {}): Promise => { // verify required parameter 'id' is not null or undefined assertParamExists('testModel', 'id', id) const localVarPath = `/api/v1/models/{id}/test` @@ -173,6 +176,10 @@ export const ModelsApiAxiosParamCreator = function (configuration?: Configuratio localVarQueryParameter['mode'] = mode; } + if (reasoningEffort !== undefined) { + localVarQueryParameter['reasoning_effort'] = reasoningEffort; + } + localVarHeaderParameter['Accept'] = 'application/json'; setSearchParams(localVarUrlObj, localVarQueryParameter); @@ -308,11 +315,12 @@ export const ModelsApiFp = function(configuration?: Configuration) { * @param {string} id The canonical model ID or an alias. * @param {string} [provider] Pin the test to this provider\'s offering. When omitted, the server selects among ready providers by catalog priority. * @param {ModelTestMode} [mode] Test mode for the single-model test endpoint. Defaults to `basic`. + * @param {ReasoningEffort} [reasoningEffort] Optional native reasoning-effort level for the model test. * @param {*} [options] Override http request option. * @throws {RequiredError} */ - async testModel(id: string, provider?: string, mode?: ModelTestMode, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise> { - const localVarAxiosArgs = await localVarAxiosParamCreator.testModel(id, provider, mode, options); + async testModel(id: string, provider?: string, mode?: ModelTestMode, reasoningEffort?: ReasoningEffort, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise> { + const localVarAxiosArgs = await localVarAxiosParamCreator.testModel(id, provider, mode, reasoningEffort, options); const localVarOperationServerIndex = configuration?.serverIndex ?? 0; const localVarOperationServerBasePath = operationServerMap['ModelsApi.testModel']?.[localVarOperationServerIndex]?.url; return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, localVarOperationServerBasePath || basePath); @@ -380,11 +388,12 @@ export const ModelsApiFactory = function (configuration?: Configuration, basePat * @param {string} id The canonical model ID or an alias. * @param {string} [provider] Pin the test to this provider\'s offering. When omitted, the server selects among ready providers by catalog priority. * @param {ModelTestMode} [mode] Test mode for the single-model test endpoint. Defaults to `basic`. + * @param {ReasoningEffort} [reasoningEffort] Optional native reasoning-effort level for the model test. * @param {*} [options] Override http request option. * @throws {RequiredError} */ - testModel(id: string, provider?: string, mode?: ModelTestMode, options?: RawAxiosRequestConfig): AxiosPromise { - return localVarFp.testModel(id, provider, mode, options).then((request) => request(axios, basePath)); + testModel(id: string, provider?: string, mode?: ModelTestMode, reasoningEffort?: ReasoningEffort, options?: RawAxiosRequestConfig): AxiosPromise { + return localVarFp.testModel(id, provider, mode, reasoningEffort, options).then((request) => request(axios, basePath)); }, /** * Validates an LLM provider API key against the server\'s effective catalog without persisting it. @@ -443,11 +452,12 @@ export class ModelsApi extends BaseAPI { * @param {string} id The canonical model ID or an alias. * @param {string} [provider] Pin the test to this provider\'s offering. When omitted, the server selects among ready providers by catalog priority. * @param {ModelTestMode} [mode] Test mode for the single-model test endpoint. Defaults to `basic`. + * @param {ReasoningEffort} [reasoningEffort] Optional native reasoning-effort level for the model test. * @param {*} [options] Override http request option. * @throws {RequiredError} */ - public testModel(id: string, provider?: string, mode?: ModelTestMode, options?: RawAxiosRequestConfig) { - return ModelsApiFp(this.configuration).testModel(id, provider, mode, options).then((request) => request(this.axios, this.basePath)); + public testModel(id: string, provider?: string, mode?: ModelTestMode, reasoningEffort?: ReasoningEffort, options?: RawAxiosRequestConfig) { + return ModelsApiFp(this.configuration).testModel(id, provider, mode, reasoningEffort, options).then((request) => request(this.axios, this.basePath)); } /**