From 073f38eb9a9da024edf1833982e1013915f17fdb Mon Sep 17 00:00:00 2001 From: Scott Werner Date: Mon, 2 Nov 2026 17:09:41 -0500 Subject: [PATCH] Return lithos-llm probe reports from model tests --- apps/fabro-web/app/routes/settings-models.tsx | 19 +- docs/public/api-reference/fabro-api.yaml | 124 +++-- docs/public/core-concepts/models.mdx | 19 + lib/apps/fabro-cli/src/commands/model.rs | 422 ++++++++---------- lib/apps/fabro-cli/tests/it/cmd/model_test.rs | 195 +++++--- .../fabro-server/src/server/handler/models.rs | 97 +--- lib/apps/fabro-server/src/server/tests.rs | 134 +++++- .../fabro-server/tests/it/scenario/dry_run.rs | 16 +- lib/components/fabro-llm/src/probe.rs | 34 +- lib/foundation/fabro-api/build.rs | 2 + lib/foundation/fabro-api/src/lib.rs | 5 +- .../tests/probe_report_round_trip.rs | 77 ++++ lib/foundation/fabro-client/src/client.rs | 3 +- .../src/.openapi-generator/FILES | 7 +- .../fabro-api-client/src/api/models-api.ts | 24 +- .../fabro-api-client/src/models/index.ts | 7 +- .../src/models/llm-error-data.ts | 42 ++ .../src/models/model-test-result.ts | 45 -- .../src/models/probe-failed.ts | 29 ++ .../src/models/probe-incorrect.ts | 26 ++ .../src/models/probe-outcome.ts | 32 ++ .../src/models/probe-passed.ts | 25 ++ .../src/models/probe-report.ts | 38 ++ 23 files changed, 872 insertions(+), 550 deletions(-) create mode 100644 lib/foundation/fabro-api/tests/probe_report_round_trip.rs create mode 100644 lib/packages/fabro-api-client/src/models/llm-error-data.ts delete mode 100644 lib/packages/fabro-api-client/src/models/model-test-result.ts create mode 100644 lib/packages/fabro-api-client/src/models/probe-failed.ts create mode 100644 lib/packages/fabro-api-client/src/models/probe-incorrect.ts create mode 100644 lib/packages/fabro-api-client/src/models/probe-outcome.ts create mode 100644 lib/packages/fabro-api-client/src/models/probe-passed.ts create mode 100644 lib/packages/fabro-api-client/src/models/probe-report.ts diff --git a/apps/fabro-web/app/routes/settings-models.tsx b/apps/fabro-web/app/routes/settings-models.tsx index fde1fe380..80007d986 100644 --- a/apps/fabro-web/app/routes/settings-models.tsx +++ b/apps/fabro-web/app/routes/settings-models.tsx @@ -310,22 +310,19 @@ function ModelsSection({ providers }: { providers: Provider[] }) { modelsApi.testModel(...modelOfferingTestArgs(offering)), ); if ( - result.provider !== offering.provider || - result.model_id !== offering.id + result.route !== null && + result.route !== `${offering.provider}/${offering.id}` ) { outcome = { - phase: "error", - message: `Server tested unexpected offering ${result.provider}/${result.model_id}`, + phase: "error", + message: `Server tested unexpected offering ${result.route}`, }; - } else if (result.status === "ok") { + } else if (result.outcome.status === "passed") { outcome = { phase: "ok" }; - } else if (result.status === "error") { - outcome = { - phase: "error", - message: result.error_message ?? "Failed", - }; + } else if (result.outcome.status === "failed") { + outcome = { phase: "error", message: result.outcome.error.message }; } else { - outcome = { phase: "error", message: "provider not configured" }; + outcome = { phase: "error", message: result.outcome.detail }; } } catch (err) { outcome = { diff --git a/docs/public/api-reference/fabro-api.yaml b/docs/public/api-reference/fabro-api.yaml index 985ab7793..e261d635b 100644 --- a/docs/public/api-reference/fabro-api.yaml +++ b/docs/public/api-reference/fabro-api.yaml @@ -5911,24 +5911,28 @@ paths: operationId: testModel tags: [Models] summary: Test Model - description: Tests a model by sending a simple prompt and reporting pass/fail. + description: >- + Runs the lithos-llm probe and returns its versioned report, including the + canonical route, outcome, elapsed time and token usage. Unknown or + unavailable models and provider failures produce HTTP 200 reports with + a failed outcome. Invalid query options remain HTTP 400 errors. parameters: - name: id in: path required: true schema: type: string - description: The canonical model ID or an alias. + description: A model selector resolved by lithos-llm (canonical ID, alias or provider-qualified selector). - $ref: "#/components/parameters/ModelTestProviderParam" - $ref: "#/components/parameters/ModelTestModeParam" - $ref: "#/components/parameters/ModelTestReasoningEffortParam" responses: "200": - description: Test result + description: Probe report (passed, failed or incorrect) content: application/json: schema: - $ref: "#/components/schemas/ModelTestResult" + $ref: "#/components/schemas/ProbeReport" "400": description: Invalid test mode or reasoning effort headers: @@ -5938,15 +5942,6 @@ paths: application/json: schema: $ref: "#/components/schemas/ErrorResponse" - "404": - description: Model not found - headers: - x-request-id: - $ref: "#/components/headers/XRequestId" - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" /api/v1/providers: get: @@ -8875,30 +8870,99 @@ components: key. Does NOT imply authentication or model requests will succeed; call `POST /models/{id}/test` to test usability. - ModelTestResult: - description: Result of testing a model in `basic` or `deep` mode. + ProbeReport: + description: >- + The lithos-llm version-1 probe report. The route is the canonical + provider/model selected by the probe, or null if selection failed. + Probe failures are findings in a successful HTTP response; API + authentication, malformed options and server setup failures remain HTTP errors. + Provider errors are redacted at the server disclosure boundary. type: object - required: - - model_id - - provider - - status + required: [version, route, outcome, latency_ms, usage] properties: - model_id: + version: + type: integer + enum: [1] + route: + type: [string, "null"] + pattern: '^[^/]+/.+$' + outcome: + $ref: "#/components/schemas/ProbeOutcome" + latency_ms: + type: integer + format: uint64 + minimum: 0 + usage: + $ref: "#/components/schemas/TokenCounts" + + ProbeOutcome: + oneOf: + - $ref: "#/components/schemas/ProbePassed" + - $ref: "#/components/schemas/ProbeFailed" + - $ref: "#/components/schemas/ProbeIncorrect" + discriminator: + propertyName: status + mapping: + passed: "#/components/schemas/ProbePassed" + failed: "#/components/schemas/ProbeFailed" + incorrect: "#/components/schemas/ProbeIncorrect" + + ProbePassed: + type: object + required: [status] + properties: + status: + type: string + enum: [passed] + + ProbeFailed: + type: object + required: [status, error] + properties: + status: + type: string + enum: [failed] + error: + $ref: "#/components/schemas/LlmErrorData" + + ProbeIncorrect: + type: object + required: [status, detail] + properties: + status: + type: string + enum: [incorrect] + detail: + type: string + + LlmErrorData: + description: Structured lithos-llm failure, subject to the server's disclosure policy. + type: object + required: [kind, message, retry] + properties: + kind: + $ref: "#/components/schemas/LlmErrorKind" + message: type: string - description: The canonical model ID that was tested. - example: "claude-opus-4-6" provider: $ref: "#/components/schemas/ProviderId" status: + type: integer + minimum: 0 + maximum: 65535 + provider_code: type: string - enum: - - ok - - error - - skip - description: Whether the model responded successfully, failed, or was skipped because its provider is not configured. - error_message: - type: ["string", "null"] - description: Error details when status is "error". + retry: + $ref: "#/components/schemas/LlmRetryClassification" + raw_data: + description: Provider response data; omitted by the model-test endpoint. + provider_retry_after_millis: + type: integer + format: uint64 + minimum: 0 + source_message: + type: string + description: Internal source detail; omitted by the model-test endpoint. ModelTestMode: description: Single-model test mode. diff --git a/docs/public/core-concepts/models.mdx b/docs/public/core-concepts/models.mdx index 49d175eb4..d6e5fcb20 100644 --- a/docs/public/core-concepts/models.mdx +++ b/docs/public/core-concepts/models.mdx @@ -344,3 +344,22 @@ fabro model test --provider openai ``` This is useful for confirming connectivity after setup or when adding a new provider key. + +With `--model`, Fabro sends the selector directly to the server's lithos-llm +probe. It displays the canonical provider/model selected by the probe, the +outcome, elapsed milliseconds, and input/output token usage. Add `--tools` to +check a multi-turn tool exchange. + +`fabro model test --model --json` returns the version-1 lithos-llm +probe report directly: `version`, `route`, `outcome`, `latency_ms`, and `usage`. +The outcome is `passed`, `failed` (with a structured `error`), or `incorrect` +(with a `detail` explaining the model's behavior). A selection failure has a +null route. Failed and incorrect probes print their report and exit nonzero. +This replaces the previous single-model JSON summary and `ok/error/skip` API +result. + +Without `--model`, Fabro lists models and tests configured offerings with the +requested concurrency. Bulk JSON retains `total`, `failures`, `skipped`, and +`results`; each completed probe's row also contains its full `report`. +Unconfigured offerings are skipped before probing. An offering that becomes +unavailable after listing is a failed probe. diff --git a/lib/apps/fabro-cli/src/commands/model.rs b/lib/apps/fabro-cli/src/commands/model.rs index 47385fb86..031b2499d 100644 --- a/lib/apps/fabro-cli/src/commands/model.rs +++ b/lib/apps/fabro-cli/src/commands/model.rs @@ -1,11 +1,11 @@ -use anyhow::{Context, Result, bail}; +use anyhow::{Result, bail}; use cli_table::format::{Border, Justify, Separator}; use cli_table::{Cell, CellStruct, Color, Style, Table}; -use fabro_api::types as api_types; use fabro_types::{Model, ModelTestMode}; use fabro_util::terminal::Styles; use futures::{StreamExt, stream}; use lithos_llm::catalog::ProviderId; +use lithos_llm::client::{ProbeOutcome, ProbeReport}; use serde::Serialize; use crate::args::{ModelListArgs, ModelTestArgs, ModelsCommand}; @@ -29,6 +29,8 @@ struct ModelTestRow { detail: Option, #[serde(skip_serializing_if = "Option::is_none")] error: Option, + #[serde(skip_serializing_if = "Option::is_none")] + report: Option, } #[derive(Serialize)] @@ -44,24 +46,7 @@ struct CompletedModelTest { model: Model, result_color: Color, status: String, -} - -fn model_matches_selector(model: &Model, selector: &str) -> bool { - model.id.as_str() == selector || model.aliases.iter().any(|alias| alias == selector) -} - -fn find_model_by_id_or_alias( - models: &[Model], - id: &str, - provider: Option<&ProviderId>, -) -> Option { - models - .iter() - .find(|model| { - provider.is_none_or(|provider| &model.provider == provider) - && model_matches_selector(model, id) - }) - .cloned() + report: Option, } pub(crate) async fn execute( @@ -183,34 +168,37 @@ fn print_models_table(models: &[Model], styles: &Styles) { ); } -fn configured_model_test_status( - expected: &Model, - result: Result, -) -> (Color, String, bool) { - match result { - Ok(resp) if resp.provider != expected.provider || resp.model_id != expected.id.as_str() => { - ( - Color::Red, - format!( - "error: server tested unexpected offering {}/{}", - resp.provider, resp.model_id - ), - true, - ) - } - Ok(resp) if resp.status == api_types::ModelTestResultStatus::Ok => { - (Color::Green, "ok".to_string(), false) - } - Ok(resp) if resp.status == api_types::ModelTestResultStatus::Skip => ( +fn probe_status(report: &ProbeReport) -> (Color, String, bool) { + match &report.outcome { + ProbeOutcome::Passed => (Color::Green, "ok".to_string(), false), + ProbeOutcome::Failed(error) => ( Color::Red, - "error: provider became unconfigured after listing".to_string(), + format!("error: {}: {}", error.kind.as_str(), error.message), true, ), - Ok(resp) => { - let message = resp - .error_message - .unwrap_or_else(|| "unknown error".to_string()); - (Color::Red, format!("error: {message}"), true) + ProbeOutcome::Incorrect { detail } => (Color::Red, format!("incorrect: {detail}"), true), + _ => (Color::Red, "error: unknown probe outcome".to_string(), true), + } +} + +fn configured_model_test_status( + expected: &Model, + result: &Result, +) -> (Color, String, bool) { + match result { + Ok(report) => { + // A bulk job names an exact offering. Keep this correlation check; + // a single selector is resolved exclusively by the server's probe. + if let Some(route) = &report.route { + if route.provider() != &expected.provider || route.model() != &expected.id { + return ( + Color::Red, + format!("error: server tested unexpected offering {route}"), + true, + ); + } + } + probe_status(report) } Err(err) => (Color::Red, format!("error: {err}"), true), } @@ -225,6 +213,7 @@ fn model_test_row_from_status(model: &Model, status: &str, result_color: Color) result: ModelTestResultKind::Pass, detail: None, error: None, + report: None, }, Color::Yellow => ModelTestRow { model: model.id.to_string(), @@ -232,12 +221,14 @@ fn model_test_row_from_status(model: &Model, status: &str, result_color: Color) result: ModelTestResultKind::Skip, detail: Some(trimmed.to_string()), error: None, + report: None, }, _ => ModelTestRow { model: model.id.to_string(), provider: model.provider.clone(), result: ModelTestResultKind::Fail, detail: None, + report: None, error: Some( trimmed .strip_prefix("error: ") @@ -265,6 +256,44 @@ async fn test_models_via_server( let reasoning_effort = args.reasoning_effort; let request_mode = args.tools.then_some(ModelTestMode::Deep); + if let Some(selector) = model { + let requested_provider = provider.map(ProviderId::new); + if !json_output { + eprint!("Testing {selector}..."); + } + let result = client + .test_model( + selector, + requested_provider.as_ref(), + request_mode, + reasoning_effort, + ) + .await; + if !json_output { + eprintln!(" done"); + } + let report = result?; + if json_output { + println!("{}", serde_json::to_string_pretty(&report)?); + } else { + let route = report + .route + .as_ref() + .map_or_else(|| selector.to_string(), ToString::to_string); + let (_, status, _) = probe_status(&report); + println!( + "{route}: {status} ({} ms; {} input, {} output tokens)", + report.latency.as_millis(), + report.usage.input, + report.usage.output + ); + } + if !report.passed() { + bail!("1 model(s) failed"); + } + return Ok(()); + } + let use_color = styles.use_color; let mut title = models_title(use_color); title.push("RESULT".cell().bold(use_color)); @@ -274,162 +303,81 @@ async fn test_models_via_server( let mut failures = 0u32; let mut skipped = 0u32; let mut skipped_providers: Vec = Vec::new(); - if let Some(model_id) = model { - let requested_provider = provider.map(ProviderId::new); - let listed_models = client.list_models(provider, Some(model_id)).await?; - let listed_info = find_model_by_id_or_alias(&listed_models, model_id, None); - if !json_output { - eprint!("Testing {model_id}..."); - } - let has_configured_match = listed_models - .iter() - .any(|model| model.configured && model_matches_selector(model, model_id)); - let result = - if requested_provider.is_none() && listed_info.is_some() && !has_configured_match { - None - } else { - Some( - client - .test_model( - model_id, - requested_provider.as_ref(), - request_mode, - reasoning_effort, - ) - .await, - ) - }; - if !json_output { - eprintln!(" done"); - } + let models_to_test = client.list_models(provider, None).await?; + if models_to_test.is_empty() { + bail!("No models found"); + } - let (info, result_color, status) = match result { - None => { - let info = listed_info.with_context(|| format!("Unknown model: {model_id}"))?; - failures += 1; - skipped += 1; - (info, Color::Yellow, "not configured".to_string()) - } - Some(Ok(resp)) => { - let info = - find_model_by_id_or_alias(&listed_models, &resp.model_id, Some(&resp.provider)) - .with_context(|| { - format!( - "Unknown model returned by server: {}/{}", - resp.provider, resp.model_id - ) - })?; - if resp.status == api_types::ModelTestResultStatus::Ok { - (info, Color::Green, "ok".to_string()) - } else if resp.status == api_types::ModelTestResultStatus::Skip { - failures += 1; - skipped += 1; - (info, Color::Yellow, "not configured".to_string()) - } else { - failures += 1; - let message = resp - .error_message - .unwrap_or_else(|| "unknown error".to_string()); - (info, Color::Red, format!("error: {message}")) + let (configured, unconfigured): (Vec<_>, Vec<_>) = models_to_test + .into_iter() + .partition(|model| model.configured); + + for info in &unconfigured { + skipped += 1; + let provider_name = info.provider.to_string(); + if !skipped_providers.contains(&provider_name) { + skipped_providers.push(provider_name); + } + if json_output { + json_rows.push(model_test_row_from_status( + info, + "not configured", + Color::Yellow, + )); + } + } + + let mut completed = stream::iter(configured.into_iter().enumerate()) + .map(|(index, info)| { + let client = client.clone(); + async move { + let result = client + .test_model( + info.id.as_str(), + Some(&info.provider), + request_mode, + reasoning_effort, + ) + .await; + if !json_output { + eprintln!("Testing {}... done", info.id); } + let (result_color, status, failed) = configured_model_test_status(&info, &result); + ( + CompletedModelTest { + index, + model: info, + result_color, + status, + report: result.ok(), + }, + failed, + ) } - Some(Err(err)) if err.to_string().contains("Model not found") => { - bail!("Unknown model: {model_id}"); - } - Some(Err(err)) => { - let info = listed_info.with_context(|| format!("Unknown model: {model_id}"))?; - failures += 1; - (info, Color::Red, format!("error: {err}")) - } - }; + }) + .buffer_unordered(jobs) + .collect::>() + .await; - let mut row = model_row(&info, use_color); + completed.sort_by_key(|(completed, _)| completed.index); + + for (completed, failed) in completed { + if failed { + failures += 1; + } + + let mut row = model_row(&completed.model, use_color); + let mut json_row = + model_test_row_from_status(&completed.model, &completed.status, completed.result_color); + json_row.report = completed.report; + json_rows.push(json_row); row.push( - status - .clone() + completed + .status .cell() - .foreground_color(color_if(use_color, result_color)), + .foreground_color(color_if(use_color, completed.result_color)), ); rows.push(row); - json_rows.push(model_test_row_from_status(&info, &status, result_color)); - } else { - let models_to_test = client.list_models(provider, None).await?; - if models_to_test.is_empty() { - bail!("No models found"); - } - - let (configured, unconfigured): (Vec<_>, Vec<_>) = models_to_test - .into_iter() - .partition(|model| model.configured); - - for info in &unconfigured { - skipped += 1; - let provider_name = info.provider.to_string(); - if !skipped_providers.contains(&provider_name) { - skipped_providers.push(provider_name); - } - if json_output { - json_rows.push(model_test_row_from_status( - info, - "not configured", - Color::Yellow, - )); - } - } - - let mut completed = stream::iter(configured.into_iter().enumerate()) - .map(|(index, info)| { - let client = client.clone(); - async move { - let result = client - .test_model( - info.id.as_str(), - Some(&info.provider), - request_mode, - reasoning_effort, - ) - .await; - if !json_output { - eprintln!("Testing {}... done", info.id); - } - let (result_color, status, failed) = - configured_model_test_status(&info, result); - ( - CompletedModelTest { - index, - model: info, - result_color, - status, - }, - failed, - ) - } - }) - .buffer_unordered(jobs) - .collect::>() - .await; - - completed.sort_by_key(|(completed, _)| completed.index); - - for (completed, failed) in completed { - if failed { - failures += 1; - } - - let mut row = model_row(&completed.model, use_color); - json_rows.push(model_test_row_from_status( - &completed.model, - &completed.status, - completed.result_color, - )); - row.push( - completed - .status - .cell() - .foreground_color(color_if(use_color, completed.result_color)), - ); - rows.push(row); - } } if json_output { @@ -458,7 +406,7 @@ async fn test_models_via_server( println!("{}", table.display()?); } - if skipped > 0 && model.is_none() { + if skipped > 0 { eprintln!( "Skipped {} model(s) (no credentials: {})", skipped, @@ -516,7 +464,7 @@ impl Default for ModelsCommand { mod tests { use fabro_types::{ModelControls, ModelCosts, ModelFeatures, ModelLimits}; use lithos_llm::catalog::builtin; - use lithos_llm::types::ReasoningEffort; + use lithos_llm::types::{ErrorKind, ReasoningEffort, TokenCounts}; use super::*; @@ -657,9 +605,11 @@ mod tests { .header("Content-Type", "application/json") .body( serde_json::json!({ - "model_id": "test-model", - "provider": "anthropic", - "status": "ok" + "version": 1, + "route": "anthropic/test-model", + "outcome": {"status": "passed"}, + "latency_ms": 12, + "usage": TokenCounts::default() }) .to_string(), ); @@ -672,8 +622,7 @@ mod tests { .await .unwrap(); - assert_eq!(response.status, api_types::ModelTestResultStatus::Ok); - assert!(response.error_message.is_none()); + assert!(response.passed()); } #[tokio::test] @@ -689,10 +638,11 @@ mod tests { .header("Content-Type", "application/json") .body( serde_json::json!({ - "model_id": "test-model", - "provider": "anthropic", - "status": "error", - "error_message": "timeout" + "version": 1, + "route": "anthropic/test-model", + "outcome": {"status": "failed", "error": {"kind": "timeout", "message": "timeout", "retry": {"type": "never"}}}, + "latency_ms": 12, + "usage": TokenCounts::default() }) .to_string(), ); @@ -710,12 +660,13 @@ mod tests { .await .unwrap(); - assert_eq!(response.status, api_types::ModelTestResultStatus::Error); - assert_eq!(response.error_message.as_deref(), Some("timeout")); + assert!( + matches!(response.outcome, ProbeOutcome::Failed(error) if error.message == "timeout") + ); } #[tokio::test] - async fn test_model_via_server_parses_skip() { + async fn test_model_via_server_parses_selection_failure() { let server = httpmock::MockServer::start_async().await; server .mock_async(|when, then| { @@ -724,9 +675,11 @@ mod tests { .header("Content-Type", "application/json") .body( serde_json::json!({ - "model_id": "kimi-k2.5", - "provider": "moonshot", - "status": "skip" + "version": 1, + "route": "moonshot/kimi-k2.5", + "outcome": {"status": "failed", "error": {"kind": "model_selection", "message": "provider unavailable", "retry": {"type": "never"}}}, + "latency_ms": 12, + "usage": TokenCounts::default() }) .to_string(), ); @@ -739,8 +692,9 @@ mod tests { .await .unwrap(); - assert_eq!(response.status, api_types::ModelTestResultStatus::Skip); - assert!(response.error_message.is_none()); + assert!( + matches!(response.outcome, ProbeOutcome::Failed(error) if error.kind == ErrorKind::ModelSelection) + ); } #[tokio::test] @@ -767,26 +721,8 @@ mod tests { } #[tokio::test] - async fn single_model_test_uses_server_model_metadata_for_custom_models() { + async fn single_model_test_uses_probe_identity_without_catalog_metadata() { let server = httpmock::MockServer::start_async().await; - server - .mock_async(|when, then| { - when.method("GET") - .path("/api/v1/models") - .query_param("page[limit]", "100") - .query_param("page[offset]", "0") - .query_param("query", "venice-large"); - then.status(200) - .header("Content-Type", "application/json") - .body( - serde_json::json!({ - "data": [custom_model_json("venice-large", "venice")], - "meta": { "has_more": false } - }) - .to_string(), - ); - }) - .await; server .mock_async(|when, then| { when.method("POST").path("/api/v1/models/venice-large/test"); @@ -794,9 +730,11 @@ mod tests { .header("Content-Type", "application/json") .body( serde_json::json!({ - "model_id": "venice-large", - "provider": "venice", - "status": "ok" + "version": 1, + "route": "venice/venice-large", + "outcome": {"status": "passed"}, + "latency_ms": 12, + "usage": TokenCounts::default() }) .to_string(), ); @@ -847,9 +785,11 @@ mod tests { .header("Content-Type", "application/json") .body( serde_json::json!({ - "model_id": "portable-model", - "provider": "openai", - "status": "ok" + "version": 1, + "route": "openai/portable-model", + "outcome": {"status": "passed"}, + "latency_ms": 12, + "usage": TokenCounts::default() }) .to_string(), ); @@ -864,9 +804,11 @@ mod tests { .header("Content-Type", "application/json") .body( serde_json::json!({ - "model_id": "portable-model", - "provider": "openrouter", - "status": "ok" + "version": 1, + "route": "openrouter/portable-model", + "outcome": {"status": "passed"}, + "latency_ms": 12, + "usage": TokenCounts::default() }) .to_string(), ); diff --git a/lib/apps/fabro-cli/tests/it/cmd/model_test.rs b/lib/apps/fabro-cli/tests/it/cmd/model_test.rs index 64c48781b..9400f3362 100644 --- a/lib/apps/fabro-cli/tests/it/cmd/model_test.rs +++ b/lib/apps/fabro-cli/tests/it/cmd/model_test.rs @@ -10,6 +10,7 @@ use axum::routing::{get, post}; use axum::{Json, Router}; use fabro_test::{fabro_snapshot, test_context}; use httpmock::MockServer; +use lithos_llm::types::TokenCounts; use tokio::net::TcpListener; use tokio::sync::{Semaphore, oneshot}; @@ -129,9 +130,11 @@ fn assert_model_test_forwards(cli_args: &[&str], expected_query: &[(&str, &str)] then.status(200) .header("Content-Type", "application/json") .json_body(serde_json::json!({ - "model_id": "test-model", - "provider": "anthropic", - "status": "ok" + "version": 1, + "route": "anthropic/test-model", + "outcome": {"status": "passed"}, + "latency_ms": 12, + "usage": TokenCounts::default() })); }); @@ -146,7 +149,7 @@ fn assert_model_test_forwards(cli_args: &[&str], expected_query: &[(&str, &str)] String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr) ); - list.assert(); + list.assert_calls(0); test.assert(); } @@ -164,38 +167,101 @@ fn model_test_deep_remains_an_alias_for_tools() { } #[test] -fn model_test_unknown_model_errors() { +fn single_model_reports_canonical_alias_without_listing() { let context = test_context!(); + let server = MockServer::start(); + context.set_http_target(&server.base_url()); + let listing = server.mock(|when, then| { + when.method("GET").path("/api/v1/models"); + then.status(500); + }); + let report = serde_json::json!({ + "version": 1, + "route": "anthropic/claude-sonnet-4.6", + "outcome": {"status": "passed"}, + "latency_ms": 12, + "usage": TokenCounts::default(), + }); + let probe = server.mock(|when, then| { + when.method("POST") + .path("/api/v1/models/claude-sonnet-4-6/test") + .query_param("provider", "anthropic"); + then.status(200).json_body(report.clone()); + }); let mut cmd = context.command(); - cmd.args(["model", "test", "--model", "nonexistent-model-xyz"]); + cmd.args([ + "model", + "test", + "--model", + "claude-sonnet-4-6", + "--provider", + "anthropic", + "--json", + ]); + let output = cmd + .assert() + .success() + .stderr("") + .get_output() + .stdout + .clone(); + assert_eq!( + serde_json::from_slice::(&output).unwrap(), + report + ); + listing.assert_calls(0); + probe.assert_calls(1); + let mut cmd = context.command(); + cmd.args([ + "model", + "test", + "--model", + "claude-sonnet-4-6", + "--provider", + "anthropic", + ]); fabro_snapshot!(context.filters(), cmd, @" - success: false - exit_code: 1 + success: true + exit_code: 0 ----- stdout ----- + anthropic/claude-sonnet-4.6: ok (12 ms; 0 input, 0 output tokens) ----- stderr ----- - Testing nonexistent-model-xyz... done - × Unknown model: nonexistent-model-xyz + Testing claude-sonnet-4-6... done "); + listing.assert_calls(0); + probe.assert_calls(2); } #[test] -fn single_model_skip_exits_nonzero() { - let context = test_context!(); - let mut cmd = context.command(); - cmd.args(["model", "test", "--model", "gemini-3.1-pro-preview"]); - remove_provider_env(&mut cmd); - - fabro_snapshot!(context.filters(), cmd, @" - success: false - exit_code: 1 - ----- stdout ----- - MODEL PROVIDER ALIASES CONTEXT COST SPEED RESULT - gemini-3.1-pro-preview gemini gemini-pro 1m $2.0 / $12.0 85 tok/s not configured - ----- stderr ----- - Testing gemini-3.1-pro-preview... done - × 1 model(s) failed - "); +fn single_model_preserves_failed_and_incorrect_findings() { + for outcome in [ + serde_json::json!({"status": "failed", "error": {"kind": "model_selection", "message": "unknown model", "retry": {"type": "never"}}}), + serde_json::json!({"status": "incorrect", "detail": "model did not call the tool"}), + ] { + let context = test_context!(); + let server = MockServer::start(); + context.set_http_target(&server.base_url()); + let report = serde_json::json!({ + "version": 1, + "route": null, + "outcome": outcome, + "latency_ms": 0, + "usage": TokenCounts::default(), + }); + let probe = server.mock(|when, then| { + when.method("POST").path("/api/v1/models/test-model/test"); + then.status(200).json_body(report.clone()); + }); + let mut cmd = context.command(); + cmd.args(["model", "test", "--model", "test-model", "--json"]); + let output = cmd.assert().failure().get_output().stdout.clone(); + assert_eq!( + serde_json::from_slice::(&output).unwrap(), + report + ); + probe.assert_calls(1); + } } #[test] @@ -224,34 +290,6 @@ fn bulk_skip_exits_zero_and_prints_summary() { ); } -#[test] -fn json_output_includes_skipped_models() { - let context = test_context!(); - let mut cmd = context.command(); - cmd.args([ - "model", - "test", - "--model", - "gemini-3.1-pro-preview", - "--json", - ]); - remove_provider_env(&mut cmd); - - let output = cmd.output().expect("failed to execute model test"); - assert!( - !output.status.success(), - "expected single-model skip to exit non-zero:\nstdout:\n{}\nstderr:\n{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); - let stdout = String::from_utf8_lossy(&output.stdout); - let json: serde_json::Value = serde_json::from_str(&stdout).expect("invalid JSON output"); - - assert_eq!(json["failures"], 1); - assert_eq!(json["skipped"], 1); - assert_eq!(json["results"][0]["result"], "skip"); -} - #[test] fn model_test_does_not_announce_unconfigured() { let context = test_context!(); @@ -267,9 +305,11 @@ fn model_test_does_not_announce_unconfigured() { then.status(200) .header("Content-Type", "application/json") .json_body(serde_json::json!({ - "model_id": "claude-opus-4-7", - "provider": "anthropic", - "status": "ok" + "version": 1, + "route": "anthropic/claude-opus-4-7", + "outcome": {"status": "passed"}, + "latency_ms": 12, + "usage": TokenCounts::default() })); }); let unconfigured_test = server.mock(|when, then| { @@ -317,9 +357,11 @@ fn model_test_skipped_footer_sources_from_listing() { then.status(200) .header("Content-Type", "application/json") .json_body(serde_json::json!({ - "model_id": "claude-opus-4-7", - "provider": "anthropic", - "status": "ok" + "version": 1, + "route": "anthropic/claude-opus-4-7", + "outcome": {"status": "passed"}, + "latency_ms": 12, + "usage": TokenCounts::default() })); }); @@ -349,9 +391,11 @@ fn model_test_post_list_race_is_a_failure() { then.status(200) .header("Content-Type", "application/json") .json_body(serde_json::json!({ - "model_id": "claude-opus-4-7", - "provider": "anthropic", - "status": "skip" + "version": 1, + "route": "anthropic/claude-opus-4-7", + "outcome": {"status": "failed", "error": {"kind": "model_selection", "message": "provider unavailable", "retry": {"type": "never"}}}, + "latency_ms": 12, + "usage": TokenCounts::default() })); }); @@ -367,7 +411,7 @@ fn model_test_post_list_race_is_a_failure() { ); let stdout = String::from_utf8_lossy(&output.stdout); let stderr = String::from_utf8_lossy(&output.stderr); - assert!(stdout.contains("provider became unconfigured after listing")); + assert!(stdout.contains("model_selection: provider unavailable")); assert!(stderr.contains("1 model(s) failed")); assert!(!stderr.contains("Skipped")); } @@ -391,9 +435,11 @@ fn model_test_json_partitions_skip_and_fail() { then.status(200) .header("Content-Type", "application/json") .json_body(serde_json::json!({ - "model_id": "claude-opus-4-7", - "provider": "anthropic", - "status": "skip" + "version": 1, + "route": "anthropic/claude-opus-4-7", + "outcome": {"status": "failed", "error": {"kind": "model_selection", "message": "provider unavailable", "retry": {"type": "never"}}}, + "latency_ms": 12, + "usage": TokenCounts::default() })); }); @@ -420,8 +466,13 @@ fn model_test_json_partitions_skip_and_fail() { assert_eq!(json["results"][1]["result"], "fail"); assert_eq!( json["results"][1]["error"], - "provider became unconfigured after listing" + "model_selection: provider unavailable" ); + assert_eq!( + json["results"][1]["report"]["outcome"]["error"]["kind"], + "model_selection" + ); + assert!(json["results"][0].get("report").is_none()); } #[derive(Clone)] @@ -598,9 +649,11 @@ async fn concurrent_test_model( state.gate.exit(); Json(serde_json::json!({ - "model_id": id, - "provider": "anthropic", - "status": "ok" + "version": 1, + "route": format!("anthropic/{id}"), + "outcome": {"status": "passed"}, + "latency_ms": 12, + "usage": TokenCounts::default() })) } diff --git a/lib/apps/fabro-server/src/server/handler/models.rs b/lib/apps/fabro-server/src/server/handler/models.rs index 53070acfc..7b275e7fc 100644 --- a/lib/apps/fabro-server/src/server/handler/models.rs +++ b/lib/apps/fabro-server/src/server/handler/models.rs @@ -2,10 +2,11 @@ use std::collections::HashSet; use std::sync::Arc; use std::time::Duration; +use fabro_llm::api; use fabro_llm::lithos_catalog::Catalog; use fabro_llm::probe::{self, ApiKeyProbeError, ModelTestStatus}; -use fabro_llm::{ModelSelectionError, api, selection}; use fabro_redact::redact_string; +use lithos_llm::client::{ProbeOptions, ProbeOutcome}; use lithos_llm::types::ReasoningEffort; use super::super::{ @@ -228,85 +229,25 @@ async fn test_model( .into_response(); } }; - let catalog = state.catalog(); - let eligible = llm_result - .provider_ids() - .into_iter() - .collect::>(); - let explicit_provider = params.provider.map(ProviderId::new); - let info = if let Some(provider) = explicit_provider.as_ref() { - match selection::resolve_on_provider(&catalog, provider, &id) { - Ok(info) => info, - Err(error) => return model_selection_response(&error), - } - } else { - match selection::select(&catalog, &id, None, &eligible) { - Ok(info) => info, - Err(error) => return model_selection_response(&error), - } - }; - let provider_id = info.provider.id().clone(); - let model_id = info.model.id().clone(); - if let Some((_, issue)) = llm_result - .auth_issues - .iter() - .find(|(provider, _)| provider == &provider_id) - { - return ApiError::bad_request(issue.to_string()).into_response(); - } - if !llm_result.has_provider(&provider_id) { - return Json(serde_json::json!({ - "model_id": model_id, - "provider": provider_id, - "status": "skip", - })) - .into_response(); - } + let selector = params + .provider + .map_or_else(|| id.clone(), |provider| format!("{provider}/{id}")); + let mut options = ProbeOptions::new().tools(mode == ModelTestMode::Deep); if let Some(effort) = reasoning_effort { - let capabilities = info.model.capabilities(); - if !capabilities.reasoning_effort(effort).is_supported() { - let allowed = ReasoningEffort::ALL - .into_iter() - .filter(|candidate| capabilities.reasoning_effort(*candidate).is_supported()) - .map(ReasoningEffort::as_str) - .collect::>() - .join(", "); - return ApiError::bad_request(format!( - "model '{model_id}' does not support reasoning_effort '{effort}'; allowed values: {allowed}" - )) - .into_response(); - } + options = options.reasoning_effort(effort); } - - let outcome = probe::run_model_test( - &llm_result.client, - &format!("{provider_id}/{model_id}"), - mode, - reasoning_effort, - None, - ) - .await; - Json(serde_json::json!({ - "model_id": model_id, - "provider": provider_id, - "status": <&'static str>::from(outcome.status), - "error_message": outcome.error_message, - })) - .into_response() -} - -fn model_selection_response(error: &ModelSelectionError) -> Response { - match error { - ModelSelectionError::UnknownProvider { .. } - | ModelSelectionError::UnknownSelector { .. } - | ModelSelectionError::UnknownSelectorOnProvider { .. } - | ModelSelectionError::ProviderUsedAsModel { .. } => { - ApiError::not_found(error.to_string()).into_response() - } - ModelSelectionError::ProviderUnavailable { .. } - | ModelSelectionError::NoEligibleOffering { .. } - | ModelSelectionError::NoDefaultModel { .. } => { - ApiError::bad_request(error.to_string()).into_response() + let mut report = llm_result.client.probe(selector, options).await; + // Model selection and probe findings belong to lithos. Fabro only owns + // disclosure of provider diagnostics across its HTTP boundary. + match &mut report.outcome { + ProbeOutcome::Failed(error) => { + error.message = redact_string(&error.message); + error.provider_code = error.provider_code.as_deref().map(redact_string); + error.raw_data = None; + error.source_message = None; } + ProbeOutcome::Incorrect { detail } => *detail = redact_string(detail), + _ => {} } + Json(report).into_response() } diff --git a/lib/apps/fabro-server/src/server/tests.rs b/lib/apps/fabro-server/src/server/tests.rs index 7353d2587..104502f2d 100644 --- a/lib/apps/fabro-server/src/server/tests.rs +++ b/lib/apps/fabro-server/src/server/tests.rs @@ -5194,7 +5194,7 @@ async fn pr_test_app_with_minimal_run( } #[tokio::test] -async fn test_model_unknown_returns_404() { +async fn test_model_unknown_returns_a_probe_finding() { let app = test_app_with(); let req = Request::builder() @@ -5205,11 +5205,14 @@ async fn test_model_unknown_returns_404() { .unwrap(); let response = app.clone().oneshot(req).await.unwrap(); - assert_status!(response, StatusCode::NOT_FOUND).await; + let body = response_json!(response, StatusCode::OK).await; + assert_eq!(body["outcome"]["status"], "failed"); + assert_eq!(body["outcome"]["error"]["kind"], "model_selection"); + assert!(body["route"].is_null()); } #[tokio::test] -async fn test_model_explicit_provider_alias_returns_canonical_model_id_when_unavailable() { +async fn test_model_explicit_provider_unavailable_returns_a_probe_finding() { let state = test_app_state_with_env_lookup( default_test_server_settings(), RunLayer::default(), @@ -5227,9 +5230,8 @@ async fn test_model_explicit_provider_alias_returns_canonical_model_id_when_unav let response = app.oneshot(req).await.unwrap(); let body = response_json!(response, StatusCode::OK).await; - assert_eq!(body["model_id"], "claude-sonnet-5"); - assert_eq!(body["provider"], "anthropic"); - assert_eq!(body["status"], "skip"); + assert_eq!(body["outcome"]["status"], "failed"); + assert_eq!(body["outcome"]["error"]["kind"], "model_selection"); } #[tokio::test] @@ -5253,7 +5255,9 @@ async fn test_model_unqualified_known_alias_requires_a_ready_provider() { .await .unwrap(); - assert_status!(response, StatusCode::BAD_REQUEST).await; + let body = response_json!(response, StatusCode::OK).await; + assert_eq!(body["outcome"]["status"], "failed"); + assert_eq!(body["outcome"]["error"]["kind"], "model_selection"); } #[tokio::test] @@ -5388,30 +5392,118 @@ capabilities = {{ text = true }} assert_eq!(filtered["data"].as_array().unwrap().len(), 1); assert_eq!(filtered["data"][0]["provider"], "aggregator"); - for (query, expected_provider) in [ - ("?provider=direct", "direct"), - ("?provider=aggregator", "aggregator"), - ("", "direct"), + for (selector, query, expected_provider) in [ + ("portable", "?provider=direct", "direct"), + ("portable", "?provider=aggregator", "aggregator"), + ("portable", "", "direct"), + ( + "vendor%2Fportable-model", + "?provider=aggregator", + "aggregator", + ), ] { let response = app .clone() .oneshot( Request::builder() .method("POST") - .uri(api(&format!("/models/portable/test{query}"))) + .uri(api(&format!("/models/{selector}/test{query}"))) .body(Body::empty()) .unwrap(), ) .await .unwrap(); let body = response_json!(response, StatusCode::OK).await; - assert_eq!(body["model_id"], "portable-model"); - assert_eq!(body["provider"], expected_provider); - assert_eq!(body["status"], "ok"); + assert_eq!(body["route"], format!("{expected_provider}/portable-model")); + assert_eq!(body["outcome"]["status"], "passed"); + assert_eq!(body["usage"]["input"], 1); } direct_probe.assert_calls(2); - aggregator_probe.assert_calls(1); + aggregator_probe.assert_calls(2); +} + +#[tokio::test] +async fn test_model_preserves_error_classification_and_redacts_provider_details() { + let leaked_key = "sk-proj-abcdefghijklmnopqrstuvwxyz0123456789"; + let upstream = MockServer::start(); + let probe = upstream.mock(|when, then| { + when.method(POST).path("/v1/chat/completions"); + then.status(401).json_body(json!({ + "error": { + "message": format!("invalid api key {leaked_key}"), + "code": format!("invalid_key_{leaked_key}"), + "type": "invalid_request_error" + }, + "private_detail": "must not cross the API boundary" + })); + }); + let state = TestAppStateBuilder::new() + .llm_overlay_toml(&acme_overlay(&upstream.base_url())) + .vault_entries([("ACME_API_KEY", leaked_key)]) + .build(); + let app = crate::test_support::build_test_router(state); + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri(api("/models/acme-large/test?provider=acme")) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let body = response_json!(response, StatusCode::OK).await; + assert_eq!(body["version"], 1); + assert_eq!(body["route"], "acme/acme-large"); + assert_eq!(body["outcome"]["status"], "failed"); + let error = &body["outcome"]["error"]; + assert_eq!(error["kind"], "authentication"); + assert_eq!(error["status"], 401); + assert_eq!(error["provider"], "acme"); + assert!(error.get("raw_data").is_none()); + assert!(error.get("source_message").is_none()); + let serialized = body.to_string(); + assert!(!serialized.contains(leaked_key)); + assert!(!serialized.contains("must not cross")); + assert!(serialized.contains("REDACTED")); + probe.assert_calls(1); +} + +#[tokio::test] +async fn test_model_tool_probe_preserves_incorrect_outcome_and_usage() { + let upstream = MockServer::start(); + let probe = upstream.mock(|when, then| { + when.method(POST).path("/v1/chat/completions"); + then.status(200).json_body(json!({ + "id": "chatcmpl-test", "model": "acme-large", + "choices": [{"message": {"role": "assistant", "content": "84 is even"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 12, "completion_tokens": 7, "total_tokens": 19} + })); + }); + let state = TestAppStateBuilder::new() + .llm_overlay_toml(&acme_overlay(&upstream.base_url())) + .vault_entries([("ACME_API_KEY", "test-key")]) + .build(); + let app = crate::test_support::build_test_router(state); + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri(api("/models/acme-large/test?provider=acme&mode=deep")) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let body = response_json!(response, StatusCode::OK).await; + assert_eq!(body["route"], "acme/acme-large"); + assert_eq!(body["outcome"]["status"], "incorrect"); + assert!(!body["outcome"]["detail"].as_str().unwrap().is_empty()); + assert_eq!(body["usage"]["input"], 12); + assert_eq!(body["usage"]["output"], 7); + assert!(body["latency_ms"].is_u64()); + probe.assert_calls(1); } #[tokio::test] @@ -5516,7 +5608,7 @@ protocol_options = {{ reasoning_effort_levels = true }} let response = app.clone().oneshot(req).await.unwrap(); let body = response_json!(response, StatusCode::OK).await; - assert_eq!(body["status"], "ok"); + assert_eq!(body["outcome"]["status"], "passed"); let unsupported = Request::builder() .method("POST") @@ -5526,11 +5618,9 @@ protocol_options = {{ reasoning_effort_levels = true }} .body(Body::empty()) .unwrap(); let response = app.oneshot(unsupported).await.unwrap(); - let body = response_json!(response, StatusCode::BAD_REQUEST).await; - assert_eq!( - body["errors"][0]["detail"], - "model 'acme-reasoner' does not support reasoning_effort 'medium'; allowed values: low, high" - ); + let body = response_json!(response, StatusCode::OK).await; + assert_eq!(body["outcome"]["status"], "failed"); + assert_eq!(body["outcome"]["error"]["kind"], "invalid_request"); completion.assert_calls(1); } diff --git a/lib/apps/fabro-server/tests/it/scenario/dry_run.rs b/lib/apps/fabro-server/tests/it/scenario/dry_run.rs index e7b3312c2..e08573830 100644 --- a/lib/apps/fabro-server/tests/it/scenario/dry_run.rs +++ b/lib/apps/fabro-server/tests/it/scenario/dry_run.rs @@ -58,7 +58,7 @@ async fn dry_run_serve_starts_and_runs_workflow() { } #[tokio::test] -async fn test_model_known_but_unavailable_returns_bad_request() { +async fn test_model_known_but_unavailable_returns_a_probe_finding() { let app = test_app_with_no_providers(); let req = Request::builder() @@ -71,15 +71,12 @@ async fn test_model_known_but_unavailable_returns_bad_request() { let response = app.oneshot(req).await.unwrap(); let body = response_json( response, - StatusCode::BAD_REQUEST, + StatusCode::OK, "POST /api/v1/models/claude-opus-4.6/test", ) .await; - assert!( - body["errors"][0]["detail"] - .as_str() - .is_some_and(|detail| detail.contains("no offering on an eligible provider")) - ); + assert_eq!(body["outcome"]["status"], "failed"); + assert_eq!(body["outcome"]["error"]["kind"], "model_selection"); } #[tokio::test] @@ -94,12 +91,13 @@ async fn test_model_unknown_via_full_router() { .unwrap(); let response = app.oneshot(req).await.unwrap(); - response_status( + let body = response_json( response, - StatusCode::NOT_FOUND, + StatusCode::OK, "POST /api/v1/models/nonexistent-model-xyz/test", ) .await; + assert_eq!(body["outcome"]["error"]["kind"], "model_selection"); } #[tokio::test] diff --git a/lib/components/fabro-llm/src/probe.rs b/lib/components/fabro-llm/src/probe.rs index 9c44c5395..f28467a9c 100644 --- a/lib/components/fabro-llm/src/probe.rs +++ b/lib/components/fabro-llm/src/probe.rs @@ -4,10 +4,8 @@ use std::sync::Arc; use std::time::Duration; use fabro_auth::ApiKeyCredentialSource; -use fabro_types::ModelTestMode; use lithos_llm::catalog::{Catalog, ProviderId}; use lithos_llm::client::{Client, ProbeOptions, ProbeOutcome}; -use lithos_llm::types::ReasoningEffort; use strum::IntoStaticStr; use crate::client::{ClientOptions, LlmSetupError, build_client}; @@ -43,25 +41,16 @@ impl ModelTestOutcome { } } -/// Probes `selector` (a `provider/model` route or bare selector) in `mode`. -/// -/// `Basic` asks for one word; `Deep` runs a two-step tool exchange and checks -/// the total. The probe resolves, authenticates, and retries exactly as a -/// real request would. -pub async fn run_model_test( +/// A credential-health projection of the library's basic probe. +/// Model-test endpoints return the full library report instead. +pub async fn run_basic_probe( client: &Client, selector: &str, - mode: ModelTestMode, - reasoning_effort: Option, - timeout: Option, + timeout: Duration, ) -> ModelTestOutcome { - let mut options = ProbeOptions::new() - .tools(mode == ModelTestMode::Deep) - .timeout(timeout.unwrap_or_else(|| Duration::from_secs(mode.timeout_secs()))); - if let Some(effort) = reasoning_effort { - options = options.reasoning_effort(effort); - } - let report = client.probe(selector, options).await; + let report = client + .probe(selector, ProbeOptions::new().timeout(timeout)) + .await; match report.outcome { ProbeOutcome::Passed => ModelTestOutcome::ok(), ProbeOutcome::Failed(data) => ModelTestOutcome::error(data.message), @@ -70,15 +59,6 @@ pub async fn run_model_test( } } -/// A basic probe of `selector` bounded by `timeout`. -pub async fn run_basic_probe( - client: &Client, - selector: &str, - timeout: Duration, -) -> ModelTestOutcome { - run_model_test(client, selector, ModelTestMode::Basic, None, Some(timeout)).await -} - /// Why an API key could not be probed. #[derive(Debug, thiserror::Error)] pub enum ApiKeyProbeError { diff --git a/lib/foundation/fabro-api/build.rs b/lib/foundation/fabro-api/build.rs index 551968fc1..92f832d90 100644 --- a/lib/foundation/fabro-api/build.rs +++ b/lib/foundation/fabro-api/build.rs @@ -606,6 +606,8 @@ fn main() { &[], ), ("TokenCounts", "lithos_llm::types::TokenCounts", &[]), + ("ProbeReport", "lithos_llm::client::ProbeReport", &[]), + ("LlmErrorData", "lithos_llm::types::ErrorData", &[]), ("Usage", "lithos_llm::types::Usage", &[]), ("UsageModelRef", "fabro_types::ModelRef", &[]), ("Speed", "lithos_llm::types::Speed", &[]), diff --git a/lib/foundation/fabro-api/src/lib.rs b/lib/foundation/fabro-api/src/lib.rs index 8f4707739..d29d93bb3 100644 --- a/lib/foundation/fabro-api/src/lib.rs +++ b/lib/foundation/fabro-api/src/lib.rs @@ -72,9 +72,10 @@ pub mod types { WorkflowVersionId, }; pub use lithos_llm::catalog::{ModelHandle, ProviderId}; + pub use lithos_llm::client::ProbeReport; pub use lithos_llm::types::{ - ContentPart, Cost, CostSource, ErrorKind as LlmErrorKind, Message, ReasoningEffort, - ReasoningOutput, ResponseFormat as CompletionResponseFormat, + ContentPart, Cost, CostSource, ErrorData as LlmErrorData, ErrorKind as LlmErrorKind, + Message, ReasoningEffort, ReasoningOutput, ResponseFormat as CompletionResponseFormat, RetryClassification as LlmRetryClassification, Role, Speed, TokenCounts, ToolChoice as CompletionToolChoice, ToolDefinition as CompletionToolDefinition, ToolDefinitionKind as CompletionToolDefinitionKind, Usage, diff --git a/lib/foundation/fabro-api/tests/probe_report_round_trip.rs b/lib/foundation/fabro-api/tests/probe_report_round_trip.rs new file mode 100644 index 000000000..d073efb2c --- /dev/null +++ b/lib/foundation/fabro-api/tests/probe_report_round_trip.rs @@ -0,0 +1,77 @@ +use std::any::TypeId; + +use fabro_api::types; +use lithos_llm::client::ProbeReport; +use lithos_llm::types::{ErrorData, TokenCounts}; +use serde_json::{Value, json}; + +fn validator(name: &str) -> jsonschema::Validator { + let yaml: serde_yaml::Value = serde_yaml::from_str(include_str!( + "../../../../docs/public/api-reference/fabro-api.yaml" + )) + .unwrap(); + let mut spec = serde_json::to_value(yaml).unwrap(); + spec["$ref"] = json!(format!("#/components/schemas/{name}")); + jsonschema::validator_for(&spec).unwrap() +} + +#[test] +fn probe_api_reuses_library_types_and_preserves_every_outcome() { + assert_eq!( + TypeId::of::(), + TypeId::of::() + ); + assert_eq!( + TypeId::of::(), + TypeId::of::() + ); + let error = json!({ + "kind": "authentication", "message": "invalid key", "provider": "acme", + "status": 401, "provider_code": "invalid_key", "retry": {"type": "never"}, + "raw_data": {"error": "detail"}, "source_message": "source", + "provider_retry_after_millis": 12 + }); + let parsed: types::LlmErrorData = serde_json::from_value(error.clone()).unwrap(); + assert_eq!(serde_json::to_value(parsed).unwrap(), error); + assert!(validator("LlmErrorData").is_valid(&error)); + + let schema = validator("ProbeReport"); + for (route, outcome) in [ + (json!("acme/vendor/model"), json!({"status": "passed"})), + (Value::Null, json!({"status": "failed", "error": error})), + ( + json!("acme/model"), + json!({"status": "incorrect", "detail": "no tool call"}), + ), + ] { + let value = json!({ + "version": 1, "route": route, "outcome": outcome, "latency_ms": 123, + "usage": TokenCounts { input: 12, output: 7, ..TokenCounts::default() } + }); + let report: types::ProbeReport = serde_json::from_value(value.clone()).unwrap(); + assert_eq!(serde_json::to_value(report).unwrap(), value); + assert!(schema.is_valid(&value), "{value}"); + } +} + +#[test] +fn probe_schema_and_rust_reject_unsupported_versions_routes_and_outcomes() { + let schema = validator("ProbeReport"); + let valid = json!({ + "version": 1, "route": "acme/model", "outcome": {"status": "passed"}, + "latency_ms": 0, "usage": TokenCounts::default() + }); + for (field, invalid) in [ + ("version", json!(2)), + ("route", json!("model")), + ("route", json!("acme/")), + ("outcome", json!({"status": "skip"})), + ("outcome", json!({"status": "failed"})), + ("latency_ms", json!(-1)), + ] { + let mut value = valid.clone(); + value[field] = invalid; + assert!(!schema.is_valid(&value), "{value}"); + assert!(serde_json::from_value::(value).is_err()); + } +} diff --git a/lib/foundation/fabro-client/src/client.rs b/lib/foundation/fabro-client/src/client.rs index 16c11a194..4042db33d 100644 --- a/lib/foundation/fabro-client/src/client.rs +++ b/lib/foundation/fabro-client/src/client.rs @@ -20,6 +20,7 @@ use fabro_util::exit::{ErrorExt, ExitClass}; use futures::future::BoxFuture; use futures::{Stream, StreamExt}; use lithos_llm::catalog::ProviderId; +use lithos_llm::client::ProbeReport; use lithos_llm::types::ReasoningEffort; use serde::Deserialize; use tokio::sync::Mutex; @@ -912,7 +913,7 @@ impl Client { provider: Option<&ProviderId>, mode: Option, reasoning_effort: Option, - ) -> Result { + ) -> Result { let response = self .send_api(|client| async move { let mut request = client.test_model().id(id.to_string()); diff --git a/lib/packages/fabro-api-client/src/.openapi-generator/FILES b/lib/packages/fabro-api-client/src/.openapi-generator/FILES index 066907eb7..ba70dce2b 100644 --- a/lib/packages/fabro-api-client/src/.openapi-generator/FILES +++ b/lib/packages/fabro-api-client/src/.openapi-generator/FILES @@ -214,6 +214,7 @@ models/interview-option.ts models/interview-provider-settings.ts models/interview-question-record.ts models/link-run-pull-request-request.ts +models/llm-error-data.ts models/llm-output-kind.ts models/llm-retry-classification-after.ts models/llm-retry-classification-never.ts @@ -252,7 +253,6 @@ models/model-handle.ts models/model-limits.ts models/model-reference.ts models/model-test-mode.ts -models/model-test-result.ts models/model-usage.ts models/model.ts models/none-run-target.ts @@ -321,6 +321,11 @@ models/principal-user.ts models/principal-webhook.ts models/principal-worker.ts models/principal.ts +models/probe-failed.ts +models/probe-incorrect.ts +models/probe-outcome.ts +models/probe-passed.ts +models/probe-report.ts models/project-namespace.ts models/provider-credential-test-request.ts models/provider-credential-test-response.ts diff --git a/lib/packages/fabro-api-client/src/api/models-api.ts b/lib/packages/fabro-api-client/src/api/models-api.ts index 7e883d91d..167896df0 100644 --- a/lib/packages/fabro-api-client/src/api/models-api.ts +++ b/lib/packages/fabro-api-client/src/api/models-api.ts @@ -26,10 +26,10 @@ import type { ErrorResponse } from '../models'; // @ts-ignore import type { ModelTestMode } from '../models'; // @ts-ignore -import type { ModelTestResult } from '../models'; -// @ts-ignore import type { PaginatedModelList } from '../models'; // @ts-ignore +import type { ProbeReport } from '../models'; +// @ts-ignore import type { ProviderCredentialTestRequest } from '../models'; // @ts-ignore import type { ProviderCredentialTestResponse } from '../models'; @@ -137,9 +137,9 @@ export const ModelsApiAxiosParamCreator = function (configuration?: Configuratio }; }, /** - * Tests a model by sending a simple prompt and reporting pass/fail. + * Runs the lithos-llm probe and returns its versioned report, including the canonical route, outcome, elapsed time and token usage. Unknown or unavailable models and provider failures produce HTTP 200 reports with a failed outcome. Invalid query options remain HTTP 400 errors. * @summary Test Model - * @param {string} id The canonical model ID or an alias. + * @param {string} id A model selector resolved by lithos-llm (canonical ID, alias or provider-qualified selector). * @param {string} [provider] Pin the test to this provider\'s offering. When omitted, the server selects among ready providers by catalog priority. * @param {ModelTestMode} [mode] Test mode for the single-model test endpoint. Defaults to `basic`. * @param {ReasoningEffort} [reasoningEffort] Optional native reasoning-effort level for the model test. @@ -310,16 +310,16 @@ export const ModelsApiFp = function(configuration?: Configuration) { return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, localVarOperationServerBasePath || basePath); }, /** - * Tests a model by sending a simple prompt and reporting pass/fail. + * Runs the lithos-llm probe and returns its versioned report, including the canonical route, outcome, elapsed time and token usage. Unknown or unavailable models and provider failures produce HTTP 200 reports with a failed outcome. Invalid query options remain HTTP 400 errors. * @summary Test Model - * @param {string} id The canonical model ID or an alias. + * @param {string} id A model selector resolved by lithos-llm (canonical ID, alias or provider-qualified selector). * @param {string} [provider] Pin the test to this provider\'s offering. When omitted, the server selects among ready providers by catalog priority. * @param {ModelTestMode} [mode] Test mode for the single-model test endpoint. Defaults to `basic`. * @param {ReasoningEffort} [reasoningEffort] Optional native reasoning-effort level for the model test. * @param {*} [options] Override http request option. * @throws {RequiredError} */ - async testModel(id: string, provider?: string, mode?: ModelTestMode, reasoningEffort?: ReasoningEffort, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise> { + async testModel(id: string, provider?: string, mode?: ModelTestMode, reasoningEffort?: ReasoningEffort, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise> { const localVarAxiosArgs = await localVarAxiosParamCreator.testModel(id, provider, mode, reasoningEffort, options); const localVarOperationServerIndex = configuration?.serverIndex ?? 0; const localVarOperationServerBasePath = operationServerMap['ModelsApi.testModel']?.[localVarOperationServerIndex]?.url; @@ -383,16 +383,16 @@ export const ModelsApiFactory = function (configuration?: Configuration, basePat return localVarFp.listProviders(options).then((request) => request(axios, basePath)); }, /** - * Tests a model by sending a simple prompt and reporting pass/fail. + * Runs the lithos-llm probe and returns its versioned report, including the canonical route, outcome, elapsed time and token usage. Unknown or unavailable models and provider failures produce HTTP 200 reports with a failed outcome. Invalid query options remain HTTP 400 errors. * @summary Test Model - * @param {string} id The canonical model ID or an alias. + * @param {string} id A model selector resolved by lithos-llm (canonical ID, alias or provider-qualified selector). * @param {string} [provider] Pin the test to this provider\'s offering. When omitted, the server selects among ready providers by catalog priority. * @param {ModelTestMode} [mode] Test mode for the single-model test endpoint. Defaults to `basic`. * @param {ReasoningEffort} [reasoningEffort] Optional native reasoning-effort level for the model test. * @param {*} [options] Override http request option. * @throws {RequiredError} */ - testModel(id: string, provider?: string, mode?: ModelTestMode, reasoningEffort?: ReasoningEffort, options?: RawAxiosRequestConfig): AxiosPromise { + testModel(id: string, provider?: string, mode?: ModelTestMode, reasoningEffort?: ReasoningEffort, options?: RawAxiosRequestConfig): AxiosPromise { return localVarFp.testModel(id, provider, mode, reasoningEffort, options).then((request) => request(axios, basePath)); }, /** @@ -447,9 +447,9 @@ export class ModelsApi extends BaseAPI { } /** - * Tests a model by sending a simple prompt and reporting pass/fail. + * Runs the lithos-llm probe and returns its versioned report, including the canonical route, outcome, elapsed time and token usage. Unknown or unavailable models and provider failures produce HTTP 200 reports with a failed outcome. Invalid query options remain HTTP 400 errors. * @summary Test Model - * @param {string} id The canonical model ID or an alias. + * @param {string} id A model selector resolved by lithos-llm (canonical ID, alias or provider-qualified selector). * @param {string} [provider] Pin the test to this provider\'s offering. When omitted, the server selects among ready providers by catalog priority. * @param {ModelTestMode} [mode] Test mode for the single-model test endpoint. Defaults to `basic`. * @param {ReasoningEffort} [reasoningEffort] Optional native reasoning-effort level for the model test. diff --git a/lib/packages/fabro-api-client/src/models/index.ts b/lib/packages/fabro-api-client/src/models/index.ts index abadb050c..ac10b48a5 100644 --- a/lib/packages/fabro-api-client/src/models/index.ts +++ b/lib/packages/fabro-api-client/src/models/index.ts @@ -184,6 +184,7 @@ export * from './interview-option'; export * from './interview-provider-settings'; export * from './interview-question-record'; export * from './link-run-pull-request-request'; +export * from './llm-error-data'; export * from './llm-output-kind'; export * from './llm-retry-classification'; export * from './llm-retry-classification-after'; @@ -223,7 +224,6 @@ export * from './model-handle'; export * from './model-limits'; export * from './model-reference'; export * from './model-test-mode'; -export * from './model-test-result'; export * from './model-usage'; export * from './none-run-target'; export * from './notification-provider-settings'; @@ -291,6 +291,11 @@ export * from './principal-system'; export * from './principal-user'; export * from './principal-webhook'; export * from './principal-worker'; +export * from './probe-failed'; +export * from './probe-incorrect'; +export * from './probe-outcome'; +export * from './probe-passed'; +export * from './probe-report'; export * from './project-namespace'; export * from './provider'; export * from './provider-credential-test-request'; diff --git a/lib/packages/fabro-api-client/src/models/llm-error-data.ts b/lib/packages/fabro-api-client/src/models/llm-error-data.ts new file mode 100644 index 000000000..931c41cf9 --- /dev/null +++ b/lib/packages/fabro-api-client/src/models/llm-error-data.ts @@ -0,0 +1,42 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * Fabro Run API + * HTTP API for managing Fabro workflow run executions. + * + * The version of the OpenAPI document: 0.2.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + +// May contain unused imports in some cases +// @ts-ignore +import type { LlmRetryClassification } from './llm-retry-classification'; + +/** + * Structured lithos-llm failure, subject to the server\'s disclosure policy. + */ +export interface LlmErrorData { + /** + * The model layer\'s stable failure category. Known values are `configuration`, `model_selection`, `authentication`, `access_denied`, `not_found`, `invalid_request`, `context_length`, `rate_limit`, `quota_exceeded`, `content_filter`, `server`, `provider`, `network`, `timeout`, `stream_decode`, `response_decode`, `resource_limit`, `middleware`, and `cancelled`. A category written by a newer model layer is carried as its own spelling. + */ + 'kind': string; + 'message': string; + /** + * LLM provider identifier. + */ + 'provider'?: string; + 'status'?: number; + 'provider_code'?: string; + 'retry': LlmRetryClassification; + 'raw_data'?: any; + 'provider_retry_after_millis'?: number; + /** + * Internal source detail; omitted by the model-test endpoint. + */ + 'source_message'?: string; +} diff --git a/lib/packages/fabro-api-client/src/models/model-test-result.ts b/lib/packages/fabro-api-client/src/models/model-test-result.ts deleted file mode 100644 index 8d6bbe596..000000000 --- a/lib/packages/fabro-api-client/src/models/model-test-result.ts +++ /dev/null @@ -1,45 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ -/** - * Fabro Run API - * HTTP API for managing Fabro workflow run executions. - * - * The version of the OpenAPI document: 0.2.0 - * - * - * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). - * https://openapi-generator.tech - * Do not edit the class manually. - */ - - - -/** - * Result of testing a model in `basic` or `deep` mode. - */ -export interface ModelTestResult { - /** - * The canonical model ID that was tested. - */ - 'model_id': string; - /** - * LLM provider identifier. - */ - 'provider': string; - /** - * Whether the model responded successfully, failed, or was skipped because its provider is not configured. - */ - 'status': ModelTestResultStatusEnum; - /** - * Error details when status is \"error\". - */ - 'error_message'?: string | null; -} - -export const ModelTestResultStatusEnum = { - OK: 'ok', - ERROR: 'error', - SKIP: 'skip' -} as const; - -export type ModelTestResultStatusEnum = typeof ModelTestResultStatusEnum[keyof typeof ModelTestResultStatusEnum]; diff --git a/lib/packages/fabro-api-client/src/models/probe-failed.ts b/lib/packages/fabro-api-client/src/models/probe-failed.ts new file mode 100644 index 000000000..2d8d44653 --- /dev/null +++ b/lib/packages/fabro-api-client/src/models/probe-failed.ts @@ -0,0 +1,29 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * Fabro Run API + * HTTP API for managing Fabro workflow run executions. + * + * The version of the OpenAPI document: 0.2.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + +// May contain unused imports in some cases +// @ts-ignore +import type { LlmErrorData } from './llm-error-data'; + +export interface ProbeFailed { + 'status': ProbeFailedStatusEnum; + 'error': LlmErrorData; +} + +export const ProbeFailedStatusEnum = { + FAILED: 'failed' +} as const; + +export type ProbeFailedStatusEnum = typeof ProbeFailedStatusEnum[keyof typeof ProbeFailedStatusEnum]; diff --git a/lib/packages/fabro-api-client/src/models/probe-incorrect.ts b/lib/packages/fabro-api-client/src/models/probe-incorrect.ts new file mode 100644 index 000000000..21b547b07 --- /dev/null +++ b/lib/packages/fabro-api-client/src/models/probe-incorrect.ts @@ -0,0 +1,26 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * Fabro Run API + * HTTP API for managing Fabro workflow run executions. + * + * The version of the OpenAPI document: 0.2.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + + +export interface ProbeIncorrect { + 'status': ProbeIncorrectStatusEnum; + 'detail': string; +} + +export const ProbeIncorrectStatusEnum = { + INCORRECT: 'incorrect' +} as const; + +export type ProbeIncorrectStatusEnum = typeof ProbeIncorrectStatusEnum[keyof typeof ProbeIncorrectStatusEnum]; diff --git a/lib/packages/fabro-api-client/src/models/probe-outcome.ts b/lib/packages/fabro-api-client/src/models/probe-outcome.ts new file mode 100644 index 000000000..dad95686e --- /dev/null +++ b/lib/packages/fabro-api-client/src/models/probe-outcome.ts @@ -0,0 +1,32 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * Fabro Run API + * HTTP API for managing Fabro workflow run executions. + * + * The version of the OpenAPI document: 0.2.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + +// May contain unused imports in some cases +// @ts-ignore +import type { LlmErrorData } from './llm-error-data'; +// May contain unused imports in some cases +// @ts-ignore +import type { ProbeFailed } from './probe-failed'; +// May contain unused imports in some cases +// @ts-ignore +import type { ProbeIncorrect } from './probe-incorrect'; +// May contain unused imports in some cases +// @ts-ignore +import type { ProbePassed } from './probe-passed'; + +/** + * @type ProbeOutcome + */ +export type ProbeOutcome = { status: 'failed' } & ProbeFailed | { status: 'incorrect' } & ProbeIncorrect | { status: 'passed' } & ProbePassed; diff --git a/lib/packages/fabro-api-client/src/models/probe-passed.ts b/lib/packages/fabro-api-client/src/models/probe-passed.ts new file mode 100644 index 000000000..25b8b7c81 --- /dev/null +++ b/lib/packages/fabro-api-client/src/models/probe-passed.ts @@ -0,0 +1,25 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * Fabro Run API + * HTTP API for managing Fabro workflow run executions. + * + * The version of the OpenAPI document: 0.2.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + + +export interface ProbePassed { + 'status': ProbePassedStatusEnum; +} + +export const ProbePassedStatusEnum = { + PASSED: 'passed' +} as const; + +export type ProbePassedStatusEnum = typeof ProbePassedStatusEnum[keyof typeof ProbePassedStatusEnum]; diff --git a/lib/packages/fabro-api-client/src/models/probe-report.ts b/lib/packages/fabro-api-client/src/models/probe-report.ts new file mode 100644 index 000000000..181e70f66 --- /dev/null +++ b/lib/packages/fabro-api-client/src/models/probe-report.ts @@ -0,0 +1,38 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * Fabro Run API + * HTTP API for managing Fabro workflow run executions. + * + * The version of the OpenAPI document: 0.2.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + +// May contain unused imports in some cases +// @ts-ignore +import type { ProbeOutcome } from './probe-outcome'; +// May contain unused imports in some cases +// @ts-ignore +import type { TokenCounts } from './token-counts'; + +/** + * The lithos-llm version-1 probe report. The route is the canonical provider/model selected by the probe, or null if selection failed. Probe failures are findings in a successful HTTP response; API authentication, malformed options and server setup failures remain HTTP errors. Provider errors are redacted at the server disclosure boundary. + */ +export interface ProbeReport { + 'version': ProbeReportVersionEnum; + 'route': string | null; + 'outcome': ProbeOutcome; + 'latency_ms': number; + 'usage': TokenCounts; +} + +export const ProbeReportVersionEnum = { + NUMBER_1: 1 +} as const; + +export type ProbeReportVersionEnum = typeof ProbeReportVersionEnum[keyof typeof ProbeReportVersionEnum];