From 073f38eb9a9da024edf1833982e1013915f17fdb Mon Sep 17 00:00:00 2001 From: Scott Werner Date: Mon, 2 Nov 2026 17:09:41 -0500 Subject: [PATCH 1/6] Return lithos-llm probe reports from model tests --- apps/fabro-web/app/routes/settings-models.tsx | 19 +- docs/public/api-reference/fabro-api.yaml | 124 +++-- docs/public/core-concepts/models.mdx | 19 + lib/apps/fabro-cli/src/commands/model.rs | 422 ++++++++---------- lib/apps/fabro-cli/tests/it/cmd/model_test.rs | 195 +++++--- .../fabro-server/src/server/handler/models.rs | 97 +--- lib/apps/fabro-server/src/server/tests.rs | 134 +++++- .../fabro-server/tests/it/scenario/dry_run.rs | 16 +- lib/components/fabro-llm/src/probe.rs | 34 +- lib/foundation/fabro-api/build.rs | 2 + lib/foundation/fabro-api/src/lib.rs | 5 +- .../tests/probe_report_round_trip.rs | 77 ++++ lib/foundation/fabro-client/src/client.rs | 3 +- .../src/.openapi-generator/FILES | 7 +- .../fabro-api-client/src/api/models-api.ts | 24 +- .../fabro-api-client/src/models/index.ts | 7 +- .../src/models/llm-error-data.ts | 42 ++ .../src/models/model-test-result.ts | 45 -- .../src/models/probe-failed.ts | 29 ++ .../src/models/probe-incorrect.ts | 26 ++ .../src/models/probe-outcome.ts | 32 ++ .../src/models/probe-passed.ts | 25 ++ .../src/models/probe-report.ts | 38 ++ 23 files changed, 872 insertions(+), 550 deletions(-) create mode 100644 lib/foundation/fabro-api/tests/probe_report_round_trip.rs create mode 100644 lib/packages/fabro-api-client/src/models/llm-error-data.ts delete mode 100644 lib/packages/fabro-api-client/src/models/model-test-result.ts create mode 100644 lib/packages/fabro-api-client/src/models/probe-failed.ts create mode 100644 lib/packages/fabro-api-client/src/models/probe-incorrect.ts create mode 100644 lib/packages/fabro-api-client/src/models/probe-outcome.ts create mode 100644 lib/packages/fabro-api-client/src/models/probe-passed.ts create mode 100644 lib/packages/fabro-api-client/src/models/probe-report.ts diff --git a/apps/fabro-web/app/routes/settings-models.tsx b/apps/fabro-web/app/routes/settings-models.tsx index fde1fe380..80007d986 100644 --- a/apps/fabro-web/app/routes/settings-models.tsx +++ b/apps/fabro-web/app/routes/settings-models.tsx @@ -310,22 +310,19 @@ function ModelsSection({ providers }: { providers: Provider[] }) { modelsApi.testModel(...modelOfferingTestArgs(offering)), ); if ( - result.provider !== offering.provider || - result.model_id !== offering.id + result.route !== null && + result.route !== `${offering.provider}/${offering.id}` ) { outcome = { - phase: "error", - message: `Server tested unexpected offering ${result.provider}/${result.model_id}`, + phase: "error", + message: `Server tested unexpected offering ${result.route}`, }; - } else if (result.status === "ok") { + } else if (result.outcome.status === "passed") { outcome = { phase: "ok" }; - } else if (result.status === "error") { - outcome = { - phase: "error", - message: result.error_message ?? "Failed", - }; + } else if (result.outcome.status === "failed") { + outcome = { phase: "error", message: result.outcome.error.message }; } else { - outcome = { phase: "error", message: "provider not configured" }; + outcome = { phase: "error", message: result.outcome.detail }; } } catch (err) { outcome = { diff --git a/docs/public/api-reference/fabro-api.yaml b/docs/public/api-reference/fabro-api.yaml index 985ab7793..e261d635b 100644 --- a/docs/public/api-reference/fabro-api.yaml +++ b/docs/public/api-reference/fabro-api.yaml @@ -5911,24 +5911,28 @@ paths: operationId: testModel tags: [Models] summary: Test Model - description: Tests a model by sending a simple prompt and reporting pass/fail. + description: >- + Runs the lithos-llm probe and returns its versioned report, including the + canonical route, outcome, elapsed time and token usage. Unknown or + unavailable models and provider failures produce HTTP 200 reports with + a failed outcome. Invalid query options remain HTTP 400 errors. parameters: - name: id in: path required: true schema: type: string - description: The canonical model ID or an alias. + description: A model selector resolved by lithos-llm (canonical ID, alias or provider-qualified selector). - $ref: "#/components/parameters/ModelTestProviderParam" - $ref: "#/components/parameters/ModelTestModeParam" - $ref: "#/components/parameters/ModelTestReasoningEffortParam" responses: "200": - description: Test result + description: Probe report (passed, failed or incorrect) content: application/json: schema: - $ref: "#/components/schemas/ModelTestResult" + $ref: "#/components/schemas/ProbeReport" "400": description: Invalid test mode or reasoning effort headers: @@ -5938,15 +5942,6 @@ paths: application/json: schema: $ref: "#/components/schemas/ErrorResponse" - "404": - description: Model not found - headers: - x-request-id: - $ref: "#/components/headers/XRequestId" - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" /api/v1/providers: get: @@ -8875,30 +8870,99 @@ components: key. Does NOT imply authentication or model requests will succeed; call `POST /models/{id}/test` to test usability. - ModelTestResult: - description: Result of testing a model in `basic` or `deep` mode. + ProbeReport: + description: >- + The lithos-llm version-1 probe report. The route is the canonical + provider/model selected by the probe, or null if selection failed. + Probe failures are findings in a successful HTTP response; API + authentication, malformed options and server setup failures remain HTTP errors. + Provider errors are redacted at the server disclosure boundary. type: object - required: - - model_id - - provider - - status + required: [version, route, outcome, latency_ms, usage] properties: - model_id: + version: + type: integer + enum: [1] + route: + type: [string, "null"] + pattern: '^[^/]+/.+$' + outcome: + $ref: "#/components/schemas/ProbeOutcome" + latency_ms: + type: integer + format: uint64 + minimum: 0 + usage: + $ref: "#/components/schemas/TokenCounts" + + ProbeOutcome: + oneOf: + - $ref: "#/components/schemas/ProbePassed" + - $ref: "#/components/schemas/ProbeFailed" + - $ref: "#/components/schemas/ProbeIncorrect" + discriminator: + propertyName: status + mapping: + passed: "#/components/schemas/ProbePassed" + failed: "#/components/schemas/ProbeFailed" + incorrect: "#/components/schemas/ProbeIncorrect" + + ProbePassed: + type: object + required: [status] + properties: + status: + type: string + enum: [passed] + + ProbeFailed: + type: object + required: [status, error] + properties: + status: + type: string + enum: [failed] + error: + $ref: "#/components/schemas/LlmErrorData" + + ProbeIncorrect: + type: object + required: [status, detail] + properties: + status: + type: string + enum: [incorrect] + detail: + type: string + + LlmErrorData: + description: Structured lithos-llm failure, subject to the server's disclosure policy. + type: object + required: [kind, message, retry] + properties: + kind: + $ref: "#/components/schemas/LlmErrorKind" + message: type: string - description: The canonical model ID that was tested. - example: "claude-opus-4-6" provider: $ref: "#/components/schemas/ProviderId" status: + type: integer + minimum: 0 + maximum: 65535 + provider_code: type: string - enum: - - ok - - error - - skip - description: Whether the model responded successfully, failed, or was skipped because its provider is not configured. - error_message: - type: ["string", "null"] - description: Error details when status is "error". + retry: + $ref: "#/components/schemas/LlmRetryClassification" + raw_data: + description: Provider response data; omitted by the model-test endpoint. + provider_retry_after_millis: + type: integer + format: uint64 + minimum: 0 + source_message: + type: string + description: Internal source detail; omitted by the model-test endpoint. ModelTestMode: description: Single-model test mode. diff --git a/docs/public/core-concepts/models.mdx b/docs/public/core-concepts/models.mdx index 49d175eb4..d6e5fcb20 100644 --- a/docs/public/core-concepts/models.mdx +++ b/docs/public/core-concepts/models.mdx @@ -344,3 +344,22 @@ fabro model test --provider openai ``` This is useful for confirming connectivity after setup or when adding a new provider key. + +With `--model`, Fabro sends the selector directly to the server's lithos-llm +probe. It displays the canonical provider/model selected by the probe, the +outcome, elapsed milliseconds, and input/output token usage. Add `--tools` to +check a multi-turn tool exchange. + +`fabro model test --model --json` returns the version-1 lithos-llm +probe report directly: `version`, `route`, `outcome`, `latency_ms`, and `usage`. +The outcome is `passed`, `failed` (with a structured `error`), or `incorrect` +(with a `detail` explaining the model's behavior). A selection failure has a +null route. Failed and incorrect probes print their report and exit nonzero. +This replaces the previous single-model JSON summary and `ok/error/skip` API +result. + +Without `--model`, Fabro lists models and tests configured offerings with the +requested concurrency. Bulk JSON retains `total`, `failures`, `skipped`, and +`results`; each completed probe's row also contains its full `report`. +Unconfigured offerings are skipped before probing. An offering that becomes +unavailable after listing is a failed probe. diff --git a/lib/apps/fabro-cli/src/commands/model.rs b/lib/apps/fabro-cli/src/commands/model.rs index 47385fb86..031b2499d 100644 --- a/lib/apps/fabro-cli/src/commands/model.rs +++ b/lib/apps/fabro-cli/src/commands/model.rs @@ -1,11 +1,11 @@ -use anyhow::{Context, Result, bail}; +use anyhow::{Result, bail}; use cli_table::format::{Border, Justify, Separator}; use cli_table::{Cell, CellStruct, Color, Style, Table}; -use fabro_api::types as api_types; use fabro_types::{Model, ModelTestMode}; use fabro_util::terminal::Styles; use futures::{StreamExt, stream}; use lithos_llm::catalog::ProviderId; +use lithos_llm::client::{ProbeOutcome, ProbeReport}; use serde::Serialize; use crate::args::{ModelListArgs, ModelTestArgs, ModelsCommand}; @@ -29,6 +29,8 @@ struct ModelTestRow { detail: Option, #[serde(skip_serializing_if = "Option::is_none")] error: Option, + #[serde(skip_serializing_if = "Option::is_none")] + report: Option, } #[derive(Serialize)] @@ -44,24 +46,7 @@ struct CompletedModelTest { model: Model, result_color: Color, status: String, -} - -fn model_matches_selector(model: &Model, selector: &str) -> bool { - model.id.as_str() == selector || model.aliases.iter().any(|alias| alias == selector) -} - -fn find_model_by_id_or_alias( - models: &[Model], - id: &str, - provider: Option<&ProviderId>, -) -> Option { - models - .iter() - .find(|model| { - provider.is_none_or(|provider| &model.provider == provider) - && model_matches_selector(model, id) - }) - .cloned() + report: Option, } pub(crate) async fn execute( @@ -183,34 +168,37 @@ fn print_models_table(models: &[Model], styles: &Styles) { ); } -fn configured_model_test_status( - expected: &Model, - result: Result, -) -> (Color, String, bool) { - match result { - Ok(resp) if resp.provider != expected.provider || resp.model_id != expected.id.as_str() => { - ( - Color::Red, - format!( - "error: server tested unexpected offering {}/{}", - resp.provider, resp.model_id - ), - true, - ) - } - Ok(resp) if resp.status == api_types::ModelTestResultStatus::Ok => { - (Color::Green, "ok".to_string(), false) - } - Ok(resp) if resp.status == api_types::ModelTestResultStatus::Skip => ( +fn probe_status(report: &ProbeReport) -> (Color, String, bool) { + match &report.outcome { + ProbeOutcome::Passed => (Color::Green, "ok".to_string(), false), + ProbeOutcome::Failed(error) => ( Color::Red, - "error: provider became unconfigured after listing".to_string(), + format!("error: {}: {}", error.kind.as_str(), error.message), true, ), - Ok(resp) => { - let message = resp - .error_message - .unwrap_or_else(|| "unknown error".to_string()); - (Color::Red, format!("error: {message}"), true) + ProbeOutcome::Incorrect { detail } => (Color::Red, format!("incorrect: {detail}"), true), + _ => (Color::Red, "error: unknown probe outcome".to_string(), true), + } +} + +fn configured_model_test_status( + expected: &Model, + result: &Result, +) -> (Color, String, bool) { + match result { + Ok(report) => { + // A bulk job names an exact offering. Keep this correlation check; + // a single selector is resolved exclusively by the server's probe. + if let Some(route) = &report.route { + if route.provider() != &expected.provider || route.model() != &expected.id { + return ( + Color::Red, + format!("error: server tested unexpected offering {route}"), + true, + ); + } + } + probe_status(report) } Err(err) => (Color::Red, format!("error: {err}"), true), } @@ -225,6 +213,7 @@ fn model_test_row_from_status(model: &Model, status: &str, result_color: Color) result: ModelTestResultKind::Pass, detail: None, error: None, + report: None, }, Color::Yellow => ModelTestRow { model: model.id.to_string(), @@ -232,12 +221,14 @@ fn model_test_row_from_status(model: &Model, status: &str, result_color: Color) result: ModelTestResultKind::Skip, detail: Some(trimmed.to_string()), error: None, + report: None, }, _ => ModelTestRow { model: model.id.to_string(), provider: model.provider.clone(), result: ModelTestResultKind::Fail, detail: None, + report: None, error: Some( trimmed .strip_prefix("error: ") @@ -265,6 +256,44 @@ async fn test_models_via_server( let reasoning_effort = args.reasoning_effort; let request_mode = args.tools.then_some(ModelTestMode::Deep); + if let Some(selector) = model { + let requested_provider = provider.map(ProviderId::new); + if !json_output { + eprint!("Testing {selector}..."); + } + let result = client + .test_model( + selector, + requested_provider.as_ref(), + request_mode, + reasoning_effort, + ) + .await; + if !json_output { + eprintln!(" done"); + } + let report = result?; + if json_output { + println!("{}", serde_json::to_string_pretty(&report)?); + } else { + let route = report + .route + .as_ref() + .map_or_else(|| selector.to_string(), ToString::to_string); + let (_, status, _) = probe_status(&report); + println!( + "{route}: {status} ({} ms; {} input, {} output tokens)", + report.latency.as_millis(), + report.usage.input, + report.usage.output + ); + } + if !report.passed() { + bail!("1 model(s) failed"); + } + return Ok(()); + } + let use_color = styles.use_color; let mut title = models_title(use_color); title.push("RESULT".cell().bold(use_color)); @@ -274,162 +303,81 @@ async fn test_models_via_server( let mut failures = 0u32; let mut skipped = 0u32; let mut skipped_providers: Vec = Vec::new(); - if let Some(model_id) = model { - let requested_provider = provider.map(ProviderId::new); - let listed_models = client.list_models(provider, Some(model_id)).await?; - let listed_info = find_model_by_id_or_alias(&listed_models, model_id, None); - if !json_output { - eprint!("Testing {model_id}..."); - } - let has_configured_match = listed_models - .iter() - .any(|model| model.configured && model_matches_selector(model, model_id)); - let result = - if requested_provider.is_none() && listed_info.is_some() && !has_configured_match { - None - } else { - Some( - client - .test_model( - model_id, - requested_provider.as_ref(), - request_mode, - reasoning_effort, - ) - .await, - ) - }; - if !json_output { - eprintln!(" done"); - } + let models_to_test = client.list_models(provider, None).await?; + if models_to_test.is_empty() { + bail!("No models found"); + } - let (info, result_color, status) = match result { - None => { - let info = listed_info.with_context(|| format!("Unknown model: {model_id}"))?; - failures += 1; - skipped += 1; - (info, Color::Yellow, "not configured".to_string()) - } - Some(Ok(resp)) => { - let info = - find_model_by_id_or_alias(&listed_models, &resp.model_id, Some(&resp.provider)) - .with_context(|| { - format!( - "Unknown model returned by server: {}/{}", - resp.provider, resp.model_id - ) - })?; - if resp.status == api_types::ModelTestResultStatus::Ok { - (info, Color::Green, "ok".to_string()) - } else if resp.status == api_types::ModelTestResultStatus::Skip { - failures += 1; - skipped += 1; - (info, Color::Yellow, "not configured".to_string()) - } else { - failures += 1; - let message = resp - .error_message - .unwrap_or_else(|| "unknown error".to_string()); - (info, Color::Red, format!("error: {message}")) + let (configured, unconfigured): (Vec<_>, Vec<_>) = models_to_test + .into_iter() + .partition(|model| model.configured); + + for info in &unconfigured { + skipped += 1; + let provider_name = info.provider.to_string(); + if !skipped_providers.contains(&provider_name) { + skipped_providers.push(provider_name); + } + if json_output { + json_rows.push(model_test_row_from_status( + info, + "not configured", + Color::Yellow, + )); + } + } + + let mut completed = stream::iter(configured.into_iter().enumerate()) + .map(|(index, info)| { + let client = client.clone(); + async move { + let result = client + .test_model( + info.id.as_str(), + Some(&info.provider), + request_mode, + reasoning_effort, + ) + .await; + if !json_output { + eprintln!("Testing {}... done", info.id); } + let (result_color, status, failed) = configured_model_test_status(&info, &result); + ( + CompletedModelTest { + index, + model: info, + result_color, + status, + report: result.ok(), + }, + failed, + ) } - Some(Err(err)) if err.to_string().contains("Model not found") => { - bail!("Unknown model: {model_id}"); - } - Some(Err(err)) => { - let info = listed_info.with_context(|| format!("Unknown model: {model_id}"))?; - failures += 1; - (info, Color::Red, format!("error: {err}")) - } - }; + }) + .buffer_unordered(jobs) + .collect::>() + .await; - let mut row = model_row(&info, use_color); + completed.sort_by_key(|(completed, _)| completed.index); + + for (completed, failed) in completed { + if failed { + failures += 1; + } + + let mut row = model_row(&completed.model, use_color); + let mut json_row = + model_test_row_from_status(&completed.model, &completed.status, completed.result_color); + json_row.report = completed.report; + json_rows.push(json_row); row.push( - status - .clone() + completed + .status .cell() - .foreground_color(color_if(use_color, result_color)), + .foreground_color(color_if(use_color, completed.result_color)), ); rows.push(row); - json_rows.push(model_test_row_from_status(&info, &status, result_color)); - } else { - let models_to_test = client.list_models(provider, None).await?; - if models_to_test.is_empty() { - bail!("No models found"); - } - - let (configured, unconfigured): (Vec<_>, Vec<_>) = models_to_test - .into_iter() - .partition(|model| model.configured); - - for info in &unconfigured { - skipped += 1; - let provider_name = info.provider.to_string(); - if !skipped_providers.contains(&provider_name) { - skipped_providers.push(provider_name); - } - if json_output { - json_rows.push(model_test_row_from_status( - info, - "not configured", - Color::Yellow, - )); - } - } - - let mut completed = stream::iter(configured.into_iter().enumerate()) - .map(|(index, info)| { - let client = client.clone(); - async move { - let result = client - .test_model( - info.id.as_str(), - Some(&info.provider), - request_mode, - reasoning_effort, - ) - .await; - if !json_output { - eprintln!("Testing {}... done", info.id); - } - let (result_color, status, failed) = - configured_model_test_status(&info, result); - ( - CompletedModelTest { - index, - model: info, - result_color, - status, - }, - failed, - ) - } - }) - .buffer_unordered(jobs) - .collect::>() - .await; - - completed.sort_by_key(|(completed, _)| completed.index); - - for (completed, failed) in completed { - if failed { - failures += 1; - } - - let mut row = model_row(&completed.model, use_color); - json_rows.push(model_test_row_from_status( - &completed.model, - &completed.status, - completed.result_color, - )); - row.push( - completed - .status - .cell() - .foreground_color(color_if(use_color, completed.result_color)), - ); - rows.push(row); - } } if json_output { @@ -458,7 +406,7 @@ async fn test_models_via_server( println!("{}", table.display()?); } - if skipped > 0 && model.is_none() { + if skipped > 0 { eprintln!( "Skipped {} model(s) (no credentials: {})", skipped, @@ -516,7 +464,7 @@ impl Default for ModelsCommand { mod tests { use fabro_types::{ModelControls, ModelCosts, ModelFeatures, ModelLimits}; use lithos_llm::catalog::builtin; - use lithos_llm::types::ReasoningEffort; + use lithos_llm::types::{ErrorKind, ReasoningEffort, TokenCounts}; use super::*; @@ -657,9 +605,11 @@ mod tests { .header("Content-Type", "application/json") .body( serde_json::json!({ - "model_id": "test-model", - "provider": "anthropic", - "status": "ok" + "version": 1, + "route": "anthropic/test-model", + "outcome": {"status": "passed"}, + "latency_ms": 12, + "usage": TokenCounts::default() }) .to_string(), ); @@ -672,8 +622,7 @@ mod tests { .await .unwrap(); - assert_eq!(response.status, api_types::ModelTestResultStatus::Ok); - assert!(response.error_message.is_none()); + assert!(response.passed()); } #[tokio::test] @@ -689,10 +638,11 @@ mod tests { .header("Content-Type", "application/json") .body( serde_json::json!({ - "model_id": "test-model", - "provider": "anthropic", - "status": "error", - "error_message": "timeout" + "version": 1, + "route": "anthropic/test-model", + "outcome": {"status": "failed", "error": {"kind": "timeout", "message": "timeout", "retry": {"type": "never"}}}, + "latency_ms": 12, + "usage": TokenCounts::default() }) .to_string(), ); @@ -710,12 +660,13 @@ mod tests { .await .unwrap(); - assert_eq!(response.status, api_types::ModelTestResultStatus::Error); - assert_eq!(response.error_message.as_deref(), Some("timeout")); + assert!( + matches!(response.outcome, ProbeOutcome::Failed(error) if error.message == "timeout") + ); } #[tokio::test] - async fn test_model_via_server_parses_skip() { + async fn test_model_via_server_parses_selection_failure() { let server = httpmock::MockServer::start_async().await; server .mock_async(|when, then| { @@ -724,9 +675,11 @@ mod tests { .header("Content-Type", "application/json") .body( serde_json::json!({ - "model_id": "kimi-k2.5", - "provider": "moonshot", - "status": "skip" + "version": 1, + "route": "moonshot/kimi-k2.5", + "outcome": {"status": "failed", "error": {"kind": "model_selection", "message": "provider unavailable", "retry": {"type": "never"}}}, + "latency_ms": 12, + "usage": TokenCounts::default() }) .to_string(), ); @@ -739,8 +692,9 @@ mod tests { .await .unwrap(); - assert_eq!(response.status, api_types::ModelTestResultStatus::Skip); - assert!(response.error_message.is_none()); + assert!( + matches!(response.outcome, ProbeOutcome::Failed(error) if error.kind == ErrorKind::ModelSelection) + ); } #[tokio::test] @@ -767,26 +721,8 @@ mod tests { } #[tokio::test] - async fn single_model_test_uses_server_model_metadata_for_custom_models() { + async fn single_model_test_uses_probe_identity_without_catalog_metadata() { let server = httpmock::MockServer::start_async().await; - server - .mock_async(|when, then| { - when.method("GET") - .path("/api/v1/models") - .query_param("page[limit]", "100") - .query_param("page[offset]", "0") - .query_param("query", "venice-large"); - then.status(200) - .header("Content-Type", "application/json") - .body( - serde_json::json!({ - "data": [custom_model_json("venice-large", "venice")], - "meta": { "has_more": false } - }) - .to_string(), - ); - }) - .await; server .mock_async(|when, then| { when.method("POST").path("/api/v1/models/venice-large/test"); @@ -794,9 +730,11 @@ mod tests { .header("Content-Type", "application/json") .body( serde_json::json!({ - "model_id": "venice-large", - "provider": "venice", - "status": "ok" + "version": 1, + "route": "venice/venice-large", + "outcome": {"status": "passed"}, + "latency_ms": 12, + "usage": TokenCounts::default() }) .to_string(), ); @@ -847,9 +785,11 @@ mod tests { .header("Content-Type", "application/json") .body( serde_json::json!({ - "model_id": "portable-model", - "provider": "openai", - "status": "ok" + "version": 1, + "route": "openai/portable-model", + "outcome": {"status": "passed"}, + "latency_ms": 12, + "usage": TokenCounts::default() }) .to_string(), ); @@ -864,9 +804,11 @@ mod tests { .header("Content-Type", "application/json") .body( serde_json::json!({ - "model_id": "portable-model", - "provider": "openrouter", - "status": "ok" + "version": 1, + "route": "openrouter/portable-model", + "outcome": {"status": "passed"}, + "latency_ms": 12, + "usage": TokenCounts::default() }) .to_string(), ); diff --git a/lib/apps/fabro-cli/tests/it/cmd/model_test.rs b/lib/apps/fabro-cli/tests/it/cmd/model_test.rs index 64c48781b..9400f3362 100644 --- a/lib/apps/fabro-cli/tests/it/cmd/model_test.rs +++ b/lib/apps/fabro-cli/tests/it/cmd/model_test.rs @@ -10,6 +10,7 @@ use axum::routing::{get, post}; use axum::{Json, Router}; use fabro_test::{fabro_snapshot, test_context}; use httpmock::MockServer; +use lithos_llm::types::TokenCounts; use tokio::net::TcpListener; use tokio::sync::{Semaphore, oneshot}; @@ -129,9 +130,11 @@ fn assert_model_test_forwards(cli_args: &[&str], expected_query: &[(&str, &str)] then.status(200) .header("Content-Type", "application/json") .json_body(serde_json::json!({ - "model_id": "test-model", - "provider": "anthropic", - "status": "ok" + "version": 1, + "route": "anthropic/test-model", + "outcome": {"status": "passed"}, + "latency_ms": 12, + "usage": TokenCounts::default() })); }); @@ -146,7 +149,7 @@ fn assert_model_test_forwards(cli_args: &[&str], expected_query: &[(&str, &str)] String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr) ); - list.assert(); + list.assert_calls(0); test.assert(); } @@ -164,38 +167,101 @@ fn model_test_deep_remains_an_alias_for_tools() { } #[test] -fn model_test_unknown_model_errors() { +fn single_model_reports_canonical_alias_without_listing() { let context = test_context!(); + let server = MockServer::start(); + context.set_http_target(&server.base_url()); + let listing = server.mock(|when, then| { + when.method("GET").path("/api/v1/models"); + then.status(500); + }); + let report = serde_json::json!({ + "version": 1, + "route": "anthropic/claude-sonnet-4.6", + "outcome": {"status": "passed"}, + "latency_ms": 12, + "usage": TokenCounts::default(), + }); + let probe = server.mock(|when, then| { + when.method("POST") + .path("/api/v1/models/claude-sonnet-4-6/test") + .query_param("provider", "anthropic"); + then.status(200).json_body(report.clone()); + }); let mut cmd = context.command(); - cmd.args(["model", "test", "--model", "nonexistent-model-xyz"]); + cmd.args([ + "model", + "test", + "--model", + "claude-sonnet-4-6", + "--provider", + "anthropic", + "--json", + ]); + let output = cmd + .assert() + .success() + .stderr("") + .get_output() + .stdout + .clone(); + assert_eq!( + serde_json::from_slice::(&output).unwrap(), + report + ); + listing.assert_calls(0); + probe.assert_calls(1); + let mut cmd = context.command(); + cmd.args([ + "model", + "test", + "--model", + "claude-sonnet-4-6", + "--provider", + "anthropic", + ]); fabro_snapshot!(context.filters(), cmd, @" - success: false - exit_code: 1 + success: true + exit_code: 0 ----- stdout ----- + anthropic/claude-sonnet-4.6: ok (12 ms; 0 input, 0 output tokens) ----- stderr ----- - Testing nonexistent-model-xyz... done - × Unknown model: nonexistent-model-xyz + Testing claude-sonnet-4-6... done "); + listing.assert_calls(0); + probe.assert_calls(2); } #[test] -fn single_model_skip_exits_nonzero() { - let context = test_context!(); - let mut cmd = context.command(); - cmd.args(["model", "test", "--model", "gemini-3.1-pro-preview"]); - remove_provider_env(&mut cmd); - - fabro_snapshot!(context.filters(), cmd, @" - success: false - exit_code: 1 - ----- stdout ----- - MODEL PROVIDER ALIASES CONTEXT COST SPEED RESULT - gemini-3.1-pro-preview gemini gemini-pro 1m $2.0 / $12.0 85 tok/s not configured - ----- stderr ----- - Testing gemini-3.1-pro-preview... done - × 1 model(s) failed - "); +fn single_model_preserves_failed_and_incorrect_findings() { + for outcome in [ + serde_json::json!({"status": "failed", "error": {"kind": "model_selection", "message": "unknown model", "retry": {"type": "never"}}}), + serde_json::json!({"status": "incorrect", "detail": "model did not call the tool"}), + ] { + let context = test_context!(); + let server = MockServer::start(); + context.set_http_target(&server.base_url()); + let report = serde_json::json!({ + "version": 1, + "route": null, + "outcome": outcome, + "latency_ms": 0, + "usage": TokenCounts::default(), + }); + let probe = server.mock(|when, then| { + when.method("POST").path("/api/v1/models/test-model/test"); + then.status(200).json_body(report.clone()); + }); + let mut cmd = context.command(); + cmd.args(["model", "test", "--model", "test-model", "--json"]); + let output = cmd.assert().failure().get_output().stdout.clone(); + assert_eq!( + serde_json::from_slice::(&output).unwrap(), + report + ); + probe.assert_calls(1); + } } #[test] @@ -224,34 +290,6 @@ fn bulk_skip_exits_zero_and_prints_summary() { ); } -#[test] -fn json_output_includes_skipped_models() { - let context = test_context!(); - let mut cmd = context.command(); - cmd.args([ - "model", - "test", - "--model", - "gemini-3.1-pro-preview", - "--json", - ]); - remove_provider_env(&mut cmd); - - let output = cmd.output().expect("failed to execute model test"); - assert!( - !output.status.success(), - "expected single-model skip to exit non-zero:\nstdout:\n{}\nstderr:\n{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); - let stdout = String::from_utf8_lossy(&output.stdout); - let json: serde_json::Value = serde_json::from_str(&stdout).expect("invalid JSON output"); - - assert_eq!(json["failures"], 1); - assert_eq!(json["skipped"], 1); - assert_eq!(json["results"][0]["result"], "skip"); -} - #[test] fn model_test_does_not_announce_unconfigured() { let context = test_context!(); @@ -267,9 +305,11 @@ fn model_test_does_not_announce_unconfigured() { then.status(200) .header("Content-Type", "application/json") .json_body(serde_json::json!({ - "model_id": "claude-opus-4-7", - "provider": "anthropic", - "status": "ok" + "version": 1, + "route": "anthropic/claude-opus-4-7", + "outcome": {"status": "passed"}, + "latency_ms": 12, + "usage": TokenCounts::default() })); }); let unconfigured_test = server.mock(|when, then| { @@ -317,9 +357,11 @@ fn model_test_skipped_footer_sources_from_listing() { then.status(200) .header("Content-Type", "application/json") .json_body(serde_json::json!({ - "model_id": "claude-opus-4-7", - "provider": "anthropic", - "status": "ok" + "version": 1, + "route": "anthropic/claude-opus-4-7", + "outcome": {"status": "passed"}, + "latency_ms": 12, + "usage": TokenCounts::default() })); }); @@ -349,9 +391,11 @@ fn model_test_post_list_race_is_a_failure() { then.status(200) .header("Content-Type", "application/json") .json_body(serde_json::json!({ - "model_id": "claude-opus-4-7", - "provider": "anthropic", - "status": "skip" + "version": 1, + "route": "anthropic/claude-opus-4-7", + "outcome": {"status": "failed", "error": {"kind": "model_selection", "message": "provider unavailable", "retry": {"type": "never"}}}, + "latency_ms": 12, + "usage": TokenCounts::default() })); }); @@ -367,7 +411,7 @@ fn model_test_post_list_race_is_a_failure() { ); let stdout = String::from_utf8_lossy(&output.stdout); let stderr = String::from_utf8_lossy(&output.stderr); - assert!(stdout.contains("provider became unconfigured after listing")); + assert!(stdout.contains("model_selection: provider unavailable")); assert!(stderr.contains("1 model(s) failed")); assert!(!stderr.contains("Skipped")); } @@ -391,9 +435,11 @@ fn model_test_json_partitions_skip_and_fail() { then.status(200) .header("Content-Type", "application/json") .json_body(serde_json::json!({ - "model_id": "claude-opus-4-7", - "provider": "anthropic", - "status": "skip" + "version": 1, + "route": "anthropic/claude-opus-4-7", + "outcome": {"status": "failed", "error": {"kind": "model_selection", "message": "provider unavailable", "retry": {"type": "never"}}}, + "latency_ms": 12, + "usage": TokenCounts::default() })); }); @@ -420,8 +466,13 @@ fn model_test_json_partitions_skip_and_fail() { assert_eq!(json["results"][1]["result"], "fail"); assert_eq!( json["results"][1]["error"], - "provider became unconfigured after listing" + "model_selection: provider unavailable" ); + assert_eq!( + json["results"][1]["report"]["outcome"]["error"]["kind"], + "model_selection" + ); + assert!(json["results"][0].get("report").is_none()); } #[derive(Clone)] @@ -598,9 +649,11 @@ async fn concurrent_test_model( state.gate.exit(); Json(serde_json::json!({ - "model_id": id, - "provider": "anthropic", - "status": "ok" + "version": 1, + "route": format!("anthropic/{id}"), + "outcome": {"status": "passed"}, + "latency_ms": 12, + "usage": TokenCounts::default() })) } diff --git a/lib/apps/fabro-server/src/server/handler/models.rs b/lib/apps/fabro-server/src/server/handler/models.rs index 53070acfc..7b275e7fc 100644 --- a/lib/apps/fabro-server/src/server/handler/models.rs +++ b/lib/apps/fabro-server/src/server/handler/models.rs @@ -2,10 +2,11 @@ use std::collections::HashSet; use std::sync::Arc; use std::time::Duration; +use fabro_llm::api; use fabro_llm::lithos_catalog::Catalog; use fabro_llm::probe::{self, ApiKeyProbeError, ModelTestStatus}; -use fabro_llm::{ModelSelectionError, api, selection}; use fabro_redact::redact_string; +use lithos_llm::client::{ProbeOptions, ProbeOutcome}; use lithos_llm::types::ReasoningEffort; use super::super::{ @@ -228,85 +229,25 @@ async fn test_model( .into_response(); } }; - let catalog = state.catalog(); - let eligible = llm_result - .provider_ids() - .into_iter() - .collect::>(); - let explicit_provider = params.provider.map(ProviderId::new); - let info = if let Some(provider) = explicit_provider.as_ref() { - match selection::resolve_on_provider(&catalog, provider, &id) { - Ok(info) => info, - Err(error) => return model_selection_response(&error), - } - } else { - match selection::select(&catalog, &id, None, &eligible) { - Ok(info) => info, - Err(error) => return model_selection_response(&error), - } - }; - let provider_id = info.provider.id().clone(); - let model_id = info.model.id().clone(); - if let Some((_, issue)) = llm_result - .auth_issues - .iter() - .find(|(provider, _)| provider == &provider_id) - { - return ApiError::bad_request(issue.to_string()).into_response(); - } - if !llm_result.has_provider(&provider_id) { - return Json(serde_json::json!({ - "model_id": model_id, - "provider": provider_id, - "status": "skip", - })) - .into_response(); - } + let selector = params + .provider + .map_or_else(|| id.clone(), |provider| format!("{provider}/{id}")); + let mut options = ProbeOptions::new().tools(mode == ModelTestMode::Deep); if let Some(effort) = reasoning_effort { - let capabilities = info.model.capabilities(); - if !capabilities.reasoning_effort(effort).is_supported() { - let allowed = ReasoningEffort::ALL - .into_iter() - .filter(|candidate| capabilities.reasoning_effort(*candidate).is_supported()) - .map(ReasoningEffort::as_str) - .collect::>() - .join(", "); - return ApiError::bad_request(format!( - "model '{model_id}' does not support reasoning_effort '{effort}'; allowed values: {allowed}" - )) - .into_response(); - } + options = options.reasoning_effort(effort); } - - let outcome = probe::run_model_test( - &llm_result.client, - &format!("{provider_id}/{model_id}"), - mode, - reasoning_effort, - None, - ) - .await; - Json(serde_json::json!({ - "model_id": model_id, - "provider": provider_id, - "status": <&'static str>::from(outcome.status), - "error_message": outcome.error_message, - })) - .into_response() -} - -fn model_selection_response(error: &ModelSelectionError) -> Response { - match error { - ModelSelectionError::UnknownProvider { .. } - | ModelSelectionError::UnknownSelector { .. } - | ModelSelectionError::UnknownSelectorOnProvider { .. } - | ModelSelectionError::ProviderUsedAsModel { .. } => { - ApiError::not_found(error.to_string()).into_response() - } - ModelSelectionError::ProviderUnavailable { .. } - | ModelSelectionError::NoEligibleOffering { .. } - | ModelSelectionError::NoDefaultModel { .. } => { - ApiError::bad_request(error.to_string()).into_response() + let mut report = llm_result.client.probe(selector, options).await; + // Model selection and probe findings belong to lithos. Fabro only owns + // disclosure of provider diagnostics across its HTTP boundary. + match &mut report.outcome { + ProbeOutcome::Failed(error) => { + error.message = redact_string(&error.message); + error.provider_code = error.provider_code.as_deref().map(redact_string); + error.raw_data = None; + error.source_message = None; } + ProbeOutcome::Incorrect { detail } => *detail = redact_string(detail), + _ => {} } + Json(report).into_response() } diff --git a/lib/apps/fabro-server/src/server/tests.rs b/lib/apps/fabro-server/src/server/tests.rs index 7353d2587..104502f2d 100644 --- a/lib/apps/fabro-server/src/server/tests.rs +++ b/lib/apps/fabro-server/src/server/tests.rs @@ -5194,7 +5194,7 @@ async fn pr_test_app_with_minimal_run( } #[tokio::test] -async fn test_model_unknown_returns_404() { +async fn test_model_unknown_returns_a_probe_finding() { let app = test_app_with(); let req = Request::builder() @@ -5205,11 +5205,14 @@ async fn test_model_unknown_returns_404() { .unwrap(); let response = app.clone().oneshot(req).await.unwrap(); - assert_status!(response, StatusCode::NOT_FOUND).await; + let body = response_json!(response, StatusCode::OK).await; + assert_eq!(body["outcome"]["status"], "failed"); + assert_eq!(body["outcome"]["error"]["kind"], "model_selection"); + assert!(body["route"].is_null()); } #[tokio::test] -async fn test_model_explicit_provider_alias_returns_canonical_model_id_when_unavailable() { +async fn test_model_explicit_provider_unavailable_returns_a_probe_finding() { let state = test_app_state_with_env_lookup( default_test_server_settings(), RunLayer::default(), @@ -5227,9 +5230,8 @@ async fn test_model_explicit_provider_alias_returns_canonical_model_id_when_unav let response = app.oneshot(req).await.unwrap(); let body = response_json!(response, StatusCode::OK).await; - assert_eq!(body["model_id"], "claude-sonnet-5"); - assert_eq!(body["provider"], "anthropic"); - assert_eq!(body["status"], "skip"); + assert_eq!(body["outcome"]["status"], "failed"); + assert_eq!(body["outcome"]["error"]["kind"], "model_selection"); } #[tokio::test] @@ -5253,7 +5255,9 @@ async fn test_model_unqualified_known_alias_requires_a_ready_provider() { .await .unwrap(); - assert_status!(response, StatusCode::BAD_REQUEST).await; + let body = response_json!(response, StatusCode::OK).await; + assert_eq!(body["outcome"]["status"], "failed"); + assert_eq!(body["outcome"]["error"]["kind"], "model_selection"); } #[tokio::test] @@ -5388,30 +5392,118 @@ capabilities = {{ text = true }} assert_eq!(filtered["data"].as_array().unwrap().len(), 1); assert_eq!(filtered["data"][0]["provider"], "aggregator"); - for (query, expected_provider) in [ - ("?provider=direct", "direct"), - ("?provider=aggregator", "aggregator"), - ("", "direct"), + for (selector, query, expected_provider) in [ + ("portable", "?provider=direct", "direct"), + ("portable", "?provider=aggregator", "aggregator"), + ("portable", "", "direct"), + ( + "vendor%2Fportable-model", + "?provider=aggregator", + "aggregator", + ), ] { let response = app .clone() .oneshot( Request::builder() .method("POST") - .uri(api(&format!("/models/portable/test{query}"))) + .uri(api(&format!("/models/{selector}/test{query}"))) .body(Body::empty()) .unwrap(), ) .await .unwrap(); let body = response_json!(response, StatusCode::OK).await; - assert_eq!(body["model_id"], "portable-model"); - assert_eq!(body["provider"], expected_provider); - assert_eq!(body["status"], "ok"); + assert_eq!(body["route"], format!("{expected_provider}/portable-model")); + assert_eq!(body["outcome"]["status"], "passed"); + assert_eq!(body["usage"]["input"], 1); } direct_probe.assert_calls(2); - aggregator_probe.assert_calls(1); + aggregator_probe.assert_calls(2); +} + +#[tokio::test] +async fn test_model_preserves_error_classification_and_redacts_provider_details() { + let leaked_key = "sk-proj-abcdefghijklmnopqrstuvwxyz0123456789"; + let upstream = MockServer::start(); + let probe = upstream.mock(|when, then| { + when.method(POST).path("/v1/chat/completions"); + then.status(401).json_body(json!({ + "error": { + "message": format!("invalid api key {leaked_key}"), + "code": format!("invalid_key_{leaked_key}"), + "type": "invalid_request_error" + }, + "private_detail": "must not cross the API boundary" + })); + }); + let state = TestAppStateBuilder::new() + .llm_overlay_toml(&acme_overlay(&upstream.base_url())) + .vault_entries([("ACME_API_KEY", leaked_key)]) + .build(); + let app = crate::test_support::build_test_router(state); + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri(api("/models/acme-large/test?provider=acme")) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let body = response_json!(response, StatusCode::OK).await; + assert_eq!(body["version"], 1); + assert_eq!(body["route"], "acme/acme-large"); + assert_eq!(body["outcome"]["status"], "failed"); + let error = &body["outcome"]["error"]; + assert_eq!(error["kind"], "authentication"); + assert_eq!(error["status"], 401); + assert_eq!(error["provider"], "acme"); + assert!(error.get("raw_data").is_none()); + assert!(error.get("source_message").is_none()); + let serialized = body.to_string(); + assert!(!serialized.contains(leaked_key)); + assert!(!serialized.contains("must not cross")); + assert!(serialized.contains("REDACTED")); + probe.assert_calls(1); +} + +#[tokio::test] +async fn test_model_tool_probe_preserves_incorrect_outcome_and_usage() { + let upstream = MockServer::start(); + let probe = upstream.mock(|when, then| { + when.method(POST).path("/v1/chat/completions"); + then.status(200).json_body(json!({ + "id": "chatcmpl-test", "model": "acme-large", + "choices": [{"message": {"role": "assistant", "content": "84 is even"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 12, "completion_tokens": 7, "total_tokens": 19} + })); + }); + let state = TestAppStateBuilder::new() + .llm_overlay_toml(&acme_overlay(&upstream.base_url())) + .vault_entries([("ACME_API_KEY", "test-key")]) + .build(); + let app = crate::test_support::build_test_router(state); + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri(api("/models/acme-large/test?provider=acme&mode=deep")) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let body = response_json!(response, StatusCode::OK).await; + assert_eq!(body["route"], "acme/acme-large"); + assert_eq!(body["outcome"]["status"], "incorrect"); + assert!(!body["outcome"]["detail"].as_str().unwrap().is_empty()); + assert_eq!(body["usage"]["input"], 12); + assert_eq!(body["usage"]["output"], 7); + assert!(body["latency_ms"].is_u64()); + probe.assert_calls(1); } #[tokio::test] @@ -5516,7 +5608,7 @@ protocol_options = {{ reasoning_effort_levels = true }} let response = app.clone().oneshot(req).await.unwrap(); let body = response_json!(response, StatusCode::OK).await; - assert_eq!(body["status"], "ok"); + assert_eq!(body["outcome"]["status"], "passed"); let unsupported = Request::builder() .method("POST") @@ -5526,11 +5618,9 @@ protocol_options = {{ reasoning_effort_levels = true }} .body(Body::empty()) .unwrap(); let response = app.oneshot(unsupported).await.unwrap(); - let body = response_json!(response, StatusCode::BAD_REQUEST).await; - assert_eq!( - body["errors"][0]["detail"], - "model 'acme-reasoner' does not support reasoning_effort 'medium'; allowed values: low, high" - ); + let body = response_json!(response, StatusCode::OK).await; + assert_eq!(body["outcome"]["status"], "failed"); + assert_eq!(body["outcome"]["error"]["kind"], "invalid_request"); completion.assert_calls(1); } diff --git a/lib/apps/fabro-server/tests/it/scenario/dry_run.rs b/lib/apps/fabro-server/tests/it/scenario/dry_run.rs index e7b3312c2..e08573830 100644 --- a/lib/apps/fabro-server/tests/it/scenario/dry_run.rs +++ b/lib/apps/fabro-server/tests/it/scenario/dry_run.rs @@ -58,7 +58,7 @@ async fn dry_run_serve_starts_and_runs_workflow() { } #[tokio::test] -async fn test_model_known_but_unavailable_returns_bad_request() { +async fn test_model_known_but_unavailable_returns_a_probe_finding() { let app = test_app_with_no_providers(); let req = Request::builder() @@ -71,15 +71,12 @@ async fn test_model_known_but_unavailable_returns_bad_request() { let response = app.oneshot(req).await.unwrap(); let body = response_json( response, - StatusCode::BAD_REQUEST, + StatusCode::OK, "POST /api/v1/models/claude-opus-4.6/test", ) .await; - assert!( - body["errors"][0]["detail"] - .as_str() - .is_some_and(|detail| detail.contains("no offering on an eligible provider")) - ); + assert_eq!(body["outcome"]["status"], "failed"); + assert_eq!(body["outcome"]["error"]["kind"], "model_selection"); } #[tokio::test] @@ -94,12 +91,13 @@ async fn test_model_unknown_via_full_router() { .unwrap(); let response = app.oneshot(req).await.unwrap(); - response_status( + let body = response_json( response, - StatusCode::NOT_FOUND, + StatusCode::OK, "POST /api/v1/models/nonexistent-model-xyz/test", ) .await; + assert_eq!(body["outcome"]["error"]["kind"], "model_selection"); } #[tokio::test] diff --git a/lib/components/fabro-llm/src/probe.rs b/lib/components/fabro-llm/src/probe.rs index 9c44c5395..f28467a9c 100644 --- a/lib/components/fabro-llm/src/probe.rs +++ b/lib/components/fabro-llm/src/probe.rs @@ -4,10 +4,8 @@ use std::sync::Arc; use std::time::Duration; use fabro_auth::ApiKeyCredentialSource; -use fabro_types::ModelTestMode; use lithos_llm::catalog::{Catalog, ProviderId}; use lithos_llm::client::{Client, ProbeOptions, ProbeOutcome}; -use lithos_llm::types::ReasoningEffort; use strum::IntoStaticStr; use crate::client::{ClientOptions, LlmSetupError, build_client}; @@ -43,25 +41,16 @@ impl ModelTestOutcome { } } -/// Probes `selector` (a `provider/model` route or bare selector) in `mode`. -/// -/// `Basic` asks for one word; `Deep` runs a two-step tool exchange and checks -/// the total. The probe resolves, authenticates, and retries exactly as a -/// real request would. -pub async fn run_model_test( +/// A credential-health projection of the library's basic probe. +/// Model-test endpoints return the full library report instead. +pub async fn run_basic_probe( client: &Client, selector: &str, - mode: ModelTestMode, - reasoning_effort: Option, - timeout: Option, + timeout: Duration, ) -> ModelTestOutcome { - let mut options = ProbeOptions::new() - .tools(mode == ModelTestMode::Deep) - .timeout(timeout.unwrap_or_else(|| Duration::from_secs(mode.timeout_secs()))); - if let Some(effort) = reasoning_effort { - options = options.reasoning_effort(effort); - } - let report = client.probe(selector, options).await; + let report = client + .probe(selector, ProbeOptions::new().timeout(timeout)) + .await; match report.outcome { ProbeOutcome::Passed => ModelTestOutcome::ok(), ProbeOutcome::Failed(data) => ModelTestOutcome::error(data.message), @@ -70,15 +59,6 @@ pub async fn run_model_test( } } -/// A basic probe of `selector` bounded by `timeout`. -pub async fn run_basic_probe( - client: &Client, - selector: &str, - timeout: Duration, -) -> ModelTestOutcome { - run_model_test(client, selector, ModelTestMode::Basic, None, Some(timeout)).await -} - /// Why an API key could not be probed. #[derive(Debug, thiserror::Error)] pub enum ApiKeyProbeError { diff --git a/lib/foundation/fabro-api/build.rs b/lib/foundation/fabro-api/build.rs index 551968fc1..92f832d90 100644 --- a/lib/foundation/fabro-api/build.rs +++ b/lib/foundation/fabro-api/build.rs @@ -606,6 +606,8 @@ fn main() { &[], ), ("TokenCounts", "lithos_llm::types::TokenCounts", &[]), + ("ProbeReport", "lithos_llm::client::ProbeReport", &[]), + ("LlmErrorData", "lithos_llm::types::ErrorData", &[]), ("Usage", "lithos_llm::types::Usage", &[]), ("UsageModelRef", "fabro_types::ModelRef", &[]), ("Speed", "lithos_llm::types::Speed", &[]), diff --git a/lib/foundation/fabro-api/src/lib.rs b/lib/foundation/fabro-api/src/lib.rs index 8f4707739..d29d93bb3 100644 --- a/lib/foundation/fabro-api/src/lib.rs +++ b/lib/foundation/fabro-api/src/lib.rs @@ -72,9 +72,10 @@ pub mod types { WorkflowVersionId, }; pub use lithos_llm::catalog::{ModelHandle, ProviderId}; + pub use lithos_llm::client::ProbeReport; pub use lithos_llm::types::{ - ContentPart, Cost, CostSource, ErrorKind as LlmErrorKind, Message, ReasoningEffort, - ReasoningOutput, ResponseFormat as CompletionResponseFormat, + ContentPart, Cost, CostSource, ErrorData as LlmErrorData, ErrorKind as LlmErrorKind, + Message, ReasoningEffort, ReasoningOutput, ResponseFormat as CompletionResponseFormat, RetryClassification as LlmRetryClassification, Role, Speed, TokenCounts, ToolChoice as CompletionToolChoice, ToolDefinition as CompletionToolDefinition, ToolDefinitionKind as CompletionToolDefinitionKind, Usage, diff --git a/lib/foundation/fabro-api/tests/probe_report_round_trip.rs b/lib/foundation/fabro-api/tests/probe_report_round_trip.rs new file mode 100644 index 000000000..d073efb2c --- /dev/null +++ b/lib/foundation/fabro-api/tests/probe_report_round_trip.rs @@ -0,0 +1,77 @@ +use std::any::TypeId; + +use fabro_api::types; +use lithos_llm::client::ProbeReport; +use lithos_llm::types::{ErrorData, TokenCounts}; +use serde_json::{Value, json}; + +fn validator(name: &str) -> jsonschema::Validator { + let yaml: serde_yaml::Value = serde_yaml::from_str(include_str!( + "../../../../docs/public/api-reference/fabro-api.yaml" + )) + .unwrap(); + let mut spec = serde_json::to_value(yaml).unwrap(); + spec["$ref"] = json!(format!("#/components/schemas/{name}")); + jsonschema::validator_for(&spec).unwrap() +} + +#[test] +fn probe_api_reuses_library_types_and_preserves_every_outcome() { + assert_eq!( + TypeId::of::(), + TypeId::of::() + ); + assert_eq!( + TypeId::of::(), + TypeId::of::() + ); + let error = json!({ + "kind": "authentication", "message": "invalid key", "provider": "acme", + "status": 401, "provider_code": "invalid_key", "retry": {"type": "never"}, + "raw_data": {"error": "detail"}, "source_message": "source", + "provider_retry_after_millis": 12 + }); + let parsed: types::LlmErrorData = serde_json::from_value(error.clone()).unwrap(); + assert_eq!(serde_json::to_value(parsed).unwrap(), error); + assert!(validator("LlmErrorData").is_valid(&error)); + + let schema = validator("ProbeReport"); + for (route, outcome) in [ + (json!("acme/vendor/model"), json!({"status": "passed"})), + (Value::Null, json!({"status": "failed", "error": error})), + ( + json!("acme/model"), + json!({"status": "incorrect", "detail": "no tool call"}), + ), + ] { + let value = json!({ + "version": 1, "route": route, "outcome": outcome, "latency_ms": 123, + "usage": TokenCounts { input: 12, output: 7, ..TokenCounts::default() } + }); + let report: types::ProbeReport = serde_json::from_value(value.clone()).unwrap(); + assert_eq!(serde_json::to_value(report).unwrap(), value); + assert!(schema.is_valid(&value), "{value}"); + } +} + +#[test] +fn probe_schema_and_rust_reject_unsupported_versions_routes_and_outcomes() { + let schema = validator("ProbeReport"); + let valid = json!({ + "version": 1, "route": "acme/model", "outcome": {"status": "passed"}, + "latency_ms": 0, "usage": TokenCounts::default() + }); + for (field, invalid) in [ + ("version", json!(2)), + ("route", json!("model")), + ("route", json!("acme/")), + ("outcome", json!({"status": "skip"})), + ("outcome", json!({"status": "failed"})), + ("latency_ms", json!(-1)), + ] { + let mut value = valid.clone(); + value[field] = invalid; + assert!(!schema.is_valid(&value), "{value}"); + assert!(serde_json::from_value::(value).is_err()); + } +} diff --git a/lib/foundation/fabro-client/src/client.rs b/lib/foundation/fabro-client/src/client.rs index 16c11a194..4042db33d 100644 --- a/lib/foundation/fabro-client/src/client.rs +++ b/lib/foundation/fabro-client/src/client.rs @@ -20,6 +20,7 @@ use fabro_util::exit::{ErrorExt, ExitClass}; use futures::future::BoxFuture; use futures::{Stream, StreamExt}; use lithos_llm::catalog::ProviderId; +use lithos_llm::client::ProbeReport; use lithos_llm::types::ReasoningEffort; use serde::Deserialize; use tokio::sync::Mutex; @@ -912,7 +913,7 @@ impl Client { provider: Option<&ProviderId>, mode: Option, reasoning_effort: Option, - ) -> Result { + ) -> Result { let response = self .send_api(|client| async move { let mut request = client.test_model().id(id.to_string()); diff --git a/lib/packages/fabro-api-client/src/.openapi-generator/FILES b/lib/packages/fabro-api-client/src/.openapi-generator/FILES index 066907eb7..ba70dce2b 100644 --- a/lib/packages/fabro-api-client/src/.openapi-generator/FILES +++ b/lib/packages/fabro-api-client/src/.openapi-generator/FILES @@ -214,6 +214,7 @@ models/interview-option.ts models/interview-provider-settings.ts models/interview-question-record.ts models/link-run-pull-request-request.ts +models/llm-error-data.ts models/llm-output-kind.ts models/llm-retry-classification-after.ts models/llm-retry-classification-never.ts @@ -252,7 +253,6 @@ models/model-handle.ts models/model-limits.ts models/model-reference.ts models/model-test-mode.ts -models/model-test-result.ts models/model-usage.ts models/model.ts models/none-run-target.ts @@ -321,6 +321,11 @@ models/principal-user.ts models/principal-webhook.ts models/principal-worker.ts models/principal.ts +models/probe-failed.ts +models/probe-incorrect.ts +models/probe-outcome.ts +models/probe-passed.ts +models/probe-report.ts models/project-namespace.ts models/provider-credential-test-request.ts models/provider-credential-test-response.ts diff --git a/lib/packages/fabro-api-client/src/api/models-api.ts b/lib/packages/fabro-api-client/src/api/models-api.ts index 7e883d91d..167896df0 100644 --- a/lib/packages/fabro-api-client/src/api/models-api.ts +++ b/lib/packages/fabro-api-client/src/api/models-api.ts @@ -26,10 +26,10 @@ import type { ErrorResponse } from '../models'; // @ts-ignore import type { ModelTestMode } from '../models'; // @ts-ignore -import type { ModelTestResult } from '../models'; -// @ts-ignore import type { PaginatedModelList } from '../models'; // @ts-ignore +import type { ProbeReport } from '../models'; +// @ts-ignore import type { ProviderCredentialTestRequest } from '../models'; // @ts-ignore import type { ProviderCredentialTestResponse } from '../models'; @@ -137,9 +137,9 @@ export const ModelsApiAxiosParamCreator = function (configuration?: Configuratio }; }, /** - * Tests a model by sending a simple prompt and reporting pass/fail. + * Runs the lithos-llm probe and returns its versioned report, including the canonical route, outcome, elapsed time and token usage. Unknown or unavailable models and provider failures produce HTTP 200 reports with a failed outcome. Invalid query options remain HTTP 400 errors. * @summary Test Model - * @param {string} id The canonical model ID or an alias. + * @param {string} id A model selector resolved by lithos-llm (canonical ID, alias or provider-qualified selector). * @param {string} [provider] Pin the test to this provider\'s offering. When omitted, the server selects among ready providers by catalog priority. * @param {ModelTestMode} [mode] Test mode for the single-model test endpoint. Defaults to `basic`. * @param {ReasoningEffort} [reasoningEffort] Optional native reasoning-effort level for the model test. @@ -310,16 +310,16 @@ export const ModelsApiFp = function(configuration?: Configuration) { return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, localVarOperationServerBasePath || basePath); }, /** - * Tests a model by sending a simple prompt and reporting pass/fail. + * Runs the lithos-llm probe and returns its versioned report, including the canonical route, outcome, elapsed time and token usage. Unknown or unavailable models and provider failures produce HTTP 200 reports with a failed outcome. Invalid query options remain HTTP 400 errors. * @summary Test Model - * @param {string} id The canonical model ID or an alias. + * @param {string} id A model selector resolved by lithos-llm (canonical ID, alias or provider-qualified selector). * @param {string} [provider] Pin the test to this provider\'s offering. When omitted, the server selects among ready providers by catalog priority. * @param {ModelTestMode} [mode] Test mode for the single-model test endpoint. Defaults to `basic`. * @param {ReasoningEffort} [reasoningEffort] Optional native reasoning-effort level for the model test. * @param {*} [options] Override http request option. * @throws {RequiredError} */ - async testModel(id: string, provider?: string, mode?: ModelTestMode, reasoningEffort?: ReasoningEffort, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise> { + async testModel(id: string, provider?: string, mode?: ModelTestMode, reasoningEffort?: ReasoningEffort, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise> { const localVarAxiosArgs = await localVarAxiosParamCreator.testModel(id, provider, mode, reasoningEffort, options); const localVarOperationServerIndex = configuration?.serverIndex ?? 0; const localVarOperationServerBasePath = operationServerMap['ModelsApi.testModel']?.[localVarOperationServerIndex]?.url; @@ -383,16 +383,16 @@ export const ModelsApiFactory = function (configuration?: Configuration, basePat return localVarFp.listProviders(options).then((request) => request(axios, basePath)); }, /** - * Tests a model by sending a simple prompt and reporting pass/fail. + * Runs the lithos-llm probe and returns its versioned report, including the canonical route, outcome, elapsed time and token usage. Unknown or unavailable models and provider failures produce HTTP 200 reports with a failed outcome. Invalid query options remain HTTP 400 errors. * @summary Test Model - * @param {string} id The canonical model ID or an alias. + * @param {string} id A model selector resolved by lithos-llm (canonical ID, alias or provider-qualified selector). * @param {string} [provider] Pin the test to this provider\'s offering. When omitted, the server selects among ready providers by catalog priority. * @param {ModelTestMode} [mode] Test mode for the single-model test endpoint. Defaults to `basic`. * @param {ReasoningEffort} [reasoningEffort] Optional native reasoning-effort level for the model test. * @param {*} [options] Override http request option. * @throws {RequiredError} */ - testModel(id: string, provider?: string, mode?: ModelTestMode, reasoningEffort?: ReasoningEffort, options?: RawAxiosRequestConfig): AxiosPromise { + testModel(id: string, provider?: string, mode?: ModelTestMode, reasoningEffort?: ReasoningEffort, options?: RawAxiosRequestConfig): AxiosPromise { return localVarFp.testModel(id, provider, mode, reasoningEffort, options).then((request) => request(axios, basePath)); }, /** @@ -447,9 +447,9 @@ export class ModelsApi extends BaseAPI { } /** - * Tests a model by sending a simple prompt and reporting pass/fail. + * Runs the lithos-llm probe and returns its versioned report, including the canonical route, outcome, elapsed time and token usage. Unknown or unavailable models and provider failures produce HTTP 200 reports with a failed outcome. Invalid query options remain HTTP 400 errors. * @summary Test Model - * @param {string} id The canonical model ID or an alias. + * @param {string} id A model selector resolved by lithos-llm (canonical ID, alias or provider-qualified selector). * @param {string} [provider] Pin the test to this provider\'s offering. When omitted, the server selects among ready providers by catalog priority. * @param {ModelTestMode} [mode] Test mode for the single-model test endpoint. Defaults to `basic`. * @param {ReasoningEffort} [reasoningEffort] Optional native reasoning-effort level for the model test. diff --git a/lib/packages/fabro-api-client/src/models/index.ts b/lib/packages/fabro-api-client/src/models/index.ts index abadb050c..ac10b48a5 100644 --- a/lib/packages/fabro-api-client/src/models/index.ts +++ b/lib/packages/fabro-api-client/src/models/index.ts @@ -184,6 +184,7 @@ export * from './interview-option'; export * from './interview-provider-settings'; export * from './interview-question-record'; export * from './link-run-pull-request-request'; +export * from './llm-error-data'; export * from './llm-output-kind'; export * from './llm-retry-classification'; export * from './llm-retry-classification-after'; @@ -223,7 +224,6 @@ export * from './model-handle'; export * from './model-limits'; export * from './model-reference'; export * from './model-test-mode'; -export * from './model-test-result'; export * from './model-usage'; export * from './none-run-target'; export * from './notification-provider-settings'; @@ -291,6 +291,11 @@ export * from './principal-system'; export * from './principal-user'; export * from './principal-webhook'; export * from './principal-worker'; +export * from './probe-failed'; +export * from './probe-incorrect'; +export * from './probe-outcome'; +export * from './probe-passed'; +export * from './probe-report'; export * from './project-namespace'; export * from './provider'; export * from './provider-credential-test-request'; diff --git a/lib/packages/fabro-api-client/src/models/llm-error-data.ts b/lib/packages/fabro-api-client/src/models/llm-error-data.ts new file mode 100644 index 000000000..931c41cf9 --- /dev/null +++ b/lib/packages/fabro-api-client/src/models/llm-error-data.ts @@ -0,0 +1,42 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * Fabro Run API + * HTTP API for managing Fabro workflow run executions. + * + * The version of the OpenAPI document: 0.2.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + +// May contain unused imports in some cases +// @ts-ignore +import type { LlmRetryClassification } from './llm-retry-classification'; + +/** + * Structured lithos-llm failure, subject to the server\'s disclosure policy. + */ +export interface LlmErrorData { + /** + * The model layer\'s stable failure category. Known values are `configuration`, `model_selection`, `authentication`, `access_denied`, `not_found`, `invalid_request`, `context_length`, `rate_limit`, `quota_exceeded`, `content_filter`, `server`, `provider`, `network`, `timeout`, `stream_decode`, `response_decode`, `resource_limit`, `middleware`, and `cancelled`. A category written by a newer model layer is carried as its own spelling. + */ + 'kind': string; + 'message': string; + /** + * LLM provider identifier. + */ + 'provider'?: string; + 'status'?: number; + 'provider_code'?: string; + 'retry': LlmRetryClassification; + 'raw_data'?: any; + 'provider_retry_after_millis'?: number; + /** + * Internal source detail; omitted by the model-test endpoint. + */ + 'source_message'?: string; +} diff --git a/lib/packages/fabro-api-client/src/models/model-test-result.ts b/lib/packages/fabro-api-client/src/models/model-test-result.ts deleted file mode 100644 index 8d6bbe596..000000000 --- a/lib/packages/fabro-api-client/src/models/model-test-result.ts +++ /dev/null @@ -1,45 +0,0 @@ -/* tslint:disable */ -/* eslint-disable */ -/** - * Fabro Run API - * HTTP API for managing Fabro workflow run executions. - * - * The version of the OpenAPI document: 0.2.0 - * - * - * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). - * https://openapi-generator.tech - * Do not edit the class manually. - */ - - - -/** - * Result of testing a model in `basic` or `deep` mode. - */ -export interface ModelTestResult { - /** - * The canonical model ID that was tested. - */ - 'model_id': string; - /** - * LLM provider identifier. - */ - 'provider': string; - /** - * Whether the model responded successfully, failed, or was skipped because its provider is not configured. - */ - 'status': ModelTestResultStatusEnum; - /** - * Error details when status is \"error\". - */ - 'error_message'?: string | null; -} - -export const ModelTestResultStatusEnum = { - OK: 'ok', - ERROR: 'error', - SKIP: 'skip' -} as const; - -export type ModelTestResultStatusEnum = typeof ModelTestResultStatusEnum[keyof typeof ModelTestResultStatusEnum]; diff --git a/lib/packages/fabro-api-client/src/models/probe-failed.ts b/lib/packages/fabro-api-client/src/models/probe-failed.ts new file mode 100644 index 000000000..2d8d44653 --- /dev/null +++ b/lib/packages/fabro-api-client/src/models/probe-failed.ts @@ -0,0 +1,29 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * Fabro Run API + * HTTP API for managing Fabro workflow run executions. + * + * The version of the OpenAPI document: 0.2.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + +// May contain unused imports in some cases +// @ts-ignore +import type { LlmErrorData } from './llm-error-data'; + +export interface ProbeFailed { + 'status': ProbeFailedStatusEnum; + 'error': LlmErrorData; +} + +export const ProbeFailedStatusEnum = { + FAILED: 'failed' +} as const; + +export type ProbeFailedStatusEnum = typeof ProbeFailedStatusEnum[keyof typeof ProbeFailedStatusEnum]; diff --git a/lib/packages/fabro-api-client/src/models/probe-incorrect.ts b/lib/packages/fabro-api-client/src/models/probe-incorrect.ts new file mode 100644 index 000000000..21b547b07 --- /dev/null +++ b/lib/packages/fabro-api-client/src/models/probe-incorrect.ts @@ -0,0 +1,26 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * Fabro Run API + * HTTP API for managing Fabro workflow run executions. + * + * The version of the OpenAPI document: 0.2.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + + +export interface ProbeIncorrect { + 'status': ProbeIncorrectStatusEnum; + 'detail': string; +} + +export const ProbeIncorrectStatusEnum = { + INCORRECT: 'incorrect' +} as const; + +export type ProbeIncorrectStatusEnum = typeof ProbeIncorrectStatusEnum[keyof typeof ProbeIncorrectStatusEnum]; diff --git a/lib/packages/fabro-api-client/src/models/probe-outcome.ts b/lib/packages/fabro-api-client/src/models/probe-outcome.ts new file mode 100644 index 000000000..dad95686e --- /dev/null +++ b/lib/packages/fabro-api-client/src/models/probe-outcome.ts @@ -0,0 +1,32 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * Fabro Run API + * HTTP API for managing Fabro workflow run executions. + * + * The version of the OpenAPI document: 0.2.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + +// May contain unused imports in some cases +// @ts-ignore +import type { LlmErrorData } from './llm-error-data'; +// May contain unused imports in some cases +// @ts-ignore +import type { ProbeFailed } from './probe-failed'; +// May contain unused imports in some cases +// @ts-ignore +import type { ProbeIncorrect } from './probe-incorrect'; +// May contain unused imports in some cases +// @ts-ignore +import type { ProbePassed } from './probe-passed'; + +/** + * @type ProbeOutcome + */ +export type ProbeOutcome = { status: 'failed' } & ProbeFailed | { status: 'incorrect' } & ProbeIncorrect | { status: 'passed' } & ProbePassed; diff --git a/lib/packages/fabro-api-client/src/models/probe-passed.ts b/lib/packages/fabro-api-client/src/models/probe-passed.ts new file mode 100644 index 000000000..25b8b7c81 --- /dev/null +++ b/lib/packages/fabro-api-client/src/models/probe-passed.ts @@ -0,0 +1,25 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * Fabro Run API + * HTTP API for managing Fabro workflow run executions. + * + * The version of the OpenAPI document: 0.2.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + + +export interface ProbePassed { + 'status': ProbePassedStatusEnum; +} + +export const ProbePassedStatusEnum = { + PASSED: 'passed' +} as const; + +export type ProbePassedStatusEnum = typeof ProbePassedStatusEnum[keyof typeof ProbePassedStatusEnum]; diff --git a/lib/packages/fabro-api-client/src/models/probe-report.ts b/lib/packages/fabro-api-client/src/models/probe-report.ts new file mode 100644 index 000000000..181e70f66 --- /dev/null +++ b/lib/packages/fabro-api-client/src/models/probe-report.ts @@ -0,0 +1,38 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * Fabro Run API + * HTTP API for managing Fabro workflow run executions. + * + * The version of the OpenAPI document: 0.2.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + +// May contain unused imports in some cases +// @ts-ignore +import type { ProbeOutcome } from './probe-outcome'; +// May contain unused imports in some cases +// @ts-ignore +import type { TokenCounts } from './token-counts'; + +/** + * The lithos-llm version-1 probe report. The route is the canonical provider/model selected by the probe, or null if selection failed. Probe failures are findings in a successful HTTP response; API authentication, malformed options and server setup failures remain HTTP errors. Provider errors are redacted at the server disclosure boundary. + */ +export interface ProbeReport { + 'version': ProbeReportVersionEnum; + 'route': string | null; + 'outcome': ProbeOutcome; + 'latency_ms': number; + 'usage': TokenCounts; +} + +export const ProbeReportVersionEnum = { + NUMBER_1: 1 +} as const; + +export type ProbeReportVersionEnum = typeof ProbeReportVersionEnum[keyof typeof ProbeReportVersionEnum]; From 645c000a74b564f1c624e75674c1b82364b157f1 Mon Sep 17 00:00:00 2001 From: Scott Werner Date: Mon, 2 Nov 2026 17:10:38 -0500 Subject: [PATCH 2/6] Refresh live usage with bounded run-stream batching --- apps/fabro-web/app/lib/cross-tab-sse.ts | 19 +++- apps/fabro-web/app/lib/run-events.test.tsx | 95 +++++++++++++++- apps/fabro-web/app/lib/run-events.ts | 8 +- apps/fabro-web/app/lib/sse.ts | 24 ++++- .../app/routes/run-usage.live.test.tsx | 102 ++++++++++++++++++ 5 files changed, 238 insertions(+), 10 deletions(-) create mode 100644 apps/fabro-web/app/routes/run-usage.live.test.tsx diff --git a/apps/fabro-web/app/lib/cross-tab-sse.ts b/apps/fabro-web/app/lib/cross-tab-sse.ts index 31f0fc6a3..31e2f466a 100644 --- a/apps/fabro-web/app/lib/cross-tab-sse.ts +++ b/apps/fabro-web/app/lib/cross-tab-sse.ts @@ -116,6 +116,8 @@ interface SubscribeOptions { resyncKeys: () => KeyOrMatcher[]; fallbackSubscribe: () => () => void; debounceMs?: number; + /** Flush pending keys within this interval, even while events keep arriving. */ + maxWaitMs?: number; } export interface SubscribeToCrossTabSseOptions @@ -136,6 +138,8 @@ interface LocalSubscription { pendingKeys: Map; debounceTimer: ReturnType | null; debounceMs: number; + maxWaitMs?: number; + maxWaitTimer: ReturnType | null; resolveInvalidation: (payload: EventPayload) => EventInvalidation; resyncKeys: () => KeyOrMatcher[]; } @@ -295,6 +299,8 @@ export class CrossTabSseCoordinator { pendingKeys: new Map(), debounceTimer: null, debounceMs: options.debounceMs ?? 300, + maxWaitMs: options.maxWaitMs, + maxWaitTimer: null, resolveInvalidation: options.resolveInvalidation as (payload: EventPayload) => EventInvalidation, resyncKeys: options.resyncKeys, }; @@ -304,6 +310,7 @@ export class CrossTabSseCoordinator { options.resolveInvalidation as (payload: EventPayload) => EventInvalidation; subscription.resyncKeys = options.resyncKeys; subscription.debounceMs = options.debounceMs ?? subscription.debounceMs; + subscription.maxWaitMs = options.maxWaitMs ?? subscription.maxWaitMs; } subscription.refcount += 1; @@ -702,6 +709,12 @@ export class CrossTabSseCoordinator { return; } + if (subscription.maxWaitMs !== undefined && subscription.maxWaitTimer === null) { + subscription.maxWaitTimer = setTimeout( + () => this.flushInvalidations(subscription), + subscription.maxWaitMs, + ); + } if (subscription.debounceTimer) { clearTimeout(subscription.debounceTimer); } @@ -712,6 +725,7 @@ export class CrossTabSseCoordinator { } private flushInvalidations(subscription: LocalSubscription) { + this.clearSubscriptionTimer(subscription); if (subscription.pendingKeys.size === 0) return; const keys = [...subscription.pendingKeys.values()]; subscription.pendingKeys.clear(); @@ -940,9 +954,10 @@ export class CrossTabSseCoordinator { } private clearSubscriptionTimer(subscription: LocalSubscription) { - if (!subscription.debounceTimer) return; - clearTimeout(subscription.debounceTimer); + if (subscription.debounceTimer !== null) clearTimeout(subscription.debounceTimer); + if (subscription.maxWaitTimer !== null) clearTimeout(subscription.maxWaitTimer); subscription.debounceTimer = null; + subscription.maxWaitTimer = null; } private resetIdleState() { diff --git a/apps/fabro-web/app/lib/run-events.test.tsx b/apps/fabro-web/app/lib/run-events.test.tsx index 21d4a2f4f..b76b0ea9c 100644 --- a/apps/fabro-web/app/lib/run-events.test.tsx +++ b/apps/fabro-web/app/lib/run-events.test.tsx @@ -1,5 +1,5 @@ -import { describe, expect, test } from "bun:test"; -import type { Key } from "swr"; +import { describe, expect, jest, test } from "bun:test"; +import { unstable_serialize, type Key } from "swr"; import { loadPetriFixture } from "./petri-fixtures"; import { streamItemName } from "./petri-stream"; @@ -431,3 +431,94 @@ async function waitFor(condition: () => boolean, timeoutMs = 200) { } throw new Error("condition did not become true before timeout"); } + +for (const mode of ["coordinated", "fallback"] as const) { + describe(`${mode} live usage batching`, () => { + function subscribe() { + const source = new FakeEventSource(); + const coordinator = mode === "coordinated" + ? createCoordinator(() => source) + : createFallbackCoordinator(); + const keys: Key[] = []; + const usageKey = unstable_serialize(queryKeys.runs.usage("run-live")); + const cleanup = subscribeToRunEvents("run-live", (key) => { + keys.push(key); + return Promise.resolve(); + }, () => source, { coordinator }); + jest.advanceTimersByTime(100); + expect(source.onmessage).not.toBeNull(); + keys.length = 0; + let seq = 0; + return { + source, keys, cleanup, + close: () => { cleanup(); coordinator.close(); }, + usageRefreshes: () => keys.filter((key) => unstable_serialize(key) === usageKey).length, + progress: () => source.emit(makePetriItem(++seq, { event: "step.progress.recorded" }, { + run_id: "run-live", stage: { name: "agent" }, + })), + }; + } + + test("refreshes bursts after 300ms and continuous progress at least once per second", () => { + jest.useFakeTimers(); + const subscription = subscribe(); + try { + subscription.progress(); + subscription.progress(); + jest.advanceTimersByTime(299); + expect(subscription.usageRefreshes()).toBe(0); + jest.advanceTimersByTime(1); + expect(subscription.usageRefreshes()).toBe(1); + // No new event means no polling or leftover maximum-wait flush. + jest.advanceTimersByTime(1_000); + expect(subscription.usageRefreshes()).toBe(1); + + for (let batch = 0; batch < 2; batch += 1) { + for (let event = 0; event < 10; event += 1) { + subscription.progress(); + expect(subscription.usageRefreshes()).toBe(batch + 1); + jest.advanceTimersByTime(100); + } + expect(subscription.usageRefreshes()).toBe(batch + 2); + } + } finally { + subscription.close(); + jest.useRealTimers(); + } + }); + + test("flushes terminal usage immediately without a later duplicate refresh", () => { + jest.useFakeTimers(); + const subscription = subscribe(); + try { + subscription.progress(); + subscription.source.emit(makePlatformItem(2, { + kind: "run.lifecycle", transition: "succeeded", status: { kind: "succeeded" }, + }, { run_id: "run-live" })); + expect(subscription.usageRefreshes()).toBe(1); + const count = subscription.keys.length; + jest.advanceTimersByTime(1_000); + expect(subscription.keys).toHaveLength(count); + expect(subscription.source.closed).toBe(mode === "fallback"); + } finally { + subscription.close(); + jest.useRealTimers(); + } + }); + + test("unsubscribing discards both pending timers", () => { + jest.useFakeTimers(); + const subscription = subscribe(); + try { + subscription.progress(); + subscription.cleanup(); + jest.advanceTimersByTime(2_000); + expect(subscription.keys).toEqual([]); + expect(subscription.source.closed).toBe(true); + } finally { + subscription.close(); + jest.useRealTimers(); + } + }); + }); +} diff --git a/apps/fabro-web/app/lib/run-events.ts b/apps/fabro-web/app/lib/run-events.ts index 0a356c091..bb5567580 100644 --- a/apps/fabro-web/app/lib/run-events.ts +++ b/apps/fabro-web/app/lib/run-events.ts @@ -40,6 +40,8 @@ export interface RunEventPayload extends EventPayload { interface RunEventOptions { debounceMs?: number; + /** Keep live queries updating even when progress never pauses. */ + maxWaitMs?: number; coordinator?: CrossTabSseCoordinator; /** Called with every well-formed stream item of the run, before invalidation. */ onItem?: (item: RunStreamItem) => void; @@ -131,7 +133,7 @@ export function queryKeysForStreamItem( immediate: false, }; } - return { keys: [queryKeys.runs.state(runId), stream, ...stageKeys], immediate: false }; + return { keys: [queryKeys.runs.state(runId), queryKeys.runs.usage(runId), stream, ...stageKeys], immediate: false }; } case "control.requested": return { @@ -196,13 +198,14 @@ export function subscribeToRunEvents( runId: string, mutate: MutateFn, eventSourceFactory: (url: string) => EventSourceLike = createBrowserEventSource, - { debounceMs = 300, coordinator, onItem }: RunEventOptions = {}, + { debounceMs = 300, maxWaitMs = 1_000, coordinator, onItem }: RunEventOptions = {}, ): () => void { return subscribeToCrossTabSse({ coordinator, subscriptionKey: `run:${runId}`, mutate, debounceMs, + maxWaitMs, resyncKeys: () => resyncKeysForRun(runId), resolveInvalidation: (payload) => { if (payload.run_id !== runId) return { keys: [] }; @@ -216,6 +219,7 @@ export function subscribeToRunEvents( mutate, eventSourceFactory, debounceMs, + maxWaitMs, resolveInvalidation: (payload) => { const result = runInvalidation(runId, payload, onItem); return { ...result, close: result.immediate }; diff --git a/apps/fabro-web/app/lib/sse.ts b/apps/fabro-web/app/lib/sse.ts index 7ee4161af..2f3590038 100644 --- a/apps/fabro-web/app/lib/sse.ts +++ b/apps/fabro-web/app/lib/sse.ts @@ -29,6 +29,7 @@ export interface SharedEventSubscription { resolvers: Map; pendingKeys: Map; debounceTimer: ReturnType | null; + maxWaitTimer: ReturnType | null; } export function sseKeyDedupeId(key: KeyOrMatcher): string { @@ -47,6 +48,7 @@ export function subscribeToSharedEventSource({ resolveInvalidation, eventSourceFactory = createBrowserEventSource, debounceMs = 300, + maxWaitMs, }: { subscriptions: Map; subscriptionKey: string; @@ -55,6 +57,8 @@ export function subscribeToSharedEventSource({ resolveInvalidation: (payload: TPayload) => EventInvalidation; eventSourceFactory?: (url: string) => EventSourceLike; debounceMs?: number; + /** Flush pending keys within this interval, even while events keep arriving. */ + maxWaitMs?: number; }): () => void { let subscription = subscriptions.get(subscriptionKey); if (!subscription) { @@ -66,6 +70,7 @@ export function subscribeToSharedEventSource({ resolvers: new Map(), pendingKeys: new Map(), debounceTimer: null, + maxWaitTimer: null, }; subscriptions.set(subscriptionKey, subscription); @@ -92,7 +97,7 @@ export function subscribeToSharedEventSource({ immediate ||= Boolean(invalidation.immediate); } - queueInvalidations(current, [...keys.values()], { debounceMs, immediate }); + queueInvalidations(current, [...keys.values()], { debounceMs, maxWaitMs, immediate }); if (close) { closeSharedEventSource(subscriptions, subscriptionKey, { flushPending: true }); @@ -133,9 +138,11 @@ function queueInvalidations( keys: KeyOrMatcher[], { debounceMs, + maxWaitMs, immediate, }: { debounceMs: number; + maxWaitMs?: number; immediate?: boolean; }, ) { @@ -149,6 +156,9 @@ function queueInvalidations( return; } + if (maxWaitMs !== undefined && subscription.maxWaitTimer === null) { + subscription.maxWaitTimer = setTimeout(() => flushInvalidations(subscription), maxWaitMs); + } if (subscription.debounceTimer) { clearTimeout(subscription.debounceTimer); } @@ -159,6 +169,7 @@ function queueInvalidations( } function flushInvalidations(subscription: SharedEventSubscription) { + clearInvalidationTimers(subscription); if (subscription.pendingKeys.size === 0) return; const keys = [...subscription.pendingKeys.values()]; subscription.pendingKeys.clear(); @@ -181,13 +192,18 @@ function closeSharedEventSource( if (flushPending) { flushInvalidations(subscription); } - if (subscription.debounceTimer) { - clearTimeout(subscription.debounceTimer); - } + clearInvalidationTimers(subscription); subscription.source.close(); subscriptions.delete(subscriptionKey); } +function clearInvalidationTimers(subscription: SharedEventSubscription) { + if (subscription.debounceTimer !== null) clearTimeout(subscription.debounceTimer); + if (subscription.maxWaitTimer !== null) clearTimeout(subscription.maxWaitTimer); + subscription.debounceTimer = null; + subscription.maxWaitTimer = null; +} + function stringifyKeyValue(value: unknown): string { if (Array.isArray(value)) { return `[${value.map((item) => stringifyKeyValue(item)).join(",")}]`; diff --git a/apps/fabro-web/app/routes/run-usage.live.test.tsx b/apps/fabro-web/app/routes/run-usage.live.test.tsx new file mode 100644 index 000000000..986701f2a --- /dev/null +++ b/apps/fabro-web/app/routes/run-usage.live.test.tsx @@ -0,0 +1,102 @@ +import { expect, jest, spyOn, test } from "bun:test"; +import TestRenderer, { act } from "react-test-renderer"; +import { SWRConfig, useSWRConfig } from "swr"; +import { AxiosHeaders } from "axios"; +import type { RunUsage } from "@qltysh/fabro-api-client"; + +import { runOutputsApi } from "../lib/api-client"; +import { createCrossTabSseCoordinator } from "../lib/cross-tab-sse"; +import { subscribeToRunEvents } from "../lib/run-events"; +import type { EventSourceLike, MutateFn } from "../lib/sse"; +import { makeUsage } from "../lib/test-fixtures"; +import { makePebbleItem, setupReactTestEnv, textContent } from "../lib/test-utils"; +import RunUsageRoute from "./run-usage"; + +function usage(responses: number): RunUsage { + const model = { provider: "anthropic", model_id: "claude-sonnet-5" }; + const totals = makeUsage({ input: responses * 10_000, output: responses * 8_000 }, responses * 108_500); + const timing = { wall_time_ms: 0, inference_time_ms: 0, tool_time_ms: 0, active_time_ms: 0 }; + return { + stages: [{ + stage: { id: "agent", name: "agent" }, model, usage: totals, timing, + state: "running", started_at: "2026-10-08T16:57:29Z", + }], + by_model: [{ model, stages: 1, usage: totals }], + totals: { usage: totals, timing }, + }; +} + +for (const mode of ["coordinated", "fallback"] as const) { + test(`${mode} progress updates the mounted Usage page through its real query cache`, async () => { + jest.useFakeTimers(); + const teardown = setupReactTestEnv(); + let remote = usage(1); + const retrieve = spyOn(runOutputsApi, "retrieveRunUsage").mockImplementation(async () => ({ + data: remote, status: 200, statusText: "OK", headers: new AxiosHeaders(), + config: { headers: new AxiosHeaders() }, + })); + const source: EventSourceLike = { onmessage: null, close() {} }; + const coordinator = createCrossTabSseCoordinator({ + channelFactory: () => { + if (mode === "fallback") throw new Error("BroadcastChannel unavailable"); + return { onmessage: null, postMessage() {}, close() {} }; + }, + eventSourceFactory: () => source, + addVisibilityChangeListener: () => () => {}, + addPagehideListener: () => () => {}, + timing: { electionJitterMs: 0 }, + }); + let mutate!: MutateFn; + function CacheAccess() { + mutate = useSWRConfig().mutate as MutateFn; + return null; + } + let renderer: TestRenderer.ReactTestRenderer | undefined; + let cleanup: (() => void) | undefined; + try { + await act(async () => { + renderer = TestRenderer.create( + new Map(), dedupingInterval: 0, + revalidateOnFocus: false, revalidateOnReconnect: false, shouldRetryOnError: false, + }}> + + + , + ); + }); + expect(retrieve).toHaveBeenCalledTimes(1); + expect(textContent(renderer!.root)).toContain("$0.11"); + await act(async () => { + cleanup = subscribeToRunEvents("run-1", mutate, () => source, { coordinator }); + jest.advanceTimersByTime(100); + }); + expect(source.onmessage).not.toBeNull(); + const fetchesBeforeProgress = retrieve.mock.calls.length; + remote = usage(2); + await act(async () => { + source.onmessage!({ data: JSON.stringify(makePebbleItem( + 1, "2026-10-08T16:57:43Z", { name: "agent" }, "AssistantMessage", {}, + )) }); + jest.advanceTimersByTime(300); + }); + expect(retrieve).toHaveBeenCalledTimes(fetchesBeforeProgress + 1); + // Stage row, page total, model row and model total all use the new snapshot. + const rows = renderer!.root.findAllByType("tr").filter((row) => textContent(row).includes("$0.22")); + expect(rows).toHaveLength(4); + for (const row of rows) { + expect(textContent(row)).toContain("20.0k"); + expect(textContent(row)).toContain("16.0k"); + } + expect(textContent(renderer!.root)).not.toContain("$0.11"); + expect(remote.stages[0].state).toBe("running"); + } finally { + cleanup?.(); + coordinator.close(); + act(() => renderer?.unmount()); + retrieve.mockRestore(); + teardown(); + jest.useRealTimers(); + } + }); +} From 72ba3cc4e888f33e87f5c77d6afdc284b0f824b7 Mon Sep 17 00:00:00 2001 From: Scott Werner Date: Mon, 2 Nov 2026 17:39:16 -0500 Subject: [PATCH 3/6] Update lithos-llm to the merged probe report API --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 740a05df5..a6489eaa6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4303,7 +4303,7 @@ checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" [[package]] name = "lithos-llm" version = "0.1.0" -source = "git+https://github.com/lithoscomputer/lithos-llm?branch=main#e8fd3ecf593fa943646ad5cd7bf584fa117dfca9" +source = "git+https://github.com/lithoscomputer/lithos-llm?branch=main#2e3d3b3ff7f72b121263651f3622e38ee34f47e4" dependencies = [ "async-trait", "aws-config", From 717c5b941cad5a40d779478286c4e219c4706103 Mon Sep 17 00:00:00 2001 From: Scott Werner Date: Tue, 3 Nov 2026 10:42:04 -0500 Subject: [PATCH 4/6] Restore the single-model test table presentation --- docs/public/core-concepts/models.mdx | 8 +- lib/apps/fabro-cli/src/commands/model.rs | 69 ++++++++-- lib/apps/fabro-cli/tests/it/cmd/model_test.rs | 124 ++++++++++++++++-- 3 files changed, 173 insertions(+), 28 deletions(-) diff --git a/docs/public/core-concepts/models.mdx b/docs/public/core-concepts/models.mdx index d6e5fcb20..9cde6db00 100644 --- a/docs/public/core-concepts/models.mdx +++ b/docs/public/core-concepts/models.mdx @@ -346,9 +346,11 @@ fabro model test --provider openai This is useful for confirming connectivity after setup or when adding a new provider key. With `--model`, Fabro sends the selector directly to the server's lithos-llm -probe. It displays the canonical provider/model selected by the probe, the -outcome, elapsed milliseconds, and input/output token usage. Add `--tools` to -check a multi-turn tool exchange. +probe. Normal output retains the model table: model, provider, aliases, context, +cost, speed, and result. Catalog metadata is fetched only for presentation; +unavailable or missing metadata appears as `—` and does not change the probe's +result. Add `--tools` to check a multi-turn tool exchange. Timing and token usage +remain available in `--json`, which does not fetch catalog metadata. `fabro model test --model --json` returns the version-1 lithos-llm probe report directly: `version`, `route`, `outcome`, `latency_ms`, and `usage`. diff --git a/lib/apps/fabro-cli/src/commands/model.rs b/lib/apps/fabro-cli/src/commands/model.rs index 031b2499d..98743c4a7 100644 --- a/lib/apps/fabro-cli/src/commands/model.rs +++ b/lib/apps/fabro-cli/src/commands/model.rs @@ -168,6 +168,31 @@ fn print_models_table(models: &[Model], styles: &Styles) { ); } +/// Catalog data decorates the report; it never selects a model or changes the +/// finding. +async fn probe_display_metadata( + client: &server_client::Client, + report: &ProbeReport, + selector: &str, + provider: Option<&str>, +) -> Option { + let (provider, query) = report.route.as_ref().map_or((provider, selector), |route| { + (Some(route.provider().as_str()), route.model().as_str()) + }); + // Missing or unavailable metadata must not turn a successful probe into a + // failure. + let models = client.list_models(provider, Some(query)).await.ok()?; + let mut matches = models.into_iter().filter(|model| { + provider.is_none_or(|provider| model.provider.as_str() == provider) + && (model.id.as_str() == query + || (report.route.is_none() && model.aliases.iter().any(|alias| alias == query))) + }); + let model = matches.next()?; + // An unresolved selector can name multiple offerings. Do not invent a selected + // route. + matches.next().is_none().then_some(model) +} + fn probe_status(report: &ProbeReport) -> (Color, String, bool) { match &report.outcome { ProbeOutcome::Passed => (Color::Green, "ok".to_string(), false), @@ -276,17 +301,39 @@ async fn test_models_via_server( if json_output { println!("{}", serde_json::to_string_pretty(&report)?); } else { - let route = report - .route - .as_ref() - .map_or_else(|| selector.to_string(), ToString::to_string); - let (_, status, _) = probe_status(&report); - println!( - "{route}: {status} ({} ms; {} input, {} output tokens)", - report.latency.as_millis(), - report.usage.input, - report.usage.output - ); + let use_color = styles.use_color; + let metadata = probe_display_metadata(client, &report, selector, provider).await; + let mut row = if let Some(metadata) = metadata { + model_row(&metadata, use_color) + } else { + let (model, provider) = report + .route + .as_ref() + .map_or((selector, provider.unwrap_or("—")), |route| { + (route.model().as_str(), route.provider().as_str()) + }); + vec![ + model.cell().bold(use_color), + provider + .cell() + .foreground_color(color_if(use_color, Color::Ansi256(8))), + "—".cell(), + "—".cell().justify(Justify::Right), + "—".cell().justify(Justify::Right), + "—".cell().justify(Justify::Right), + ] + }; + let (color, status, _) = probe_status(&report); + row.push(status.cell().foreground_color(color_if(use_color, color))); + let mut title = models_title(use_color); + title.push("RESULT".cell().bold(use_color)); + let table = vec![row] + .table() + .title(title) + .color_choice(color_choice(use_color)) + .border(Border::builder().build()) + .separator(Separator::builder().build()); + println!("{}", table.display()?); } if !report.passed() { bail!("1 model(s) failed"); diff --git a/lib/apps/fabro-cli/tests/it/cmd/model_test.rs b/lib/apps/fabro-cli/tests/it/cmd/model_test.rs index 9400f3362..5dab6c633 100644 --- a/lib/apps/fabro-cli/tests/it/cmd/model_test.rs +++ b/lib/apps/fabro-cli/tests/it/cmd/model_test.rs @@ -149,7 +149,7 @@ fn assert_model_test_forwards(cli_args: &[&str], expected_query: &[(&str, &str)] String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr) ); - list.assert_calls(0); + list.assert_calls(1); test.assert(); } @@ -167,7 +167,7 @@ fn model_test_deep_remains_an_alias_for_tools() { } #[test] -fn single_model_reports_canonical_alias_without_listing() { +fn single_model_json_reports_canonical_alias_without_listing() { let context = test_context!(); let server = MockServer::start(); context.set_http_target(&server.base_url()); @@ -211,26 +211,122 @@ fn single_model_reports_canonical_alias_without_listing() { ); listing.assert_calls(0); probe.assert_calls(1); +} +#[test] +fn single_model_text_restores_the_metadata_table() { + let context = test_context!(); + let server = MockServer::start(); + context.set_http_target(&server.base_url()); + let mut model = model_json("test-model", "anthropic", false); + model["aliases"] = serde_json::json!(["tm"]); + let listing = server.mock(|when, then| { + when.method("GET") + .path("/api/v1/models") + .query_param("provider", "anthropic") + .query_param("query", "test-model"); + then.status(200).json_body(serde_json::json!({ + "data": [model_json("test-model", "other-provider", true), model], + "meta": {"has_more": false} + })); + }); + let probe = server.mock(|when, then| { + when.method("POST").path("/api/v1/models/tm/test"); + then.status(200).json_body(serde_json::json!({ + "version": 1, "route": "anthropic/test-model", "outcome": {"status": "passed"}, + "latency_ms": 12, "usage": TokenCounts::default() + })); + }); let mut cmd = context.command(); - cmd.args([ - "model", - "test", - "--model", - "claude-sonnet-4-6", - "--provider", - "anthropic", - ]); + cmd.args(["model", "test", "--model", "tm"]); fabro_snapshot!(context.filters(), cmd, @" success: true exit_code: 0 ----- stdout ----- - anthropic/claude-sonnet-4.6: ok (12 ms; 0 input, 0 output tokens) + MODEL PROVIDER ALIASES CONTEXT COST SPEED RESULT + test-model anthropic tm 131k $1.0 / $2.0 42 tok/s ok ----- stderr ----- - Testing claude-sonnet-4-6... done + Testing tm... done "); - listing.assert_calls(0); - probe.assert_calls(2); + listing.assert_calls(1); + probe.assert_calls(1); +} + +#[test] +fn single_model_text_preserves_success_when_metadata_is_missing_or_unavailable() { + for (status, models) in [ + (500, vec![]), + (200, vec![]), + (200, vec![model_json("test-model", "other-provider", true)]), + ] { + let context = test_context!(); + let server = MockServer::start(); + context.set_http_target(&server.base_url()); + let listing = server.mock(|when, then| { + when.method("GET").path("/api/v1/models"); + then.status(status).json_body(serde_json::json!({ + "data": models, "meta": {"has_more": false} + })); + }); + let probe = server.mock(|when, then| { + when.method("POST").path("/api/v1/models/tm/test"); + then.status(200).json_body(serde_json::json!({ + "version": 1, "route": "anthropic/test-model", "outcome": {"status": "passed"}, + "latency_ms": 12, "usage": TokenCounts::default() + })); + }); + let mut cmd = context.command(); + cmd.args(["model", "test", "--model", "tm"]); + insta::allow_duplicates! { + fabro_snapshot!(context.filters(), cmd, @" + success: true + exit_code: 0 + ----- stdout ----- + MODEL PROVIDER ALIASES CONTEXT COST SPEED RESULT + test-model anthropic — — — — ok + ----- stderr ----- + Testing tm... done + "); + } + listing.assert_calls(1); + probe.assert_calls(1); + } +} + +#[test] +fn single_model_text_preserves_failure_when_metadata_is_unavailable() { + let context = test_context!(); + let server = MockServer::start(); + context.set_http_target(&server.base_url()); + let listing = server.mock(|when, then| { + when.method("GET").path("/api/v1/models"); + then.status(500); + }); + let probe = server.mock(|when, then| { + when.method("POST") + .path("/api/v1/models/unknown-model/test"); + then.status(200).json_body(serde_json::json!({ + "version": 1, "route": null, + "outcome": {"status": "failed", "error": { + "kind": "model_selection", "message": "unknown model", "retry": {"type": "never"} + }}, + "latency_ms": 0, "usage": TokenCounts::default() + })); + }); + let mut cmd = context.command(); + cmd.args(["model", "test", "--model", "unknown-model"]); + fabro_snapshot!(context.filters(), cmd, @" + success: false + exit_code: 1 + ----- stdout ----- + MODEL PROVIDER ALIASES CONTEXT COST SPEED RESULT + unknown-model — — — — — error: model_selection: unknown model + ----- stderr ----- + Testing unknown-model... done + × 1 model(s) failed + "); + listing.assert_calls(1); + probe.assert_calls(1); } #[test] From 0bef5b00f0e08b6d28cc86dd44f7f94c788079a5 Mon Sep 17 00:00:00 2001 From: "fabro-releases[bot]" Date: Wed, 4 Nov 2026 09:30:18 +0000 Subject: [PATCH 5/6] Bump version to 0.407.0-nightly.0 --- Cargo.lock | 90 +++++++++++++++++++++++++++--------------------------- Cargo.toml | 2 +- 2 files changed, 46 insertions(+), 46 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b591b5d20..a75421225 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1992,7 +1992,7 @@ dependencies = [ [[package]] name = "fabro-api" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" dependencies = [ "chrono", "fabro-automation", @@ -2018,7 +2018,7 @@ dependencies = [ [[package]] name = "fabro-auth" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" dependencies = [ "anyhow", "async-trait", @@ -2043,7 +2043,7 @@ dependencies = [ [[package]] name = "fabro-automation" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" dependencies = [ "anyhow", "chrono", @@ -2064,11 +2064,11 @@ dependencies = [ [[package]] name = "fabro-build-support" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" [[package]] name = "fabro-checkpoint" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" dependencies = [ "fabro-config", "fabro-types", @@ -2076,7 +2076,7 @@ dependencies = [ [[package]] name = "fabro-cli" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" dependencies = [ "anyhow", "assert_cmd", @@ -2172,7 +2172,7 @@ dependencies = [ [[package]] name = "fabro-client" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" dependencies = [ "anyhow", "bytes", @@ -2201,7 +2201,7 @@ dependencies = [ [[package]] name = "fabro-config" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" dependencies = [ "anyhow", "chrono", @@ -2230,7 +2230,7 @@ dependencies = [ [[package]] name = "fabro-db" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" dependencies = [ "anyhow", "chrono", @@ -2243,7 +2243,7 @@ dependencies = [ [[package]] name = "fabro-dev" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" dependencies = [ "anyhow", "assert_cmd", @@ -2262,7 +2262,7 @@ dependencies = [ [[package]] name = "fabro-dot" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" dependencies = [ "fabro-template", "fabro-types", @@ -2274,7 +2274,7 @@ dependencies = [ [[package]] name = "fabro-dump" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" dependencies = [ "anyhow", "bytes", @@ -2288,7 +2288,7 @@ dependencies = [ [[package]] name = "fabro-environment" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" dependencies = [ "anyhow", "chrono", @@ -2310,7 +2310,7 @@ dependencies = [ [[package]] name = "fabro-github" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" dependencies = [ "anyhow", "async-trait", @@ -2335,7 +2335,7 @@ dependencies = [ [[package]] name = "fabro-graphviz" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" dependencies = [ "anyhow", "fabro-dot", @@ -2345,7 +2345,7 @@ dependencies = [ [[package]] name = "fabro-http" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" dependencies = [ "fabro-static", "http 1.4.0", @@ -2355,7 +2355,7 @@ dependencies = [ [[package]] name = "fabro-install" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" dependencies = [ "anyhow", "fabro-config", @@ -2372,7 +2372,7 @@ dependencies = [ [[package]] name = "fabro-interview" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" dependencies = [ "async-trait", "fabro-types", @@ -2384,7 +2384,7 @@ dependencies = [ [[package]] name = "fabro-llm" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" dependencies = [ "async-trait", "bytes", @@ -2409,7 +2409,7 @@ dependencies = [ [[package]] name = "fabro-macros" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" dependencies = [ "clap", "fabro-options-metadata", @@ -2420,7 +2420,7 @@ dependencies = [ [[package]] name = "fabro-manifest" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" dependencies = [ "anyhow", "async-trait", @@ -2449,7 +2449,7 @@ dependencies = [ [[package]] name = "fabro-mcp-server" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" dependencies = [ "anyhow", "fabro-client", @@ -2469,7 +2469,7 @@ dependencies = [ [[package]] name = "fabro-mcp-store" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" dependencies = [ "chrono", "fabro-db", @@ -2487,7 +2487,7 @@ dependencies = [ [[package]] name = "fabro-oauth" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" dependencies = [ "anyhow", "axum", @@ -2509,7 +2509,7 @@ dependencies = [ [[package]] name = "fabro-options-metadata" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" dependencies = [ "serde", "serde_json", @@ -2517,7 +2517,7 @@ dependencies = [ [[package]] name = "fabro-petri" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" dependencies = [ "anyhow", "async-trait", @@ -2572,7 +2572,7 @@ dependencies = [ [[package]] name = "fabro-proc" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" dependencies = [ "libc", "tempfile", @@ -2584,7 +2584,7 @@ dependencies = [ [[package]] name = "fabro-redact" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" dependencies = [ "aho-corasick", "pebble-coding-agent", @@ -2602,7 +2602,7 @@ dependencies = [ [[package]] name = "fabro-server" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" dependencies = [ "anyhow", "async-trait", @@ -2697,7 +2697,7 @@ dependencies = [ [[package]] name = "fabro-slack" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" dependencies = [ "fabro-http", "fabro-interview", @@ -2718,18 +2718,18 @@ dependencies = [ [[package]] name = "fabro-spa" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" dependencies = [ "rust-embed", ] [[package]] name = "fabro-static" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" [[package]] name = "fabro-store" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" dependencies = [ "bytes", "chrono", @@ -2757,7 +2757,7 @@ dependencies = [ [[package]] name = "fabro-telemetry" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" dependencies = [ "anyhow", "base64", @@ -2783,7 +2783,7 @@ dependencies = [ [[package]] name = "fabro-template" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" dependencies = [ "anyhow", "fabro-types", @@ -2797,7 +2797,7 @@ dependencies = [ [[package]] name = "fabro-test" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" dependencies = [ "anyhow", "assert_cmd", @@ -2822,7 +2822,7 @@ dependencies = [ [[package]] name = "fabro-tool" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" dependencies = [ "anyhow", "async-trait", @@ -2846,7 +2846,7 @@ dependencies = [ [[package]] name = "fabro-tracker" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" dependencies = [ "anyhow", "async-trait", @@ -2860,7 +2860,7 @@ dependencies = [ [[package]] name = "fabro-types" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" dependencies = [ "chrono", "clap", @@ -2887,7 +2887,7 @@ dependencies = [ [[package]] name = "fabro-util" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" dependencies = [ "anyhow", "console 0.15.11", @@ -2910,7 +2910,7 @@ dependencies = [ [[package]] name = "fabro-variable" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" dependencies = [ "anyhow", "chrono", @@ -2927,7 +2927,7 @@ dependencies = [ [[package]] name = "fabro-vault" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" dependencies = [ "anyhow", "chrono", @@ -2946,7 +2946,7 @@ dependencies = [ [[package]] name = "fabro-workflow" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" dependencies = [ "anyhow", "async-trait", @@ -2991,7 +2991,7 @@ dependencies = [ [[package]] name = "fabro-workflow-version" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" dependencies = [ "fabro-config", "fabro-dot", @@ -7931,7 +7931,7 @@ dependencies = [ [[package]] name = "twin-github" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" dependencies = [ "axum", "base64", diff --git a/Cargo.toml b/Cargo.toml index 41e8b188f..05431fd29 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ resolver = "2" [workspace.package] edition = "2021" -version = "0.403.0-nightly.0" +version = "0.407.0-nightly.0" license = "MIT" [workspace.dependencies] From 28c726333bcb268eb2b1d16a71520cafa100d579 Mon Sep 17 00:00:00 2001 From: Scott Werner Date: Wed, 4 Nov 2026 08:03:31 -0500 Subject: [PATCH 6/6] Retry Homebrew publication after transient push failures --- .github/workflows/installer.yml | 29 ++++++ .github/workflows/release.yml | 10 +- installer/push-homebrew.sh | 18 ++++ installer/tests/test_push_homebrew.py | 131 ++++++++++++++++++++++++++ 4 files changed, 186 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/installer.yml create mode 100644 installer/push-homebrew.sh create mode 100644 installer/tests/test_push_homebrew.py diff --git a/.github/workflows/installer.yml b/.github/workflows/installer.yml new file mode 100644 index 000000000..8aa22d040 --- /dev/null +++ b/.github/workflows/installer.yml @@ -0,0 +1,29 @@ +name: Installer + +on: + push: + branches: [main] + paths: + - "installer/**" + - ".github/workflows/installer.yml" + - ".github/workflows/release.yml" + pull_request: + branches: [main] + paths: + - "installer/**" + - ".github/workflows/installer.yml" + - ".github/workflows/release.yml" + +permissions: {} + +jobs: + homebrew-push: + name: Homebrew push retries + runs-on: ubuntu-24.04 + permissions: + contents: read + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - run: python3 -m unittest discover -s installer/tests -v diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a1e35eee2..e25b3272d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -319,7 +319,10 @@ jobs: git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" git add Formula/fabro.rb versions.json - git diff --cached --quiet || (git commit -m "Update fabro to ${TAG_NAME}" && git push) + if ! git diff --cached --quiet; then + git commit -m "Update fabro to ${TAG_NAME}" + bash ../installer/push-homebrew.sh + fi update-homebrew-nightly: name: Update Homebrew Nightly Formula @@ -368,4 +371,7 @@ jobs: git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" git add Formula/fabro-nightly.rb versions.json - git diff --cached --quiet || (git commit -m "Update fabro-nightly to ${TAG_NAME}" && git push) + if ! git diff --cached --quiet; then + git commit -m "Update fabro-nightly to ${TAG_NAME}" + bash ../installer/push-homebrew.sh + fi diff --git a/installer/push-homebrew.sh b/installer/push-homebrew.sh new file mode 100644 index 000000000..88d5834e0 --- /dev/null +++ b/installer/push-homebrew.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Run from the tap checkout after committing the formula and versions.json. +# Retry the same commit: if GitHub accepted it but lost the response, the next +# push is a no-op. Never force or rebase over another publisher's update. +for delay in 0 10 30; do + if (( delay > 0 )); then + echo "Homebrew push failed; retrying in ${delay}s" >&2 + sleep "$delay" + fi + if git push origin HEAD:refs/heads/main; then + exit 0 + fi +done + +echo "::error::Homebrew publication failed after 3 push attempts" >&2 +exit 1 diff --git a/installer/tests/test_push_homebrew.py b/installer/tests/test_push_homebrew.py new file mode 100644 index 000000000..538ba4318 --- /dev/null +++ b/installer/tests/test_push_homebrew.py @@ -0,0 +1,131 @@ +"""Exercise publication against local Git repositories; no network or credentials.""" + +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + + +SCRIPT = Path(__file__).resolve().parents[1] / "push-homebrew.sh" +REAL_GIT = shutil.which("git") + + +class HomebrewPushTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.tap = self.root / "tap" + self.remote = self.root / "remote.git" + self.bin = self.root / "bin" + self.bin.mkdir() + self.env = { + **os.environ, + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_TERMINAL_PROMPT": "0", + "REAL_GIT": REAL_GIT, + "TEST_ROOT": str(self.root), + } + for key in list(self.env): + if key.startswith(("GIT_CONFIG_KEY_", "GIT_CONFIG_VALUE_")) or key in ( + "GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_CONFIG_COUNT" + ): + del self.env[key] + self.git("init", "--bare", "--initial-branch=main", str(self.remote)) + self.git("init", "--initial-branch=main", str(self.tap)) + self.git("config", "user.name", "Test") + self.git("config", "user.email", "test@example.invalid") + self.git("remote", "add", "origin", str(self.remote)) + self.commit("old version") + self.git("push", "origin", "HEAD:main") + self.before = self.git("rev-parse", "HEAD") + self.commit("new version") + self.expected = self.git("rev-parse", "HEAD") + # Keep tests fast and record the backoff without changing production code. + self.write_executable(self.bin / "sleep", '#!/bin/sh\necho "$1" >> "$TEST_ROOT/delays"\n') + self.write_executable(self.bin / "git", '''#!/bin/sh +printf 'attempt\n' >> "$TEST_ROOT/attempts" +"$REAL_GIT" "$@" +status=$? +if [ "$status" -eq 0 ] && [ -f "$TEST_ROOT/lose-response" ]; then + rm "$TEST_ROOT/lose-response" + exit 1 +fi +exit "$status" +''') + + def git(self, *args): + result = subprocess.run( + [REAL_GIT, *args], cwd=self.tap if self.tap.exists() else self.root, + env=self.env, text=True, capture_output=True, check=True, + ) + return result.stdout.strip() + + def commit(self, value): + (self.tap / "formula").write_text(value) + self.git("add", "formula") + self.git("commit", "-m", value) + + def write_executable(self, path, content): + path.write_text(content) + path.chmod(0o755) + + def reject_pushes(self, failures): + self.write_executable(self.remote / "hooks" / "pre-receive", f'''#!/bin/sh +count=0 +if [ -f reject-count ]; then count=$(cat reject-count); fi +count=$((count + 1)) +echo "$count" > reject-count +if [ "$count" -le {failures} ]; then + echo "Internal Server Error" >&2 + exit 1 +fi +''') + + def run_push(self, success, attempts, delays): + result = subprocess.run( + ["bash", str(SCRIPT)], cwd=self.tap, text=True, capture_output=True, + env={**self.env, "PATH": str(self.bin) + os.pathsep + os.environ["PATH"]}, + timeout=10, + ) + self.assertEqual(result.returncode == 0, success, result.stderr) + self.assertEqual((self.root / "attempts").read_text().splitlines(), ["attempt"] * attempts) + delay_file = self.root / "delays" + self.assertEqual(delay_file.read_text().splitlines() if delay_file.exists() else [], delays) + return self.git("--git-dir", str(self.remote), "rev-parse", "main") + + def test_success_needs_no_retry(self): + self.assertEqual(self.run_push(True, 1, []), self.expected) + + def test_transient_server_rejection_recovers(self): + self.reject_pushes(2) + self.assertEqual(self.run_push(True, 3, ["10", "30"]), self.expected) + + def test_persistent_failure_remains_failure(self): + self.reject_pushes(99) + self.assertEqual(self.run_push(False, 3, ["10", "30"]), self.before) + + def test_lost_success_response_is_idempotent(self): + (self.root / "lose-response").touch() + self.assertEqual(self.run_push(True, 2, ["10"]), self.expected) + self.assertEqual(self.git("rev-parse", "HEAD"), self.expected) + + def test_already_published_commit_is_noop(self): + self.git("push", "origin", "HEAD:main") + self.assertEqual(self.run_push(True, 1, []), self.expected) + + def test_concurrent_publication_is_not_overwritten(self): + self.git("checkout", "--detach", self.before) + self.commit("competing version") + competing = self.git("rev-parse", "HEAD") + self.git("push", "origin", "HEAD:main") + self.git("checkout", "main") + self.assertEqual(self.run_push(False, 3, ["10", "30"]), competing) + self.assertEqual(self.git("rev-parse", "HEAD"), self.expected) + + +if __name__ == "__main__": + unittest.main()