scriptoriumadmin/fabro
1
0
Fork 0
mirror of https://github.com/fabro-sh/fabro.git synced 2026-11-15 09:11:22 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Return lithos-llm probe reports from model tests

Browse source
This commit is contained in:
Scott Werner 2026-11-02 17:09:41 -05:00
parent 9d96890cbd
commit 073f38eb9a
23 changed files with 872 additions and 550 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
apps/fabro-web/app/routes/settings-models.tsx
View file

@ -310,22 +310,19 @@ function ModelsSection({ providers }: { providers: Provider[] }) {
modelsApi.testModel(...modelOfferingTestArgs(offering)),
);
if (
result.provider !== offering.provider ||
result.model_id !== offering.id
result.route !== null &&
result.route !== `${offering.provider}/${offering.id}`
) {
outcome = {
phase: "error",
message: `Server tested unexpected offering ${result.provider}/${result.model_id}`,
phase: "error",
message: `Server tested unexpected offering ${result.route}`,
};
} else if (result.status === "ok") {
} else if (result.outcome.status === "passed") {
outcome = { phase: "ok" };
} else if (result.status === "error") {
outcome = {
phase: "error",
message: result.error_message ?? "Failed",
};
} else if (result.outcome.status === "failed") {
outcome = { phase: "error", message: result.outcome.error.message };
} else {
outcome = { phase: "error", message: "provider not configured" };
outcome = { phase: "error", message: result.outcome.detail };
}
} catch (err) {
outcome = {

124
docs/public/api-reference/fabro-api.yaml
View file

@ -5911,24 +5911,28 @@ paths:
operationId: testModel
tags: [Models]
summary: Test Model
description: Tests a model by sending a simple prompt and reporting pass/fail.
description: >-
Runs the lithos-llm probe and returns its versioned report, including the
canonical route, outcome, elapsed time and token usage. Unknown or
unavailable models and provider failures produce HTTP 200 reports with
a failed outcome. Invalid query options remain HTTP 400 errors.
parameters:
- name: id
in: path
required: true
schema:
type: string
description: The canonical model ID or an alias.
description: A model selector resolved by lithos-llm (canonical ID, alias or provider-qualified selector).
- $ref: "#/components/parameters/ModelTestProviderParam"
- $ref: "#/components/parameters/ModelTestModeParam"
- $ref: "#/components/parameters/ModelTestReasoningEffortParam"
responses:
"200":
description: Test result
description: Probe report (passed, failed or incorrect)
content:
application/json:
schema:
$ref: "#/components/schemas/ModelTestResult"
$ref: "#/components/schemas/ProbeReport"
"400":
description: Invalid test mode or reasoning effort
headers:
@ -5938,15 +5942,6 @@ paths:
application/json:
schema:
$ref: "#/components/schemas/ErrorResponse"
"404":
description: Model not found
headers:
x-request-id:
$ref: "#/components/headers/XRequestId"
content:
application/json:
schema:
$ref: "#/components/schemas/ErrorResponse"
/api/v1/providers:
get:
@ -8875,30 +8870,99 @@ components:
key. Does NOT imply authentication or model requests will succeed;
call `POST /models/{id}/test` to test usability.
ModelTestResult:
description: Result of testing a model in `basic` or `deep` mode.
ProbeReport:
description: >-
The lithos-llm version-1 probe report. The route is the canonical
provider/model selected by the probe, or null if selection failed.
Probe failures are findings in a successful HTTP response; API
authentication, malformed options and server setup failures remain HTTP errors.
Provider errors are redacted at the server disclosure boundary.
type: object
required:
- model_id
- provider
- status
required: [version, route, outcome, latency_ms, usage]
properties:
model_id:
version:
type: integer
enum: [1]
route:
type: [string, "null"]
pattern: '^[^/]+/.+$'
outcome:
$ref: "#/components/schemas/ProbeOutcome"
latency_ms:
type: integer
format: uint64
minimum: 0
usage:
$ref: "#/components/schemas/TokenCounts"
ProbeOutcome:
oneOf:
- $ref: "#/components/schemas/ProbePassed"
- $ref: "#/components/schemas/ProbeFailed"
- $ref: "#/components/schemas/ProbeIncorrect"
discriminator:
propertyName: status
mapping:
passed: "#/components/schemas/ProbePassed"
failed: "#/components/schemas/ProbeFailed"
incorrect: "#/components/schemas/ProbeIncorrect"
ProbePassed:
type: object
required: [status]
properties:
status:
type: string
enum: [passed]
ProbeFailed:
type: object
required: [status, error]
properties:
status:
type: string
enum: [failed]
error:
$ref: "#/components/schemas/LlmErrorData"
ProbeIncorrect:
type: object
required: [status, detail]
properties:
status:
type: string
enum: [incorrect]
detail:
type: string
LlmErrorData:
description: Structured lithos-llm failure, subject to the server's disclosure policy.
type: object
required: [kind, message, retry]
properties:
kind:
$ref: "#/components/schemas/LlmErrorKind"
message:
type: string
description: The canonical model ID that was tested.
example: "claude-opus-4-6"
provider:
$ref: "#/components/schemas/ProviderId"
status:
type: integer
minimum: 0
maximum: 65535
provider_code:
type: string
enum:
- ok
- error
- skip
description: Whether the model responded successfully, failed, or was skipped because its provider is not configured.
error_message:
type: ["string", "null"]
description: Error details when status is "error".
retry:
$ref: "#/components/schemas/LlmRetryClassification"
raw_data:
description: Provider response data; omitted by the model-test endpoint.
provider_retry_after_millis:
type: integer
format: uint64
minimum: 0
source_message:
type: string
description: Internal source detail; omitted by the model-test endpoint.
ModelTestMode:
description: Single-model test mode.

19
docs/public/core-concepts/models.mdx
View file

@ -344,3 +344,22 @@ fabro model test --provider openai
```
This is useful for confirming connectivity after setup or when adding a new provider key.
With `--model`, Fabro sends the selector directly to the server's lithos-llm
probe. It displays the canonical provider/model selected by the probe, the
outcome, elapsed milliseconds, and input/output token usage. Add `--tools` to
check a multi-turn tool exchange.
`fabro model test --model <selector> --json` returns the version-1 lithos-llm
probe report directly: `version`, `route`, `outcome`, `latency_ms`, and `usage`.
The outcome is `passed`, `failed` (with a structured `error`), or `incorrect`
(with a `detail` explaining the model's behavior). A selection failure has a
null route. Failed and incorrect probes print their report and exit nonzero.
This replaces the previous single-model JSON summary and `ok/error/skip` API
result.
Without `--model`, Fabro lists models and tests configured offerings with the
requested concurrency. Bulk JSON retains `total`, `failures`, `skipped`, and
`results`; each completed probe's row also contains its full `report`.
Unconfigured offerings are skipped before probing. An offering that becomes
unavailable after listing is a failed probe.

422
lib/apps/fabro-cli/src/commands/model.rs
View file

@ -1,11 +1,11 @@
use anyhow::{Context, Result, bail};
use anyhow::{Result, bail};
use cli_table::format::{Border, Justify, Separator};
use cli_table::{Cell, CellStruct, Color, Style, Table};
use fabro_api::types as api_types;
use fabro_types::{Model, ModelTestMode};
use fabro_util::terminal::Styles;
use futures::{StreamExt, stream};
use lithos_llm::catalog::ProviderId;
use lithos_llm::client::{ProbeOutcome, ProbeReport};
use serde::Serialize;
use crate::args::{ModelListArgs, ModelTestArgs, ModelsCommand};
@ -29,6 +29,8 @@ struct ModelTestRow {
detail: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
error: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
report: Option<ProbeReport>,
}
#[derive(Serialize)]
@ -44,24 +46,7 @@ struct CompletedModelTest {
model: Model,
result_color: Color,
status: String,
}
fn model_matches_selector(model: &Model, selector: &str) -> bool {
model.id.as_str() == selector || model.aliases.iter().any(|alias| alias == selector)
}
fn find_model_by_id_or_alias(
models: &[Model],
id: &str,
provider: Option<&ProviderId>,
) -> Option<Model> {
models
.iter()
.find(|model| {
provider.is_none_or(|provider| &model.provider == provider)
&& model_matches_selector(model, id)
})
.cloned()
report: Option<ProbeReport>,
}
pub(crate) async fn execute(
@ -183,34 +168,37 @@ fn print_models_table(models: &[Model], styles: &Styles) {
);
}
fn configured_model_test_status(
expected: &Model,
result: Result<api_types::ModelTestResult>,
) -> (Color, String, bool) {
match result {
Ok(resp) if resp.provider != expected.provider || resp.model_id != expected.id.as_str() => {
(
Color::Red,
format!(
"error: server tested unexpected offering {}/{}",
resp.provider, resp.model_id
),
true,
)
}
Ok(resp) if resp.status == api_types::ModelTestResultStatus::Ok => {
(Color::Green, "ok".to_string(), false)
}
Ok(resp) if resp.status == api_types::ModelTestResultStatus::Skip => (
fn probe_status(report: &ProbeReport) -> (Color, String, bool) {
match &report.outcome {
ProbeOutcome::Passed => (Color::Green, "ok".to_string(), false),
ProbeOutcome::Failed(error) => (
Color::Red,
"error: provider became unconfigured after listing".to_string(),
format!("error: {}: {}", error.kind.as_str(), error.message),
true,
),
Ok(resp) => {
let message = resp
.error_message
.unwrap_or_else(|| "unknown error".to_string());
(Color::Red, format!("error: {message}"), true)
ProbeOutcome::Incorrect { detail } => (Color::Red, format!("incorrect: {detail}"), true),
_ => (Color::Red, "error: unknown probe outcome".to_string(), true),
}
}
fn configured_model_test_status(
expected: &Model,
result: &Result<ProbeReport>,
) -> (Color, String, bool) {
match result {
Ok(report) => {
// A bulk job names an exact offering. Keep this correlation check;
// a single selector is resolved exclusively by the server's probe.
if let Some(route) = &report.route {
if route.provider() != &expected.provider || route.model() != &expected.id {
return (
Color::Red,
format!("error: server tested unexpected offering {route}"),
true,
);
}
}
probe_status(report)
}
Err(err) => (Color::Red, format!("error: {err}"), true),
}
@ -225,6 +213,7 @@ fn model_test_row_from_status(model: &Model, status: &str, result_color: Color)
result: ModelTestResultKind::Pass,
detail: None,
error: None,
report: None,
},
Color::Yellow => ModelTestRow {
model: model.id.to_string(),
@ -232,12 +221,14 @@ fn model_test_row_from_status(model: &Model, status: &str, result_color: Color)
result: ModelTestResultKind::Skip,
detail: Some(trimmed.to_string()),
error: None,
report: None,
},
_ => ModelTestRow {
model: model.id.to_string(),
provider: model.provider.clone(),
result: ModelTestResultKind::Fail,
detail: None,
report: None,
error: Some(
trimmed
.strip_prefix("error: ")
@ -265,6 +256,44 @@ async fn test_models_via_server(
let reasoning_effort = args.reasoning_effort;
let request_mode = args.tools.then_some(ModelTestMode::Deep);
if let Some(selector) = model {
let requested_provider = provider.map(ProviderId::new);
if !json_output {
eprint!("Testing {selector}...");
}
let result = client
.test_model(
selector,
requested_provider.as_ref(),
request_mode,
reasoning_effort,
)
.await;
if !json_output {
eprintln!(" done");
}
let report = result?;
if json_output {
println!("{}", serde_json::to_string_pretty(&report)?);
} else {
let route = report
.route
.as_ref()
.map_or_else(|| selector.to_string(), ToString::to_string);
let (_, status, _) = probe_status(&report);
println!(
"{route}: {status} ({} ms; {} input, {} output tokens)",
report.latency.as_millis(),
report.usage.input,
report.usage.output
);
}
if !report.passed() {
bail!("1 model(s) failed");
}
return Ok(());
}
let use_color = styles.use_color;
let mut title = models_title(use_color);
title.push("RESULT".cell().bold(use_color));
@ -274,162 +303,81 @@ async fn test_models_via_server(
let mut failures = 0u32;
let mut skipped = 0u32;
let mut skipped_providers: Vec<String> = Vec::new();
if let Some(model_id) = model {
let requested_provider = provider.map(ProviderId::new);
let listed_models = client.list_models(provider, Some(model_id)).await?;
let listed_info = find_model_by_id_or_alias(&listed_models, model_id, None);
if !json_output {
eprint!("Testing {model_id}...");
}
let has_configured_match = listed_models
.iter()
.any(|model| model.configured && model_matches_selector(model, model_id));
let result =
if requested_provider.is_none() && listed_info.is_some() && !has_configured_match {
None
} else {
Some(
client
.test_model(
model_id,
requested_provider.as_ref(),
request_mode,
reasoning_effort,
)
.await,
)
};
if !json_output {
eprintln!(" done");
}
let models_to_test = client.list_models(provider, None).await?;
if models_to_test.is_empty() {
bail!("No models found");
}
let (info, result_color, status) = match result {
None => {
let info = listed_info.with_context(|| format!("Unknown model: {model_id}"))?;
failures += 1;
skipped += 1;
(info, Color::Yellow, "not configured".to_string())
}
Some(Ok(resp)) => {
let info =
find_model_by_id_or_alias(&listed_models, &resp.model_id, Some(&resp.provider))
.with_context(|| {
format!(
"Unknown model returned by server: {}/{}",
resp.provider, resp.model_id
)
})?;
if resp.status == api_types::ModelTestResultStatus::Ok {
(info, Color::Green, "ok".to_string())
} else if resp.status == api_types::ModelTestResultStatus::Skip {
failures += 1;
skipped += 1;
(info, Color::Yellow, "not configured".to_string())
} else {
failures += 1;
let message = resp
.error_message
.unwrap_or_else(|| "unknown error".to_string());
(info, Color::Red, format!("error: {message}"))
let (configured, unconfigured): (Vec<_>, Vec<_>) = models_to_test
.into_iter()
.partition(|model| model.configured);
for info in &unconfigured {
skipped += 1;
let provider_name = info.provider.to_string();
if !skipped_providers.contains(&provider_name) {
skipped_providers.push(provider_name);
}
if json_output {
json_rows.push(model_test_row_from_status(
info,
"not configured",
Color::Yellow,
));
}
}
let mut completed = stream::iter(configured.into_iter().enumerate())
.map(|(index, info)| {
let client = client.clone();
async move {
let result = client
.test_model(
info.id.as_str(),
Some(&info.provider),
request_mode,
reasoning_effort,
)
.await;
if !json_output {
eprintln!("Testing {}... done", info.id);
}
let (result_color, status, failed) = configured_model_test_status(&info, &result);
(
CompletedModelTest {
index,
model: info,
result_color,
status,
report: result.ok(),
},
failed,
)
}
Some(Err(err)) if err.to_string().contains("Model not found") => {
bail!("Unknown model: {model_id}");
}
Some(Err(err)) => {
let info = listed_info.with_context(|| format!("Unknown model: {model_id}"))?;
failures += 1;
(info, Color::Red, format!("error: {err}"))
}
};
})
.buffer_unordered(jobs)
.collect::<Vec<_>>()
.await;
let mut row = model_row(&info, use_color);
completed.sort_by_key(|(completed, _)| completed.index);
for (completed, failed) in completed {
if failed {
failures += 1;
}
let mut row = model_row(&completed.model, use_color);
let mut json_row =
model_test_row_from_status(&completed.model, &completed.status, completed.result_color);
json_row.report = completed.report;
json_rows.push(json_row);
row.push(
status
.clone()
completed
.status
.cell()
.foreground_color(color_if(use_color, result_color)),
.foreground_color(color_if(use_color, completed.result_color)),
);
rows.push(row);
json_rows.push(model_test_row_from_status(&info, &status, result_color));
} else {
let models_to_test = client.list_models(provider, None).await?;
if models_to_test.is_empty() {
bail!("No models found");
}
let (configured, unconfigured): (Vec<_>, Vec<_>) = models_to_test
.into_iter()
.partition(|model| model.configured);
for info in &unconfigured {
skipped += 1;
let provider_name = info.provider.to_string();
if !skipped_providers.contains(&provider_name) {
skipped_providers.push(provider_name);
}
if json_output {
json_rows.push(model_test_row_from_status(
info,
"not configured",
Color::Yellow,
));
}
}
let mut completed = stream::iter(configured.into_iter().enumerate())
.map(|(index, info)| {
let client = client.clone();
async move {
let result = client
.test_model(
info.id.as_str(),
Some(&info.provider),
request_mode,
reasoning_effort,
)
.await;
if !json_output {
eprintln!("Testing {}... done", info.id);
}
let (result_color, status, failed) =
configured_model_test_status(&info, result);
(
CompletedModelTest {
index,
model: info,
result_color,
status,
},
failed,
)
}
})
.buffer_unordered(jobs)
.collect::<Vec<_>>()
.await;
completed.sort_by_key(|(completed, _)| completed.index);
for (completed, failed) in completed {
if failed {
failures += 1;
}
let mut row = model_row(&completed.model, use_color);
json_rows.push(model_test_row_from_status(
&completed.model,
&completed.status,
completed.result_color,
));
row.push(
completed
.status
.cell()
.foreground_color(color_if(use_color, completed.result_color)),
);
rows.push(row);
}
}
if json_output {
@ -458,7 +406,7 @@ async fn test_models_via_server(
println!("{}", table.display()?);
}
if skipped > 0 && model.is_none() {
if skipped > 0 {
eprintln!(
"Skipped {} model(s) (no credentials: {})",
skipped,
@ -516,7 +464,7 @@ impl Default for ModelsCommand {
mod tests {
use fabro_types::{ModelControls, ModelCosts, ModelFeatures, ModelLimits};
use lithos_llm::catalog::builtin;
use lithos_llm::types::ReasoningEffort;
use lithos_llm::types::{ErrorKind, ReasoningEffort, TokenCounts};
use super::*;
@ -657,9 +605,11 @@ mod tests {
.header("Content-Type", "application/json")
.body(
serde_json::json!({
"model_id": "test-model",
"provider": "anthropic",
"status": "ok"
"version": 1,
"route": "anthropic/test-model",
"outcome": {"status": "passed"},
"latency_ms": 12,
"usage": TokenCounts::default()
})
.to_string(),
);
@ -672,8 +622,7 @@ mod tests {
.await
.unwrap();
assert_eq!(response.status, api_types::ModelTestResultStatus::Ok);
assert!(response.error_message.is_none());
assert!(response.passed());
}
#[tokio::test]
@ -689,10 +638,11 @@ mod tests {
.header("Content-Type", "application/json")
.body(
serde_json::json!({
"model_id": "test-model",
"provider": "anthropic",
"status": "error",
"error_message": "timeout"
"version": 1,
"route": "anthropic/test-model",
"outcome": {"status": "failed", "error": {"kind": "timeout", "message": "timeout", "retry": {"type": "never"}}},
"latency_ms": 12,
"usage": TokenCounts::default()
})
.to_string(),
);
@ -710,12 +660,13 @@ mod tests {
.await
.unwrap();
assert_eq!(response.status, api_types::ModelTestResultStatus::Error);
assert_eq!(response.error_message.as_deref(), Some("timeout"));
assert!(
matches!(response.outcome, ProbeOutcome::Failed(error) if error.message == "timeout")
);
}
#[tokio::test]
async fn test_model_via_server_parses_skip() {
async fn test_model_via_server_parses_selection_failure() {
let server = httpmock::MockServer::start_async().await;
server
.mock_async(|when, then| {
@ -724,9 +675,11 @@ mod tests {
.header("Content-Type", "application/json")
.body(
serde_json::json!({
"model_id": "kimi-k2.5",
"provider": "moonshot",
"status": "skip"
"version": 1,
"route": "moonshot/kimi-k2.5",
"outcome": {"status": "failed", "error": {"kind": "model_selection", "message": "provider unavailable", "retry": {"type": "never"}}},
"latency_ms": 12,
"usage": TokenCounts::default()
})
.to_string(),
);
@ -739,8 +692,9 @@ mod tests {
.await
.unwrap();
assert_eq!(response.status, api_types::ModelTestResultStatus::Skip);
assert!(response.error_message.is_none());
assert!(
matches!(response.outcome, ProbeOutcome::Failed(error) if error.kind == ErrorKind::ModelSelection)
);
}
#[tokio::test]
@ -767,26 +721,8 @@ mod tests {
}
#[tokio::test]
async fn single_model_test_uses_server_model_metadata_for_custom_models() {
async fn single_model_test_uses_probe_identity_without_catalog_metadata() {
let server = httpmock::MockServer::start_async().await;
server
.mock_async(|when, then| {
when.method("GET")
.path("/api/v1/models")
.query_param("page[limit]", "100")
.query_param("page[offset]", "0")
.query_param("query", "venice-large");
then.status(200)
.header("Content-Type", "application/json")
.body(
serde_json::json!({
"data": [custom_model_json("venice-large", "venice")],
"meta": { "has_more": false }
})
.to_string(),
);
})
.await;
server
.mock_async(|when, then| {
when.method("POST").path("/api/v1/models/venice-large/test");
@ -794,9 +730,11 @@ mod tests {
.header("Content-Type", "application/json")
.body(
serde_json::json!({
"model_id": "venice-large",
"provider": "venice",
"status": "ok"
"version": 1,
"route": "venice/venice-large",
"outcome": {"status": "passed"},
"latency_ms": 12,
"usage": TokenCounts::default()
})
.to_string(),
);
@ -847,9 +785,11 @@ mod tests {
.header("Content-Type", "application/json")
.body(
serde_json::json!({
"model_id": "portable-model",
"provider": "openai",
"status": "ok"
"version": 1,
"route": "openai/portable-model",
"outcome": {"status": "passed"},
"latency_ms": 12,
"usage": TokenCounts::default()
})
.to_string(),
);
@ -864,9 +804,11 @@ mod tests {
.header("Content-Type", "application/json")
.body(
serde_json::json!({
"model_id": "portable-model",
"provider": "openrouter",
"status": "ok"
"version": 1,
"route": "openrouter/portable-model",
"outcome": {"status": "passed"},
"latency_ms": 12,
"usage": TokenCounts::default()
})
.to_string(),
);

195
lib/apps/fabro-cli/tests/it/cmd/model_test.rs
View file

@ -10,6 +10,7 @@ use axum::routing::{get, post};
use axum::{Json, Router};
use fabro_test::{fabro_snapshot, test_context};
use httpmock::MockServer;
use lithos_llm::types::TokenCounts;
use tokio::net::TcpListener;
use tokio::sync::{Semaphore, oneshot};
@ -129,9 +130,11 @@ fn assert_model_test_forwards(cli_args: &[&str], expected_query: &[(&str, &str)]
then.status(200)
.header("Content-Type", "application/json")
.json_body(serde_json::json!({
"model_id": "test-model",
"provider": "anthropic",
"status": "ok"
"version": 1,
"route": "anthropic/test-model",
"outcome": {"status": "passed"},
"latency_ms": 12,
"usage": TokenCounts::default()
}));
});
@ -146,7 +149,7 @@ fn assert_model_test_forwards(cli_args: &[&str], expected_query: &[(&str, &str)]
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
list.assert();
list.assert_calls(0);
test.assert();
}
@ -164,38 +167,101 @@ fn model_test_deep_remains_an_alias_for_tools() {
}
#[test]
fn model_test_unknown_model_errors() {
fn single_model_reports_canonical_alias_without_listing() {
let context = test_context!();
let server = MockServer::start();
context.set_http_target(&server.base_url());
let listing = server.mock(|when, then| {
when.method("GET").path("/api/v1/models");
then.status(500);
});
let report = serde_json::json!({
"version": 1,
"route": "anthropic/claude-sonnet-4.6",
"outcome": {"status": "passed"},
"latency_ms": 12,
"usage": TokenCounts::default(),
});
let probe = server.mock(|when, then| {
when.method("POST")
.path("/api/v1/models/claude-sonnet-4-6/test")
.query_param("provider", "anthropic");
then.status(200).json_body(report.clone());
});
let mut cmd = context.command();
cmd.args(["model", "test", "--model", "nonexistent-model-xyz"]);
cmd.args([
"model",
"test",
"--model",
"claude-sonnet-4-6",
"--provider",
"anthropic",
"--json",
]);
let output = cmd
.assert()
.success()
.stderr("")
.get_output()
.stdout
.clone();
assert_eq!(
serde_json::from_slice::<serde_json::Value>(&output).unwrap(),
report
);
listing.assert_calls(0);
probe.assert_calls(1);
let mut cmd = context.command();
cmd.args([
"model",
"test",
"--model",
"claude-sonnet-4-6",
"--provider",
"anthropic",
]);
fabro_snapshot!(context.filters(), cmd, @"
success: false
exit_code: 1
success: true
exit_code: 0
----- stdout -----
anthropic/claude-sonnet-4.6: ok (12 ms; 0 input, 0 output tokens)
----- stderr -----
Testing nonexistent-model-xyz... done
× Unknown model: nonexistent-model-xyz
Testing claude-sonnet-4-6... done
");
listing.assert_calls(0);
probe.assert_calls(2);
}
#[test]
fn single_model_skip_exits_nonzero() {
let context = test_context!();
let mut cmd = context.command();
cmd.args(["model", "test", "--model", "gemini-3.1-pro-preview"]);
remove_provider_env(&mut cmd);
fabro_snapshot!(context.filters(), cmd, @"
success: false
exit_code: 1
----- stdout -----
MODEL PROVIDER ALIASES CONTEXT COST SPEED RESULT
gemini-3.1-pro-preview gemini gemini-pro 1m $2.0 / $12.0 85 tok/s not configured
----- stderr -----
Testing gemini-3.1-pro-preview... done
× 1 model(s) failed
");
fn single_model_preserves_failed_and_incorrect_findings() {
for outcome in [
serde_json::json!({"status": "failed", "error": {"kind": "model_selection", "message": "unknown model", "retry": {"type": "never"}}}),
serde_json::json!({"status": "incorrect", "detail": "model did not call the tool"}),
] {
let context = test_context!();
let server = MockServer::start();
context.set_http_target(&server.base_url());
let report = serde_json::json!({
"version": 1,
"route": null,
"outcome": outcome,
"latency_ms": 0,
"usage": TokenCounts::default(),
});
let probe = server.mock(|when, then| {
when.method("POST").path("/api/v1/models/test-model/test");
then.status(200).json_body(report.clone());
});
let mut cmd = context.command();
cmd.args(["model", "test", "--model", "test-model", "--json"]);
let output = cmd.assert().failure().get_output().stdout.clone();
assert_eq!(
serde_json::from_slice::<serde_json::Value>(&output).unwrap(),
report
);
probe.assert_calls(1);
}
}
#[test]
@ -224,34 +290,6 @@ fn bulk_skip_exits_zero_and_prints_summary() {
);
}
#[test]
fn json_output_includes_skipped_models() {
let context = test_context!();
let mut cmd = context.command();
cmd.args([
"model",
"test",
"--model",
"gemini-3.1-pro-preview",
"--json",
]);
remove_provider_env(&mut cmd);
let output = cmd.output().expect("failed to execute model test");
assert!(
!output.status.success(),
"expected single-model skip to exit non-zero:\nstdout:\n{}\nstderr:\n{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
let stdout = String::from_utf8_lossy(&output.stdout);
let json: serde_json::Value = serde_json::from_str(&stdout).expect("invalid JSON output");
assert_eq!(json["failures"], 1);
assert_eq!(json["skipped"], 1);
assert_eq!(json["results"][0]["result"], "skip");
}
#[test]
fn model_test_does_not_announce_unconfigured() {
let context = test_context!();
@ -267,9 +305,11 @@ fn model_test_does_not_announce_unconfigured() {
then.status(200)
.header("Content-Type", "application/json")
.json_body(serde_json::json!({
"model_id": "claude-opus-4-7",
"provider": "anthropic",
"status": "ok"
"version": 1,
"route": "anthropic/claude-opus-4-7",
"outcome": {"status": "passed"},
"latency_ms": 12,
"usage": TokenCounts::default()
}));
});
let unconfigured_test = server.mock(|when, then| {
@ -317,9 +357,11 @@ fn model_test_skipped_footer_sources_from_listing() {
then.status(200)
.header("Content-Type", "application/json")
.json_body(serde_json::json!({
"model_id": "claude-opus-4-7",
"provider": "anthropic",
"status": "ok"
"version": 1,
"route": "anthropic/claude-opus-4-7",
"outcome": {"status": "passed"},
"latency_ms": 12,
"usage": TokenCounts::default()
}));
});
@ -349,9 +391,11 @@ fn model_test_post_list_race_is_a_failure() {
then.status(200)
.header("Content-Type", "application/json")
.json_body(serde_json::json!({
"model_id": "claude-opus-4-7",
"provider": "anthropic",
"status": "skip"
"version": 1,
"route": "anthropic/claude-opus-4-7",
"outcome": {"status": "failed", "error": {"kind": "model_selection", "message": "provider unavailable", "retry": {"type": "never"}}},
"latency_ms": 12,
"usage": TokenCounts::default()
}));
});
@ -367,7 +411,7 @@ fn model_test_post_list_race_is_a_failure() {
);
let stdout = String::from_utf8_lossy(&output.stdout);
let stderr = String::from_utf8_lossy(&output.stderr);
assert!(stdout.contains("provider became unconfigured after listing"));
assert!(stdout.contains("model_selection: provider unavailable"));
assert!(stderr.contains("1 model(s) failed"));
assert!(!stderr.contains("Skipped"));
}
@ -391,9 +435,11 @@ fn model_test_json_partitions_skip_and_fail() {
then.status(200)
.header("Content-Type", "application/json")
.json_body(serde_json::json!({
"model_id": "claude-opus-4-7",
"provider": "anthropic",
"status": "skip"
"version": 1,
"route": "anthropic/claude-opus-4-7",
"outcome": {"status": "failed", "error": {"kind": "model_selection", "message": "provider unavailable", "retry": {"type": "never"}}},
"latency_ms": 12,
"usage": TokenCounts::default()
}));
});
@ -420,8 +466,13 @@ fn model_test_json_partitions_skip_and_fail() {
assert_eq!(json["results"][1]["result"], "fail");
assert_eq!(
json["results"][1]["error"],
"provider became unconfigured after listing"
"model_selection: provider unavailable"
);
assert_eq!(
json["results"][1]["report"]["outcome"]["error"]["kind"],
"model_selection"
);
assert!(json["results"][0].get("report").is_none());
}
#[derive(Clone)]
@ -598,9 +649,11 @@ async fn concurrent_test_model(
state.gate.exit();
Json(serde_json::json!({
"model_id": id,
"provider": "anthropic",
"status": "ok"
"version": 1,
"route": format!("anthropic/{id}"),
"outcome": {"status": "passed"},
"latency_ms": 12,
"usage": TokenCounts::default()
}))
}

97
lib/apps/fabro-server/src/server/handler/models.rs
View file

@ -2,10 +2,11 @@ use std::collections::HashSet;
use std::sync::Arc;
use std::time::Duration;
use fabro_llm::api;
use fabro_llm::lithos_catalog::Catalog;
use fabro_llm::probe::{self, ApiKeyProbeError, ModelTestStatus};
use fabro_llm::{ModelSelectionError, api, selection};
use fabro_redact::redact_string;
use lithos_llm::client::{ProbeOptions, ProbeOutcome};
use lithos_llm::types::ReasoningEffort;
use super::super::{
@ -228,85 +229,25 @@ async fn test_model(
.into_response();
}
};
let catalog = state.catalog();
let eligible = llm_result
.provider_ids()
.into_iter()
.collect::<HashSet<_>>();
let explicit_provider = params.provider.map(ProviderId::new);
let info = if let Some(provider) = explicit_provider.as_ref() {
match selection::resolve_on_provider(&catalog, provider, &id) {
Ok(info) => info,
Err(error) => return model_selection_response(&error),
}
} else {
match selection::select(&catalog, &id, None, &eligible) {
Ok(info) => info,
Err(error) => return model_selection_response(&error),
}
};
let provider_id = info.provider.id().clone();
let model_id = info.model.id().clone();
if let Some((_, issue)) = llm_result
.auth_issues
.iter()
.find(|(provider, _)| provider == &provider_id)
{
return ApiError::bad_request(issue.to_string()).into_response();
}
if !llm_result.has_provider(&provider_id) {
return Json(serde_json::json!({
"model_id": model_id,
"provider": provider_id,
"status": "skip",
}))
.into_response();
}
let selector = params
.provider
.map_or_else(|| id.clone(), |provider| format!("{provider}/{id}"));
let mut options = ProbeOptions::new().tools(mode == ModelTestMode::Deep);
if let Some(effort) = reasoning_effort {
let capabilities = info.model.capabilities();
if !capabilities.reasoning_effort(effort).is_supported() {
let allowed = ReasoningEffort::ALL
.into_iter()
.filter(|candidate| capabilities.reasoning_effort(*candidate).is_supported())
.map(ReasoningEffort::as_str)
.collect::<Vec<_>>()
.join(", ");
return ApiError::bad_request(format!(
"model '{model_id}' does not support reasoning_effort '{effort}'; allowed values: {allowed}"
))
.into_response();
}
options = options.reasoning_effort(effort);
}
let outcome = probe::run_model_test(
&llm_result.client,
&format!("{provider_id}/{model_id}"),
mode,
reasoning_effort,
None,
)
.await;
Json(serde_json::json!({
"model_id": model_id,
"provider": provider_id,
"status": <&'static str>::from(outcome.status),
"error_message": outcome.error_message,
}))
.into_response()
}
fn model_selection_response(error: &ModelSelectionError) -> Response {
match error {
ModelSelectionError::UnknownProvider { .. }
| ModelSelectionError::UnknownSelector { .. }
| ModelSelectionError::UnknownSelectorOnProvider { .. }
| ModelSelectionError::ProviderUsedAsModel { .. } => {
ApiError::not_found(error.to_string()).into_response()
}
ModelSelectionError::ProviderUnavailable { .. }
| ModelSelectionError::NoEligibleOffering { .. }
| ModelSelectionError::NoDefaultModel { .. } => {
ApiError::bad_request(error.to_string()).into_response()
let mut report = llm_result.client.probe(selector, options).await;
// Model selection and probe findings belong to lithos. Fabro only owns
// disclosure of provider diagnostics across its HTTP boundary.
match &mut report.outcome {
ProbeOutcome::Failed(error) => {
error.message = redact_string(&error.message);
error.provider_code = error.provider_code.as_deref().map(redact_string);
error.raw_data = None;
error.source_message = None;
}
ProbeOutcome::Incorrect { detail } => *detail = redact_string(detail),
_ => {}
}
Json(report).into_response()
}

134
lib/apps/fabro-server/src/server/tests.rs
View file

@ -5194,7 +5194,7 @@ async fn pr_test_app_with_minimal_run(
}
#[tokio::test]
async fn test_model_unknown_returns_404() {
async fn test_model_unknown_returns_a_probe_finding() {
let app = test_app_with();
let req = Request::builder()
@ -5205,11 +5205,14 @@ async fn test_model_unknown_returns_404() {
.unwrap();
let response = app.clone().oneshot(req).await.unwrap();
assert_status!(response, StatusCode::NOT_FOUND).await;
let body = response_json!(response, StatusCode::OK).await;
assert_eq!(body["outcome"]["status"], "failed");
assert_eq!(body["outcome"]["error"]["kind"], "model_selection");
assert!(body["route"].is_null());
}
#[tokio::test]
async fn test_model_explicit_provider_alias_returns_canonical_model_id_when_unavailable() {
async fn test_model_explicit_provider_unavailable_returns_a_probe_finding() {
let state = test_app_state_with_env_lookup(
default_test_server_settings(),
RunLayer::default(),
@ -5227,9 +5230,8 @@ async fn test_model_explicit_provider_alias_returns_canonical_model_id_when_unav
let response = app.oneshot(req).await.unwrap();
let body = response_json!(response, StatusCode::OK).await;
assert_eq!(body["model_id"], "claude-sonnet-5");
assert_eq!(body["provider"], "anthropic");
assert_eq!(body["status"], "skip");
assert_eq!(body["outcome"]["status"], "failed");
assert_eq!(body["outcome"]["error"]["kind"], "model_selection");
}
#[tokio::test]
@ -5253,7 +5255,9 @@ async fn test_model_unqualified_known_alias_requires_a_ready_provider() {
.await
.unwrap();
assert_status!(response, StatusCode::BAD_REQUEST).await;
let body = response_json!(response, StatusCode::OK).await;
assert_eq!(body["outcome"]["status"], "failed");
assert_eq!(body["outcome"]["error"]["kind"], "model_selection");
}
#[tokio::test]
@ -5388,30 +5392,118 @@ capabilities = {{ text = true }}
assert_eq!(filtered["data"].as_array().unwrap().len(), 1);
assert_eq!(filtered["data"][0]["provider"], "aggregator");
for (query, expected_provider) in [
("?provider=direct", "direct"),
("?provider=aggregator", "aggregator"),
("", "direct"),
for (selector, query, expected_provider) in [
("portable", "?provider=direct", "direct"),
("portable", "?provider=aggregator", "aggregator"),
("portable", "", "direct"),
(
"vendor%2Fportable-model",
"?provider=aggregator",
"aggregator",
),
] {
let response = app
.clone()
.oneshot(
Request::builder()
.method("POST")
.uri(api(&format!("/models/portable/test{query}")))
.uri(api(&format!("/models/{selector}/test{query}")))
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
let body = response_json!(response, StatusCode::OK).await;
assert_eq!(body["model_id"], "portable-model");
assert_eq!(body["provider"], expected_provider);
assert_eq!(body["status"], "ok");
assert_eq!(body["route"], format!("{expected_provider}/portable-model"));
assert_eq!(body["outcome"]["status"], "passed");
assert_eq!(body["usage"]["input"], 1);
}
direct_probe.assert_calls(2);
aggregator_probe.assert_calls(1);
aggregator_probe.assert_calls(2);
}
#[tokio::test]
async fn test_model_preserves_error_classification_and_redacts_provider_details() {
let leaked_key = "sk-proj-abcdefghijklmnopqrstuvwxyz0123456789";
let upstream = MockServer::start();
let probe = upstream.mock(|when, then| {
when.method(POST).path("/v1/chat/completions");
then.status(401).json_body(json!({
"error": {
"message": format!("invalid api key {leaked_key}"),
"code": format!("invalid_key_{leaked_key}"),
"type": "invalid_request_error"
},
"private_detail": "must not cross the API boundary"
}));
});
let state = TestAppStateBuilder::new()
.llm_overlay_toml(&acme_overlay(&upstream.base_url()))
.vault_entries([("ACME_API_KEY", leaked_key)])
.build();
let app = crate::test_support::build_test_router(state);
let response = app
.oneshot(
Request::builder()
.method("POST")
.uri(api("/models/acme-large/test?provider=acme"))
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
let body = response_json!(response, StatusCode::OK).await;
assert_eq!(body["version"], 1);
assert_eq!(body["route"], "acme/acme-large");
assert_eq!(body["outcome"]["status"], "failed");
let error = &body["outcome"]["error"];
assert_eq!(error["kind"], "authentication");
assert_eq!(error["status"], 401);
assert_eq!(error["provider"], "acme");
assert!(error.get("raw_data").is_none());
assert!(error.get("source_message").is_none());
let serialized = body.to_string();
assert!(!serialized.contains(leaked_key));
assert!(!serialized.contains("must not cross"));
assert!(serialized.contains("REDACTED"));
probe.assert_calls(1);
}
#[tokio::test]
async fn test_model_tool_probe_preserves_incorrect_outcome_and_usage() {
let upstream = MockServer::start();
let probe = upstream.mock(|when, then| {
when.method(POST).path("/v1/chat/completions");
then.status(200).json_body(json!({
"id": "chatcmpl-test", "model": "acme-large",
"choices": [{"message": {"role": "assistant", "content": "84 is even"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 12, "completion_tokens": 7, "total_tokens": 19}
}));
});
let state = TestAppStateBuilder::new()
.llm_overlay_toml(&acme_overlay(&upstream.base_url()))
.vault_entries([("ACME_API_KEY", "test-key")])
.build();
let app = crate::test_support::build_test_router(state);
let response = app
.oneshot(
Request::builder()
.method("POST")
.uri(api("/models/acme-large/test?provider=acme&mode=deep"))
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
let body = response_json!(response, StatusCode::OK).await;
assert_eq!(body["route"], "acme/acme-large");
assert_eq!(body["outcome"]["status"], "incorrect");
assert!(!body["outcome"]["detail"].as_str().unwrap().is_empty());
assert_eq!(body["usage"]["input"], 12);
assert_eq!(body["usage"]["output"], 7);
assert!(body["latency_ms"].is_u64());
probe.assert_calls(1);
}
#[tokio::test]
@ -5516,7 +5608,7 @@ protocol_options = {{ reasoning_effort_levels = true }}
let response = app.clone().oneshot(req).await.unwrap();
let body = response_json!(response, StatusCode::OK).await;
assert_eq!(body["status"], "ok");
assert_eq!(body["outcome"]["status"], "passed");
let unsupported = Request::builder()
.method("POST")
@ -5526,11 +5618,9 @@ protocol_options = {{ reasoning_effort_levels = true }}
.body(Body::empty())
.unwrap();
let response = app.oneshot(unsupported).await.unwrap();
let body = response_json!(response, StatusCode::BAD_REQUEST).await;
assert_eq!(
body["errors"][0]["detail"],
"model 'acme-reasoner' does not support reasoning_effort 'medium'; allowed values: low, high"
);
let body = response_json!(response, StatusCode::OK).await;
assert_eq!(body["outcome"]["status"], "failed");
assert_eq!(body["outcome"]["error"]["kind"], "invalid_request");
completion.assert_calls(1);
}

16
lib/apps/fabro-server/tests/it/scenario/dry_run.rs
View file

@ -58,7 +58,7 @@ async fn dry_run_serve_starts_and_runs_workflow() {
}
#[tokio::test]
async fn test_model_known_but_unavailable_returns_bad_request() {
async fn test_model_known_but_unavailable_returns_a_probe_finding() {
let app = test_app_with_no_providers();
let req = Request::builder()
@ -71,15 +71,12 @@ async fn test_model_known_but_unavailable_returns_bad_request() {
let response = app.oneshot(req).await.unwrap();
let body = response_json(
response,
StatusCode::BAD_REQUEST,
StatusCode::OK,
"POST /api/v1/models/claude-opus-4.6/test",
)
.await;
assert!(
body["errors"][0]["detail"]
.as_str()
.is_some_and(|detail| detail.contains("no offering on an eligible provider"))
);
assert_eq!(body["outcome"]["status"], "failed");
assert_eq!(body["outcome"]["error"]["kind"], "model_selection");
}
#[tokio::test]
@ -94,12 +91,13 @@ async fn test_model_unknown_via_full_router() {
.unwrap();
let response = app.oneshot(req).await.unwrap();
response_status(
let body = response_json(
response,
StatusCode::NOT_FOUND,
StatusCode::OK,
"POST /api/v1/models/nonexistent-model-xyz/test",
)
.await;
assert_eq!(body["outcome"]["error"]["kind"], "model_selection");
}
#[tokio::test]

34
lib/components/fabro-llm/src/probe.rs
View file

@ -4,10 +4,8 @@ use std::sync::Arc;
use std::time::Duration;
use fabro_auth::ApiKeyCredentialSource;
use fabro_types::ModelTestMode;
use lithos_llm::catalog::{Catalog, ProviderId};
use lithos_llm::client::{Client, ProbeOptions, ProbeOutcome};
use lithos_llm::types::ReasoningEffort;
use strum::IntoStaticStr;
use crate::client::{ClientOptions, LlmSetupError, build_client};
@ -43,25 +41,16 @@ impl ModelTestOutcome {
}
}
/// Probes `selector` (a `provider/model` route or bare selector) in `mode`.
///
/// `Basic` asks for one word; `Deep` runs a two-step tool exchange and checks
/// the total. The probe resolves, authenticates, and retries exactly as a
/// real request would.
pub async fn run_model_test(
/// A credential-health projection of the library's basic probe.
/// Model-test endpoints return the full library report instead.
pub async fn run_basic_probe(
client: &Client,
selector: &str,
mode: ModelTestMode,
reasoning_effort: Option<ReasoningEffort>,
timeout: Option<Duration>,
timeout: Duration,
) -> ModelTestOutcome {
let mut options = ProbeOptions::new()
.tools(mode == ModelTestMode::Deep)
.timeout(timeout.unwrap_or_else(|| Duration::from_secs(mode.timeout_secs())));
if let Some(effort) = reasoning_effort {
options = options.reasoning_effort(effort);
}
let report = client.probe(selector, options).await;
let report = client
.probe(selector, ProbeOptions::new().timeout(timeout))
.await;
match report.outcome {
ProbeOutcome::Passed => ModelTestOutcome::ok(),
ProbeOutcome::Failed(data) => ModelTestOutcome::error(data.message),
@ -70,15 +59,6 @@ pub async fn run_model_test(
}
}
/// A basic probe of `selector` bounded by `timeout`.
pub async fn run_basic_probe(
client: &Client,
selector: &str,
timeout: Duration,
) -> ModelTestOutcome {
run_model_test(client, selector, ModelTestMode::Basic, None, Some(timeout)).await
}
/// Why an API key could not be probed.
#[derive(Debug, thiserror::Error)]
pub enum ApiKeyProbeError {

2
lib/foundation/fabro-api/build.rs
View file

@ -606,6 +606,8 @@ fn main() {
&[],
),
("TokenCounts", "lithos_llm::types::TokenCounts", &[]),
("ProbeReport", "lithos_llm::client::ProbeReport", &[]),
("LlmErrorData", "lithos_llm::types::ErrorData", &[]),
("Usage", "lithos_llm::types::Usage", &[]),
("UsageModelRef", "fabro_types::ModelRef", &[]),
("Speed", "lithos_llm::types::Speed", &[]),

5
lib/foundation/fabro-api/src/lib.rs
View file

@ -72,9 +72,10 @@ pub mod types {
WorkflowVersionId,
};
pub use lithos_llm::catalog::{ModelHandle, ProviderId};
pub use lithos_llm::client::ProbeReport;
pub use lithos_llm::types::{
ContentPart, Cost, CostSource, ErrorKind as LlmErrorKind, Message, ReasoningEffort,
ReasoningOutput, ResponseFormat as CompletionResponseFormat,
ContentPart, Cost, CostSource, ErrorData as LlmErrorData, ErrorKind as LlmErrorKind,
Message, ReasoningEffort, ReasoningOutput, ResponseFormat as CompletionResponseFormat,
RetryClassification as LlmRetryClassification, Role, Speed, TokenCounts,
ToolChoice as CompletionToolChoice, ToolDefinition as CompletionToolDefinition,
ToolDefinitionKind as CompletionToolDefinitionKind, Usage,

77
lib/foundation/fabro-api/tests/probe_report_round_trip.rs Normal file
View file

@ -0,0 +1,77 @@
use std::any::TypeId;
use fabro_api::types;
use lithos_llm::client::ProbeReport;
use lithos_llm::types::{ErrorData, TokenCounts};
use serde_json::{Value, json};
fn validator(name: &str) -> jsonschema::Validator {
let yaml: serde_yaml::Value = serde_yaml::from_str(include_str!(
"../../../../docs/public/api-reference/fabro-api.yaml"
))
.unwrap();
let mut spec = serde_json::to_value(yaml).unwrap();
spec["$ref"] = json!(format!("#/components/schemas/{name}"));
jsonschema::validator_for(&spec).unwrap()
}
#[test]
fn probe_api_reuses_library_types_and_preserves_every_outcome() {
assert_eq!(
TypeId::of::<types::ProbeReport>(),
TypeId::of::<ProbeReport>()
);
assert_eq!(
TypeId::of::<types::LlmErrorData>(),
TypeId::of::<ErrorData>()
);
let error = json!({
"kind": "authentication", "message": "invalid key", "provider": "acme",
"status": 401, "provider_code": "invalid_key", "retry": {"type": "never"},
"raw_data": {"error": "detail"}, "source_message": "source",
"provider_retry_after_millis": 12
});
let parsed: types::LlmErrorData = serde_json::from_value(error.clone()).unwrap();
assert_eq!(serde_json::to_value(parsed).unwrap(), error);
assert!(validator("LlmErrorData").is_valid(&error));
let schema = validator("ProbeReport");
for (route, outcome) in [
(json!("acme/vendor/model"), json!({"status": "passed"})),
(Value::Null, json!({"status": "failed", "error": error})),
(
json!("acme/model"),
json!({"status": "incorrect", "detail": "no tool call"}),
),
] {
let value = json!({
"version": 1, "route": route, "outcome": outcome, "latency_ms": 123,
"usage": TokenCounts { input: 12, output: 7, ..TokenCounts::default() }
});
let report: types::ProbeReport = serde_json::from_value(value.clone()).unwrap();
assert_eq!(serde_json::to_value(report).unwrap(), value);
assert!(schema.is_valid(&value), "{value}");
}
}
#[test]
fn probe_schema_and_rust_reject_unsupported_versions_routes_and_outcomes() {
let schema = validator("ProbeReport");
let valid = json!({
"version": 1, "route": "acme/model", "outcome": {"status": "passed"},
"latency_ms": 0, "usage": TokenCounts::default()
});
for (field, invalid) in [
("version", json!(2)),
("route", json!("model")),
("route", json!("acme/")),
("outcome", json!({"status": "skip"})),
("outcome", json!({"status": "failed"})),
("latency_ms", json!(-1)),
] {
let mut value = valid.clone();
value[field] = invalid;
assert!(!schema.is_valid(&value), "{value}");
assert!(serde_json::from_value::<types::ProbeReport>(value).is_err());
}
}

3
lib/foundation/fabro-client/src/client.rs
View file

@ -20,6 +20,7 @@ use fabro_util::exit::{ErrorExt, ExitClass};
use futures::future::BoxFuture;
use futures::{Stream, StreamExt};
use lithos_llm::catalog::ProviderId;
use lithos_llm::client::ProbeReport;
use lithos_llm::types::ReasoningEffort;
use serde::Deserialize;
use tokio::sync::Mutex;
@ -912,7 +913,7 @@ impl Client {
provider: Option<&ProviderId>,
mode: Option<ModelTestMode>,
reasoning_effort: Option<ReasoningEffort>,
) -> Result<types::ModelTestResult> {
) -> Result<ProbeReport> {
let response = self
.send_api(|client| async move {
let mut request = client.test_model().id(id.to_string());

7
lib/packages/fabro-api-client/src/.openapi-generator/FILES generated
View file

@ -214,6 +214,7 @@ models/interview-option.ts
models/interview-provider-settings.ts
models/interview-question-record.ts
models/link-run-pull-request-request.ts
models/llm-error-data.ts
models/llm-output-kind.ts
models/llm-retry-classification-after.ts
models/llm-retry-classification-never.ts
@ -252,7 +253,6 @@ models/model-handle.ts
models/model-limits.ts
models/model-reference.ts
models/model-test-mode.ts
models/model-test-result.ts
models/model-usage.ts
models/model.ts
models/none-run-target.ts
@ -321,6 +321,11 @@ models/principal-user.ts
models/principal-webhook.ts
models/principal-worker.ts
models/principal.ts
models/probe-failed.ts
models/probe-incorrect.ts
models/probe-outcome.ts
models/probe-passed.ts
models/probe-report.ts
models/project-namespace.ts
models/provider-credential-test-request.ts
models/provider-credential-test-response.ts

24
lib/packages/fabro-api-client/src/api/models-api.ts generated
View file

@ -26,10 +26,10 @@ import type { ErrorResponse } from '../models';
// @ts-ignore
import type { ModelTestMode } from '../models';
// @ts-ignore
import type { ModelTestResult } from '../models';
// @ts-ignore
import type { PaginatedModelList } from '../models';
// @ts-ignore
import type { ProbeReport } from '../models';
// @ts-ignore
import type { ProviderCredentialTestRequest } from '../models';
// @ts-ignore
import type { ProviderCredentialTestResponse } from '../models';
@ -137,9 +137,9 @@ export const ModelsApiAxiosParamCreator = function (configuration?: Configuratio
};
},
/**
* Tests a model by sending a simple prompt and reporting pass/fail.
* Runs the lithos-llm probe and returns its versioned report, including the canonical route, outcome, elapsed time and token usage. Unknown or unavailable models and provider failures produce HTTP 200 reports with a failed outcome. Invalid query options remain HTTP 400 errors.
* @summary Test Model
* @param {string} id The canonical model ID or an alias.
* @param {string} id A model selector resolved by lithos-llm (canonical ID, alias or provider-qualified selector).
* @param {string} [provider] Pin the test to this provider\&#39;s offering. When omitted, the server selects among ready providers by catalog priority.
* @param {ModelTestMode} [mode] Test mode for the single-model test endpoint. Defaults to &#x60;basic&#x60;.
* @param {ReasoningEffort} [reasoningEffort] Optional native reasoning-effort level for the model test.
@ -310,16 +310,16 @@ export const ModelsApiFp = function(configuration?: Configuration) {
return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, localVarOperationServerBasePath || basePath);
},
/**
* Tests a model by sending a simple prompt and reporting pass/fail.
* Runs the lithos-llm probe and returns its versioned report, including the canonical route, outcome, elapsed time and token usage. Unknown or unavailable models and provider failures produce HTTP 200 reports with a failed outcome. Invalid query options remain HTTP 400 errors.
* @summary Test Model
* @param {string} id The canonical model ID or an alias.
* @param {string} id A model selector resolved by lithos-llm (canonical ID, alias or provider-qualified selector).
* @param {string} [provider] Pin the test to this provider\&#39;s offering. When omitted, the server selects among ready providers by catalog priority.
* @param {ModelTestMode} [mode] Test mode for the single-model test endpoint. Defaults to &#x60;basic&#x60;.
* @param {ReasoningEffort} [reasoningEffort] Optional native reasoning-effort level for the model test.
* @param {*} [options] Override http request option.
* @throws {RequiredError}
*/
async testModel(id: string, provider?: string, mode?: ModelTestMode, reasoningEffort?: ReasoningEffort, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise<ModelTestResult>> {
async testModel(id: string, provider?: string, mode?: ModelTestMode, reasoningEffort?: ReasoningEffort, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise<ProbeReport>> {
const localVarAxiosArgs = await localVarAxiosParamCreator.testModel(id, provider, mode, reasoningEffort, options);
const localVarOperationServerIndex = configuration?.serverIndex ?? 0;
const localVarOperationServerBasePath = operationServerMap['ModelsApi.testModel']?.[localVarOperationServerIndex]?.url;
@ -383,16 +383,16 @@ export const ModelsApiFactory = function (configuration?: Configuration, basePat
return localVarFp.listProviders(options).then((request) => request(axios, basePath));
},
/**
* Tests a model by sending a simple prompt and reporting pass/fail.
* Runs the lithos-llm probe and returns its versioned report, including the canonical route, outcome, elapsed time and token usage. Unknown or unavailable models and provider failures produce HTTP 200 reports with a failed outcome. Invalid query options remain HTTP 400 errors.
* @summary Test Model
* @param {string} id The canonical model ID or an alias.
* @param {string} id A model selector resolved by lithos-llm (canonical ID, alias or provider-qualified selector).
* @param {string} [provider] Pin the test to this provider\&#39;s offering. When omitted, the server selects among ready providers by catalog priority.
* @param {ModelTestMode} [mode] Test mode for the single-model test endpoint. Defaults to &#x60;basic&#x60;.
* @param {ReasoningEffort} [reasoningEffort] Optional native reasoning-effort level for the model test.
* @param {*} [options] Override http request option.
* @throws {RequiredError}
*/
testModel(id: string, provider?: string, mode?: ModelTestMode, reasoningEffort?: ReasoningEffort, options?: RawAxiosRequestConfig): AxiosPromise<ModelTestResult> {
testModel(id: string, provider?: string, mode?: ModelTestMode, reasoningEffort?: ReasoningEffort, options?: RawAxiosRequestConfig): AxiosPromise<ProbeReport> {
return localVarFp.testModel(id, provider, mode, reasoningEffort, options).then((request) => request(axios, basePath));
},
/**
@ -447,9 +447,9 @@ export class ModelsApi extends BaseAPI {
}
/**
* Tests a model by sending a simple prompt and reporting pass/fail.
* Runs the lithos-llm probe and returns its versioned report, including the canonical route, outcome, elapsed time and token usage. Unknown or unavailable models and provider failures produce HTTP 200 reports with a failed outcome. Invalid query options remain HTTP 400 errors.
* @summary Test Model
* @param {string} id The canonical model ID or an alias.
* @param {string} id A model selector resolved by lithos-llm (canonical ID, alias or provider-qualified selector).
* @param {string} [provider] Pin the test to this provider\&#39;s offering. When omitted, the server selects among ready providers by catalog priority.
* @param {ModelTestMode} [mode] Test mode for the single-model test endpoint. Defaults to &#x60;basic&#x60;.
* @param {ReasoningEffort} [reasoningEffort] Optional native reasoning-effort level for the model test.

7
lib/packages/fabro-api-client/src/models/index.ts generated
View file

@ -184,6 +184,7 @@ export * from './interview-option';
export * from './interview-provider-settings';
export * from './interview-question-record';
export * from './link-run-pull-request-request';
export * from './llm-error-data';
export * from './llm-output-kind';
export * from './llm-retry-classification';
export * from './llm-retry-classification-after';
@ -223,7 +224,6 @@ export * from './model-handle';
export * from './model-limits';
export * from './model-reference';
export * from './model-test-mode';
export * from './model-test-result';
export * from './model-usage';
export * from './none-run-target';
export * from './notification-provider-settings';
@ -291,6 +291,11 @@ export * from './principal-system';
export * from './principal-user';
export * from './principal-webhook';
export * from './principal-worker';
export * from './probe-failed';
export * from './probe-incorrect';
export * from './probe-outcome';
export * from './probe-passed';
export * from './probe-report';
export * from './project-namespace';
export * from './provider';
export * from './provider-credential-test-request';

42
lib/packages/fabro-api-client/src/models/llm-error-data.ts generated Normal file
View file

@ -0,0 +1,42 @@
/* tslint:disable */
/* eslint-disable */
/**
* Fabro Run API
* HTTP API for managing Fabro workflow run executions.
*
* The version of the OpenAPI document: 0.2.0
*
*
* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
* https://openapi-generator.tech
* Do not edit the class manually.
*/
// May contain unused imports in some cases
// @ts-ignore
import type { LlmRetryClassification } from './llm-retry-classification';
/**
* Structured lithos-llm failure, subject to the server\'s disclosure policy.
*/
export interface LlmErrorData {
/**
* The model layer\'s stable failure category. Known values are `configuration`, `model_selection`, `authentication`, `access_denied`, `not_found`, `invalid_request`, `context_length`, `rate_limit`, `quota_exceeded`, `content_filter`, `server`, `provider`, `network`, `timeout`, `stream_decode`, `response_decode`, `resource_limit`, `middleware`, and `cancelled`. A category written by a newer model layer is carried as its own spelling.
*/
'kind': string;
'message': string;
/**
* LLM provider identifier.
*/
'provider'?: string;
'status'?: number;
'provider_code'?: string;
'retry': LlmRetryClassification;
'raw_data'?: any;
'provider_retry_after_millis'?: number;
/**
* Internal source detail; omitted by the model-test endpoint.
*/
'source_message'?: string;
}

45
lib/packages/fabro-api-client/src/models/model-test-result.ts generated
View file

@ -1,45 +0,0 @@
/* tslint:disable */
/* eslint-disable */
/**
* Fabro Run API
* HTTP API for managing Fabro workflow run executions.
*
* The version of the OpenAPI document: 0.2.0
*
*
* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
* https://openapi-generator.tech
* Do not edit the class manually.
*/
/**
* Result of testing a model in `basic` or `deep` mode.
*/
export interface ModelTestResult {
/**
* The canonical model ID that was tested.
*/
'model_id': string;
/**
* LLM provider identifier.
*/
'provider': string;
/**
* Whether the model responded successfully, failed, or was skipped because its provider is not configured.
*/
'status': ModelTestResultStatusEnum;
/**
* Error details when status is \"error\".
*/
'error_message'?: string | null;
}
export const ModelTestResultStatusEnum = {
OK: 'ok',
ERROR: 'error',
SKIP: 'skip'
} as const;
export type ModelTestResultStatusEnum = typeof ModelTestResultStatusEnum[keyof typeof ModelTestResultStatusEnum];

29
lib/packages/fabro-api-client/src/models/probe-failed.ts generated Normal file
View file

@ -0,0 +1,29 @@
/* tslint:disable */
/* eslint-disable */
/**
* Fabro Run API
* HTTP API for managing Fabro workflow run executions.
*
* The version of the OpenAPI document: 0.2.0
*
*
* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
* https://openapi-generator.tech
* Do not edit the class manually.
*/
// May contain unused imports in some cases
// @ts-ignore
import type { LlmErrorData } from './llm-error-data';
export interface ProbeFailed {
'status': ProbeFailedStatusEnum;
'error': LlmErrorData;
}
export const ProbeFailedStatusEnum = {
FAILED: 'failed'
} as const;
export type ProbeFailedStatusEnum = typeof ProbeFailedStatusEnum[keyof typeof ProbeFailedStatusEnum];

26
lib/packages/fabro-api-client/src/models/probe-incorrect.ts generated Normal file
View file

@ -0,0 +1,26 @@
/* tslint:disable */
/* eslint-disable */
/**
* Fabro Run API
* HTTP API for managing Fabro workflow run executions.
*
* The version of the OpenAPI document: 0.2.0
*
*
* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
* https://openapi-generator.tech
* Do not edit the class manually.
*/
export interface ProbeIncorrect {
'status': ProbeIncorrectStatusEnum;
'detail': string;
}
export const ProbeIncorrectStatusEnum = {
INCORRECT: 'incorrect'
} as const;
export type ProbeIncorrectStatusEnum = typeof ProbeIncorrectStatusEnum[keyof typeof ProbeIncorrectStatusEnum];

32
lib/packages/fabro-api-client/src/models/probe-outcome.ts generated Normal file
View file

@ -0,0 +1,32 @@
/* tslint:disable */
/* eslint-disable */
/**
* Fabro Run API
* HTTP API for managing Fabro workflow run executions.
*
* The version of the OpenAPI document: 0.2.0
*
*
* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
* https://openapi-generator.tech
* Do not edit the class manually.
*/
// May contain unused imports in some cases
// @ts-ignore
import type { LlmErrorData } from './llm-error-data';
// May contain unused imports in some cases
// @ts-ignore
import type { ProbeFailed } from './probe-failed';
// May contain unused imports in some cases
// @ts-ignore
import type { ProbeIncorrect } from './probe-incorrect';
// May contain unused imports in some cases
// @ts-ignore
import type { ProbePassed } from './probe-passed';
/**
* @type ProbeOutcome
*/
export type ProbeOutcome = { status: 'failed' } & ProbeFailed | { status: 'incorrect' } & ProbeIncorrect | { status: 'passed' } & ProbePassed;

25
lib/packages/fabro-api-client/src/models/probe-passed.ts generated Normal file
View file

@ -0,0 +1,25 @@
/* tslint:disable */
/* eslint-disable */
/**
* Fabro Run API
* HTTP API for managing Fabro workflow run executions.
*
* The version of the OpenAPI document: 0.2.0
*
*
* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
* https://openapi-generator.tech
* Do not edit the class manually.
*/
export interface ProbePassed {
'status': ProbePassedStatusEnum;
}
export const ProbePassedStatusEnum = {
PASSED: 'passed'
} as const;
export type ProbePassedStatusEnum = typeof ProbePassedStatusEnum[keyof typeof ProbePassedStatusEnum];

38
lib/packages/fabro-api-client/src/models/probe-report.ts generated Normal file
View file

@ -0,0 +1,38 @@
/* tslint:disable */
/* eslint-disable */
/**
* Fabro Run API
* HTTP API for managing Fabro workflow run executions.
*
* The version of the OpenAPI document: 0.2.0
*
*
* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
* https://openapi-generator.tech
* Do not edit the class manually.
*/
// May contain unused imports in some cases
// @ts-ignore
import type { ProbeOutcome } from './probe-outcome';
// May contain unused imports in some cases
// @ts-ignore
import type { TokenCounts } from './token-counts';
/**
* The lithos-llm version-1 probe report. The route is the canonical provider/model selected by the probe, or null if selection failed. Probe failures are findings in a successful HTTP response; API authentication, malformed options and server setup failures remain HTTP errors. Provider errors are redacted at the server disclosure boundary.
*/
export interface ProbeReport {
'version': ProbeReportVersionEnum;
'route': string | null;
'outcome': ProbeOutcome;
'latency_ms': number;
'usage': TokenCounts;
}
export const ProbeReportVersionEnum = {
NUMBER_1: 1
} as const;
export type ProbeReportVersionEnum = typeof ProbeReportVersionEnum[keyof typeof ProbeReportVersionEnum];