refactor(cli): move model CLI out of fabro-llm

Move the model command surface into fabro-cli and delete the dead
fabro-llm CLI module now that prompt/chat/model CLI entrypoints are gone.
This also removes the now-unused fabro-llm CLI-only dependencies.
This commit is contained in:
Bryan Helmkamp 2026-04-05 14:26:05 -04:00
parent b2ef792a85
commit 6af4943f4c
No known key found for this signature in database
8 changed files with 795 additions and 1770 deletions

6
Cargo.lock generated
View file

@ -1714,10 +1714,6 @@ dependencies = [
"async-trait",
"base64",
"bytes",
"clap",
"cli-table",
"dialoguer",
"fabro-api",
"fabro-macros",
"fabro-model",
"fabro-test",
@ -1725,9 +1721,7 @@ dependencies = [
"futures",
"http",
"httpmock",
"indicatif",
"insta",
"progenitor-client",
"rand 0.8.5",
"reqwest 0.13.2",
"serde",

View file

@ -4,7 +4,6 @@ use std::path::PathBuf;
use clap::{Args, Subcommand, ValueEnum};
use fabro_agent::cli::AgentArgs;
use fabro_graphviz::render::GraphFormat;
use fabro_llm::cli::ModelsCommand;
pub(crate) const LONG_VERSION: &str = concat!(
env!("CARGO_PKG_VERSION"),
@ -729,6 +728,35 @@ impl RunsCommands {
}
}
#[derive(Subcommand)]
pub(crate) enum ModelsCommand {
/// List available models
List {
/// Filter by provider
#[arg(short, long)]
provider: Option<String>,
/// Search for models matching this string
#[arg(short, long)]
query: Option<String>,
},
/// Test model availability by sending a simple prompt
Test {
/// Filter by provider
#[arg(short, long)]
provider: Option<String>,
/// Test a specific model
#[arg(short, long)]
model: Option<String>,
/// Run a multi-turn tool-use test (catches reasoning round-trip bugs)
#[arg(long)]
deep: bool,
},
}
#[derive(Subcommand)]
pub(crate) enum Commands {
/// Run an agentic coding session

View file

@ -1,10 +1,42 @@
use anyhow::Result;
use fabro_llm::cli::{ModelsCommand, run_models};
use anyhow::{Context, Result, anyhow, bail};
use cli_table::format::{Border, Justify, Separator};
use cli_table::{Cell, CellStruct, Color, Style, Table};
use fabro_api::{self, types as api_types};
use fabro_model::{Catalog, Model, Provider};
use fabro_util::terminal::Styles;
use serde::Serialize;
use serde::de::DeserializeOwned;
use crate::args::GlobalArgs;
use crate::args::{GlobalArgs, ModelsCommand};
use crate::server_client;
use crate::user_config;
#[derive(Serialize)]
#[serde(rename_all = "snake_case")]
enum ModelTestResultKind {
Pass,
Fail,
Skip,
}
#[derive(Serialize)]
struct ModelTestRow {
model: String,
provider: Provider,
result: ModelTestResultKind,
#[serde(skip_serializing_if = "Option::is_none")]
detail: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
error: Option<String>,
}
#[derive(Serialize)]
struct ModelTestOutput {
results: Vec<ModelTestRow>,
total: usize,
failures: u32,
}
pub(crate) async fn execute(command: Option<ModelsCommand>, globals: &GlobalArgs) -> Result<()> {
let cli_settings = user_config::load_user_settings_with_globals(globals)?;
let client = match globals.server_url.as_deref() {
@ -20,3 +52,729 @@ pub(crate) async fn execute(command: Option<ModelsCommand>, globals: &GlobalArgs
run_models(command, client, globals.json).await
}
fn format_context_window(tokens: i64) -> String {
let rounded = ((tokens + 500) / 1_000) * 1_000;
if rounded >= 1_000_000 {
format!("{}m", rounded / 1_000_000)
} else if rounded >= 1_000 {
format!("{}k", rounded / 1_000)
} else {
tokens.to_string()
}
}
fn format_cost(cost: Option<f64>) -> String {
match cost {
None => "-".to_string(),
Some(c) => format!("${c:.1}"),
}
}
fn format_speed(tps: Option<f64>) -> String {
match tps {
None => "-".to_string(),
#[allow(clippy::cast_possible_truncation)]
Some(t) => format!("{} tok/s", t as i64),
}
}
fn color_if(use_color: bool, color: Color) -> Option<Color> {
if use_color { Some(color) } else { None }
}
fn color_choice(use_color: bool) -> cli_table::ColorChoice {
if use_color {
cli_table::ColorChoice::Auto
} else {
cli_table::ColorChoice::Never
}
}
fn model_row(model: &Model, use_color: bool) -> Vec<CellStruct> {
let aliases = model.aliases.join(", ");
let cost = format!(
"{} / {}",
format_cost(model.costs.input_cost_per_mtok),
format_cost(model.costs.output_cost_per_mtok),
);
vec![
model.id.clone().cell().bold(use_color),
model
.provider
.cell()
.foreground_color(color_if(use_color, Color::Ansi256(8))),
aliases
.cell()
.foreground_color(color_if(use_color, Color::Ansi256(8))),
format_context_window(model.limits.context_window)
.cell()
.justify(Justify::Right),
cost.cell().justify(Justify::Right),
format_speed(model.estimated_output_tps)
.cell()
.justify(Justify::Right)
.foreground_color(color_if(use_color, Color::Cyan)),
]
}
fn models_title(use_color: bool) -> Vec<CellStruct> {
vec![
"MODEL".cell().bold(use_color),
"PROVIDER".cell().bold(use_color),
"ALIASES".cell().bold(use_color),
"CONTEXT".cell().bold(use_color).justify(Justify::Right),
"COST".cell().bold(use_color).justify(Justify::Right),
"SPEED".cell().bold(use_color).justify(Justify::Right),
]
}
#[allow(clippy::print_stdout)]
fn print_models_table(models: &[Model], styles: &Styles) {
let use_color = styles.use_color;
let rows: Vec<Vec<CellStruct>> = models
.iter()
.map(|model| model_row(model, use_color))
.collect();
let table = rows
.table()
.title(models_title(use_color))
.color_choice(color_choice(use_color))
.border(Border::builder().build())
.separator(Separator::builder().build());
println!("{}", table.display().unwrap());
}
fn model_test_row_from_status(model: &Model, status: &str, result_color: Color) -> ModelTestRow {
let trimmed = status.trim();
match result_color {
Color::Green => ModelTestRow {
model: model.id.clone(),
provider: model.provider,
result: ModelTestResultKind::Pass,
detail: None,
error: None,
},
Color::Yellow => ModelTestRow {
model: model.id.clone(),
provider: model.provider,
result: ModelTestResultKind::Skip,
detail: Some(trimmed.to_string()),
error: None,
},
_ => ModelTestRow {
model: model.id.clone(),
provider: model.provider,
result: ModelTestResultKind::Fail,
detail: None,
error: Some(
trimmed
.strip_prefix("error: ")
.unwrap_or(trimmed)
.to_string(),
),
},
}
}
fn map_api_error<E>(err: progenitor_client::Error<E>) -> anyhow::Error
where
E: serde::Serialize + std::fmt::Debug,
{
match err {
progenitor_client::Error::ErrorResponse(response) => {
let status = response.status();
if let Ok(value) = serde_json::to_value(response.into_inner()) {
if let Some(detail) = value
.get("errors")
.and_then(serde_json::Value::as_array)
.and_then(|errors| errors.first())
.and_then(|entry| entry.get("detail"))
.and_then(serde_json::Value::as_str)
{
return anyhow!("{detail}");
}
}
anyhow!("request failed with status {status}")
}
progenitor_client::Error::UnexpectedResponse(response) => {
anyhow!("request failed with status {}", response.status())
}
other => anyhow!("{other}"),
}
}
fn convert_type<TInput, TOutput>(value: TInput) -> Result<TOutput>
where
TInput: serde::Serialize,
TOutput: DeserializeOwned,
{
serde_json::from_value(serde_json::to_value(value)?).map_err(Into::into)
}
async fn fetch_models_from_server(
client: &fabro_api::Client,
provider: Option<&str>,
query: Option<&str>,
) -> Result<Vec<Model>> {
let mut offset = 0u64;
let mut models = Vec::new();
loop {
let mut request = client.list_models().page_limit(100u64).page_offset(offset);
if let Some(provider) = provider {
request = request.provider(provider.to_string());
}
if let Some(query) = query {
request = request.query(query.to_string());
}
let response = request.send().await.map_err(map_api_error)?;
let parsed = response.into_inner();
let count = parsed.data.len() as u64;
models.extend(convert_type::<_, Vec<Model>>(parsed.data)?);
if !parsed.meta.has_more {
break;
}
offset += count;
}
Ok(models)
}
async fn test_model_via_server(
client: &fabro_api::Client,
model_id: &str,
mode: Option<api_types::ModelTestMode>,
) -> Result<api_types::ModelTestResult> {
let mut request = client.test_model().id(model_id.to_string());
if let Some(mode) = mode {
request = request.mode(mode);
}
let response = request.send().await.map_err(map_api_error)?;
Ok(response.into_inner())
}
#[allow(clippy::print_stdout, clippy::print_stderr)]
async fn test_models_via_server(
client: &fabro_api::Client,
provider: Option<&str>,
model: Option<&str>,
deep: bool,
styles: &Styles,
json_output: bool,
) -> Result<()> {
let request_mode = deep.then_some(api_types::ModelTestMode::Deep);
let use_color = styles.use_color;
let mut title = models_title(use_color);
title.push("RESULT".cell().bold(use_color));
let mut rows: Vec<Vec<CellStruct>> = Vec::new();
let mut json_rows = Vec::new();
let mut failures = 0u32;
if let Some(model_id) = model {
if !json_output {
eprint!("Testing {model_id}...");
}
let result = test_model_via_server(client, model_id, request_mode).await;
if !json_output {
eprintln!(" done");
}
let (info, result_color, status) = match result {
Ok(resp) => {
let info = Catalog::builtin()
.get(&resp.model_id)
.cloned()
.with_context(|| {
format!("Unknown model returned by server: {}", resp.model_id)
})?;
if resp.status == api_types::ModelTestResultStatus::Ok {
(info, Color::Green, "ok".to_string())
} else {
failures += 1;
let message = resp
.error_message
.unwrap_or_else(|| "unknown error".to_string());
(info, Color::Red, format!("error: {message}"))
}
}
Err(err) if err.to_string().contains("Model not found") => {
bail!("Unknown model: {model_id}");
}
Err(err) => {
let info = Catalog::builtin()
.get(model_id)
.cloned()
.with_context(|| format!("Unknown model: {model_id}"))?;
failures += 1;
(info, Color::Red, format!("error: {err}"))
}
};
let mut row = model_row(&info, use_color);
row.push(
status
.clone()
.cell()
.foreground_color(color_if(use_color, result_color)),
);
rows.push(row);
json_rows.push(model_test_row_from_status(&info, &status, result_color));
} else {
let models_to_test = fetch_models_from_server(client, provider, None).await?;
if models_to_test.is_empty() {
bail!("No models found");
}
for info in &models_to_test {
if !json_output {
eprint!("Testing {}...", info.id);
}
let result = test_model_via_server(client, &info.id, request_mode).await;
if !json_output {
eprintln!(" done");
}
let (result_color, status) = match result {
Ok(resp) if resp.status == api_types::ModelTestResultStatus::Ok => {
(Color::Green, "ok".to_string())
}
Ok(resp) => {
failures += 1;
let message = resp
.error_message
.unwrap_or_else(|| "unknown error".to_string());
(Color::Red, format!("error: {message}"))
}
Err(err) => {
failures += 1;
(Color::Red, format!("error: {err}"))
}
};
let mut row = model_row(info, use_color);
row.push(
status
.clone()
.cell()
.foreground_color(color_if(use_color, result_color)),
);
rows.push(row);
json_rows.push(model_test_row_from_status(info, &status, result_color));
}
}
if json_output {
println!(
"{}",
serde_json::to_string_pretty(&ModelTestOutput {
total: json_rows.len(),
failures,
results: json_rows,
})?
);
if failures > 0 {
bail!("{failures} model(s) failed");
}
return Ok(());
}
let table = rows
.table()
.title(title)
.color_choice(color_choice(use_color))
.border(Border::builder().build())
.separator(Separator::builder().build());
println!("{}", table.display()?);
if failures > 0 {
bail!("{failures} model(s) failed");
}
Ok(())
}
#[allow(clippy::print_stdout)]
async fn run_models(
command: Option<ModelsCommand>,
client: fabro_api::Client,
json_output: bool,
) -> Result<()> {
let command = command.unwrap_or(ModelsCommand::List {
provider: None,
query: None,
});
let styles = Styles::detect_stdout();
match command {
ModelsCommand::List { provider, query } => {
let models =
fetch_models_from_server(&client, provider.as_deref(), query.as_deref()).await?;
if json_output {
println!("{}", serde_json::to_string_pretty(&models)?);
} else {
print_models_table(&models, &styles);
}
}
ModelsCommand::Test {
provider,
model,
deep,
} => {
test_models_via_server(
&client,
provider.as_deref(),
model.as_deref(),
deep,
&styles,
json_output,
)
.await?;
}
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use fabro_model::{ModelCosts, ModelFeatures, ModelLimits};
fn test_http_client() -> reqwest::Client {
reqwest::Client::builder().no_proxy().build().unwrap()
}
fn test_api_client(base_url: &str) -> fabro_api::Client {
fabro_api::Client::new_with_client(base_url, test_http_client())
}
fn test_model_json(id: &str, provider: Provider) -> serde_json::Value {
serde_json::to_value(Model {
id: id.to_string(),
provider,
family: "test".to_string(),
display_name: format!("{id} display"),
limits: ModelLimits {
context_window: 128_000,
max_output: Some(4096),
},
training: None,
knowledge_cutoff: None,
features: ModelFeatures {
tools: true,
vision: false,
reasoning: false,
effort: false,
},
costs: ModelCosts {
input_cost_per_mtok: Some(1.0),
output_cost_per_mtok: Some(2.0),
cache_input_cost_per_mtok: None,
},
estimated_output_tps: Some(100.0),
aliases: vec!["tm".to_string()],
default: false,
})
.unwrap()
}
#[test]
fn format_context_window_millions() {
assert_eq!(format_context_window(1_000_000), "1m");
}
#[test]
fn format_context_window_thousands() {
assert_eq!(format_context_window(128_000), "128k");
}
#[test]
fn format_context_window_small() {
assert_eq!(format_context_window(400), "400");
}
#[test]
fn format_context_window_rounds_up() {
assert_eq!(format_context_window(1500), "2k");
}
#[test]
fn format_context_window_rounds_down() {
assert_eq!(format_context_window(1499), "1k");
}
#[test]
fn format_context_window_zero() {
assert_eq!(format_context_window(0), "0");
}
#[test]
fn format_cost_none() {
assert_eq!(format_cost(None), "-");
}
#[test]
fn format_cost_some() {
assert_eq!(format_cost(Some(3.0)), "$3.0");
}
#[test]
fn format_cost_fractional() {
assert_eq!(format_cost(Some(15.75)), "$15.8");
}
#[test]
fn format_speed_none() {
assert_eq!(format_speed(None), "-");
}
#[test]
fn format_speed_some() {
assert_eq!(format_speed(Some(85.5)), "85 tok/s");
}
#[tokio::test]
async fn test_model_via_server_parses_ok() {
let server = httpmock::MockServer::start_async().await;
server
.mock_async(|when, then| {
when.method("POST").path("/api/v1/models/test-model/test");
then.status(200)
.header("Content-Type", "application/json")
.body(
serde_json::json!({
"model_id": "test-model",
"status": "ok"
})
.to_string(),
);
})
.await;
let client = test_api_client(&server.url(""));
let response = test_model_via_server(&client, "test-model", None)
.await
.unwrap();
assert_eq!(response.status, api_types::ModelTestResultStatus::Ok);
assert!(response.error_message.is_none());
}
#[tokio::test]
async fn test_model_via_server_passes_mode_and_parses_error() {
let server = httpmock::MockServer::start_async().await;
server
.mock_async(|when, then| {
when.method("POST")
.path("/api/v1/models/test-model/test")
.query_param("mode", "deep");
then.status(200)
.header("Content-Type", "application/json")
.body(
serde_json::json!({
"model_id": "test-model",
"status": "error",
"error_message": "timeout"
})
.to_string(),
);
})
.await;
let client = test_api_client(&server.url(""));
let response =
test_model_via_server(&client, "test-model", Some(api_types::ModelTestMode::Deep))
.await
.unwrap();
assert_eq!(response.status, api_types::ModelTestResultStatus::Error);
assert_eq!(response.error_message.as_deref(), Some("timeout"));
}
#[tokio::test]
async fn test_model_via_server_404() {
let server = httpmock::MockServer::start_async().await;
server
.mock_async(|when, then| {
when.method("POST").path("/api/v1/models/bad-model/test");
then.status(404)
.header("Content-Type", "application/json")
.body(
serde_json::json!({
"errors": [{"status": "404", "title": "Not Found", "detail": "Model not found"}]
})
.to_string(),
);
})
.await;
let client = test_api_client(&server.url(""));
let result = test_model_via_server(&client, "bad-model", None).await;
assert!(result.is_err());
assert!(result.unwrap_err().to_string().contains("Model not found"));
}
#[tokio::test]
async fn fetch_models_from_server_parses_response() {
let server = httpmock::MockServer::start_async().await;
let mock = server
.mock_async(|when, then| {
when.method("GET")
.path("/api/v1/models")
.query_param("page[limit]", "100")
.query_param("page[offset]", "0");
then.status(200)
.header("Content-Type", "application/json")
.body(
serde_json::json!({
"data": [test_model_json("test-model", Provider::Anthropic)],
"meta": { "has_more": false }
})
.to_string(),
);
})
.await;
let client = test_api_client(&server.url(""));
let models = fetch_models_from_server(&client, None, None).await.unwrap();
mock.assert_async().await;
assert_eq!(models.len(), 1);
assert_eq!(models[0].id, "test-model");
assert_eq!(models[0].provider, Provider::Anthropic);
}
#[tokio::test]
async fn fetch_models_from_server_filters_by_provider() {
let server = httpmock::MockServer::start_async().await;
server
.mock_async(|when, then| {
when.method("GET")
.path("/api/v1/models")
.query_param("page[limit]", "100")
.query_param("page[offset]", "0")
.query_param("provider", "anthropic");
then.status(200)
.header("Content-Type", "application/json")
.body(
serde_json::json!({
"data": [test_model_json("model-a", Provider::Anthropic)],
"meta": { "has_more": false }
})
.to_string(),
);
})
.await;
let client = test_api_client(&server.url(""));
let models = fetch_models_from_server(&client, Some("anthropic"), None)
.await
.unwrap();
assert_eq!(models.len(), 1);
assert_eq!(models[0].id, "model-a");
}
#[tokio::test]
async fn fetch_models_from_server_passes_query_param() {
let server = httpmock::MockServer::start_async().await;
let mock = server
.mock_async(|when, then| {
when.method("GET")
.path("/api/v1/models")
.query_param("page[limit]", "100")
.query_param("page[offset]", "0")
.query_param("query", "sonnet");
then.status(200)
.header("Content-Type", "application/json")
.body(
serde_json::json!({
"data": [test_model_json("claude-sonnet-4-5", Provider::Anthropic)],
"meta": { "has_more": false }
})
.to_string(),
);
})
.await;
let client = test_api_client(&server.url(""));
let models = fetch_models_from_server(&client, None, Some("sonnet"))
.await
.unwrap();
mock.assert_async().await;
assert_eq!(models.len(), 1);
assert_eq!(models[0].id, "claude-sonnet-4-5");
}
#[tokio::test]
async fn fetch_models_from_server_follows_pagination() {
let server = httpmock::MockServer::start_async().await;
let first_page = server
.mock_async(|when, then| {
when.method("GET")
.path("/api/v1/models")
.query_param("page[limit]", "100")
.query_param("page[offset]", "0");
then.status(200)
.header("Content-Type", "application/json")
.body(
serde_json::json!({
"data": [test_model_json("model-a", Provider::Anthropic)],
"meta": { "has_more": true }
})
.to_string(),
);
})
.await;
let second_page = server
.mock_async(|when, then| {
when.method("GET")
.path("/api/v1/models")
.query_param("page[limit]", "100")
.query_param("page[offset]", "1");
then.status(200)
.header("Content-Type", "application/json")
.body(
serde_json::json!({
"data": [test_model_json("model-b", Provider::OpenAi)],
"meta": { "has_more": false }
})
.to_string(),
);
})
.await;
let client = test_api_client(&server.url(""));
let models = fetch_models_from_server(&client, None, None).await.unwrap();
first_page.assert_async().await;
second_page.assert_async().await;
assert_eq!(models.len(), 2);
assert_eq!(models[0].id, "model-a");
assert_eq!(models[1].id, "model-b");
}
#[tokio::test]
async fn fetch_models_from_server_error_on_failure() {
let server = httpmock::MockServer::start_async().await;
server
.mock_async(|when, then| {
when.method("GET")
.path("/api/v1/models")
.query_param("page[limit]", "100")
.query_param("page[offset]", "0");
then.status(500).body("internal error");
})
.await;
let client = test_api_client(&server.url(""));
let result = fetch_models_from_server(&client, None, None).await;
assert!(result.is_err());
}
}

View file

@ -31,13 +31,7 @@ reqwest.workspace = true
base64.workspace = true
bytes.workspace = true
tokio-util.workspace = true
cli-table.workspace = true
indicatif.workspace = true
clap.workspace = true
dialoguer.workspace = true
tracing.workspace = true
progenitor-client = "0.13"
fabro-api = { path = "../fabro-api" }
fabro-model = { path = "../fabro-model" }
fabro-util = { path = "../fabro-util" }

File diff suppressed because it is too large Load diff

View file

@ -1,4 +1,3 @@
pub mod cli;
pub mod client;
pub mod error;
pub mod generate;

View file

@ -5,8 +5,7 @@ use tokio::time;
use crate::generate::{self, GenerateParams};
use crate::tools::Tool;
use crate::types::GenerateResult;
use crate::types::ReasoningEffort;
use crate::types::{GenerateResult, ReasoningEffort};
use fabro_model::Model;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
@ -198,6 +197,7 @@ fn validate_deep_result(result: &GenerateResult) -> Result<(), String> {
#[cfg(test)]
mod tests {
use super::*;
use crate::types::ToolResult;
use crate::types::{FinishReason, Message, Response, StepResult, Usage};
use fabro_model::{ModelCosts, ModelFeatures, ModelLimits, Provider};
@ -259,10 +259,7 @@ mod tests {
#[test]
fn validate_deep_result_does_not_fail_only_for_missing_reasoning() {
let tool_results = vec![crate::types::ToolResult::success(
"call_1",
serde_json::json!(42),
)];
let tool_results = vec![ToolResult::success("call_1", serde_json::json!(42))];
let first_step = StepResult {
response: response_with_text("tool step"),
tool_results: tool_results.clone(),

View file

@ -21,11 +21,11 @@ impl HttpApi {
fn build_client(timeout: AdapterTimeout) -> reqwest::Client {
#[cfg(test)]
{
return reqwest::Client::builder()
reqwest::Client::builder()
.connect_timeout(Duration::from_secs_f64(timeout.connect))
.no_proxy()
.build()
.unwrap_or_default();
.unwrap_or_default()
}
#[cfg(not(test))]
{