scriptoriumadmin/fabro
1
0
Fork 0
mirror of https://github.com/fabro-sh/fabro.git synced 2026-10-11 03:40:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4
fabro/lib/crates/fabro-cli/tests/it/cmd/model.rs
Bryan Helmkamp 74aaed2611
Merge origin/main into feat/vertex-adapter 
Resolve conflicts introduced by the provider-identity refactor on main (#280)
while preserving the Vertex adapter intent from this branch:

- Adopt main's typed `AdapterKind` enum; add `Vertex` variant and `VERTEX`
  metadata entry rather than reintroducing the deleted `Provider` enum.
- Replace `AdapterMetadata.api_key_header: ApiKeyHeaderPolicy` with
  `auth_strategy: AdapterAuthStrategy` (`ApiKey(policy)` | `GoogleApplicationDefault`)
  so credential-resolution code can short-circuit for adapter-managed auth.
- Update `fabro_auth::{env_source,resolve}` to use catalog-driven adapter
  metadata and look up Vertex via `ProviderId::new("vertex")`.
- Wire Vertex into `fabro_llm::adapter_registry::factory_for`.
- Route `fabro_workflow::handler::llm::cli` and `fabro_agent::cli`/`memory`
  off `AgentProfileKind` rather than the closed Provider enum so Vertex
  inherits the Anthropic profile via catalog metadata.
- Map Vertex to the Anthropic billing/pricing algorithm.
- Drop `lib/crates/fabro-model/src/provider.rs` (deleted by main).
- Drop the legacy `provider_key_url` / `unsupported_install_provider_error`
  helpers; vertex.toml now drives provider catalog behavior.

cargo build --workspace, cargo nextest run -p fabro-model -p fabro-auth
-p fabro-llm -p fabro-agent -p fabro-workflow -p fabro-server -p fabro-cli,
cargo +nightly-2026-04-14 fmt --check --all, and cargo +nightly-2026-04-14
clippy --workspace --all-targets -- -D warnings all pass.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-17 08:57:55 -04:00

284 lines
10 KiB
Rust
Raw Permalink Blame History

#![expect(
clippy::disallowed_methods,
reason = "integration tests stage fixtures with sync std::fs; test infrastructure, not Tokio-hot path"
)]
use fabro_test::{fabro_snapshot, test_context};
use httpmock::MockServer;
fn successful_stdout(mut cmd: assert_cmd::Command) -> String {
let assert = cmd.assert().success().stderr("");
String::from_utf8(assert.get_output().stdout.clone())
.expect("model list stdout should be UTF-8")
}
fn assert_model_list_table(stdout: &str) {
let mut lines = stdout.lines();
let header = lines.next().expect("model list should render a header");
assert!(header.contains("MODEL"), "missing MODEL column: {header}");
assert!(
header.contains("PROVIDER"),
"missing PROVIDER column: {header}"
);
assert!(
header.contains("CONTEXT"),
"missing CONTEXT column: {header}"
);
assert!(
lines.any(|line| !line.trim().is_empty()),
"model list should render at least one model row"
);
}
#[test]
fn help() {
let context = test_context!();
let mut cmd = context.model();
cmd.arg("--help");
fabro_snapshot!(context.filters(), cmd, @"
success: true
exit_code: 0
----- stdout -----
List and test LLM models
Usage: fabro model [OPTIONS] [COMMAND]
Commands:
list List available models
test Test model availability by sending a simple prompt
help Print this message or the help of the given subcommand(s)
Options:
--json Output as JSON [env: FABRO_JSON=]
--debug Enable DEBUG-level logging (default is INFO) [env: FABRO_DEBUG=]
--no-upgrade-check Disable automatic upgrade check [env: FABRO_NO_UPGRADE_CHECK=true]
--quiet Suppress non-essential output [env: FABRO_QUIET=]
--verbose Enable verbose output [env: FABRO_VERBOSE=]
-h, --help Print help
----- stderr -----
");
}
#[test]
fn bare() {
let context = test_context!();
let stdout = successful_stdout(context.model());
assert_model_list_table(&stdout);
}
#[test]
fn list() {
let context = test_context!();
let mut cmd = context.model();
cmd.arg("list");
let stdout = successful_stdout(cmd);
assert_model_list_table(&stdout);
}
#[test]
fn list_provider() {
let context = test_context!();
let mut cmd = context.model();
cmd.args(["list", "--provider", "anthropic"]);
fabro_snapshot!(context.filters(), cmd, @"
success: true
exit_code: 0
----- stdout -----
MODEL PROVIDER ALIASES CONTEXT COST SPEED
claude-haiku-4-5 anthropic haiku, claude-haiku 200k $0.8 / $4.0 100 tok/s
claude-opus-4-6 anthropic 1m $5.0 / $25.0 25 tok/s
claude-opus-4-7 anthropic opus, claude-opus 1m $5.0 / $25.0 25 tok/s
claude-sonnet-4-5 anthropic 200k $3.0 / $15.0 50 tok/s
claude-sonnet-4-6 anthropic sonnet, claude-sonnet 200k $3.0 / $15.0 50 tok/s
----- stderr -----
");
}
#[test]
fn list_query() {
let context = test_context!();
let mut cmd = context.model();
cmd.args(["list", "--query", "opus"]);
fabro_snapshot!(context.filters(), cmd, @"
success: true
exit_code: 0
----- stdout -----
MODEL PROVIDER ALIASES CONTEXT COST SPEED
claude-opus-4-6 anthropic 1m $5.0 / $25.0 25 tok/s
claude-opus-4-7 anthropic opus, claude-opus 1m $5.0 / $25.0 25 tok/s
vertex-claude-opus-4-1 vertex 200k - / - 25 tok/s
vertex-claude-opus-4-5 vertex 200k - / - 25 tok/s
vertex-claude-opus-4-6 vertex 1m - / - 25 tok/s
vertex-claude-opus-4-7 vertex 1m - / - 25 tok/s
----- stderr -----
");
}
#[test]
fn list_query_aliases() {
let context = test_context!();
let mut cmd = context.model();
cmd.args(["list", "--query", "codex"]);
fabro_snapshot!(context.filters(), cmd, @"
success: true
exit_code: 0
----- stdout -----
MODEL PROVIDER ALIASES CONTEXT COST SPEED
gpt-5.2-codex openai 1m $1.8 / $14.0 100 tok/s
gpt-5.3-codex openai codex 1m $1.8 / $14.0 100 tok/s
gpt-5.3-codex-spark openai codex-spark 131k - / - 1000 tok/s
----- stderr -----
");
}
#[test]
fn list_query_case_insensitive() {
let context = test_context!();
let mut cmd = context.model();
cmd.args(["list", "--query", "OPUS"]);
fabro_snapshot!(context.filters(), cmd, @"
success: true
exit_code: 0
----- stdout -----
MODEL PROVIDER ALIASES CONTEXT COST SPEED
claude-opus-4-6 anthropic 1m $5.0 / $25.0 25 tok/s
claude-opus-4-7 anthropic opus, claude-opus 1m $5.0 / $25.0 25 tok/s
vertex-claude-opus-4-1 vertex 200k - / - 25 tok/s
vertex-claude-opus-4-5 vertex 200k - / - 25 tok/s
vertex-claude-opus-4-6 vertex 1m - / - 25 tok/s
vertex-claude-opus-4-7 vertex 1m - / - 25 tok/s
----- stderr -----
");
}
#[test]
fn list_unknown_provider_returns_empty() {
let context = test_context!();
let mut cmd = context.model();
cmd.args(["list", "--provider", "not-a-provider"]);
fabro_snapshot!(context.filters(), cmd, @"
success: true
exit_code: 0
----- stdout -----
MODEL PROVIDER ALIASES CONTEXT COST SPEED
----- stderr -----
");
}
#[test]
fn list_uses_configured_server_target_without_server_flag() {
let context = test_context!();
let server = MockServer::start();
let mock = server.mock(|when, then| {
when.method("GET");
then.status(200)
.header("Content-Type", "application/json")
.body(
serde_json::json!({
"data": [{
"id": "remote-model",
"display_name": "Remote Model",
"provider": "openai",
"family": "test",
"aliases": ["remote"],
"limits": {
"context_window": 131_072,
"max_output": 4096
},
"training": null,
"knowledge_cutoff": null,
"features": {
"tools": true,
"vision": false,
"reasoning": false
},
"costs": {
"input_cost_per_mtok": 1.0,
"output_cost_per_mtok": 2.0,
"cache_input_cost_per_mtok": null
},
"estimated_output_tps": 42.0,
"default": false,
"configured": false
}],
"meta": { "has_more": false }
})
.to_string(),
);
});
context.set_http_target(&server.base_url());
let mut cmd = context.model();
cmd.args(["list", "--json"]);
let output = cmd.assert().success().get_output().stdout.clone();
let models: serde_json::Value =
serde_json::from_slice(&output).expect("model list json should parse");
mock.assert();
assert_eq!(models.as_array().map(Vec::len), Some(1));
assert_eq!(models[0]["id"].as_str(), Some("remote-model"));
}
#[test]
fn list_uses_fabro_config_for_machine_settings() {
let context = test_context!();
let server = MockServer::start();
let mock = server.mock(|when, then| {
when.method("GET");
then.status(200)
.header("Content-Type", "application/json")
.body(
serde_json::json!({
"data": [{
"id": "remote-model",
"display_name": "Remote Model",
"provider": "openai",
"family": "test",
"aliases": ["remote"],
"limits": {
"context_window": 131_072,
"max_output": 4096
},
"training": null,
"knowledge_cutoff": null,
"features": {
"tools": true,
"vision": false,
"reasoning": false
},
"costs": {
"input_cost_per_mtok": 1.0,
"output_cost_per_mtok": 2.0,
"cache_input_cost_per_mtok": null
},
"estimated_output_tps": 42.0,
"default": false,
"configured": false
}],
"meta": { "has_more": false }
})
.to_string(),
);
});
let config_dir = tempfile::tempdir().unwrap();
let config_path = config_dir.path().join("custom-settings.toml");
std::fs::write(
&config_path,
format!(
"_version = 1\n\n[cli.target]\ntype = \"http\"\nurl = \"{}/api/v1\"\n",
server.base_url()
),
)
.unwrap();
let mut cmd = context.model();
cmd.args(["list", "--json"]);
cmd.env("FABRO_CONFIG", &config_path);
let output = cmd.assert().success().get_output().stdout.clone();
let models: serde_json::Value =
serde_json::from_slice(&output).expect("model list json should parse");
mock.assert();
assert_eq!(models.as_array().map(Vec::len), Some(1));
assert_eq!(models[0]["id"].as_str(), Some("remote-model"));
}
Reference in a new issue View git blame Copy permalink