diff --git a/crates/arc-llm/src/catalog.json b/crates/arc-llm/src/catalog.json index ef78dd387..b39422140 100644 --- a/crates/arc-llm/src/catalog.json +++ b/crates/arc-llm/src/catalog.json @@ -10,6 +10,7 @@ "supports_reasoning": true, "input_cost_per_million": 15.0, "output_cost_per_million": 75.0, + "estimated_output_tps": 25, "aliases": ["opus", "claude-opus"] }, { @@ -23,6 +24,7 @@ "supports_reasoning": true, "input_cost_per_million": 3.0, "output_cost_per_million": 15.0, + "estimated_output_tps": 50, "aliases": ["sonnet", "claude-sonnet"] }, { @@ -36,6 +38,7 @@ "supports_reasoning": false, "input_cost_per_million": 0.8, "output_cost_per_million": 4.0, + "estimated_output_tps": 100, "aliases": ["haiku", "claude-haiku"] }, { @@ -49,6 +52,7 @@ "supports_reasoning": true, "input_cost_per_million": 1.75, "output_cost_per_million": 14.0, + "estimated_output_tps": 65, "aliases": ["gpt5"] }, { @@ -62,6 +66,7 @@ "supports_reasoning": true, "input_cost_per_million": 0.25, "output_cost_per_million": 2.0, + "estimated_output_tps": 70, "aliases": ["gpt5-mini"] }, { @@ -75,6 +80,7 @@ "supports_reasoning": true, "input_cost_per_million": 1.75, "output_cost_per_million": 14.0, + "estimated_output_tps": 100, "aliases": [] }, { @@ -88,6 +94,7 @@ "supports_reasoning": true, "input_cost_per_million": 1.75, "output_cost_per_million": 14.0, + "estimated_output_tps": 100, "aliases": ["codex"] }, { @@ -101,6 +108,7 @@ "supports_reasoning": true, "input_cost_per_million": 2.0, "output_cost_per_million": 12.0, + "estimated_output_tps": 85, "aliases": ["gemini-pro"] }, { @@ -114,6 +122,7 @@ "supports_reasoning": true, "input_cost_per_million": 0.5, "output_cost_per_million": 3.0, + "estimated_output_tps": 150, "aliases": ["gemini-flash"] }, { @@ -127,6 +136,7 @@ "supports_reasoning": false, "input_cost_per_million": 0.6, "output_cost_per_million": 3.0, + "estimated_output_tps": 50, "aliases": ["kimi"] }, { @@ -140,6 +150,7 @@ "supports_reasoning": false, "input_cost_per_million": 0.6, "output_cost_per_million": 2.2, + "estimated_output_tps": 100, "aliases": ["glm", "glm4"] }, { @@ -153,6 +164,7 @@ "supports_reasoning": false, "input_cost_per_million": 0.3, "output_cost_per_million": 1.2, + "estimated_output_tps": 45, "aliases": ["minimax"] }, { @@ -166,6 +178,7 @@ "supports_reasoning": true, "input_cost_per_million": 0.25, "output_cost_per_million": 0.75, + "estimated_output_tps": 1000, "aliases": ["mercury"] } ] diff --git a/crates/arc-llm/src/cli.rs b/crates/arc-llm/src/cli.rs index 4a86c1978..100371627 100644 --- a/crates/arc-llm/src/cli.rs +++ b/crates/arc-llm/src/cli.rs @@ -89,21 +89,29 @@ fn format_cost(cost: Option) -> String { } } +fn format_speed(tps: Option) -> String { + match tps { + None => "-".to_string(), + Some(t) => format!("{} tok/s", t as i64), + } +} + fn print_models_table(models: &[crate::types::ModelInfo]) { println!( - "{:<24} {:<12} {:<24} {:>10} {:>7} {:>7}", - "MODEL", "PROVIDER", "ALIASES", "CONTEXT", "COST", "" + "{:<24} {:<12} {:<24} {:>10} {:>7} {:>7} {:>10}", + "MODEL", "PROVIDER", "ALIASES", "CONTEXT", "COST", "", "SPEED" ); for model in models { let aliases = model.aliases.join(", "); println!( - "{:<24} {:<12} {:<24} {:>10} {:>7} / {:<7}", + "{:<24} {:<12} {:<24} {:>10} {:>7} / {:<7} {:>10}", model.id, model.provider, aliases, format_context_window(model.context_window), format_cost(model.input_cost_per_million), - format_cost(model.output_cost_per_million) + format_cost(model.output_cost_per_million), + format_speed(model.estimated_output_tps) ); } } @@ -366,8 +374,8 @@ async fn test_models(provider: Option<&str>, model: Option<&str>) -> Result<()> } println!( - "{:<24} {:<12} {:>10} {:>7} {:>7} {}", - "MODEL", "PROVIDER", "CONTEXT", "COST", "", "RESULT" + "{:<24} {:<12} {:>10} {:>7} {:>7} {:>10} {}", + "MODEL", "PROVIDER", "CONTEXT", "COST", "", "SPEED", "RESULT" ); let mut failures = 0u32; @@ -393,12 +401,13 @@ async fn test_models(provider: Option<&str>, model: Option<&str>) -> Result<()> }; println!( - "{:<24} {:<12} {:>10} {:>7} / {:<7} {status}", + "{:<24} {:<12} {:>10} {:>7} / {:<7} {:>10} {status}", info.id, info.provider, format_context_window(info.context_window), format_cost(info.input_cost_per_million), - format_cost(info.output_cost_per_million) + format_cost(info.output_cost_per_million), + format_speed(info.estimated_output_tps) ); } diff --git a/crates/arc-llm/src/types.rs b/crates/arc-llm/src/types.rs index 639e9eff3..076becbcb 100644 --- a/crates/arc-llm/src/types.rs +++ b/crates/arc-llm/src/types.rs @@ -639,6 +639,7 @@ pub struct ModelInfo { pub supports_reasoning: bool, pub input_cost_per_million: Option, pub output_cost_per_million: Option, + pub estimated_output_tps: Option, pub aliases: Vec, }