From bf6ea4d78115f73d1e8f1b10d6ee32fc83da8457 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Wed, 23 Sep 2026 19:16:12 -0700 Subject: [PATCH] fix(cost): match Python's Vertex character routing and Fireworks size parsing Vertex routes and prices characters with the provider prefix stripped, so prefixed gemini-pro keeps its fixed pricing, and Fireworks parses the parameter count as a float so oversized counts land in the above-16b tier Co-Authored-By: Claude Opus 5.5 --- .../crates/cost/src/fireworks_cost.rs | 6 ++-- litellm-rust/crates/cost/src/vertex_cost.rs | 12 +++++-- .../cost/tests/python_fireworks_cost.rs | 2 ++ .../crates/cost/tests/python_vertex_cost.rs | 36 +++++++++++++++++++ 4 files changed, 51 insertions(+), 5 deletions(-) diff --git a/litellm-rust/crates/cost/src/fireworks_cost.rs b/litellm-rust/crates/cost/src/fireworks_cost.rs index 93de86da0f9..c0344d93ef8 100644 --- a/litellm-rust/crates/cost/src/fireworks_cost.rs +++ b/litellm-rust/crates/cost/src/fireworks_cost.rs @@ -56,12 +56,12 @@ pub fn get_base_model_for_pricing( } if let Some(size) = MODEL_SIZE .captures(&name) - .and_then(|captures| captures.get(1)?.as_str().parse::().ok()) + .and_then(|captures| captures.get(1)?.as_str().parse::().ok()) { - if size <= thresholds.small { + if size <= thresholds.small as f64 { return "fireworks-ai-up-to-4b"; } - if size <= thresholds.medium { + if size <= thresholds.medium as f64 { return "fireworks-ai-4.1b-to-16b"; } return "fireworks-ai-above-16b"; diff --git a/litellm-rust/crates/cost/src/vertex_cost.rs b/litellm-rust/crates/cost/src/vertex_cost.rs index 1a9cdf03e11..dfbfcdf2ebd 100644 --- a/litellm-rust/crates/cost/src/vertex_cost.rs +++ b/litellm-rust/crates/cost/src/vertex_cost.rs @@ -140,7 +140,15 @@ pub fn vertex_cost( { return Ok(cost); } - if cost_router(request.model, request.provider.unwrap_or(""), call_type) == CostRoute::PerToken + let model_without_prefix = request + .model + .split_once('/') + .map_or(request.model, |(_, model)| model); + if cost_router( + model_without_prefix, + request.provider.unwrap_or(""), + call_type, + ) == CostRoute::PerToken { return catalog_cost_per_token(catalog, request); } @@ -148,7 +156,7 @@ pub fn vertex_cost( .entry(request.model, request.provider, request.region) .ok_or(CostError::ModelNotFound)?; Ok(cost_per_character( - request.model, + model_without_prefix, request.usage, entry, (prompt_characters, completion_characters), diff --git a/litellm-rust/crates/cost/tests/python_fireworks_cost.rs b/litellm-rust/crates/cost/tests/python_fireworks_cost.rs index 780763f9c52..4a5ad1a0925 100644 --- a/litellm-rust/crates/cost/tests/python_fireworks_cost.rs +++ b/litellm-rust/crates/cost/tests/python_fireworks_cost.rs @@ -29,6 +29,8 @@ fn usage(value: Value) -> litellm_cost::responses_usage::ChatUsage { #[case("model-8x22b", "fireworks-ai-56b-to-176b")] #[case("model-8x30b", "fireworks-ai-above-16b")] #[case("model-without-size", "fireworks-ai-default")] +#[case("foo-99999999999999999999b", "fireworks-ai-above-16b")] +#[case("99999999999999999999x2b", "fireworks-ai-up-to-4b")] fn fireworks_model_size_category_matches_python_pattern_order( #[case] model: &str, #[case] expected: &str, diff --git a/litellm-rust/crates/cost/tests/python_vertex_cost.rs b/litellm-rust/crates/cost/tests/python_vertex_cost.rs index 129bdc11206..590e376baa3 100644 --- a/litellm-rust/crates/cost/tests/python_vertex_cost.rs +++ b/litellm-rust/crates/cost/tests/python_vertex_cost.rs @@ -182,3 +182,39 @@ fn catalog_vertex_cost_routes_token_and_character_calls() { (0.01, 0.01) ); } + +#[rstest] +#[case::bare_model("gemini-pro")] +#[case::provider_prefixed_model("vertex_ai/gemini-pro")] +fn vertex_cost_matches_models_without_dynamic_pricing_after_stripping_the_prefix( + #[case] model: &str, +) { + let catalog = ModelInfoCatalog::new(HashMap::from([( + "vertex_ai/gemini-pro".to_owned(), + json!({ + "input_cost_per_character": 1e-6, + "input_cost_per_character_above_128k_tokens": 9e-6, + "output_cost_per_character": 1e-6 + }), + )])); + let usage = ChatUsage::default(); + let (prompt, _) = vertex_cost( + &catalog, + ModelCostRequest { + model, + provider: Some("vertex_ai"), + region: None, + usage: &usage, + service_tier: None, + data_residency: None, + vertex_location: None, + at: at(), + response_time_ms: None, + }, + "completion", + Some(40_000.0), + Some(0.0), + ) + .unwrap(); + assert!((prompt - 0.04).abs() < 1e-12); +}