From 29a8f4ce83f7bc6c49be5f8e05cfb05d8529563b Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Wed, 23 Sep 2026 19:13:23 -0700 Subject: [PATCH] fix(cost): pass each provider only the pricing flags Python forwards A new generic_cost_per_token mirrors Python's: it applies the provider cache read default and inclusive thresholds itself. cost_per_token forwards the service tier to anthropic, bedrock, azure, and gemini but not data residency, drops both for deepseek and tencent, and never forwards the Vertex location outside the Vertex route. Speech priced per token, transcription token usage, and realtime candidates go through generic_cost_per_token like Python Co-Authored-By: Claude Opus 5.5 --- .../crates/cost/src/cost_calculator.rs | 69 ++++++++++++------- litellm-rust/crates/cost/src/generic_cost.rs | 27 +++++++- .../cost/tests/python_provider_routes.rs | 49 +++++++++++++ .../crates/cost/tests/python_speech_cost.rs | 46 +++++++++++++ 4 files changed, 164 insertions(+), 27 deletions(-) diff --git a/litellm-rust/crates/cost/src/cost_calculator.rs b/litellm-rust/crates/cost/src/cost_calculator.rs index 033c1dfe967..a3cde184dfe 100644 --- a/litellm-rust/crates/cost/src/cost_calculator.rs +++ b/litellm-rust/crates/cost/src/cost_calculator.rs @@ -6,7 +6,6 @@ use serde_json::Value; use crate::anthropic_cost::fast_speed_multiplier; use crate::azure_ai_cost::azure_ai_cost_per_token; use crate::azure_cost::output_per_second_cost; -use crate::base_rate_selection::uses_inclusive_token_thresholds; use crate::batch::batch_cost_from_model_info; use crate::catalog::{CostCall, ModelCostRequest, ModelInfoCatalog}; use crate::completion_cost::{ @@ -20,7 +19,7 @@ use crate::error::CostError; use crate::fireworks_cost::{ FireworksThresholds, cost_per_token as fireworks_cost_per_token, get_base_model_for_pricing, }; -use crate::generic_cost::calculate_generic_cost_from_model_info_with_region; +use crate::generic_cost::{GenericCostRequest, generic_cost_per_token}; use crate::lemonade_cost::lemonade_cost_per_token; use crate::ocr_cost::ocr_cost; use crate::openai_cost::video_generation_cost; @@ -321,15 +320,25 @@ pub fn cost_per_token( if !dispatches_before_the_gate && !has_token_or_tiered_pricing(&model_info) { return Ok((0.0, 0.0)); } - let cost = calculate_generic_cost_from_model_info_with_region( - request.usage, - &model_info, - request.service_tier, - uses_inclusive_token_thresholds(request.provider), - request.data_residency, - request.vertex_location, - request.at, - ); + let (service_tier, data_residency) = match provider { + Some( + LlmProviders::ANTHROPIC + | LlmProviders::BEDROCK + | LlmProviders::AZURE + | LlmProviders::GEMINI, + ) => (request.service_tier, None), + Some(LlmProviders::DEEPSEEK | LlmProviders::TENCENT) => (None, None), + _ => (request.service_tier, request.data_residency), + }; + let cost = generic_cost_per_token(GenericCostRequest { + model_info: catalog.entry_for_key(key), + usage: request.usage, + provider: request.provider, + service_tier, + data_residency, + vertex_location: None, + at: request.at, + }); let speed = if provider == Some(LlmProviders::ANTHROPIC) { fast_speed_multiplier(&model_info, request.usage) * get_provider_specific_geo_multiplier( @@ -374,7 +383,15 @@ pub fn speech_cost( completion.unwrap_or(0.0), )) } - SpeechCostMetric::PerToken => cost_per_token(catalog, request), + SpeechCostMetric::PerToken => Ok(generic_cost_per_token(GenericCostRequest { + model_info, + usage: request.usage, + provider: request.provider, + service_tier: request.service_tier, + data_residency: request.data_residency, + vertex_location: None, + at: request.at, + })), } } @@ -387,15 +404,15 @@ pub fn transcription_cost( .entry(request.model, request.provider, request.region) .ok_or(CostError::ModelNotFound)?; if transcription_usage_has_token_details(request.usage) { - return Ok(calculate_generic_cost_from_model_info_with_region( - request.usage, + return Ok(generic_cost_per_token(GenericCostRequest { model_info, - request.service_tier, - false, - request.data_residency, - request.vertex_location, - request.at, - )); + usage: request.usage, + provider: request.provider, + service_tier: request.service_tier, + data_residency: request.data_residency, + vertex_location: None, + at: request.at, + })); } Ok(cost_per_second(model_info, duration_seconds)) } @@ -453,15 +470,15 @@ pub fn handle_realtime_stream_cost_calculation( .chain(std::iter::once(requested_model)) .find_map(|model| { let model_info = catalog.entry(model, Some(provider), None)?; - let cost = calculate_generic_cost_from_model_info_with_region( - combined_usage, + let cost = generic_cost_per_token(GenericCostRequest { model_info, - None, - false, + usage: combined_usage, + provider: Some(provider), + service_tier: None, data_residency, - None, + vertex_location: None, at, - ); + }); (cost.0 + cost.1 > 0.0 || cost_map_entry_declares_pricing(catalog, model, provider)) .then_some(cost) }) diff --git a/litellm-rust/crates/cost/src/generic_cost.rs b/litellm-rust/crates/cost/src/generic_cost.rs index 078f7edfc42..1e36255a30a 100644 --- a/litellm-rust/crates/cost/src/generic_cost.rs +++ b/litellm-rust/crates/cost/src/generic_cost.rs @@ -3,12 +3,13 @@ use serde_json::Value; use crate::base_rate_selection::{ TokenBaseRates, get_tiered_reasoning_rate, get_token_base_cost, - get_token_base_cost_without_off_peak, + get_token_base_cost_without_off_peak, uses_inclusive_token_thresholds, }; use crate::generic_input::{InputBaseRates, calculate_input_cost}; use crate::generic_output::{calculate_output_cost, resolve_reasoning_token_cost}; use crate::generic_usage::{ParsedPromptDetails, parse_prompt_tokens_details}; use crate::off_peak::{open_off_peak_block, parse_off_peak_rate}; +use crate::provider_cache::apply_provider_cache_read_default; use crate::regional_uplift::regional_totals_multiplier; use crate::responses_usage::ChatUsage; @@ -187,3 +188,27 @@ pub fn calculate_generic_cost_from_model_info_with_region( at, ) } + +#[derive(Clone, Copy, Debug)] +pub struct GenericCostRequest<'a> { + pub model_info: &'a Value, + pub usage: &'a ChatUsage, + pub provider: Option<&'a str>, + pub service_tier: Option<&'a str>, + pub data_residency: Option<&'a str>, + pub vertex_location: Option<&'a str>, + pub at: Timestamp, +} + +pub fn generic_cost_per_token(request: GenericCostRequest<'_>) -> (f64, f64) { + let model_info = apply_provider_cache_read_default(request.model_info, request.provider); + calculate_generic_cost_from_model_info_with_region( + request.usage, + &model_info, + request.service_tier, + uses_inclusive_token_thresholds(request.provider), + request.data_residency, + request.vertex_location, + request.at, + ) +} diff --git a/litellm-rust/crates/cost/tests/python_provider_routes.rs b/litellm-rust/crates/cost/tests/python_provider_routes.rs index 2055f472720..7e4f457b6d7 100644 --- a/litellm-rust/crates/cost/tests/python_provider_routes.rs +++ b/litellm-rust/crates/cost/tests/python_provider_routes.rs @@ -148,3 +148,52 @@ fn lemonade_provider_is_free_for_unmapped_and_token_priced_models() { .unwrap(); assert_eq!(elapsed, (0.1, 0.0)); } + +fn flagged_rates() -> Value { + json!({ + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "input_cost_per_token_flex": 0.5e-6, + "output_cost_per_token_flex": 1e-6, + "regional_processing_uplift_multiplier_eu": 1.5, + "regional_endpoint_uplift_multiplier": 3.0 + }) +} + +#[rstest] +#[case::openai_takes_tier_and_residency("openai", (0.5e-3 * 1.5, 1e-3 * 1.5))] +#[case::custom_provider_takes_tier_and_residency("custom", (0.5e-3 * 1.5, 1e-3 * 1.5))] +#[case::anthropic_takes_only_the_tier("anthropic", (0.5e-3, 1e-3))] +#[case::bedrock_takes_only_the_tier("bedrock", (0.5e-3, 1e-3))] +#[case::azure_takes_only_the_tier("azure", (0.5e-3, 1e-3))] +#[case::gemini_takes_only_the_tier("gemini", (0.5e-3, 1e-3))] +#[case::deepseek_takes_neither("deepseek", (1e-3, 2e-3))] +#[case::tencent_takes_neither("tencent", (1e-3, 2e-3))] +fn cost_per_token_forwards_only_the_pricing_flags_each_python_provider_passes( + #[case] provider: &str, + #[case] expected: (f64, f64), +) { + let catalog = ModelInfoCatalog::new(HashMap::from([( + format!("{provider}/model"), + flagged_rates(), + )])); + let usage = + get_usage_object(&json!({"usage": {"prompt_tokens": 1000, "completion_tokens": 1000}})) + .unwrap() + .unwrap(); + let (prompt, completion) = cost_per_token( + &catalog, + ModelCostRequest { + service_tier: Some("flex"), + data_residency: Some("eu"), + vertex_location: Some("us-east5"), + ..request("model", provider, &usage) + }, + ) + .unwrap(); + assert!((prompt - expected.0).abs() < 1e-15, "{provider}: {prompt}"); + assert!( + (completion - expected.1).abs() < 1e-15, + "{provider}: {completion}" + ); +} diff --git a/litellm-rust/crates/cost/tests/python_speech_cost.rs b/litellm-rust/crates/cost/tests/python_speech_cost.rs index 553313c2354..2a8549a58e8 100644 --- a/litellm-rust/crates/cost/tests/python_speech_cost.rs +++ b/litellm-rust/crates/cost/tests/python_speech_cost.rs @@ -174,3 +174,49 @@ fn transcription_cost_selects_token_usage_or_duration_pricing() { (0.05, 0.0) ); } + +#[rstest] +fn speech_priced_per_token_uses_generic_rates_not_the_provider_per_second_route() { + let catalog = ModelInfoCatalog::new(HashMap::from([( + "azure/tts".to_owned(), + json!({"input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6, "output_cost_per_second": 0.01}), + )])); + let usage = + get_usage_object(&json!({"usage": {"prompt_tokens": 1000, "completion_tokens": 1000}})) + .unwrap() + .unwrap(); + let (prompt, completion) = litellm_cost::cost_calculator::speech_cost( + &catalog, + ModelCostRequest { + response_time_ms: Some(1000.0), + ..request("tts", "azure", &usage) + }, + None, + ) + .unwrap(); + assert!((prompt - 1e-3).abs() < 1e-15); + assert!((completion - 2e-3).abs() < 1e-15); +} + +#[rstest] +fn transcription_token_pricing_ignores_the_vertex_location() { + let catalog = ModelInfoCatalog::new(HashMap::from([( + "openai/transcribe".to_owned(), + json!({"input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6, "regional_endpoint_uplift_multiplier": 3.0}), + )])); + let usage = + get_usage_object(&json!({"usage": {"prompt_tokens": 1000, "completion_tokens": 1000}})) + .unwrap() + .unwrap(); + let (prompt, completion) = litellm_cost::cost_calculator::transcription_cost( + &catalog, + ModelCostRequest { + vertex_location: Some("us-east5"), + ..request("transcribe", "openai", &usage) + }, + 0.0, + ) + .unwrap(); + assert!((prompt - 1e-3).abs() < 1e-15); + assert!((completion - 2e-3).abs() < 1e-15); +}