diff --git a/litellm-rust/crates/cost/src/cost_calculator.rs b/litellm-rust/crates/cost/src/cost_calculator.rs index 033c1dfe967..a3cde184dfe 100644 --- a/litellm-rust/crates/cost/src/cost_calculator.rs +++ b/litellm-rust/crates/cost/src/cost_calculator.rs @@ -6,7 +6,6 @@ use serde_json::Value; use crate::anthropic_cost::fast_speed_multiplier; use crate::azure_ai_cost::azure_ai_cost_per_token; use crate::azure_cost::output_per_second_cost; -use crate::base_rate_selection::uses_inclusive_token_thresholds; use crate::batch::batch_cost_from_model_info; use crate::catalog::{CostCall, ModelCostRequest, ModelInfoCatalog}; use crate::completion_cost::{ @@ -20,7 +19,7 @@ use crate::error::CostError; use crate::fireworks_cost::{ FireworksThresholds, cost_per_token as fireworks_cost_per_token, get_base_model_for_pricing, }; -use crate::generic_cost::calculate_generic_cost_from_model_info_with_region; +use crate::generic_cost::{GenericCostRequest, generic_cost_per_token}; use crate::lemonade_cost::lemonade_cost_per_token; use crate::ocr_cost::ocr_cost; use crate::openai_cost::video_generation_cost; @@ -321,15 +320,25 @@ pub fn cost_per_token( if !dispatches_before_the_gate && !has_token_or_tiered_pricing(&model_info) { return Ok((0.0, 0.0)); } - let cost = calculate_generic_cost_from_model_info_with_region( - request.usage, - &model_info, - request.service_tier, - uses_inclusive_token_thresholds(request.provider), - request.data_residency, - request.vertex_location, - request.at, - ); + let (service_tier, data_residency) = match provider { + Some( + LlmProviders::ANTHROPIC + | LlmProviders::BEDROCK + | LlmProviders::AZURE + | LlmProviders::GEMINI, + ) => (request.service_tier, None), + Some(LlmProviders::DEEPSEEK | LlmProviders::TENCENT) => (None, None), + _ => (request.service_tier, request.data_residency), + }; + let cost = generic_cost_per_token(GenericCostRequest { + model_info: catalog.entry_for_key(key), + usage: request.usage, + provider: request.provider, + service_tier, + data_residency, + vertex_location: None, + at: request.at, + }); let speed = if provider == Some(LlmProviders::ANTHROPIC) { fast_speed_multiplier(&model_info, request.usage) * get_provider_specific_geo_multiplier( @@ -374,7 +383,15 @@ pub fn speech_cost( completion.unwrap_or(0.0), )) } - SpeechCostMetric::PerToken => cost_per_token(catalog, request), + SpeechCostMetric::PerToken => Ok(generic_cost_per_token(GenericCostRequest { + model_info, + usage: request.usage, + provider: request.provider, + service_tier: request.service_tier, + data_residency: request.data_residency, + vertex_location: None, + at: request.at, + })), } } @@ -387,15 +404,15 @@ pub fn transcription_cost( .entry(request.model, request.provider, request.region) .ok_or(CostError::ModelNotFound)?; if transcription_usage_has_token_details(request.usage) { - return Ok(calculate_generic_cost_from_model_info_with_region( - request.usage, + return Ok(generic_cost_per_token(GenericCostRequest { model_info, - request.service_tier, - false, - request.data_residency, - request.vertex_location, - request.at, - )); + usage: request.usage, + provider: request.provider, + service_tier: request.service_tier, + data_residency: request.data_residency, + vertex_location: None, + at: request.at, + })); } Ok(cost_per_second(model_info, duration_seconds)) } @@ -453,15 +470,15 @@ pub fn handle_realtime_stream_cost_calculation( .chain(std::iter::once(requested_model)) .find_map(|model| { let model_info = catalog.entry(model, Some(provider), None)?; - let cost = calculate_generic_cost_from_model_info_with_region( - combined_usage, + let cost = generic_cost_per_token(GenericCostRequest { model_info, - None, - false, + usage: combined_usage, + provider: Some(provider), + service_tier: None, data_residency, - None, + vertex_location: None, at, - ); + }); (cost.0 + cost.1 > 0.0 || cost_map_entry_declares_pricing(catalog, model, provider)) .then_some(cost) }) diff --git a/litellm-rust/crates/cost/src/generic_cost.rs b/litellm-rust/crates/cost/src/generic_cost.rs index 078f7edfc42..1e36255a30a 100644 --- a/litellm-rust/crates/cost/src/generic_cost.rs +++ b/litellm-rust/crates/cost/src/generic_cost.rs @@ -3,12 +3,13 @@ use serde_json::Value; use crate::base_rate_selection::{ TokenBaseRates, get_tiered_reasoning_rate, get_token_base_cost, - get_token_base_cost_without_off_peak, + get_token_base_cost_without_off_peak, uses_inclusive_token_thresholds, }; use crate::generic_input::{InputBaseRates, calculate_input_cost}; use crate::generic_output::{calculate_output_cost, resolve_reasoning_token_cost}; use crate::generic_usage::{ParsedPromptDetails, parse_prompt_tokens_details}; use crate::off_peak::{open_off_peak_block, parse_off_peak_rate}; +use crate::provider_cache::apply_provider_cache_read_default; use crate::regional_uplift::regional_totals_multiplier; use crate::responses_usage::ChatUsage; @@ -187,3 +188,27 @@ pub fn calculate_generic_cost_from_model_info_with_region( at, ) } + +#[derive(Clone, Copy, Debug)] +pub struct GenericCostRequest<'a> { + pub model_info: &'a Value, + pub usage: &'a ChatUsage, + pub provider: Option<&'a str>, + pub service_tier: Option<&'a str>, + pub data_residency: Option<&'a str>, + pub vertex_location: Option<&'a str>, + pub at: Timestamp, +} + +pub fn generic_cost_per_token(request: GenericCostRequest<'_>) -> (f64, f64) { + let model_info = apply_provider_cache_read_default(request.model_info, request.provider); + calculate_generic_cost_from_model_info_with_region( + request.usage, + &model_info, + request.service_tier, + uses_inclusive_token_thresholds(request.provider), + request.data_residency, + request.vertex_location, + request.at, + ) +} diff --git a/litellm-rust/crates/cost/tests/python_provider_routes.rs b/litellm-rust/crates/cost/tests/python_provider_routes.rs index 2055f472720..7e4f457b6d7 100644 --- a/litellm-rust/crates/cost/tests/python_provider_routes.rs +++ b/litellm-rust/crates/cost/tests/python_provider_routes.rs @@ -148,3 +148,52 @@ fn lemonade_provider_is_free_for_unmapped_and_token_priced_models() { .unwrap(); assert_eq!(elapsed, (0.1, 0.0)); } + +fn flagged_rates() -> Value { + json!({ + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "input_cost_per_token_flex": 0.5e-6, + "output_cost_per_token_flex": 1e-6, + "regional_processing_uplift_multiplier_eu": 1.5, + "regional_endpoint_uplift_multiplier": 3.0 + }) +} + +#[rstest] +#[case::openai_takes_tier_and_residency("openai", (0.5e-3 * 1.5, 1e-3 * 1.5))] +#[case::custom_provider_takes_tier_and_residency("custom", (0.5e-3 * 1.5, 1e-3 * 1.5))] +#[case::anthropic_takes_only_the_tier("anthropic", (0.5e-3, 1e-3))] +#[case::bedrock_takes_only_the_tier("bedrock", (0.5e-3, 1e-3))] +#[case::azure_takes_only_the_tier("azure", (0.5e-3, 1e-3))] +#[case::gemini_takes_only_the_tier("gemini", (0.5e-3, 1e-3))] +#[case::deepseek_takes_neither("deepseek", (1e-3, 2e-3))] +#[case::tencent_takes_neither("tencent", (1e-3, 2e-3))] +fn cost_per_token_forwards_only_the_pricing_flags_each_python_provider_passes( + #[case] provider: &str, + #[case] expected: (f64, f64), +) { + let catalog = ModelInfoCatalog::new(HashMap::from([( + format!("{provider}/model"), + flagged_rates(), + )])); + let usage = + get_usage_object(&json!({"usage": {"prompt_tokens": 1000, "completion_tokens": 1000}})) + .unwrap() + .unwrap(); + let (prompt, completion) = cost_per_token( + &catalog, + ModelCostRequest { + service_tier: Some("flex"), + data_residency: Some("eu"), + vertex_location: Some("us-east5"), + ..request("model", provider, &usage) + }, + ) + .unwrap(); + assert!((prompt - expected.0).abs() < 1e-15, "{provider}: {prompt}"); + assert!( + (completion - expected.1).abs() < 1e-15, + "{provider}: {completion}" + ); +} diff --git a/litellm-rust/crates/cost/tests/python_speech_cost.rs b/litellm-rust/crates/cost/tests/python_speech_cost.rs index 553313c2354..2a8549a58e8 100644 --- a/litellm-rust/crates/cost/tests/python_speech_cost.rs +++ b/litellm-rust/crates/cost/tests/python_speech_cost.rs @@ -174,3 +174,49 @@ fn transcription_cost_selects_token_usage_or_duration_pricing() { (0.05, 0.0) ); } + +#[rstest] +fn speech_priced_per_token_uses_generic_rates_not_the_provider_per_second_route() { + let catalog = ModelInfoCatalog::new(HashMap::from([( + "azure/tts".to_owned(), + json!({"input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6, "output_cost_per_second": 0.01}), + )])); + let usage = + get_usage_object(&json!({"usage": {"prompt_tokens": 1000, "completion_tokens": 1000}})) + .unwrap() + .unwrap(); + let (prompt, completion) = litellm_cost::cost_calculator::speech_cost( + &catalog, + ModelCostRequest { + response_time_ms: Some(1000.0), + ..request("tts", "azure", &usage) + }, + None, + ) + .unwrap(); + assert!((prompt - 1e-3).abs() < 1e-15); + assert!((completion - 2e-3).abs() < 1e-15); +} + +#[rstest] +fn transcription_token_pricing_ignores_the_vertex_location() { + let catalog = ModelInfoCatalog::new(HashMap::from([( + "openai/transcribe".to_owned(), + json!({"input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6, "regional_endpoint_uplift_multiplier": 3.0}), + )])); + let usage = + get_usage_object(&json!({"usage": {"prompt_tokens": 1000, "completion_tokens": 1000}})) + .unwrap() + .unwrap(); + let (prompt, completion) = litellm_cost::cost_calculator::transcription_cost( + &catalog, + ModelCostRequest { + vertex_location: Some("us-east5"), + ..request("transcribe", "openai", &usage) + }, + 0.0, + ) + .unwrap(); + assert!((prompt - 1e-3).abs() < 1e-15); + assert!((completion - 2e-3).abs() < 1e-15); +}