diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index fd058a6dd2d..a3504690296 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -2979,6 +2979,7 @@ name = "litellm-cost" version = "0.1.0" dependencies = [ "criterion", + "fancy-regex 0.19.2", "jiff", "litellm-python-compat", "litellm-token-counter", diff --git a/litellm-rust/crates/cost/AGENTS.md b/litellm-rust/crates/cost/AGENTS.md index bc379794007..8d799a99edb 100644 --- a/litellm-rust/crates/cost/AGENTS.md +++ b/litellm-rust/crates/cost/AGENTS.md @@ -1,7 +1,7 @@ # litellm-cost port rules - This crate is a standalone port of Python's cost calculation. Prices, usage, model metadata, response kinds, and feature flags are caller-supplied; there is no Python bridge, no proxy integration, no network, and no env or secret resolution -- Every Rust function maps to a named Python function in `litellm/cost_calculator.py`, `litellm/litellm_core_utils/llm_cost_calc/`, or a provider `cost_calculator.py`/`cost_calculation.py`. Python entry points that also look up models, normalize responses, or choose providers correspond to Rust functions starting from already-selected inputs; `ModelInfoCatalog` supplies the selection layer Python does inline +- Every Rust function maps to a named Python function in `litellm/cost_calculator.py`, `litellm/litellm_core_utils/llm_cost_calc/`, or a provider `cost_calculator.py`/`cost_calculation.py`. Python entry points that also look up models, normalize responses, or choose providers correspond to Rust functions starting from already-selected inputs; `ModelInfoCatalog` supplies the selection layer Python does inline. Model lookup goes through `ModelInfoCatalog::get_model_info` (`model_info.rs`, mirroring `utils.py::_get_model_info_helper`) and provider inference through `get_llm_provider.rs` and `fallback_generalizations.rs`; never read `entries()` by a hand-built key where Python calls `get_model_info`. Known gaps: provider configs whose `get_model_info` queries a live server (ollama, lemonade) and the `ModelInfoBase` field projection are not ported - Module `foo_cost.rs` mirrors the calculation portion of the matching Python module; `tests/python_foo_cost.rs` carries the mapping evidence, with each test naming the Python test it mirrors (`tests/test_litellm/` mirrors `litellm/` per the repo-root AGENTS.md) - Preserve Python's precedence and fallback orders exactly: cache hit before provider-reported before calculated cost, discount before margin, wall-clock pricing before provider totals, quality and size key selection order, model-key selection order including duplicate provider prefixes and regional aliases - Diverge only where Python is observably wrong, and pin each divergence with a test that would fail if either side changed diff --git a/litellm-rust/crates/cost/Cargo.toml b/litellm-rust/crates/cost/Cargo.toml index 5b79276f26b..00c52c57dd7 100644 --- a/litellm-rust/crates/cost/Cargo.toml +++ b/litellm-rust/crates/cost/Cargo.toml @@ -10,6 +10,7 @@ serde.workspace = true serde_json.workspace = true litellm-python-compat = { path = "../python-compat" } litellm-token-counter = { path = "../token-counter", default-features = false } +fancy-regex.workspace = true jiff = "=0.2.37" regex = "=1.13.1" strum.workspace = true diff --git a/litellm-rust/crates/cost/src/azure_ai_cost.rs b/litellm-rust/crates/cost/src/azure_ai_cost.rs index a910bb8c228..af56dffc46d 100644 --- a/litellm-rust/crates/cost/src/azure_ai_cost.rs +++ b/litellm-rust/crates/cost/src/azure_ai_cost.rs @@ -57,6 +57,7 @@ pub fn azure_ai_cost_per_token( request_model: Option<&str>, ) -> Result<(f64, f64), CostError> { let model_info = catalog.entry(request.model, Some("azure_ai"), request.region); + let model_info = model_info.as_deref(); if let Some(cost) = model_info.and_then(|info| per_second_pricing_cost(info, request.response_time_ms)) { @@ -98,6 +99,6 @@ pub fn azure_ai_router_fee( let entry = catalog .entry(fee_entry, Some("azure_ai"), None) .ok_or(CostError::ModelNotFound)?; - let fee = calculate_azure_model_router_flat_cost(fee_name, prompt_tokens, entry); + let fee = calculate_azure_model_router_flat_cost(fee_name, prompt_tokens, &entry); Ok((fee > 0.0).then_some(fee)) } diff --git a/litellm-rust/crates/cost/src/bedrock_common_utils.rs b/litellm-rust/crates/cost/src/bedrock_common_utils.rs new file mode 100644 index 00000000000..8b643cbeb15 --- /dev/null +++ b/litellm-rust/crates/cost/src/bedrock_common_utils.rs @@ -0,0 +1,103 @@ +use std::sync::LazyLock; + +use regex::Regex; + +const ROUTING_PREFIXES: [&str; 7] = [ + "bedrock/", + "converse/", + "invoke/", + "openai/", + "mantle/", + "nova-2/", + "nova/", +]; + +const CROSS_REGION_INFERENCE_REGIONS: [&str; 7] = + ["global", "us", "eu", "apac", "jp", "au", "us-gov"]; + +const ALL_REGIONS: [&str; 23] = [ + "us-east-1", + "us-east-2", + "us-west-1", + "us-west-2", + "us-gov-east-1", + "us-gov-west-1", + "eu-west-1", + "eu-west-2", + "eu-west-3", + "eu-central-1", + "eu-central-2", + "eu-south-1", + "eu-south-2", + "eu-north-1", + "ap-northeast-1", + "ap-northeast-2", + "ap-northeast-3", + "ap-south-1", + "ap-south-2", + "ap-southeast-1", + "ap-southeast-2", + "ca-central-1", + "sa-east-1", +]; + +static THROUGHPUT_SUFFIX: LazyLock = + LazyLock::new(|| Regex::new(r"(:\d+):\d+k$").expect("valid throughput pattern")); +static CONTEXT_WINDOW_SUFFIX: LazyLock = + LazyLock::new(|| Regex::new(r"\[\w+\]$").expect("valid context window pattern")); + +pub fn is_bedrock_region(region: &str) -> bool { + ALL_REGIONS.contains(®ion) +} + +pub fn strip_bedrock_routing_prefix(model: &str) -> &str { + ROUTING_PREFIXES.iter().fold(model, |model, prefix| { + if model.starts_with(prefix) { + model.split_once('/').map_or(model, |(_, rest)| rest) + } else { + model + } + }) +} + +pub fn strip_bedrock_throughput_suffix(model: &str) -> String { + let without_throughput = THROUGHPUT_SUFFIX.replace(model, "${1}"); + CONTEXT_WINDOW_SUFFIX + .replace(&without_throughput, "") + .into_owned() +} + +pub fn extract_model_name_from_bedrock_arn(model: &str) -> &str { + if model.to_lowercase().contains("arn") { + model.rsplit('/').next().unwrap_or(model) + } else { + model + } +} + +pub fn get_bedrock_base_model(model: &str) -> String { + let spec = ["bedrock/converse/", "bedrock/", "converse/"] + .into_iter() + .find_map(|prefix| model.strip_prefix(prefix)) + .unwrap_or(model); + if spec.starts_with("nova-2/") { + return "amazon.nova-2-custom".to_owned(); + } + if spec.starts_with("nova/") { + return "amazon.nova-custom".to_owned(); + } + let model = strip_bedrock_throughput_suffix(extract_model_name_from_bedrock_arn( + strip_bedrock_routing_prefix(model), + )); + if let Some((region, rest)) = model.split_once('.') + && CROSS_REGION_INFERENCE_REGIONS.contains(®ion) + { + return rest.to_owned(); + } + if let Some((region, rest)) = model.split_once('/') + && is_bedrock_region(region) + { + return rest.to_owned(); + } + model +} diff --git a/litellm-rust/crates/cost/src/bedrock_image_cost.rs b/litellm-rust/crates/cost/src/bedrock_image_cost.rs index 6750636e20b..1593c7dd0ea 100644 --- a/litellm-rust/crates/cost/src/bedrock_image_cost.rs +++ b/litellm-rust/crates/cost/src/bedrock_image_cost.rs @@ -1,7 +1,9 @@ -use std::collections::HashMap; - use serde_json::Value; +use crate::catalog::ModelInfoCatalog; +use crate::error::CostError; +use crate::wire::py_real; + #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum BedrockImageFamily { Titan, @@ -22,57 +24,48 @@ pub fn get_config_class(model: &str) -> BedrockImageFamily { } } -pub fn stability1_pricing_key(model: &str, size: Option<&str>, optional_params: &Value) -> String { - let steps = optional_params - .get("steps") - .and_then(Value::as_f64) - .unwrap_or(50.0); +pub fn stability1_pricing_key( + model: &str, + size: Option<&str>, + optional_params: &Value, +) -> Result { + let steps = match optional_params.get("steps") { + None => 50.0, + Some(steps) => py_real(steps).ok_or(CostError::InvalidSteps)?, + }; let tier = if steps > 50.0 { "max-steps" } else { "50-steps" }; - format!("{}/{tier}/{model}", size.unwrap_or("1024-x-1024")) -} - -fn model_info<'a>( - entries: &'a HashMap, - model: &str, - family: BedrockImageFamily, - size: Option<&str>, - optional_params: &Value, -) -> Option<&'a Value> { - let bare = model.strip_prefix("bedrock/").unwrap_or(model); - match family { - BedrockImageFamily::Stability1 => { - let key = stability1_pricing_key(model, size, optional_params); - let bare_key = stability1_pricing_key(bare, size, optional_params); - [key, bare_key] - .into_iter() - .find_map(|candidate| entries.get(&candidate)) - } - _ => [model.to_owned(), format!("bedrock/{bare}"), bare.to_owned()] - .into_iter() - .find_map(|candidate| entries.get(&candidate)), - } + Ok(format!("{}/{tier}/{model}", size.unwrap_or("1024-x-1024"))) } pub fn cost_calculator( + catalog: &ModelInfoCatalog, model: &str, image_response: &Value, size: Option<&str>, optional_params: &Value, - entries: &HashMap, -) -> Option { - let family = get_config_class(model); - let info = model_info(entries, model, family, size, optional_params)?; - let rate = info +) -> Result { + let model_info = match get_config_class(model) { + BedrockImageFamily::Titan => catalog.get_model_info(model, None)?, + BedrockImageFamily::NovaCanvas | BedrockImageFamily::Stability3 => { + catalog.get_model_info(model, Some("bedrock"))? + } + BedrockImageFamily::Stability1 => catalog.get_model_info( + &stability1_pricing_key(model, size.filter(|size| !size.is_empty()), optional_params)?, + Some("bedrock"), + )?, + }; + let rate = model_info + .info .get("output_cost_per_image") - .and_then(Value::as_f64) + .and_then(py_real) .unwrap_or(0.0); let images = image_response .get("data") .and_then(Value::as_array) .map_or(0, Vec::len); - Some(rate * images as f64) + Ok(rate * images as f64) } diff --git a/litellm-rust/crates/cost/src/billed_token_rates.rs b/litellm-rust/crates/cost/src/billed_token_rates.rs index 10a4da4907a..0cbf8c7eea8 100644 --- a/litellm-rust/crates/cost/src/billed_token_rates.rs +++ b/litellm-rust/crates/cost/src/billed_token_rates.rs @@ -190,12 +190,12 @@ pub fn calculate_token_type_cost_breakdown( } fn billed_rates_request<'a>( - catalog: &'a ModelInfoCatalog, + model_info: Option<&'a Value>, request: ModelCostRequest<'a>, custom_cost_per_token: Option, ) -> BilledRatesRequest<'a> { BilledRatesRequest { - model_info: catalog.entry(request.model, request.provider, request.region), + model_info, usage: request.usage, provider: request.provider, service_tier: request.service_tier, @@ -211,8 +211,9 @@ pub fn get_billed_token_rates( request: ModelCostRequest<'_>, custom_cost_per_token: Option, ) -> Option { + let model_info = catalog.entry(request.model, request.provider, request.region); calculate_billed_token_rates(billed_rates_request( - catalog, + model_info.as_deref(), request, custom_cost_per_token, )) @@ -223,8 +224,9 @@ pub fn get_token_type_cost_breakdown( request: ModelCostRequest<'_>, custom_cost_per_token: Option, ) -> TokenTypeCostBreakdown { + let model_info = catalog.entry(request.model, request.provider, request.region); calculate_token_type_cost_breakdown(billed_rates_request( - catalog, + model_info.as_deref(), request, custom_cost_per_token, )) diff --git a/litellm-rust/crates/cost/src/catalog.rs b/litellm-rust/crates/cost/src/catalog.rs index 10792c5e062..b8085454d43 100644 --- a/litellm-rust/crates/cost/src/catalog.rs +++ b/litellm-rust/crates/cost/src/catalog.rs @@ -1,3 +1,4 @@ +use std::borrow::Cow; use std::collections::HashMap; use jiff::Timestamp; @@ -7,6 +8,11 @@ use crate::completion_input::{ CompletionInputRequest, PreparedCompletionInput, prepare_completion_input, }; use crate::error::CostError; +use crate::fallback_generalizations::FallbackGeneralizations; +use crate::get_llm_provider::{LlmProvider, ProviderModelSets, get_llm_provider}; +use crate::model_info::{ModelCost, ModelInfo, get_model_info_helper, lowercase_key_map}; + +const FALLBACK_GENERALIZATIONS_KEY: &str = "fallback_generalizations"; use crate::model_selection::{ ModelSelectionRequest, get_provider_for_cost_calc, select_model_name_for_cost_calc, }; @@ -15,6 +21,9 @@ use crate::responses_usage::ChatUsage; #[derive(Clone, Debug, Default)] pub struct ModelInfoCatalog { entries: HashMap, + lowercase_keys: HashMap, + model_sets: ProviderModelSets, + generalizations: FallbackGeneralizations, } #[derive(Clone, Copy, Debug)] @@ -63,33 +72,13 @@ pub enum CostCall<'a> { }, } -pub fn check_provider_match(model_info: &Value, provider: Option<&str>) -> bool { - let Some(provider) = provider.filter(|provider| !provider.is_empty()) else { - return true; - }; - let Some(entry_provider) = model_info.get("litellm_provider").and_then(Value::as_str) else { - return true; - }; - entry_provider == provider - || (provider == "vertex_ai" && entry_provider.starts_with("vertex_ai")) - || (provider == "fireworks_ai" && entry_provider.starts_with("fireworks_ai")) - || (provider.starts_with("bedrock") && entry_provider.starts_with("bedrock")) - || provider == "litellm_proxy" - || (provider == "azure_ai" && matches!(entry_provider, "azure" | "openai")) - || provider == "github" -} - -fn select_model_key<'a>( - entries: &'a HashMap, - model: &str, +fn cost_map_model<'m>( + entries: &HashMap, + model: &'m str, provider: Option<&str>, region: Option<&str>, -) -> Option<&'a str> { - let match_provider = provider.map(|provider| match provider { - "vertex_ai_beta" => "vertex_ai", - provider => provider, - }); - let normalized = match provider { +) -> Cow<'m, str> { + let model = match provider { Some(provider) => { let prefix = format!("{provider}/"); let mut name = model; @@ -106,44 +95,94 @@ fn select_model_key<'a>( let model_with_provider = match provider { Some(provider) => { let prefix = format!("{provider}/"); - let bare = normalized.strip_prefix(&prefix).unwrap_or(normalized); + let bare = model.strip_prefix(&prefix).unwrap_or(model); let regional = region.map(|region| format!("{provider}/{region}/{bare}")); if let Some(regional) = regional.filter(|key| entries.contains_key(key)) { - regional - } else if normalized.starts_with(&prefix) { - normalized.to_owned() + Cow::Owned(regional) + } else if model.starts_with(&prefix) { + Cow::Borrowed(model) } else { - format!("{provider}/{normalized}") + Cow::Owned(format!("{provider}/{model}")) } } - None => normalized.to_owned(), + None => Cow::Borrowed(model), }; - let without_prefix = normalized + let without_prefix = model .split_once('/') - .map_or(normalized, |(_, remainder)| remainder); - [model_with_provider.as_str(), normalized, without_prefix] - .into_iter() - .find_map(|candidate| { - entries - .get_key_value(candidate) - .filter(|(_, model_info)| check_provider_match(model_info, match_provider)) - .map(|(key, _)| key.as_str()) - }) + .map_or(model, |(_, remainder)| remainder); + if entries.contains_key(model_with_provider.as_ref()) { + model_with_provider + } else if entries.contains_key(model) { + Cow::Borrowed(model) + } else if entries.contains_key(without_prefix) { + Cow::Borrowed(without_prefix) + } else { + Cow::Borrowed(model) + } } impl ModelInfoCatalog { - pub fn new(entries: HashMap) -> Self { - Self { entries } + pub fn new(mut entries: HashMap) -> Self { + let generalizations = FallbackGeneralizations::from_block( + entries.remove(FALLBACK_GENERALIZATIONS_KEY).as_ref(), + ); + Self { + lowercase_keys: lowercase_key_map(&entries), + model_sets: ProviderModelSets::from_model_cost(&entries), + generalizations, + entries, + } + } + + fn model_cost(&self) -> ModelCost<'_> { + ModelCost { + entries: &self.entries, + lowercase_keys: &self.lowercase_keys, + sets: &self.model_sets, + generalizations: &self.generalizations, + } } pub fn entries(&self) -> &HashMap { &self.entries } - pub fn entry_for_key(&self, key: &str) -> &Value { - self.entries - .get(key) - .expect("key came from select_model_key") + pub fn get_llm_provider(&self, model: &str) -> Option { + get_llm_provider(model, &self.model_sets, &self.generalizations) + } + + pub fn get_model_info( + &self, + model: &str, + provider: Option<&str>, + ) -> Result, CostError> { + get_model_info_helper(self.model_cost(), model, provider) + } + + pub fn cost_map_model<'m>( + &self, + model: &'m str, + provider: Option<&str>, + region: Option<&str>, + ) -> Cow<'m, str> { + cost_map_model(&self.entries, model, provider, region) + } + + pub fn select_model_info( + &self, + model: &str, + provider: Option<&str>, + region: Option<&str>, + ) -> Option> { + let selected = cost_map_model(&self.entries, model, provider, region); + let inferred = provider + .is_none() + .then(|| self.get_llm_provider(model)) + .flatten(); + let provider = provider.or(inferred + .as_ref() + .map(|inferred| inferred.custom_llm_provider.as_str())); + self.get_model_info(&selected, provider).ok() } pub fn entry( @@ -151,45 +190,27 @@ impl ModelInfoCatalog { model: &str, provider: Option<&str>, region: Option<&str>, - ) -> Option<&Value> { - self.select_model_key(model, provider, region) - .and_then(|key| self.entries.get(key)) - } - - pub fn select_model_key<'a>( - &'a self, - model: &str, - provider: Option<&str>, - region: Option<&str>, - ) -> Option<&'a str> { - select_model_key(&self.entries, model, provider, region) + ) -> Option> { + self.select_model_info(model, provider, region) + .map(|model_info| model_info.info) } pub fn contains_exact_model(&self, model: &str) -> bool { self.entries.contains_key(model) } - pub fn get_provider_for_cost_calc( - &self, - model: Option<&str>, - provider: Option<&str>, - known_providers: &[&str], - ) -> Option { - get_provider_for_cost_calc(model, provider, known_providers, &self.entries) - } - pub fn select_model_name_for_cost_calc( &self, request: ModelSelectionRequest<'_>, ) -> Option { - select_model_name_for_cost_calc(request, &self.entries) + select_model_name_for_cost_calc(request, self) } pub fn pricing_entry_for_cost_calc<'a>( &'a self, request: ModelSelectionRequest<'a>, logging_details: Option<&'a Value>, - ) -> Option<(&'a str, &'a Value)> { + ) -> Option> { let deployment_info = if request.custom_pricing { request .router_model_id @@ -209,15 +230,13 @@ impl ModelInfoCatalog { if let (Some(key), Some(info)) = (request.router_model_id.or(request.model), deployment_info) { - return Some((key, info)); + return Some(ModelInfo { + key: Cow::Borrowed(key), + info: Cow::Borrowed(info), + }); } let selected = self.select_model_name_for_cost_calc(request); - let provider = get_provider_for_cost_calc( - request.model, - request.provider, - request.known_providers, - &self.entries, - ); + let provider = get_provider_for_cost_calc(request.model, request.provider, self); [ selected.as_deref(), request @@ -228,16 +247,13 @@ impl ModelInfoCatalog { ] .into_iter() .flatten() - .find_map(|model| { - self.select_model_key(model, provider.as_deref(), None) - .and_then(|key| self.entries.get(key).map(|info| (key, info))) - }) + .find_map(|model| self.select_model_info(model, provider.as_deref(), None)) } pub fn prepare_completion_input( &self, request: CompletionInputRequest<'_>, ) -> Result { - prepare_completion_input(request, &self.entries) + prepare_completion_input(request, self) } } diff --git a/litellm-rust/crates/cost/src/completion_input.rs b/litellm-rust/crates/cost/src/completion_input.rs index 87f204c496b..5dba3715d1f 100644 --- a/litellm-rust/crates/cost/src/completion_input.rs +++ b/litellm-rust/crates/cost/src/completion_input.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; +use crate::catalog::ModelInfoCatalog; use serde_json::Value; @@ -106,7 +106,7 @@ pub fn select_service_tier( pub fn prepare_completion_input( request: CompletionInputRequest<'_>, - cost_map: &HashMap, + catalog: &ModelInfoCatalog, ) -> Result { let response = request.model_selection.response; let call_type = @@ -126,7 +126,7 @@ pub fn prepare_completion_input( model, ..request.model_selection }, - cost_map, + catalog, ); let response_model = get_response_model(response).map(str::to_owned); let requested_model = model.map(str::to_owned); diff --git a/litellm-rust/crates/cost/src/completion_response.rs b/litellm-rust/crates/cost/src/completion_response.rs index b58b1b4cef3..052fae965c3 100644 --- a/litellm-rust/crates/cost/src/completion_response.rs +++ b/litellm-rust/crates/cost/src/completion_response.rs @@ -25,6 +25,7 @@ use crate::image_cost_router::{ ImageCostRouteRequest, call_type_has_image_response, route_image_generation_cost_calculator, }; use crate::mcp_cost::calculate_mcp_tool_call_cost; +use crate::model_selection::get_provider_for_cost_calc; use crate::per_second::{DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND, get_replicate_completion_pricing}; use crate::provider::LlmProviders; use crate::realtime_cost::{ @@ -647,13 +648,7 @@ pub fn completion_cost_from_response( .model_candidates .iter() .flatten() - .find_map(|model| { - catalog.get_provider_for_cost_calc( - Some(model), - None, - request.input.model_selection.known_providers, - ) - }) + .find_map(|model| get_provider_for_cost_calc(Some(model), None, catalog)) }); let provider = inferred_provider.as_deref(); let deployment_info = request diff --git a/litellm-rust/crates/cost/src/cost_calculator.rs b/litellm-rust/crates/cost/src/cost_calculator.rs index 3a6ce2e1a61..cf181d9d442 100644 --- a/litellm-rust/crates/cost/src/cost_calculator.rs +++ b/litellm-rust/crates/cost/src/cost_calculator.rs @@ -96,6 +96,16 @@ pub fn cost_per_token_for_call( catalog: &ModelInfoCatalog, request: ModelCostRequest<'_>, call: CostCall<'_>, +) -> Result<(f64, f64), CostError> { + with_inferred_provider(catalog, request, |catalog, request| { + cost_per_token_for_provider(catalog, request, call) + }) +} + +fn cost_per_token_for_provider( + catalog: &ModelInfoCatalog, + request: ModelCostRequest<'_>, + call: CostCall<'_>, ) -> Result<(f64, f64), CostError> { let provider = request .provider @@ -119,7 +129,7 @@ pub fn cost_per_token_for_call( if provider == Some(LlmProviders::AZURE_AI) { return azure_ai_cost_per_token(catalog, request, request_model); } - Ok(cost_per_token(catalog, request)?) + token_cost_per_token(catalog, request) } CostCall::Speech { prompt_characters } => speech_cost(catalog, request, prompt_characters), CostCall::Transcription { duration_seconds } => { @@ -154,7 +164,7 @@ pub fn cost_per_token_for_call( deployment_info, } => { let published = catalog.entry(request.model, request.provider, request.region); - Ok(ocr_cost(response, deployment_info, published)?) + Ok(ocr_cost(response, deployment_info, published.as_deref())?) } CostCall::Batch { deployment_info } => { let published = catalog.entry(request.model, request.provider, request.region); @@ -171,7 +181,7 @@ pub fn cost_per_token_for_call( let info = if has_deployment_rates { deployment_info } else { - published.or(deployment_info) + published.as_deref().or(deployment_info) }; let Some(info) = info else { return Ok((0.0, 0.0)); @@ -190,10 +200,39 @@ pub fn cost_per_token_for_call( pub fn cost_per_token( catalog: &ModelInfoCatalog, request: ModelCostRequest<'_>, +) -> Result<(f64, f64), CostError> { + with_inferred_provider(catalog, request, token_cost_per_token) +} + +fn with_inferred_provider( + catalog: &ModelInfoCatalog, + request: ModelCostRequest<'_>, + price: impl FnOnce(&ModelInfoCatalog, ModelCostRequest<'_>) -> Result<(f64, f64), CostError>, +) -> Result<(f64, f64), CostError> { + if request.provider.is_some() { + return price(catalog, request); + } + let inferred = catalog + .get_llm_provider(request.model) + .ok_or(CostError::MissingProvider)?; + let selected = catalog.cost_map_model(request.model, None, request.region); + price( + catalog, + ModelCostRequest { + model: &selected, + provider: Some(&inferred.custom_llm_provider), + ..request + }, + ) +} + +fn token_cost_per_token( + catalog: &ModelInfoCatalog, + request: ModelCostRequest<'_>, ) -> Result<(f64, f64), CostError> { if let Some(cost) = catalog .entry(request.model, request.provider, request.region) - .and_then(|info| per_second_pricing_cost(info, request.response_time_ms)) + .and_then(|info| per_second_pricing_cost(&info, request.response_time_ms)) { return Ok(cost); } @@ -219,18 +258,17 @@ pub fn cost_per_token( { return Ok((0.0, cost)); } - let key = catalog - .select_model_key(request.model, request.provider, request.region) + let selected = catalog + .select_model_info(request.model, request.provider, request.region) .or_else(|| { (provider == Some(LlmProviders::FIREWORKS_AI)) .then(|| get_base_model_for_pricing(request.model, FireworksThresholds::default())) .and_then(|category| { - catalog.select_model_key(category, request.provider, request.region) + catalog.select_model_info(category, request.provider, request.region) }) }) .ok_or(CostError::ModelNotFound)?; - let model_info = - apply_provider_cache_read_default(catalog.entry_for_key(key), request.provider); + let model_info = apply_provider_cache_read_default(&selected.info, request.provider); if let Some(cost) = per_second_pricing_cost(&model_info, request.response_time_ms) { return Ok(cost); } @@ -262,7 +300,7 @@ pub fn cost_per_token( Some(LlmProviders::FIREWORKS_AI) => { return Ok(fireworks_cost_per_token( request.usage, - catalog.entry_for_key(key), + &selected.info, request.at, )); } @@ -313,7 +351,7 @@ pub fn cost_per_token( _ => (request.service_tier, request.data_residency), }; let cost = generic_cost_per_token(GenericCostRequest { - model_info: catalog.entry_for_key(key), + model_info: &selected.info, usage: request.usage, provider: request.provider, service_tier, @@ -369,6 +407,7 @@ pub fn speech_cost( let model_info = catalog .entry(request.model, request.provider, request.region) .ok_or(CostError::ModelNotFound)?; + let model_info = &*model_info; match select_cost_metric_for_model(model_info)? { SpeechCostMetric::PerCharacter => { let characters = prompt_characters.ok_or(CostError::MissingPromptCharacters)?; @@ -399,6 +438,7 @@ pub fn transcription_cost( let model_info = catalog .entry(request.model, request.provider, request.region) .ok_or(CostError::ModelNotFound)?; + let model_info = &*model_info; if transcription_usage_has_token_details(request.usage) { return Ok(generic_cost_per_token(GenericCostRequest { model_info, @@ -421,6 +461,7 @@ pub fn handle_realtime_transcription_cost_calculation( ) -> f64 { let model = get_transcription_model_name_from_results(results).unwrap_or(requested_model); let model_info = catalog.entry(model, Some(provider), None); + let model_info = model_info.as_deref(); results .iter() .filter(|event| { @@ -467,7 +508,7 @@ pub fn handle_realtime_stream_cost_calculation( .find_map(|model| { let model_info = catalog.entry(model, Some(provider), None)?; let cost = generic_cost_per_token(GenericCostRequest { - model_info, + model_info: &model_info, usage: combined_usage, provider: Some(provider), service_tier: None, diff --git a/litellm-rust/crates/cost/src/databricks_cost.rs b/litellm-rust/crates/cost/src/databricks_cost.rs index be921d2e16f..299b9e30b35 100644 --- a/litellm-rust/crates/cost/src/databricks_cost.rs +++ b/litellm-rust/crates/cost/src/databricks_cost.rs @@ -34,7 +34,7 @@ pub fn databricks_cost_per_token( ) -> Result<(f64, f64), CostError> { if let Some(cost) = catalog .entry(request.model, Some("databricks"), request.region) - .and_then(|info| per_second_pricing_cost(info, request.response_time_ms)) + .and_then(|info| per_second_pricing_cost(&info, request.response_time_ms)) { return Ok(cost); } @@ -47,7 +47,7 @@ pub fn databricks_cost_per_token( .ok_or(CostError::ModelNotFound)?; Ok(calculate_generic_cost_from_model_info_with_region( request.usage, - entry, + &entry, None, false, None, diff --git a/litellm-rust/crates/cost/src/error.rs b/litellm-rust/crates/cost/src/error.rs index cfc9716c4be..26715714a6d 100644 --- a/litellm-rust/crates/cost/src/error.rs +++ b/litellm-rust/crates/cost/src/error.rs @@ -32,6 +32,8 @@ pub enum CostError { MissingProvider, #[error("no rate for the priced unit")] MissingRate, + #[error("image steps are not a number")] + InvalidSteps, #[error("no usage on the response")] MissingUsage, #[error("model is not in the cost map")] diff --git a/litellm-rust/crates/cost/src/fal_ai_image_cost.rs b/litellm-rust/crates/cost/src/fal_ai_image_cost.rs index 152cdc2449f..69d433c994a 100644 --- a/litellm-rust/crates/cost/src/fal_ai_image_cost.rs +++ b/litellm-rust/crates/cost/src/fal_ai_image_cost.rs @@ -4,6 +4,7 @@ use serde_json::Value; use crate::catalog::ModelInfoCatalog; use crate::image_response_cost::resolve_image_model_info; +use crate::wire::py_real; pub const PIXELS_PER_MEGAPIXEL: u64 = 1_048_576; @@ -136,12 +137,13 @@ pub fn fal_ai_passthrough_cost_from_model_info( } pub fn cost_calculator( + catalog: &ModelInfoCatalog, model: &str, image_response: &Value, optional_params: &Value, deployment_prices: Option<&Value>, - entries: &HashMap, ) -> Option { + let entries = catalog.entries(); let images = image_response.get("data").and_then(Value::as_array); let count = images.map_or(0, Vec::len); if let Some(rate) = deployment_prices @@ -160,15 +162,16 @@ pub fn cost_calculator( if keyed.iter().all(Option::is_some) { return Some(keyed.into_iter().flatten().sum()); } - let shared = entries.get(&format!("fal_ai/{normalized_model}")); - let resolved = resolve_image_model_info(shared, deployment_prices)?; + let shared = catalog + .get_model_info(normalized_model, Some("fal_ai")) + .ok() + .map(|model_info| model_info.info); + let resolved = resolve_image_model_info(shared.as_deref(), deployment_prices)?; let flat_rate = resolved .get("output_cost_per_image") - .and_then(Value::as_f64) + .and_then(py_real) .unwrap_or(0.0); - let pixel_rate = resolved - .get("output_cost_per_pixel") - .and_then(Value::as_f64); + let pixel_rate = resolved.get("output_cost_per_pixel").and_then(py_real); Some( images .into_iter() diff --git a/litellm-rust/crates/cost/src/fallback_generalizations.rs b/litellm-rust/crates/cost/src/fallback_generalizations.rs new file mode 100644 index 00000000000..ab7b6c5e9d6 --- /dev/null +++ b/litellm-rust/crates/cost/src/fallback_generalizations.rs @@ -0,0 +1,147 @@ +use std::collections::HashMap; + +use fancy_regex::{Regex, RegexBuilder}; +use serde_json::{Map, Value}; + +const PROVIDER_KEY: &str = "litellm_provider"; + +#[derive(Clone, Debug)] +struct RoutingRule { + pattern: Regex, + provider: String, +} + +#[derive(Clone, Debug)] +struct CapabilityRule { + pattern: Regex, + model_info: Map, +} + +#[derive(Clone, Debug, Default)] +pub struct FallbackGeneralizations { + routing_rules: Vec, + capability_rules: Vec, +} + +fn resolve_legacy_extends(rules: &[Value]) -> Vec { + let base_by_name: HashMap<&str, &Map> = rules + .iter() + .filter_map(|rule| { + Some(( + rule.get("name")?.as_str()?, + rule.get("model_info")?.as_object()?, + )) + }) + .collect(); + rules + .iter() + .map(|rule| { + let parent = rule + .get("extends") + .and_then(Value::as_str) + .and_then(|name| base_by_name.get(name)); + match (parent, rule.get("model_info").and_then(Value::as_object)) { + (Some(parent), Some(own)) => { + let mut merged = (*parent).clone(); + merged.extend(own.clone()); + let mut resolved = rule.as_object().cloned().unwrap_or_default(); + resolved.insert("model_info".to_owned(), Value::Object(merged)); + Value::Object(resolved) + } + _ => rule.clone(), + } + }) + .collect() +} + +fn fill_missing_is_malformed(rule: &Value) -> bool { + rule.get("fill_missing_for_providers") + .is_some_and(|providers| { + !providers + .as_array() + .is_some_and(|providers| providers.iter().all(Value::is_string)) + }) +} + +fn compile_rule(rule: &Value) -> (Option, Option) { + let (Some(pattern), Some(model_info)) = ( + rule.get("pattern").and_then(Value::as_str), + rule.get("model_info").and_then(Value::as_object), + ) else { + return (None, None); + }; + let Ok(pattern) = RegexBuilder::new(pattern).case_insensitive(true).build() else { + return (None, None); + }; + if fill_missing_is_malformed(rule) { + return (None, None); + } + let capability = || CapabilityRule { + pattern: pattern.clone(), + model_info: model_info.clone(), + }; + match model_info.get(PROVIDER_KEY) { + None => (None, Some(capability())), + Some(Value::String(provider)) => { + let routing = RoutingRule { + pattern: pattern.clone(), + provider: provider.clone(), + }; + let capability = (model_info.len() > 1).then(capability); + (Some(routing), capability) + } + Some(_) => (None, None), + } +} + +fn is_match(pattern: &Regex, model: &str) -> bool { + pattern.is_match(model).unwrap_or(false) +} + +impl FallbackGeneralizations { + pub fn from_block(block: Option<&Value>) -> Self { + let rules = block + .and_then(|block| block.get("rules")) + .and_then(Value::as_array) + .map(|rules| resolve_legacy_extends(rules)) + .unwrap_or_default(); + let (routing_rules, capability_rules): (Vec<_>, Vec<_>) = + rules.iter().map(compile_rule).unzip(); + Self { + routing_rules: routing_rules.into_iter().flatten().collect(), + capability_rules: capability_rules.into_iter().flatten().collect(), + } + } + + pub fn match_routing(&self, model: &str) -> Option<&str> { + if model.is_empty() { + return None; + } + self.routing_rules + .iter() + .find(|rule| is_match(&rule.pattern, model)) + .map(|rule| rule.provider.as_str()) + } + + pub fn match_capabilities(&self, model: &str) -> Option> { + if model.is_empty() { + return None; + } + let matched: Vec<&Map> = self + .capability_rules + .iter() + .filter(|rule| is_match(&rule.pattern, model)) + .map(|rule| &rule.model_info) + .collect(); + (!matched.is_empty()).then(|| { + matched + .into_iter() + .flat_map(|model_info| { + model_info + .iter() + .map(|(key, value)| (key.clone(), value.clone())) + }) + .collect() + }) + } +} diff --git a/litellm-rust/crates/cost/src/get_llm_provider.rs b/litellm-rust/crates/cost/src/get_llm_provider.rs new file mode 100644 index 00000000000..20658a58c52 --- /dev/null +++ b/litellm-rust/crates/cost/src/get_llm_provider.rs @@ -0,0 +1,291 @@ +use std::collections::{HashMap, HashSet}; +use std::str::FromStr; +use std::sync::LazyLock; + +use regex::Regex; +use serde_json::Value; + +use crate::fallback_generalizations::FallbackGeneralizations; +use crate::provider::LlmProviders; + +const OPENAI_IMAGE_GENERATION_MODELS: [&str; 2] = ["dall-e-2", "dall-e-3"]; +const OPENAI_VIDEO_GENERATION_MODELS: [&str; 1] = ["sora-2"]; +const COHERE_EMBEDDING_MODELS: [&str; 7] = [ + "embed-v4.0", + "embed-english-v3.0", + "embed-english-light-v3.0", + "embed-multilingual-v3.0", + "embed-english-v2.0", + "embed-english-light-v2.0", + "embed-multilingual-v2.0", +]; +const REPLICATE_MODEL_NAME_WITH_ID_LENGTH: usize = 64; +const PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO: [&str; 2] = ["github_copilot", "chatgpt"]; +const MODEL_PREFIX_PROVIDERS: [(&str, &str); 11] = [ + ("bytez/", "bytez"), + ("gdc/", "gdc"), + ("lemonade/", "lemonade"), + ("heroku/", "heroku"), + ("cometapi/", "cometapi"), + ("oci/", "oci"), + ("compactifai/", "compactifai"), + ("ovhcloud/", "ovhcloud"), + ("clarifai/", "clarifai"), + ("amazon_nova", "amazon_nova"), + ("sap/", "sap"), +]; + +static JSON_PROVIDERS: LazyLock> = LazyLock::new(|| { + serde_json::from_str::>(include_str!( + "../../../../litellm/llms/openai_like/providers.json" + )) + .expect("openai-like provider registry parses") + .into_iter() + .map(|(slug, _)| slug) + .collect() +}); + +static BEDROCK_PRICING_ONLY: LazyLock = LazyLock::new(|| { + Regex::new(r"^bedrock/[a-zA-Z0-9_-]+/.+$").expect("valid bedrock pricing pattern") +}); + +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub enum ModelList { + OpenAiChatCompletion, + OpenAiTextCompletion, + Anthropic, + Cohere, + CohereChat, + MistralChat, + Openrouter, + VertexText, + VertexCodeText, + VertexLanguage, + VertexVision, + VertexChat, + VertexCodeChat, + VertexEmbedding, + VertexLlama3, + VertexMistral, + VertexAi21, + VertexImage, + VertexVideo, + Ai21Chat, + Ai21, + NlpCloud, + AlephAlpha, + Bedrock, + BedrockConverse, + Watsonx, + Empower, + GradientAi, +} + +fn is_openai_finetune_model(key: &str) -> bool { + key.starts_with("ft:") && key.matches(':').count() <= 1 +} + +fn is_bedrock_pricing_only_model(key: &str) -> bool { + BEDROCK_PRICING_ONLY.is_match(key) +} + +fn vertex_member(key: &str) -> String { + key.replace("vertex_ai/", "") +} + +fn model_list_membership(key: &str, info: &Value) -> Option<(ModelList, String)> { + let provider = info.get("litellm_provider").and_then(Value::as_str)?; + let mode = info.get("mode").and_then(Value::as_str); + let list = match provider { + "openai" if !is_openai_finetune_model(key) => ModelList::OpenAiChatCompletion, + "text-completion-openai" => ModelList::OpenAiTextCompletion, + "cohere" => ModelList::Cohere, + "cohere_chat" => ModelList::CohereChat, + "mistral" => ModelList::MistralChat, + "anthropic" => ModelList::Anthropic, + "empower" => ModelList::Empower, + "openrouter" => ModelList::Openrouter, + "vertex_ai-text-models" => ModelList::VertexText, + "vertex_ai-code-text-models" => ModelList::VertexCodeText, + "vertex_ai-language-models" => ModelList::VertexLanguage, + "vertex_ai-vision-models" => ModelList::VertexVision, + "vertex_ai-chat-models" => ModelList::VertexChat, + "vertex_ai-code-chat-models" => ModelList::VertexCodeChat, + "vertex_ai-embedding-models" => ModelList::VertexEmbedding, + "vertex_ai-llama_models" => return Some((ModelList::VertexLlama3, vertex_member(key))), + "vertex_ai-mistral_models" => return Some((ModelList::VertexMistral, vertex_member(key))), + "vertex_ai-ai21_models" => return Some((ModelList::VertexAi21, vertex_member(key))), + "vertex_ai-image-models" => return Some((ModelList::VertexImage, vertex_member(key))), + "vertex_ai-video-models" => return Some((ModelList::VertexVideo, vertex_member(key))), + "ai21" if mode == Some("chat") => ModelList::Ai21Chat, + "ai21" => ModelList::Ai21, + "nlp_cloud" => ModelList::NlpCloud, + "aleph_alpha" => ModelList::AlephAlpha, + "bedrock" if mode == Some("guardrail") => return None, + "bedrock" if !is_bedrock_pricing_only_model(key) => ModelList::Bedrock, + "bedrock_converse" => ModelList::BedrockConverse, + "watsonx" => ModelList::Watsonx, + "gradient_ai" => ModelList::GradientAi, + _ => return None, + }; + Some((list, key.to_owned())) +} + +#[derive(Clone, Debug, Default)] +pub struct ProviderModelSets { + members: HashMap>, +} + +impl ProviderModelSets { + pub fn from_model_cost(model_cost: &HashMap) -> Self { + let members = model_cost + .iter() + .filter_map(|(key, info)| model_list_membership(key, info)) + .fold( + HashMap::>::new(), + |mut members, (list, name)| { + members.entry(list).or_default().insert(name); + members + }, + ); + Self { members } + } + + pub fn contains(&self, list: ModelList, model: &str) -> bool { + self.members + .get(&list) + .is_some_and(|models| models.contains(model)) + } + + fn contains_any(&self, lists: &[ModelList], model: &str) -> bool { + lists.iter().any(|list| self.contains(*list, model)) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct LlmProvider { + pub model: String, + pub custom_llm_provider: String, +} + +pub fn declared_authenticating_provider(model: &str) -> Option<&str> { + model + .split_once('/') + .map(|(declared, _)| declared) + .filter(|declared| PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO.contains(declared)) +} + +fn is_anthropic_text_model(model: &str) -> bool { + matches!(model, "claude-2" | "claude-instant-1") +} + +fn resolved(model: &str, provider: &str) -> Option { + Some(LlmProvider { + model: model.to_owned(), + custom_llm_provider: provider.to_owned(), + }) +} + +fn bare_model_provider<'a>(model: &str, sets: &ProviderModelSets) -> Option<&'a str> { + use ModelList::*; + let replicate_id = model + .split(':') + .nth(1) + .is_some_and(|id| id.len() == REPLICATE_MODEL_NAME_WITH_ID_LENGTH); + if sets.contains(OpenAiChatCompletion, model) + || model.contains("ft:gpt-3.5-turbo") + || model.contains("ft:gpt-4") + || OPENAI_IMAGE_GENERATION_MODELS.contains(&model) + || model.starts_with("gpt-image") + || OPENAI_VIDEO_GENERATION_MODELS.contains(&model) + { + Some("openai") + } else if sets.contains(OpenAiTextCompletion, model) { + Some("text-completion-openai") + } else if sets.contains(Anthropic, model) { + Some(if is_anthropic_text_model(model) { + "anthropic_text" + } else { + "anthropic" + }) + } else if sets.contains(Cohere, model) || COHERE_EMBEDDING_MODELS.contains(&model) { + Some("cohere") + } else if sets.contains(CohereChat, model) { + Some("cohere_chat") + } else if model.contains(':') && model.len() > REPLICATE_MODEL_NAME_WITH_ID_LENGTH { + replicate_id.then_some("replicate") + } else if sets.contains(Openrouter, model) { + Some("openrouter") + } else if sets.contains_any( + &[ + VertexChat, + VertexCodeChat, + VertexText, + VertexCodeText, + VertexLanguage, + VertexEmbedding, + VertexVision, + VertexImage, + VertexVideo, + ], + model, + ) { + Some("vertex_ai") + } else if sets.contains_any(&[Ai21Chat, Ai21], model) { + Some("ai21_chat") + } else if sets.contains(AlephAlpha, model) { + Some("aleph_alpha") + } else if sets.contains(NlpCloud, model) { + Some("nlp_cloud") + } else if sets.contains_any(&[Bedrock, BedrockConverse], model) { + Some("bedrock") + } else if sets.contains(Watsonx, model) { + Some("watsonx") + } else if sets.contains(Empower, model) { + Some("empower") + } else if sets.contains(GradientAi, model) { + Some("gradient_ai") + } else if model == "*" { + Some("openai") + } else { + MODEL_PREFIX_PROVIDERS + .iter() + .find(|(prefix, _)| model.starts_with(prefix)) + .map(|(_, provider)| *provider) + } +} + +pub fn get_llm_provider( + model: &str, + sets: &ProviderModelSets, + generalizations: &FallbackGeneralizations, +) -> Option { + if let Some((prefix, rest)) = model.split_once('/') { + if prefix == "azure" + && (sets.contains(ModelList::CohereChat, rest) + || sets.contains(ModelList::MistralChat, &format!("mistral/{rest}"))) + { + return resolved(model, "openai"); + } + if prefix == "cohere" && sets.contains(ModelList::CohereChat, rest) { + return resolved(rest, "cohere_chat"); + } + if prefix == "anthropic" && is_anthropic_text_model(rest) { + return resolved(rest, "anthropic_text"); + } + if JSON_PROVIDERS.contains(prefix) { + return resolved(rest, prefix); + } + if LlmProviders::from_str(prefix).is_ok() { + let provider = if prefix == "ai21" { + "ai21_chat" + } else { + prefix + }; + return resolved(rest, provider); + } + } + bare_model_provider(model, sets) + .or_else(|| generalizations.match_routing(model)) + .and_then(|provider| resolved(model, provider)) +} diff --git a/litellm-rust/crates/cost/src/image_cost_router.rs b/litellm-rust/crates/cost/src/image_cost_router.rs index 3e731e59056..3a4b4a2b2cc 100644 --- a/litellm-rust/crates/cost/src/image_cost_router.rs +++ b/litellm-rust/crates/cost/src/image_cost_router.rs @@ -282,21 +282,20 @@ pub fn route_image_generation_cost_calculator<'a>( pricing.as_ref(), ), Some(LlmProviders::FAL_AI) => fal_ai_image_cost_calculator( + catalog, request.model, request.image_response, request.optional_params, pricing.as_ref(), - catalog.entries(), ) .ok_or(CostError::ModelNotFound), Some(LlmProviders::BEDROCK) => bedrock_image_cost_calculator( + catalog, request.model, request.image_response, Some(resolved_size), request.optional_params, - catalog.entries(), - ) - .ok_or(CostError::ModelNotFound), + ), Some(provider @ (LlmProviders::OPENAI | LlmProviders::AZURE)) if request.model.to_ascii_lowercase().contains("gpt-image") => { @@ -330,7 +329,11 @@ fn resolved_model_info( supplied_model_info: Option<&Value>, ) -> Result { resolve_image_model_info( - catalog.entry(model, Some(provider), None), + catalog + .get_model_info(model, Some(provider)) + .ok() + .map(|model_info| model_info.info) + .as_deref(), supplied_model_info, ) .ok_or(CostError::ModelNotFound) @@ -374,9 +377,8 @@ pub fn google_image_edit_cost( Ok(gemini_image_edit_cost(image_response, &model_info, at)) } "vertex_ai" => catalog - .entry(model, Some(provider), None) - .map(|model_info| vertex_image_edit_cost(image_response, model_info)) - .ok_or(CostError::ModelNotFound), + .get_model_info(model, Some(provider)) + .map(|model_info| vertex_image_edit_cost(image_response, &model_info.info)), _ => Err(CostError::ModelNotFound), } } @@ -385,7 +387,8 @@ pub fn azure_ai_image_generation_cost( catalog: &ModelInfoCatalog, request: AzureAiImageCatalogRequest<'_>, ) -> Result { - let shared = catalog.entry(request.model, Some("azure_ai"), None); + let selected = catalog.get_model_info(request.model, Some("azure_ai")).ok(); + let shared = selected.as_ref().map(|selected| &*selected.info); let model_info = resolve_image_model_info(shared, request.supplied_model_info) .ok_or(CostError::ModelNotFound)?; let shared_pricing = model_info @@ -395,9 +398,9 @@ pub fn azure_ai_image_generation_cost( .or(shared); azure_ai_image_cost_calculator(AzureAiImageRequest { catalog, - model: catalog - .select_model_key(request.model, Some("azure_ai"), None) - .unwrap_or(request.model), + model: selected + .as_ref() + .map_or(request.model, |selected| &selected.key), image_response: request.image_response, model_info: &model_info, supplied_model_info: request.supplied_model_info, diff --git a/litellm-rust/crates/cost/src/lib.rs b/litellm-rust/crates/cost/src/lib.rs index ca492510fbf..7c24efb4049 100644 --- a/litellm-rust/crates/cost/src/lib.rs +++ b/litellm-rust/crates/cost/src/lib.rs @@ -15,6 +15,7 @@ pub mod background_cost_polling; pub mod base_rate_selection; #[allow(clippy::disallowed_types)] pub mod batch; +pub mod bedrock_common_utils; #[allow(clippy::disallowed_types)] pub mod bedrock_image_cost; #[allow(clippy::disallowed_types)] @@ -40,6 +41,8 @@ pub mod error; #[allow(clippy::disallowed_types)] pub mod fal_ai_image_cost; #[allow(clippy::disallowed_types)] +pub mod fallback_generalizations; +#[allow(clippy::disallowed_types)] pub mod fireworks_cost; #[allow(clippy::disallowed_types)] pub mod gemini_cost; @@ -51,6 +54,8 @@ pub mod generic_input; pub mod generic_output; pub mod generic_usage; #[allow(clippy::disallowed_types)] +pub mod get_llm_provider; +#[allow(clippy::disallowed_types)] pub mod groq_cost; #[allow(clippy::disallowed_types)] pub mod guardrail_cost; @@ -65,6 +70,8 @@ pub mod lemonade_cost; #[allow(clippy::disallowed_types)] pub mod mcp_cost; #[allow(clippy::disallowed_types)] +pub mod model_info; +#[allow(clippy::disallowed_types)] pub mod model_selection; pub mod non_token; #[allow(clippy::disallowed_types)] diff --git a/litellm-rust/crates/cost/src/model_info.rs b/litellm-rust/crates/cost/src/model_info.rs new file mode 100644 index 00000000000..7f18c7d28e4 --- /dev/null +++ b/litellm-rust/crates/cost/src/model_info.rs @@ -0,0 +1,306 @@ +use std::borrow::Cow; +use std::collections::HashMap; +use std::sync::LazyLock; + +use regex::Regex; +use serde_json::{Value, json}; + +use crate::bedrock_common_utils::{ + get_bedrock_base_model, is_bedrock_region, strip_bedrock_routing_prefix, +}; +use crate::error::CostError; +use crate::fallback_generalizations::FallbackGeneralizations; +use crate::get_llm_provider::{ + ModelList, ProviderModelSets, declared_authenticating_provider, get_llm_provider, +}; + +const AZURE_LLMS: [(&str, &str); 7] = [ + ("gpt-35-turbo", "azure/gpt-35-turbo"), + ("gpt-35-turbo-16k", "azure/gpt-35-turbo-16k"), + ("gpt-35-turbo-instruct", "azure/gpt-35-turbo-instruct"), + ("azure/gpt-41", "gpt-4.1"), + ("azure/gpt-41-mini", "gpt-4.1-mini"), + ("azure/gpt-41-nano", "gpt-4.1-nano"), + ("ada", "azure/ada"), +]; + +static STABLE_VERTEX_VERSION: LazyLock = + LazyLock::new(|| Regex::new(r"-\d+$").expect("valid vertex version pattern")); +static DATED_SNAPSHOT_SUFFIX: LazyLock = + LazyLock::new(|| Regex::new(r"-\d{4}-\d{2}-\d{2}$").expect("valid snapshot pattern")); +static OPENAI_FINETUNE_SUFFIX: LazyLock = + LazyLock::new(|| Regex::new(r"(:[^:]*){3}$").expect("valid finetune pattern")); + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct PotentialModelNames { + pub split_model: String, + pub combined_model_name: String, + pub stripped_model_name: String, + pub combined_stripped_model_name: String, + pub provider_prefixed_model_name: String, + pub custom_llm_provider: Option, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct ModelInfo<'a> { + pub key: Cow<'a, str>, + pub info: Cow<'a, Value>, +} + +#[derive(Clone, Copy, Debug)] +pub struct ModelCost<'a> { + pub entries: &'a HashMap, + pub lowercase_keys: &'a HashMap, + pub sets: &'a ProviderModelSets, + pub generalizations: &'a FallbackGeneralizations, +} + +pub fn strip_model_name(model: &str, custom_llm_provider: Option<&str>) -> String { + match custom_llm_provider { + Some("bedrock" | "bedrock_converse") => get_bedrock_base_model(model), + Some("vertex_ai" | "gemini" | "databricks") => { + STABLE_VERTEX_VERSION.replace(model, "").into_owned() + } + _ if model.contains("ft:") => OPENAI_FINETUNE_SUFFIX.replace(model, "").into_owned(), + _ => DATED_SNAPSHOT_SUFFIX.replace(model, "").into_owned(), + } +} + +fn split_mantle_region_prefix(model: &str) -> &str { + match model.split_once('/') { + Some((head, tail)) if is_bedrock_region(head) => tail, + _ => model, + } +} + +fn resolve_fireworks_resource_name(model: &str) -> String { + let stripped = model.strip_prefix("fireworks_ai/").unwrap_or(model); + if stripped.starts_with("accounts/") || stripped.starts_with("FW-") || stripped.contains('#') { + stripped.to_owned() + } else if stripped.starts_with("routers/") || stripped.starts_with("models/") { + format!("accounts/fireworks/{stripped}") + } else if stripped.ends_with("-fast") { + format!("accounts/fireworks/routers/{stripped}") + } else { + format!("accounts/fireworks/models/{stripped}") + } +} + +pub fn get_potential_model_names( + model_cost: ModelCost<'_>, + model: &str, + custom_llm_provider: Option<&str>, +) -> PotentialModelNames { + let (split_model, combined, stripped, combined_stripped, provider_prefixed, provider) = + match custom_llm_provider { + None => { + let inferred = get_llm_provider(model, model_cost.sets, model_cost.generalizations); + let provider = inferred + .as_ref() + .map(|inferred| inferred.custom_llm_provider.clone()); + let stripped = strip_model_name(model, provider.as_deref()); + ( + inferred.map_or_else(|| model.to_owned(), |inferred| inferred.model), + model.to_owned(), + stripped.clone(), + stripped, + model.to_owned(), + provider, + ) + } + Some(provider) => { + let (split, combined, provider_prefixed) = + match model.strip_prefix(&format!("{provider}/")) { + Some(split) => (split, model.to_owned(), format!("{provider}/{model}")), + None => ( + model, + format!("{provider}/{model}"), + format!("{provider}/{model}"), + ), + }; + let stripped = strip_model_name(split, Some(provider)); + let combined_stripped = format!("{provider}/{stripped}"); + ( + split.to_owned(), + combined, + stripped, + combined_stripped, + provider_prefixed, + Some(provider.to_owned()), + ) + } + }; + let split_model = match provider.as_deref() { + Some("bedrock" | "bedrock_converse") => { + strip_bedrock_routing_prefix(&split_model).to_owned() + } + _ => split_model, + }; + let (split_model, combined_stripped) = match provider.as_deref() { + Some(provider @ "bedrock_mantle") => { + let region_free = split_mantle_region_prefix(&split_model).to_owned(); + let combined_stripped = format!( + "bedrock_mantle/{}", + strip_model_name(®ion_free, Some(provider)) + ); + (region_free, combined_stripped) + } + _ => (split_model, combined_stripped), + }; + let provider_prefixed = match provider.as_deref() { + Some("fireworks_ai") => format!( + "fireworks_ai/{}", + resolve_fireworks_resource_name(&split_model) + ), + _ => provider_prefixed, + }; + PotentialModelNames { + split_model, + combined_model_name: combined, + stripped_model_name: stripped, + combined_stripped_model_name: combined_stripped, + provider_prefixed_model_name: provider_prefixed, + custom_llm_provider: provider, + } +} + +pub fn get_model_cost_key<'a>(model_cost: ModelCost<'a>, potential_key: &str) -> Option<&'a str> { + model_cost + .entries + .get_key_value(potential_key) + .map(|(key, _)| key.as_str()) + .or_else(|| { + model_cost + .lowercase_keys + .get(&potential_key.to_lowercase()) + .map(String::as_str) + }) +} + +pub fn lowercase_key_map(model_cost: &HashMap) -> HashMap { + model_cost.keys().fold(HashMap::new(), |mut map, key| { + map.entry(key.to_lowercase()) + .and_modify(|existing: &mut String| { + if key < existing { + existing.clone_from(key); + } + }) + .or_insert_with(|| key.clone()); + map + }) +} + +fn vertex_ai_model_alias(model: &str, sets: &ProviderModelSets) -> Option { + let meta = format!("meta/{model}"); + if sets.contains(ModelList::VertexLlama3, &meta) { + return Some(meta); + } + let latest = format!("{model}@latest"); + (sets.contains(ModelList::VertexMistral, &latest) + || sets.contains(ModelList::VertexAi21, &latest)) + .then_some(latest) +} + +fn get_model_info_from_generalization( + model_cost: ModelCost<'_>, + candidates: &[&str], + custom_llm_provider: Option<&str>, +) -> Option<(String, Value)> { + if candidates + .iter() + .any(|candidate| get_model_cost_key(model_cost, candidate).is_some()) + { + return None; + } + candidates.iter().find_map(|candidate| { + let mut info = model_cost.generalizations.match_capabilities(candidate)?; + if let Some(provider) = custom_llm_provider { + info.insert( + "litellm_provider".to_owned(), + Value::String(provider.to_owned()), + ); + } + Some(((*candidate).to_owned(), Value::Object(info))) + }) +} + +pub fn get_model_info_helper<'a>( + model_cost: ModelCost<'a>, + model: &str, + custom_llm_provider: Option<&str>, +) -> Result, CostError> { + let model = AZURE_LLMS + .iter() + .find(|(alias, _)| *alias == model) + .map_or(model, |(_, target)| *target); + let custom_llm_provider = custom_llm_provider.map(|provider| match provider { + "vertex_ai_beta" => "vertex_ai", + provider => provider, + }); + let model = match custom_llm_provider { + Some("vertex_ai") => { + vertex_ai_model_alias(model, model_cost.sets).map_or(Cow::Borrowed(model), Cow::Owned) + } + _ => Cow::Borrowed(model), + }; + let names = get_potential_model_names( + model_cost, + &model, + custom_llm_provider.or_else(|| declared_authenticating_provider(&model)), + ); + let provider = names.custom_llm_provider.as_deref(); + if provider == Some("huggingface") { + return Ok(ModelInfo { + key: Cow::Owned(model.into_owned()), + info: Cow::Owned(json!({ + "input_cost_per_token": 0, + "output_cost_per_token": 0, + "litellm_provider": "huggingface", + "mode": "chat" + })), + }); + } + let candidates = [ + names.combined_model_name.as_str(), + &model, + &names.split_model, + &names.combined_stripped_model_name, + &names.stripped_model_name, + &names.provider_prefixed_model_name, + ]; + let mapped = candidates + .iter() + .filter_map(|candidate| get_model_cost_key(model_cost, candidate)) + .find_map(|key| { + let info = &model_cost.entries[key]; + check_provider_match(info, provider).then_some(ModelInfo { + key: Cow::Borrowed(key), + info: Cow::Borrowed(info), + }) + }); + if let Some(mapped) = mapped { + return Ok(mapped); + } + get_model_info_from_generalization(model_cost, &candidates, provider) + .map(|(key, info)| ModelInfo { + key: Cow::Owned(key), + info: Cow::Owned(info), + }) + .ok_or(CostError::ModelNotFound) +} + +pub fn check_provider_match(model_info: &Value, provider: Option<&str>) -> bool { + let Some(provider) = provider.filter(|provider| !provider.is_empty()) else { + return true; + }; + let Some(entry_provider) = model_info.get("litellm_provider").and_then(Value::as_str) else { + return true; + }; + entry_provider == provider + || (provider == "vertex_ai" && entry_provider.starts_with("vertex_ai")) + || (provider == "fireworks_ai" && entry_provider.starts_with("fireworks_ai")) + || (provider.starts_with("bedrock") && entry_provider.starts_with("bedrock")) + || provider == "litellm_proxy" + || (provider == "azure_ai" && matches!(entry_provider, "azure" | "openai")) + || provider == "github" +} diff --git a/litellm-rust/crates/cost/src/model_selection.rs b/litellm-rust/crates/cost/src/model_selection.rs index d1c388fe5bf..8f5a8abd55d 100644 --- a/litellm-rust/crates/cost/src/model_selection.rs +++ b/litellm-rust/crates/cost/src/model_selection.rs @@ -1,7 +1,10 @@ -use std::collections::HashMap; +use std::str::FromStr; use serde_json::Value; +use crate::catalog::ModelInfoCatalog; +use crate::provider::LlmProviders; + #[derive(Clone, Copy, Debug)] pub struct ModelSelectionRequest<'a> { pub model: Option<&'a str>, @@ -12,7 +15,6 @@ pub struct ModelSelectionRequest<'a> { pub provider: Option<&'a str>, pub router_model_id: Option<&'a str>, pub region_name: Option<&'a str>, - pub known_providers: &'a [&'a str], } pub fn get_response_model(response: Option<&Value>) -> Option<&str> { @@ -29,17 +31,20 @@ pub fn get_hidden_str_for_cost_calc<'a>( .filter(|value| !value.is_empty()) } -pub fn model_contains_known_llm_provider(model: &str, known_providers: &[&str]) -> bool { - let prefix = model.split('/').next().unwrap_or(model); - known_providers.contains(&prefix) +fn is_llm_provider(name: &str) -> bool { + LlmProviders::from_str(name).is_ok() +} + +pub fn model_contains_known_llm_provider(model: &str) -> bool { + model.split('/').next().is_some_and(is_llm_provider) } pub fn strip_unregistered_leading_segments( model: &str, region_name: Option<&str>, - known_providers: &[&str], - cost_map: &HashMap, + catalog: &ModelInfoCatalog, ) -> String { + let cost_map = catalog.entries(); let segments: Vec<&str> = model.split('/').collect(); if segments.len() < 2 || cost_map.contains_key(&segments[1..].join("/")) { return model.to_owned(); @@ -55,7 +60,7 @@ pub fn strip_unregistered_leading_segments( let tail = &segments[head_len..]; let first_provider = tail .iter() - .position(|segment| known_providers.contains(segment)) + .position(|segment| is_llm_provider(segment)) .unwrap_or(tail.len()); (0..=first_provider.min(tail.len().saturating_sub(1))) .map(|start| format!("{head}/{}", tail[start..].join("/"))) @@ -76,38 +81,22 @@ fn has_explicit_pricing(entry: &Value) -> bool { pub fn get_provider_for_cost_calc( model: Option<&str>, - explicit_provider: Option<&str>, - known_providers: &[&str], - cost_map: &HashMap, + custom_llm_provider: Option<&str>, + catalog: &ModelInfoCatalog, ) -> Option { - explicit_provider - .map(str::to_owned) - .or_else(|| { - model - .and_then(|model| model.split_once('/')) - .map(|(provider, _)| provider) - .filter(|provider| known_providers.contains(provider)) - .map(str::to_owned) - }) - .or_else(|| { - model - .and_then(|model| cost_map.get(model)) - .and_then(|info| info.get("litellm_provider")) - .and_then(Value::as_str) - .map(str::to_owned) - }) + custom_llm_provider.map(str::to_owned).or_else(|| { + catalog + .get_llm_provider(model?) + .map(|resolved| resolved.custom_llm_provider) + }) } pub fn select_model_name_for_cost_calc( request: ModelSelectionRequest<'_>, - cost_map: &HashMap, + catalog: &ModelInfoCatalog, ) -> Option { - let provider = get_provider_for_cost_calc( - request.model, - request.provider, - request.known_providers, - cost_map, - ); + let cost_map = catalog.entries(); + let provider = get_provider_for_cost_calc(request.model, request.provider, catalog); let response_model = get_response_model(request.response); let private_model = get_hidden_str_for_cost_calc(request.hidden_params, "provider_response_model"); @@ -135,7 +124,7 @@ pub fn select_model_name_for_cost_calc( let Some(provider) = provider else { return Some(selected.to_owned()); }; - if model_contains_known_llm_provider(selected, request.known_providers) { + if model_contains_known_llm_provider(selected) { return Some(selected.to_owned()); } let prefix = @@ -143,7 +132,6 @@ pub fn select_model_name_for_cost_calc( Some(strip_unregistered_leading_segments( &format!("{prefix}/{selected}"), priced_region, - request.known_providers, - cost_map, + catalog, )) } diff --git a/litellm-rust/crates/cost/src/prompt_caching_savings.rs b/litellm-rust/crates/cost/src/prompt_caching_savings.rs index d694e600fdd..3fda5ed2701 100644 --- a/litellm-rust/crates/cost/src/prompt_caching_savings.rs +++ b/litellm-rust/crates/cost/src/prompt_caching_savings.rs @@ -60,7 +60,7 @@ pub fn prompt_caching_savings_for_model( let model_info = catalog.entry(request.model, request.provider, request.region)?; Some(calculate_prompt_caching_savings( PromptCachingSavingsRequest { - model_info, + model_info: &model_info, usage: request.usage, provider: request.provider, service_tier: request.service_tier, diff --git a/litellm-rust/crates/cost/src/retrieval_cost.rs b/litellm-rust/crates/cost/src/retrieval_cost.rs index 1b9462b7ec9..1dc5db99d77 100644 --- a/litellm-rust/crates/cost/src/retrieval_cost.rs +++ b/litellm-rust/crates/cost/src/retrieval_cost.rs @@ -47,7 +47,7 @@ pub fn rerank_cost( ) -> (f64, f64) { rerank_cost_from_model_info( provider, - catalog.entry(model, Some(provider), region), + catalog.entry(model, Some(provider), region).as_deref(), billed_units, ) } diff --git a/litellm-rust/crates/cost/src/search_cost.rs b/litellm-rust/crates/cost/src/search_cost.rs index b872f204e7f..f1ae382c9d3 100644 --- a/litellm-rust/crates/cost/src/search_cost.rs +++ b/litellm-rust/crates/cost/src/search_cost.rs @@ -141,7 +141,7 @@ pub fn search_provider_cost_per_query( .entry(model, provider, None) .ok_or(CostError::ModelNotFound)?; Ok(search_provider_cost_from_model_info( - model_info, + &model_info, number_of_queries, optional_params, )) diff --git a/litellm-rust/crates/cost/src/tool_cost_dispatch.rs b/litellm-rust/crates/cost/src/tool_cost_dispatch.rs index ca7501e65e0..06d14d5e1fa 100644 --- a/litellm-rust/crates/cost/src/tool_cost_dispatch.rs +++ b/litellm-rust/crates/cost/src/tool_cost_dispatch.rs @@ -45,10 +45,17 @@ fn usage_reports_web_search(usage: &ChatUsage) -> bool { ) } -#[derive(Clone, Copy, Debug, Default)] +#[derive(Clone, Debug, Default)] pub struct ResolvedModelInfo<'a> { - pub model_info: Option<&'a Value>, - pub provider: Option<&'a str>, + pub model_info: Option>, + pub provider: Option>, +} + +fn litellm_provider<'a>(model_info: &Value) -> Option> { + model_info + .get("litellm_provider") + .and_then(Value::as_str) + .map(|provider| Cow::Owned(provider.to_owned())) } pub fn resolve_model_info<'a>( @@ -57,14 +64,13 @@ pub fn resolve_model_info<'a>( provider: Option<&'a str>, region: Option<&str>, ) -> ResolvedModelInfo<'a> { - let litellm_provider = - |model_info: &'a Value| model_info.get("litellm_provider").and_then(Value::as_str); if let Some(direct) = catalog.entry(model, provider, region) { return ResolvedModelInfo { - model_info: Some(direct), provider: provider .filter(|provider| !provider.is_empty()) - .or_else(|| litellm_provider(direct)), + .map(Cow::Borrowed) + .or_else(|| litellm_provider(&direct)), + model_info: Some(direct), }; } let by_prefix = model @@ -73,12 +79,12 @@ pub fn resolve_model_info<'a>( .flatten(); match by_prefix { Some(by_prefix) => ResolvedModelInfo { + provider: litellm_provider(&by_prefix), model_info: Some(by_prefix), - provider: litellm_provider(by_prefix), }, None => ResolvedModelInfo { model_info: None, - provider, + provider: provider.map(Cow::Borrowed), }, } } @@ -158,24 +164,27 @@ fn usage_with_anthropic_web_search<'a>( Some(Cow::Owned(ChatUsage { extra, ..base })) } -fn maps_cost(usage: Option<&ChatUsage>, resolved: ResolvedModelInfo<'_>) -> f64 { +fn maps_cost(usage: Option<&ChatUsage>, resolved: &ResolvedModelInfo<'_>) -> f64 { let Some(usage) = usage.filter(|usage| google_maps_grounding_requests(Some(usage)).is_some()) else { return 0.0; }; - match resolved { - ResolvedModelInfo { - model_info: Some(model_info), - provider: Some(provider), - } => get_cost_for_google_maps_grounding_request(provider, usage, model_info).unwrap_or(0.0), + match (resolved.model_info.as_deref(), resolved.provider.as_deref()) { + (Some(model_info), Some(provider)) => { + get_cost_for_google_maps_grounding_request(provider, usage, model_info).unwrap_or(0.0) + } _ => 0.0, } } -fn web_search_cost(request: BuiltInToolCostRequest<'_>, resolved: ResolvedModelInfo<'_>) -> f64 { +fn web_search_cost(request: BuiltInToolCostRequest<'_>, resolved: &ResolvedModelInfo<'_>) -> f64 { let usage = usage_with_anthropic_web_search(request.usage, request.response, request.response_kind); - let routed = match (resolved.model_info, usage.as_deref(), resolved.provider) { + let routed = match ( + resolved.model_info.as_deref(), + usage.as_deref(), + resolved.provider.as_deref(), + ) { (Some(model_info), Some(usage), Some(provider)) => { get_cost_for_web_search_request(provider, usage, model_info, request.defaults) } @@ -184,7 +193,7 @@ fn web_search_cost(request: BuiltInToolCostRequest<'_>, resolved: ResolvedModelI routed.unwrap_or_else(|| { get_cost_for_web_search( request.params.get("web_search_options"), - resolved.model_info, + resolved.model_info.as_deref(), ) * count_web_search_calls(request.response, request.response_kind) as f64 }) } @@ -243,17 +252,17 @@ pub fn get_cost_for_built_in_tools( request: BuiltInToolCostRequest<'_>, ) -> f64 { let resolved = resolve_model_info(catalog, model, request.provider, region); - let maps = maps_cost(request.usage, resolved); + let maps = maps_cost(request.usage, &resolved); let web_search = response_object_includes_web_search_call(request.response, request.response_kind, None) || (request.response_kind != ResponseKind::Responses && request.usage.is_some_and(usage_reports_web_search)); if web_search { - return maps + web_search_cost(request, resolved); + return maps + web_search_cost(request, &resolved); } let direct = catalog.entry(model, request.provider, region); if response_object_includes_file_search_call(request.response, request.response_kind) { - return maps + file_search_cost(request, direct); + return maps + file_search_cost(request, direct.as_deref()); } - maps + azure_assistant_cost(request, direct) + maps + azure_assistant_cost(request, direct.as_deref()) } diff --git a/litellm-rust/crates/cost/src/vertex_cost.rs b/litellm-rust/crates/cost/src/vertex_cost.rs index 149f579884c..8872032aacd 100644 --- a/litellm-rust/crates/cost/src/vertex_cost.rs +++ b/litellm-rust/crates/cost/src/vertex_cost.rs @@ -142,7 +142,7 @@ pub fn vertex_cost( ) -> Result<(f64, f64), CostError> { if let Some(cost) = catalog .entry(request.model, request.provider, request.region) - .and_then(|info| per_second_pricing_cost(info, request.response_time_ms)) + .and_then(|info| per_second_pricing_cost(&info, request.response_time_ms)) { return Ok(cost); } @@ -164,7 +164,7 @@ pub fn vertex_cost( Ok(cost_per_character( model_without_prefix, request.usage, - entry, + &entry, (prompt_characters, completion_characters), request.service_tier, request.vertex_location, diff --git a/litellm-rust/crates/cost/src/zero_cost_diagnostic.rs b/litellm-rust/crates/cost/src/zero_cost_diagnostic.rs index 603442d9555..db7cdf7024e 100644 --- a/litellm-rust/crates/cost/src/zero_cost_diagnostic.rs +++ b/litellm-rust/crates/cost/src/zero_cost_diagnostic.rs @@ -103,12 +103,12 @@ pub fn zero_cost_finding( return None; } let usage = get_usage_object(response).ok().flatten()?; - let (pricing_model, pricing_entry) = + let pricing = catalog.pricing_entry_for_cost_calc(request.model_selection, request.logging_details)?; let diagnostic = diagnose_zero_cost( &usage, - pricing_model, - pricing_entry, + &pricing.key, + &pricing.info, request.calculation_failed, )?; let model = request.model_selection.model.unwrap_or("None"); diff --git a/litellm-rust/crates/cost/tests/python_bedrock_common_utils.rs b/litellm-rust/crates/cost/tests/python_bedrock_common_utils.rs new file mode 100644 index 00000000000..85fc57a23ab --- /dev/null +++ b/litellm-rust/crates/cost/tests/python_bedrock_common_utils.rs @@ -0,0 +1,44 @@ +// mirrors: test_litellm/llms/bedrock/test_bedrock_common_utils.py::test_get_bedrock_base_model +// expected values recorded from litellm/llms/bedrock/common_utils.py + +use litellm_cost::bedrock_common_utils::{get_bedrock_base_model, strip_bedrock_routing_prefix}; +use rstest::rstest; + +#[rstest] +#[case( + "bedrock/converse/us.anthropic.claude-x-v1:0", + "us.anthropic.claude-x-v1:0" +)] +#[case("converse/bedrock/m", "bedrock/m")] +#[case("openai/bedrock/m", "bedrock/m")] +#[case("nova-2/arn:aws:x", "arn:aws:x")] +#[case("bedrock/nova/arn:aws:y", "arn:aws:y")] +#[case("invoke/mantle/nova/m", "m")] +#[case("m", "m")] +fn strip_bedrock_routing_prefix_strips_each_prefix_once_in_order( + #[case] model: &str, + #[case] expected: &str, +) { + assert_eq!(strip_bedrock_routing_prefix(model), expected); +} + +#[rstest] +#[case("us.meta.llama3-v1:0", "meta.llama3-v1:0")] +#[case("bedrock/converse/model", "model")] +#[case("bedrock/nova-2/arn:aws:bedrock:x", "amazon.nova-2-custom")] +#[case("bedrock/nova/arn:aws:x", "amazon.nova-custom")] +#[case("converse/nova-2/x", "amazon.nova-2-custom")] +#[case("invoke/nova/x", "x")] +#[case("anthropic.claude-v2:0:51k", "anthropic.claude-v2:0")] +#[case("us.anthropic.claude-opus-v1[1m]", "anthropic.claude-opus-v1")] +#[case( + "arn:aws:bedrock:us-east-1:1:foundation-model/anthropic.claude-v2", + "anthropic.claude-v2" +)] +#[case("us-east-1/amazon.titan", "amazon.titan")] +#[case("ap-south-9/amazon.titan", "ap-south-9/amazon.titan")] +#[case("global.x", "x")] +#[case("mars.x", "mars.x")] +fn get_bedrock_base_model_matches_python(#[case] model: &str, #[case] expected: &str) { + assert_eq!(get_bedrock_base_model(model), expected); +} diff --git a/litellm-rust/crates/cost/tests/python_bedrock_image_cost.rs b/litellm-rust/crates/cost/tests/python_bedrock_image_cost.rs index d1796f80a42..8e6d30daa13 100644 --- a/litellm-rust/crates/cost/tests/python_bedrock_image_cost.rs +++ b/litellm-rust/crates/cost/tests/python_bedrock_image_cost.rs @@ -1,6 +1,7 @@ #![allow(clippy::disallowed_types)] // mirrors: image_gen_tests/test_bedrock_image_gen_unit_tests.py::test_cost_calculator_stability1 +// expected values recorded from litellm/llms/bedrock/image_generation/cost_calculator.py::cost_calculator use std::collections::HashMap; @@ -10,6 +11,7 @@ use litellm_cost::bedrock_image_cost::{ }; use litellm_cost::call_type::CallTypes; use litellm_cost::catalog::ModelInfoCatalog; +use litellm_cost::error::CostError; use litellm_cost::image_cost_router::{ ImageCostRouteRequest, route_image_generation_cost_calculator, }; @@ -40,52 +42,75 @@ fn get_config_class_matches_model_family( } #[rstest] -#[case(50, "50-steps")] -#[case(51, "max-steps")] -fn stability1_pricing_key_selects_step_tier(#[case] steps: u64, #[case] tier: &str) { - let key = stability1_pricing_key( - "stability.stable-diffusion-xl-v1", - Some("1024-x-1024"), - &json!({"steps": steps}), - ); +#[case::at_the_threshold(json!({"steps": 50}), "50-steps")] +#[case::above_the_threshold(json!({"steps": 51}), "max-steps")] +#[case::float_steps(json!({"steps": 50.5}), "max-steps")] +#[case::bool_steps_compare_as_int(json!({"steps": true}), "50-steps")] +#[case::default_steps(json!({}), "50-steps")] +fn stability1_pricing_key_selects_step_tier(#[case] params: Value, #[case] tier: &str) { assert_eq!( - key, - format!("1024-x-1024/{tier}/stability.stable-diffusion-xl-v1") + stability1_pricing_key("stability.sd-zz-v1", Some("1024-x-1024"), ¶ms), + Ok(format!("1024-x-1024/{tier}/stability.sd-zz-v1")) ); } +fn bedrock_image_catalog() -> ModelInfoCatalog { + let entry = |rate: f64| json!({"output_cost_per_image": rate, "litellm_provider": "bedrock", "mode": "image_generation"}); + ModelInfoCatalog::new(HashMap::from([ + ( + "1024-x-1024/50-steps/stability.sd-zz-v1".to_owned(), + entry(0.04), + ), + ( + "1024-x-1024/max-steps/stability.sd-zz-v1".to_owned(), + entry(0.08), + ), + ("amazon.nova-canvas-zz".to_owned(), entry(0.03)), + ("amazon.titan-image-zz".to_owned(), entry(0.02)), + ("stability.sd3-zz".to_owned(), entry(0.05)), + ])) +} + #[rstest] -fn bedrock_stability1_cost_uses_size_and_steps_lookup() { - let entries = HashMap::from([ - ( - "1024-x-1024/50-steps/stability.stable-diffusion-xl-v1".to_owned(), - json!({"output_cost_per_image": 0.04}), - ), - ( - "1024-x-1024/max-steps/stability.stable-diffusion-xl-v1".to_owned(), - json!({"output_cost_per_image": 0.08}), - ), - ]); - let response = json!({"data": [{}, {}]}); +#[case::stability1_default_tier("stability.sd-zz-v1", Some("1024-x-1024"), json!({"steps": 50}), Ok(0.08))] +#[case::stability1_max_tier("stability.sd-zz-v1", Some("1024-x-1024"), json!({"steps": 60}), Ok(0.16))] +#[case::stability1_missing_steps("stability.sd-zz-v1", Some("1024-x-1024"), json!({}), Ok(0.08))] +#[case::stability1_default_size("stability.sd-zz-v1", None, json!({"steps": true}), Ok(0.08))] +#[case::stability1_null_steps("stability.sd-zz-v1", Some("1024-x-1024"), json!({"steps": null}), Err(CostError::InvalidSteps))] +#[case::stability1_string_steps("stability.sd-zz-v1", Some("1024-x-1024"), json!({"steps": "60"}), Err(CostError::InvalidSteps))] +#[case::stability1_keeps_the_routing_prefix_in_the_key( + "bedrock/stability.sd-zz-v1", + Some("1024-x-1024"), + json!({"steps": 50}), + Err(CostError::ModelNotFound) +)] +#[case::stability1_key_is_case_insensitive("STABILITY.SD-ZZ-V1", Some("1024-x-1024"), json!({}), Ok(0.08))] +#[case::nova_bare("amazon.nova-canvas-zz", None, json!({}), Ok(0.06))] +#[case::nova_routing_prefix("bedrock/amazon.nova-canvas-zz", None, json!({}), Ok(0.06))] +#[case::nova_cross_region_prefix("us.amazon.nova-canvas-zz", None, json!({}), Ok(0.06))] +#[case::nova_region_path("bedrock/us-east-1/amazon.nova-canvas-zz", None, json!({}), Ok(0.06))] +#[case::nova_invoke_route("invoke/amazon.nova-canvas-zz", None, json!({}), Ok(0.06))] +#[case::nova_unmapped("amazon.nova-canvas-missing", None, json!({}), Err(CostError::ModelNotFound))] +#[case::titan_bare("amazon.titan-image-zz", None, json!({}), Ok(0.04))] +#[case::titan_routing_prefix("bedrock/amazon.titan-image-zz", None, json!({}), Ok(0.04))] +#[case::titan_infers_no_provider_for_a_cross_region_id( + "us.amazon.titan-image-zz", + None, + json!({}), + Err(CostError::ModelNotFound) +)] +#[case::stability3_bare("stability.sd3-zz", None, json!({}), Ok(0.1))] +#[case::stability3_routing_prefix("bedrock/stability.sd3-zz", None, json!({}), Ok(0.1))] +fn bedrock_image_cost_resolves_models_like_get_model_info( + #[case] model: &str, + #[case] size: Option<&str>, + #[case] params: Value, + #[case] expected: Result, +) { + let response = json!({"data": [{"url": "x"}, {"url": "y"}]}); assert_eq!( - cost_calculator( - "bedrock/stability.stable-diffusion-xl-v1", - &response, - Some("1024-x-1024"), - &json!({"steps": 50}), - &entries - ), - Some(0.08) - ); - assert_eq!( - cost_calculator( - "bedrock/stability.stable-diffusion-xl-v1", - &response, - Some("1024-x-1024"), - &json!({"steps": 60}), - &entries - ), - Some(0.16) + cost_calculator(&bedrock_image_catalog(), model, &response, size, ¶ms), + expected ); } @@ -95,8 +120,8 @@ fn bedrock_stability1_cost_uses_size_and_steps_lookup() { #[case("stability.sd3-large-v1:0", 0.04)] fn bedrock_direct_families_bill_returned_images(#[case] model: &str, #[case] rate: f64) { let catalog = ModelInfoCatalog::new(HashMap::from([( - format!("bedrock/{model}"), - json!({"output_cost_per_image": rate}), + model.to_owned(), + json!({"output_cost_per_image": rate, "litellm_provider": "bedrock"}), )])); let response = json!({"data": [{}, {}, {}]}); let supplied = json!({"output_cost_per_image": 1.0}); diff --git a/litellm-rust/crates/cost/tests/python_completion_input.rs b/litellm-rust/crates/cost/tests/python_completion_input.rs index 6f74934d3d9..e052a75e8cc 100644 --- a/litellm-rust/crates/cost/tests/python_completion_input.rs +++ b/litellm-rust/crates/cost/tests/python_completion_input.rs @@ -15,8 +15,6 @@ use litellm_cost::model_selection::ModelSelectionRequest; use rstest::rstest; use serde_json::{Value, json}; -const PROVIDERS: &[&str] = &["anthropic", "vertex_ai"]; - #[rstest] #[case("ON_DEMAND_PRIORITY", Some("priority"))] #[case("on_demand_flex", Some("flex"))] @@ -108,7 +106,6 @@ fn prepared_input_prices_served_priority_rate() { provider: Some("anthropic"), router_model_id: None, region_name: None, - known_providers: PROVIDERS, }, call_type: None, response_kind: Some(ResponseKind::Completion), diff --git a/litellm-rust/crates/cost/tests/python_completion_response.rs b/litellm-rust/crates/cost/tests/python_completion_response.rs index 8ae42dbc7da..b57966d0636 100644 --- a/litellm-rust/crates/cost/tests/python_completion_response.rs +++ b/litellm-rust/crates/cost/tests/python_completion_response.rs @@ -19,17 +19,6 @@ use litellm_token_counter::{Error as TokenCounterError, TokenCounter, Tokenizer} use rstest::rstest; use serde_json::{Value, json}; -const PROVIDERS: &[&str] = &[ - "anthropic", - "azure_ai", - "exa_ai", - "openai", - "recraft", - "replicate", - "vertex_ai", - "xai", -]; - struct CharacterTokenizer; impl Tokenizer for CharacterTokenizer { @@ -56,7 +45,6 @@ fn request<'a>( provider, router_model_id: None, region_name: None, - known_providers: PROVIDERS, }, call_type: None, response_kind: Some(ResponseKind::Completion), @@ -165,12 +153,13 @@ fn response_cost_calculator_prices_response_without_provider_override() { } #[rstest] -#[case("xai/served", json!({"input_cost_per_token": 0.01, "output_cost_per_token": 0.02}))] -#[case("served", json!({"litellm_provider": "xai", "input_cost_per_token": 0.01, "output_cost_per_token": 0.02}))] -fn response_cost_infers_provider_for_specialized_pricing(#[case] model: &str, #[case] info: Value) { - let catalog = ModelInfoCatalog::new(HashMap::from([(model.to_owned(), info)])); +fn response_cost_infers_provider_for_specialized_pricing() { + let catalog = ModelInfoCatalog::new(HashMap::from([( + "xai/served".to_owned(), + json!({"input_cost_per_token": 0.01, "output_cost_per_token": 0.02}), + )])); let response = json!({ - "model": model, + "model": "xai/served", "usage": { "prompt_tokens": 100, "completion_tokens": 10, @@ -181,12 +170,33 @@ fn response_cost_infers_provider_for_specialized_pricing(#[case] model: &str, #[ let empty = json!({}); let result = completion_cost_from_response( &catalog, - request(Some(&response), Some(model), None, &empty, &empty), + request(Some(&response), Some("xai/served"), None, &empty, &empty), ) .unwrap(); assert!((result.cost.total - (100.0 * 0.01 + 30.0 * 0.02)).abs() < 1e-12); } +#[rstest] +fn response_cost_rejects_a_bare_model_get_llm_provider_cannot_route() { + let catalog = ModelInfoCatalog::new(HashMap::from([( + "served".to_owned(), + json!({"litellm_provider": "xai", "input_cost_per_token": 0.01, "output_cost_per_token": 0.02}), + )])); + let response = json!({ + "model": "served", + "usage": {"prompt_tokens": 100, "completion_tokens": 10, "total_tokens": 110} + }); + let empty = json!({}); + assert_eq!( + completion_cost_from_response( + &catalog, + request(Some(&response), Some("served"), None, &empty, &empty), + ) + .map(|result| result.cost.total), + Err(CostError::MissingProvider) + ); +} + #[rstest] fn response_cost_infers_provider_from_priced_fallback_model() { let catalog = ModelInfoCatalog::new(HashMap::from([( diff --git a/litellm-rust/crates/cost/tests/python_fal_ai_image_cost.rs b/litellm-rust/crates/cost/tests/python_fal_ai_image_cost.rs index 3bb02cc1e54..5d5e9b16b6b 100644 --- a/litellm-rust/crates/cost/tests/python_fal_ai_image_cost.rs +++ b/litellm-rust/crates/cost/tests/python_fal_ai_image_cost.rs @@ -98,7 +98,7 @@ fn flat_cost_per_image_rounds_each_response_to_megapixels() { #[rstest] fn fal_image_cost_uses_deployment_rate_before_keyed_and_base_prices() { - let entries = HashMap::from([ + let catalog = ModelInfoCatalog::new(HashMap::from([ ( "fal_ai/high/1024-x-768/openai/model".to_owned(), json!({"output_cost_per_image": 0.10}), @@ -107,42 +107,42 @@ fn fal_image_cost_uses_deployment_rate_before_keyed_and_base_prices() { "fal_ai/openai/model".to_owned(), json!({"output_cost_per_image": 0.50}), ), - ]); + ])); let response = json!({"data": [{}, {}]}); let supplied = json!({"output_cost_per_image": 0.25}); assert_eq!( cost_calculator( + &catalog, "fal_ai/openai/model", &response, &json!({}), Some(&supplied), - &entries ), Some(0.50) ); assert_eq!( - cost_calculator("fal_ai/openai/model", &response, &json!({}), None, &entries), + cost_calculator(&catalog, "fal_ai/openai/model", &response, &json!({}), None), Some(0.20) ); } #[rstest] fn fal_image_cost_falls_back_to_base_megapixel_and_flat_rates_per_image() { - let entries = HashMap::from([( + let catalog = ModelInfoCatalog::new(HashMap::from([( "fal_ai/fal-ai/flux/dev".to_owned(), json!({"output_cost_per_image": 0.5, "output_cost_per_pixel": 0.0001}), - )]); + )])); let response = json!({"data": [ image(json!(1024), json!(1024)), image(json!(1920), json!(1080)), image(json!(true), json!(1024)) ]}); let cost = cost_calculator( + &catalog, "fal_ai/fal-ai/flux/dev", &response, &json!({}), None, - &entries, ) .unwrap(); let expected = 0.0001 * PIXELS_PER_MEGAPIXEL as f64 * 3.0 + 0.5; @@ -210,3 +210,33 @@ fn fal_passthrough_selects_resolution_rate_and_falls_back_to_base() { None ); } + +// expected values recorded from litellm/llms/fal_ai/cost_calculator.py::cost_calculator +#[rstest] +#[case::bare_key_owned_by_fal("fal_ai/flux-zz", Some(0.4))] +#[case::case_insensitive_key("FLUX-ZZ", Some(0.4))] +#[case::bool_rate_counts_as_int("fal_ai/flag-zz", Some(2.0))] +#[case::string_rate_is_not_a_number("fal_ai/str-zz", Some(0.0))] +#[case::unmapped_model("fal_ai/missing", None)] +fn fal_image_flat_fallback_resolves_through_get_model_info( + #[case] model: &str, + #[case] expected: Option, +) { + let entry = |rate: Value| json!({"output_cost_per_image": rate, "litellm_provider": "fal_ai"}); + let catalog = ModelInfoCatalog::new(HashMap::from([ + ("flux-zz".to_owned(), entry(json!(0.2))), + ("fal_ai/flag-zz".to_owned(), entry(json!(true))), + ("fal_ai/str-zz".to_owned(), entry(json!("0.3"))), + ])); + let response = json!({"data": [{"url": "x"}, {"url": "y"}]}); + assert_eq!( + cost_calculator( + &catalog, + model, + &response, + &json!({"image_size": "weird"}), + None + ), + expected + ); +} diff --git a/litellm-rust/crates/cost/tests/python_model_info.rs b/litellm-rust/crates/cost/tests/python_model_info.rs new file mode 100644 index 00000000000..4cddd308f4c --- /dev/null +++ b/litellm-rust/crates/cost/tests/python_model_info.rs @@ -0,0 +1,262 @@ +#![allow(clippy::disallowed_types)] + +// mirrors: test_litellm/test_utils.py::test_get_model_info_* (get_model_info key resolution) +// expected values recorded from litellm/utils.py::_get_model_info_helper and +// litellm/litellm_core_utils/get_llm_provider_logic.py::get_llm_provider on this catalog + +use std::collections::HashMap; + +use litellm_cost::catalog::ModelInfoCatalog; +use litellm_cost::error::CostError; +use rstest::rstest; +use serde_json::{Value, json}; + +fn catalog() -> ModelInfoCatalog { + let provider = |name: &str| json!({"litellm_provider": name}); + ModelInfoCatalog::new(HashMap::from([ + ("openai/gpt-zz".to_owned(), provider("openai")), + ("gpt-zz".to_owned(), provider("openai")), + ("gemini/gemini-zz".to_owned(), provider("gemini")), + ( + "gemini-zz-pro".to_owned(), + provider("vertex_ai-language-models"), + ), + ("claude-zz".to_owned(), provider("anthropic")), + ("ft:gpt-yy".to_owned(), provider("openai")), + ( + "vertex_ai/meta/llama-zz".to_owned(), + provider("vertex_ai-llama_models"), + ), + ( + "vertex_ai/mistral-zz@latest".to_owned(), + provider("vertex_ai-mistral_models"), + ), + ( + "anthropic.claude-zz-v1".to_owned(), + provider("bedrock_converse"), + ), + ( + "bedrock_mantle/model-zz".to_owned(), + provider("bedrock_mantle"), + ), + ( + "fireworks_ai/accounts/fireworks/models/fw-zz".to_owned(), + provider("fireworks_ai"), + ), + ("gpt-4.1".to_owned(), provider("openai")), + ("Mixed-Case-ZZ".to_owned(), provider("together_ai")), + ( + "fallback_generalizations".to_owned(), + json!({"rules": [ + {"name": "zz-route", "pattern": "^zzroute-", "model_info": {"litellm_provider": "anthropic"}}, + {"name": "zz-cap", "pattern": "zzcap-\\d+", "model_info": {"mode": "chat", "max_tokens": 10}}, + {"name": "zz-cap2", "pattern": "zzcap-9", "model_info": {"max_tokens": 20}}, + { + "name": "zz-legacy", + "pattern": "^legacy-zz", + "extends": "zz-cap", + "model_info": {"litellm_provider": "xai", "max_tokens": 30} + } + ]}), + ), + ])) +} + +type Resolved = Result<(&'static str, Option<&'static str>, Option), CostError>; + +#[rstest] +#[case::provider_prefixed_key_first("gpt-zz", Some("openai"), Ok(("openai/gpt-zz", Some("openai"), None)))] +#[case::bare_model_without_provider("gpt-zz", None, Ok(("gpt-zz", Some("openai"), None)))] +#[case::prefixed_model_without_provider("openai/gpt-zz", None, Ok(("openai/gpt-zz", Some("openai"), None)))] +#[case::case_insensitive_key("GPT-ZZ", Some("openai"), Ok(("openai/gpt-zz", Some("openai"), None)))] +#[case::dated_snapshot_suffix("gpt-zz-2025-01-01", Some("openai"), Ok(("openai/gpt-zz", Some("openai"), None)))] +#[case::dated_snapshot_without_provider("gpt-zz-2025-01-01", None, Ok(("gpt-zz", Some("openai"), None)))] +#[case::stable_version_suffix("gemini-zz-001", Some("gemini"), Ok(("gemini/gemini-zz", Some("gemini"), None)))] +#[case::vertex_family_entry( + "gemini-zz-pro", + Some("vertex_ai"), + Ok(("gemini-zz-pro", Some("vertex_ai-language-models"), None)) +)] +#[case::mismatched_provider_is_skipped( + "gemini/gemini-zz", + Some("vertex_ai"), + Err(CostError::ModelNotFound) +)] +#[case::openai_finetune("ft:gpt-yy:org:suffix:id", Some("openai"), Ok(("ft:gpt-yy", Some("openai"), None)))] +#[case::vertex_llama_alias( + "llama-zz", + Some("vertex_ai"), + Ok(("vertex_ai/meta/llama-zz", Some("vertex_ai-llama_models"), None)) +)] +#[case::vertex_beta_latest_alias( + "mistral-zz", + Some("vertex_ai_beta"), + Ok(("vertex_ai/mistral-zz@latest", Some("vertex_ai-mistral_models"), None)) +)] +#[case::bedrock_cross_region_keeps_the_version( + "us.anthropic.claude-zz-v1:0", + Some("bedrock"), + Err(CostError::ModelNotFound) +)] +#[case::bedrock_routing_prefix( + "bedrock/converse/anthropic.claude-zz-v1", + Some("bedrock"), + Ok(("anthropic.claude-zz-v1", Some("bedrock_converse"), None)) +)] +#[case::bedrock_throughput_suffix_keeps_the_version( + "anthropic.claude-zz-v1:0:51k", + Some("bedrock_converse"), + Err(CostError::ModelNotFound) +)] +#[case::mantle_region_prefix( + "bedrock_mantle/us-east-1/model-zz", + Some("bedrock_mantle"), + Ok(("bedrock_mantle/model-zz", Some("bedrock_mantle"), None)) +)] +#[case::fireworks_resource_key( + "fw-zz", + Some("fireworks_ai"), + Ok(("fireworks_ai/accounts/fireworks/models/fw-zz", Some("fireworks_ai"), None)) +)] +#[case::azure_alias("azure/gpt-41", None, Ok(("gpt-4.1", Some("openai"), None)))] +#[case::lowercase_query_finds_mixed_case_key( + "mixed-case-zz", + Some("together_ai"), + Ok(("Mixed-Case-ZZ", Some("together_ai"), None)) +)] +#[case::huggingface_is_zero_rated("any-model", Some("huggingface"), Ok(("any-model", Some("huggingface"), None)))] +#[case::other_provider_entry("claude-zz", Some("openai"), Err(CostError::ModelNotFound))] +#[case::capability_union_later_rule_wins("zzcap-9", None, Ok(("zzcap-9", None, Some(20))))] +#[case::capability_backfills_the_provider("zzcap-9", Some("openai"), Ok(("openai/zzcap-9", Some("openai"), Some(20))))] +#[case::routing_rule_alone_has_no_capabilities("zzroute-x", None, Err(CostError::ModelNotFound))] +#[case::legacy_rule_routes_and_extends("legacy-zz-1", None, Ok(("legacy-zz-1", Some("xai"), Some(30))))] +#[case::unmapped("nothing-matches", None, Err(CostError::ModelNotFound))] +fn get_model_info_resolves_keys_like_python( + #[case] model: &str, + #[case] provider: Option<&str>, + #[case] expected: Resolved, +) { + let catalog = catalog(); + let actual = catalog.get_model_info(model, provider).map(|resolved| { + ( + resolved.key.into_owned(), + resolved + .info + .get("litellm_provider") + .and_then(Value::as_str) + .map(str::to_owned), + resolved.info.get("max_tokens").and_then(Value::as_u64), + ) + }); + assert_eq!( + actual, + expected.map(|(key, provider, max_tokens)| { + (key.to_owned(), provider.map(str::to_owned), max_tokens) + }) + ); +} + +#[rstest] +fn get_model_info_prices_huggingface_at_zero() { + let catalog = catalog(); + let info = catalog + .get_model_info("any-model", Some("huggingface")) + .unwrap() + .info; + assert_eq!(info.get("input_cost_per_token"), Some(&json!(0))); + assert_eq!(info.get("output_cost_per_token"), Some(&json!(0))); +} + +#[rstest] +fn capability_rules_lose_to_an_exact_key_owned_by_another_provider() { + let catalog = ModelInfoCatalog::new(HashMap::from([ + ("zzcap-1".to_owned(), json!({"litellm_provider": "gemini"})), + ( + "fallback_generalizations".to_owned(), + json!({"rules": [{"name": "cap", "pattern": "zzcap", "model_info": {"mode": "chat"}}]}), + ), + ])); + assert_eq!( + catalog + .get_model_info("zzcap-1", Some("openai")) + .map(|info| info.key.into_owned()), + Err(CostError::ModelNotFound) + ); + assert_eq!( + catalog + .get_model_info("zzcap-2", Some("openai")) + .map(|info| info.key.into_owned()), + Ok("openai/zzcap-2".to_owned()) + ); +} + +#[rstest] +#[case::cataloged_bare("gpt-zz", Some("openai"))] +#[case::provider_prefix("gemini/anything", Some("gemini"))] +#[case::vertex_family("gemini-zz-pro", Some("vertex_ai"))] +#[case::bedrock_converse_family("anthropic.claude-zz-v1", Some("bedrock"))] +#[case::routing_rule("zzroute-x", Some("anthropic"))] +#[case::legacy_routing_rule("legacy-zz-1", Some("xai"))] +#[case::azure_prefix_stays_azure("azure/gpt-41", Some("azure"))] +#[case::json_registry_provider("gmi/any", Some("gmi"))] +#[case::static_cohere_embedding("embed-v4.0", Some("cohere"))] +#[case::static_openai_image("dall-e-3", Some("openai"))] +#[case::unroutable("GPT-ZZ", None)] +fn get_llm_provider_infers_like_python(#[case] model: &str, #[case] expected: Option<&str>) { + assert_eq!( + catalog() + .get_llm_provider(model) + .map(|resolved| resolved.custom_llm_provider) + .as_deref(), + expected + ); +} + +fn usage() -> litellm_cost::responses_usage::ChatUsage { + litellm_cost::usage_dispatch::get_usage_object( + &json!({"usage": {"prompt_tokens": 100, "completion_tokens": 10, "total_tokens": 110}}), + ) + .unwrap() + .unwrap() +} + +// expected values recorded from litellm.completion_cost for unmapped ids the shipped rules generalize +#[rstest] +#[case::routed_and_generalized("claude-newfam-9", None, Ok((0.0, 0.0)))] +#[case::generalized_under_explicit_provider("claude-newfam-9", Some("anthropic"), Ok((0.0, 0.0)))] +#[case::generalized_bedrock_id("anthropic.claude-newfam-9", Some("bedrock"), Ok((0.0, 0.0)))] +#[case::unroutable("totally-unknown", None, Err(CostError::MissingProvider))] +#[case::routable_but_unmapped("zzroute-x", None, Err(CostError::ModelNotFound))] +fn unmapped_models_price_through_capability_rules( + #[case] model: &str, + #[case] provider: Option<&str>, + #[case] expected: Result<(f64, f64), CostError>, +) { + let catalog = ModelInfoCatalog::new(HashMap::from([( + "fallback_generalizations".to_owned(), + json!({"rules": [ + {"name": "route-bedrock", "pattern": "^(?:[a-z-]+\\.)?anthropic\\.claude-", "model_info": {"litellm_provider": "bedrock"}}, + {"name": "route", "pattern": "^claude-[a-z]+-\\d+$", "model_info": {"litellm_provider": "anthropic"}}, + {"name": "route-only", "pattern": "^zzroute-", "model_info": {"litellm_provider": "anthropic"}}, + {"name": "baseline", "pattern": "claude-[a-z]+-\\d+", "model_info": {"mode": "chat"}} + ]}), + )])); + let usage = usage(); + assert_eq!( + litellm_cost::cost_calculator::cost_per_token( + &catalog, + litellm_cost::catalog::ModelCostRequest { + model, + provider, + region: None, + usage: &usage, + service_tier: None, + data_residency: None, + vertex_location: None, + at: "2026-01-01T12:00Z".parse().unwrap(), + response_time_ms: None, + }, + ), + expected + ); +} diff --git a/litellm-rust/crates/cost/tests/python_model_lookup.rs b/litellm-rust/crates/cost/tests/python_model_lookup.rs index cebcc1ad3e8..659b9160c20 100644 --- a/litellm-rust/crates/cost/tests/python_model_lookup.rs +++ b/litellm-rust/crates/cost/tests/python_model_lookup.rs @@ -7,7 +7,8 @@ use litellm_cost::error::CostError; use std::collections::HashMap; use jiff::Timestamp; -use litellm_cost::catalog::{ModelCostRequest, ModelInfoCatalog, check_provider_match}; +use litellm_cost::catalog::{ModelCostRequest, ModelInfoCatalog}; +use litellm_cost::model_info::check_provider_match; use litellm_cost::usage_dispatch::get_usage_object; use rstest::rstest; use serde_json::json; @@ -84,7 +85,10 @@ fn cost_per_token_resolves_model_key_in_python_order( #[case] expected: Option<&str>, ) { assert_eq!( - catalog().select_model_key(model, provider, region), + catalog() + .select_model_info(model, provider, region) + .map(|selected| selected.key.into_owned()) + .as_deref(), expected ); } @@ -221,7 +225,11 @@ fn model_info_catalog_applies_xai_inclusive_threshold_policy() { #[case::vertex_family_is_one_way(Some("vertex_ai"), Some("vertex_ai-language-models"), false)] #[case::fireworks_family(Some("fireworks_ai-embedding-models"), Some("fireworks_ai"), true)] #[case::fireworks_request_rejects_other_providers(Some("openai"), Some("fireworks_ai"), false)] -#[case::fireworks_entry_needs_a_fireworks_request(Some("fireworks_ai-embedding-models"), Some("together_ai"), false)] +#[case::fireworks_entry_needs_a_fireworks_request( + Some("fireworks_ai-embedding-models"), + Some("together_ai"), + false +)] #[case::bedrock_family(Some("bedrock_converse"), Some("bedrock"), true)] #[case::bedrock_request_rejects_other_providers(Some("openai"), Some("bedrock"), false)] #[case::bedrock_entry_needs_a_bedrock_request(Some("bedrock"), Some("sagemaker"), false)] @@ -252,7 +260,10 @@ fn select_model_key_skips_entries_owned_by_another_provider( json!({"litellm_provider": "gemini", "input_cost_per_token": 1e-6}), )])); assert_eq!( - catalog.select_model_key("gemini/gemini-x", provider, None), + catalog + .select_model_info("gemini/gemini-x", provider, None) + .map(|selected| selected.key.into_owned()) + .as_deref(), expected ); } @@ -264,7 +275,10 @@ fn select_model_key_lets_vertex_ai_beta_use_vertex_entries() { json!({"litellm_provider": "vertex_ai-language-models"}), )])); assert_eq!( - catalog.select_model_key("vertex_ai/gemini-x", Some("vertex_ai_beta"), None), + catalog + .select_model_info("vertex_ai/gemini-x", Some("vertex_ai_beta"), None) + .map(|selected| selected.key.into_owned()) + .as_deref(), Some("vertex_ai/gemini-x") ); } diff --git a/litellm-rust/crates/cost/tests/python_model_selection.rs b/litellm-rust/crates/cost/tests/python_model_selection.rs index 700ee66ce5b..a7c162b58ac 100644 --- a/litellm-rust/crates/cost/tests/python_model_selection.rs +++ b/litellm-rust/crates/cost/tests/python_model_selection.rs @@ -12,7 +12,15 @@ use litellm_cost::model_selection::{ use rstest::rstest; use serde_json::{Value, json}; -const PROVIDERS: &[&str] = &["openai", "bedrock", "vertex_ai", "anthropic", "dashscope"]; +fn pricing<'a>( + catalog: &'a ModelInfoCatalog, + request: ModelSelectionRequest<'a>, + logging_details: Option<&'a Value>, +) -> Option<(String, Value)> { + catalog + .pricing_entry_for_cost_calc(request, logging_details) + .map(|selected| (selected.key.into_owned(), selected.info.into_owned())) +} fn request<'a>(model: Option<&'a str>, provider: Option<&'a str>) -> ModelSelectionRequest<'a> { ModelSelectionRequest { @@ -24,7 +32,6 @@ fn request<'a>(model: Option<&'a str>, provider: Option<&'a str>) -> ModelSelect provider, router_model_id: None, region_name: None, - known_providers: PROVIDERS, } } @@ -166,31 +173,64 @@ fn model_selection_helpers_ignore_invalid_hidden_values_and_stop_at_provider_seg get_hidden_str_for_cost_calc(Some(&json!({"model": ""})), "model"), None ); - assert!(model_contains_known_llm_provider("openai/model", PROVIDERS)); - assert!(!model_contains_known_llm_provider("team/model", PROVIDERS)); - let catalog = HashMap::from([( + assert!(model_contains_known_llm_provider("openai/model")); + assert!(!model_contains_known_llm_provider("team/model")); + let catalog = ModelInfoCatalog::new(HashMap::from([( "vertex_ai/claude".to_owned(), json!({"input_cost_per_token": 1e-6}), - )]); + )])); assert_eq!( - strip_unregistered_leading_segments("vertex_ai/openai/claude", None, PROVIDERS, &catalog,), + strip_unregistered_leading_segments("vertex_ai/openai/claude", None, &catalog), "vertex_ai/openai/claude" ); } #[rstest] -#[case(Some("xai/model"), None, Some("xai"))] -#[case(Some("custom"), None, Some("xai"))] -#[case(Some("xai/model"), Some("anthropic"), Some("anthropic"))] -#[case(Some("unknown"), None, None)] -fn provider_inference_uses_explicit_prefix_or_catalog_metadata( +#[case::provider_prefix(Some("xai/model"), None, Some("xai"))] +#[case::explicit_provider_wins(Some("xai/model"), Some("anthropic"), Some("anthropic"))] +#[case::no_model(None, None, None)] +#[case::unlisted_bare_provider_is_not_inferred(Some("custom-xai"), None, None)] +#[case::unknown_bare_model(Some("unknown"), None, None)] +#[case::bedrock_converse_routes_to_bedrock(Some("converse-model"), None, Some("bedrock"))] +#[case::vertex_family_routes_to_vertex_ai(Some("vertex-model"), None, Some("vertex_ai"))] +#[case::ai21_chat(Some("ai21-model"), None, Some("ai21_chat"))] +#[case::ai21_prefix(Some("ai21/any"), None, Some("ai21_chat"))] +#[case::cohere_chat_under_cohere_prefix(Some("cohere/chat-model"), None, Some("cohere_chat"))] +#[case::azure_hosted_mistral(Some("azure/mistral-model"), None, Some("openai"))] +#[case::anthropic_text(Some("anthropic/claude-2"), None, Some("anthropic_text"))] +#[case::openai_finetune(Some("ft:gpt-4o:org::id"), None, Some("openai"))] +#[case::replicate_version_id(Some(&*format!("owner/model:{}", "a".repeat(64))), None, Some("replicate"))] +#[case::wildcard(Some("*"), None, Some("openai"))] +fn provider_inference_follows_get_llm_provider( #[case] model: Option<&str>, #[case] explicit: Option<&str>, #[case] expected: Option<&str>, ) { - let catalog = HashMap::from([("custom".to_owned(), json!({"litellm_provider": "xai"}))]); + let catalog = ModelInfoCatalog::new(HashMap::from([ + ("custom-xai".to_owned(), json!({"litellm_provider": "xai"})), + ( + "converse-model".to_owned(), + json!({"litellm_provider": "bedrock_converse"}), + ), + ( + "vertex-model".to_owned(), + json!({"litellm_provider": "vertex_ai-language-models"}), + ), + ( + "ai21-model".to_owned(), + json!({"litellm_provider": "ai21", "mode": "chat"}), + ), + ( + "chat-model".to_owned(), + json!({"litellm_provider": "cohere_chat"}), + ), + ( + "mistral/mistral-model".to_owned(), + json!({"litellm_provider": "mistral"}), + ), + ])); assert_eq!( - get_provider_for_cost_calc(model, explicit, &["xai", "anthropic"], &catalog).as_deref(), + get_provider_for_cost_calc(model, explicit, &catalog).as_deref(), expected ); } @@ -215,8 +255,11 @@ fn pricing_entry_prefers_registered_deployment_to_served_model() { ..request(Some("requested"), Some("openai")) }; assert_eq!( - catalog.pricing_entry_for_cost_calc(request, None), - Some(("router-id", &json!({"input_cost_per_token": 0.03}))) + pricing(&catalog, request, None), + Some(( + "router-id".to_owned(), + json!({"input_cost_per_token": 0.03}) + )) ); } @@ -237,18 +280,25 @@ fn pricing_entry_reads_deployment_metadata_before_published_price() { ..request(Some("requested"), Some("openai")) }; assert_eq!( - catalog.pricing_entry_for_cost_calc(selected, Some(&logging)), - Some(("requested", &json!({"input_cost_per_token": 0.02}))) + pricing(&catalog, selected, Some(&logging)), + Some(( + "requested".to_owned(), + json!({"input_cost_per_token": 0.02}) + )) ); assert_eq!( - catalog.pricing_entry_for_cost_calc( + pricing( + &catalog, ModelSelectionRequest { custom_pricing: false, ..selected }, Some(&logging) ), - Some(("openai/served", &json!({"input_cost_per_token": 0.01}))) + Some(( + "openai/served".to_owned(), + json!({"input_cost_per_token": 0.01}) + )) ); } @@ -260,7 +310,8 @@ fn pricing_entry_falls_back_from_unpriced_base_to_served_model() { )])); let response = json!({"model": "served"}); assert_eq!( - catalog.pricing_entry_for_cost_calc( + pricing( + &catalog, ModelSelectionRequest { response: Some(&response), base_model: Some("unpriced"), @@ -268,7 +319,10 @@ fn pricing_entry_falls_back_from_unpriced_base_to_served_model() { }, None, ), - Some(("openai/served", &json!({"input_cost_per_token": 0.01}))) + Some(( + "openai/served".to_owned(), + json!({"input_cost_per_token": 0.01}) + )) ); } diff --git a/litellm-rust/crates/cost/tests/python_zero_cost_diagnostic.rs b/litellm-rust/crates/cost/tests/python_zero_cost_diagnostic.rs index 01c8380fc05..cdd4712eb0d 100644 --- a/litellm-rust/crates/cost/tests/python_zero_cost_diagnostic.rs +++ b/litellm-rust/crates/cost/tests/python_zero_cost_diagnostic.rs @@ -138,7 +138,6 @@ fn finding_request<'a>(response: &'a Value) -> ZeroCostFindingRequest<'a> { provider: Some("openai"), router_model_id: None, region_name: None, - known_providers: &["openai"], }, logging_details: None, metadata: None,