diff --git a/litellm-rust/crates/cost/src/billed_token_rates.rs b/litellm-rust/crates/cost/src/billed_token_rates.rs index a2835808268..456a093b439 100644 --- a/litellm-rust/crates/cost/src/billed_token_rates.rs +++ b/litellm-rust/crates/cost/src/billed_token_rates.rs @@ -1,8 +1,8 @@ use jiff::Timestamp; use serde_json::Value; -use crate::base_rate_selection::uses_inclusive_token_thresholds; -use crate::base_rate_selection::{get_token_base_cost, tier_key}; +use crate::base_rate_selection::{get_token_base_cost, tier_key, uses_inclusive_token_thresholds}; +use crate::catalog::{ModelCostRequest, ModelInfoCatalog}; use crate::custom_pricing::CustomTokenRates; use crate::generic_cost::resolve_billed_reasoning_rate; use crate::generic_input::{calculate_cache_writing_cost, get_cost_per_unit}; @@ -72,7 +72,7 @@ fn custom_pricing_rates(custom: CustomTokenRates) -> BilledTokenRates { } } -pub fn get_billed_token_rates(request: BilledRatesRequest<'_>) -> Option { +pub fn calculate_billed_token_rates(request: BilledRatesRequest<'_>) -> Option { if let Some(custom) = request.custom_cost_per_token { return Some(custom_pricing_rates(custom)); } @@ -125,8 +125,10 @@ fn count(value: Option<&Value>) -> u64 { value.and_then(Value::as_u64).unwrap_or(0) } -pub fn get_token_type_cost_breakdown(request: BilledRatesRequest<'_>) -> TokenTypeCostBreakdown { - let Some(rates) = get_billed_token_rates(request) else { +pub fn calculate_token_type_cost_breakdown( + request: BilledRatesRequest<'_>, +) -> TokenTypeCostBreakdown { + let Some(rates) = calculate_billed_token_rates(request) else { return TokenTypeCostBreakdown { reasoning_cost: 0.0, cache_read_cost: 0.0, @@ -180,3 +182,44 @@ pub fn get_token_type_cost_breakdown(request: BilledRatesRequest<'_>) -> TokenTy rates: Some(rates), } } + +fn billed_rates_request<'a>( + catalog: &'a ModelInfoCatalog, + request: ModelCostRequest<'a>, + custom_cost_per_token: Option, +) -> BilledRatesRequest<'a> { + BilledRatesRequest { + model_info: catalog.entry(request.model, request.provider, request.region), + usage: request.usage, + provider: request.provider, + service_tier: request.service_tier, + data_residency: request.data_residency, + vertex_location: request.vertex_location, + at: request.at, + custom_cost_per_token, + } +} + +pub fn get_billed_token_rates( + catalog: &ModelInfoCatalog, + request: ModelCostRequest<'_>, + custom_cost_per_token: Option, +) -> Option { + calculate_billed_token_rates(billed_rates_request( + catalog, + request, + custom_cost_per_token, + )) +} + +pub fn get_token_type_cost_breakdown( + catalog: &ModelInfoCatalog, + request: ModelCostRequest<'_>, + custom_cost_per_token: Option, +) -> TokenTypeCostBreakdown { + calculate_token_type_cost_breakdown(billed_rates_request( + catalog, + request, + custom_cost_per_token, + )) +} diff --git a/litellm-rust/crates/cost/src/catalog.rs b/litellm-rust/crates/cost/src/catalog.rs index 67bff3340c7..7078f7b0970 100644 --- a/litellm-rust/crates/cost/src/catalog.rs +++ b/litellm-rust/crates/cost/src/catalog.rs @@ -3,58 +3,14 @@ use std::collections::HashMap; use jiff::Timestamp; use serde_json::Value; -use crate::azure_ai_image_cost::{ - AzureAiImageRequest, cost_calculator as azure_ai_image_cost_calculator, -}; -use crate::bedrock_image_cost::cost_calculator as bedrock_image_cost_calculator; -use crate::billed_token_rates::{ - BilledRatesRequest, BilledTokenRates, TokenTypeCostBreakdown, - get_billed_token_rates as calculate_billed_token_rates, - get_token_type_cost_breakdown as calculate_token_type_cost_breakdown, -}; -use crate::completion_cost::{ - CompletionCost, completion_cost, get_response_cost_from_hidden_params, -}; use crate::completion_input::{ CompletionInputRequest, PreparedCompletionInput, prepare_completion_input, }; -use crate::cost_calculator::cost_per_token; -use crate::custom_pricing::CustomTokenRates; use crate::error::CostError; -use crate::fal_ai_image_cost::{ - cost_calculator as fal_ai_image_cost_calculator, - fal_ai_passthrough_cost as calculate_fal_ai_passthrough_cost, -}; -use crate::generic_cost::calculate_generic_cost_from_model_info_with_region; -use crate::image_response_cost::{ - calculate_image_response_cost_from_usage, flat_image_cost, gemini_image_edit_cost, - gemini_image_generation_cost, resolve_image_model_info, vertex_image_edit_cost, - vertex_image_generation_cost, -}; use crate::model_selection::{ ModelSelectionRequest, get_provider_for_cost_calc, select_model_name_for_cost_calc, }; -use crate::non_token::{ImageRates, ImageUsage, calculate_image}; -use crate::openai_cost::video_generation_cost as calculate_video_generation_cost; -use crate::openai_image_cost::cost_calculator as openai_image_cost_calculator; -use crate::prompt_caching_savings::{ - PromptCachingSavingsRequest, calculate_prompt_caching_savings, -}; -use crate::realtime_cost::{ - combine_usage_objects, event_usage, get_transcription_model_name_from_results, - partition_results_by_service_tier, transcription_usage_cost, -}; use crate::responses_usage::ChatUsage; -use crate::retrieval_cost::{rerank_cost, vector_store_search_cost}; -use crate::search_cost::{ - ParallelAiPricing, effective_mode, parallel_ai_search_cost, provider_usage, - search_provider_cost_per_query, -}; -use crate::speech_cost::{ - SpeechCostMetric, cost_per_second, generic_cost_per_character, lyria_generation_cost, - select_cost_metric_for_model, transcription_usage_has_token_details, -}; -use crate::tool_cost_dispatch::{BuiltInToolCostRequest, get_cost_for_built_in_tools}; #[derive(Clone, Debug, Default)] pub struct ModelInfoCatalog { @@ -107,49 +63,6 @@ pub enum CostCall<'a> { }, } -#[derive(Clone, Copy, Debug)] -pub struct CompletionCostRequest<'a> { - pub token: ModelCostRequest<'a>, - pub built_in_tools: BuiltInToolCharge<'a>, - pub additional_costs: &'a [f64], - pub discount_config: &'a Value, - pub margin_config: &'a Value, -} - -#[derive(Clone, Copy, Debug)] -pub enum BuiltInToolCharge<'a> { - Provided(f64), - FromResponse(BuiltInToolCostRequest<'a>), -} - -#[derive(Clone, Copy, Debug)] -pub struct ResponseCostRequest<'a> { - pub completion: CompletionCostRequest<'a>, - pub cache_hit: bool, - pub hidden_params: &'a Value, -} - -#[derive(Clone, Copy, Debug)] -pub struct AzureAiImageCatalogRequest<'a> { - pub model: &'a str, - pub image_response: &'a Value, - pub size: Option<&'a str>, - pub n: Option, - pub optional_params: &'a Value, - pub supplied_model_info: Option<&'a Value>, - pub at: Timestamp, -} - -#[derive(Clone, Copy, Debug)] -pub struct DefaultImageCostRequest<'a> { - pub model: &'a str, - pub provider: Option<&'a str>, - pub quality: Option<&'a str>, - pub n: Option, - pub size: Option<&'a str>, - pub supplied_model_info: Option<&'a Value>, -} - fn select_model_key<'a, T>( entries: &'a HashMap, model: &str, @@ -306,590 +219,4 @@ impl ModelInfoCatalog { ) -> Result { prepare_completion_input(request, &self.entries) } - - pub fn speech_cost( - &self, - request: ModelCostRequest<'_>, - prompt_characters: Option, - ) -> Result<(f64, f64), CostError> { - let key = self - .select_model_key(request.model, request.provider, request.region) - .ok_or(CostError::ModelNotFound)?; - let model_info = &self.entries[key]; - if matches!(request.provider, Some("vertex_ai" | "vertex_ai_beta")) - && let Some(cost) = lyria_generation_cost(model_info) - { - return Ok((0.0, cost)); - } - match select_cost_metric_for_model(model_info)? { - SpeechCostMetric::PerCharacter => { - let characters = prompt_characters.ok_or(CostError::MissingPromptCharacters)?; - let (prompt, completion) = - generic_cost_per_character(model_info, characters, 0.0, None, Some(0.0)); - Ok(( - prompt.ok_or(CostError::MissingInputCharacterRate)?, - completion.unwrap_or(0.0), - )) - } - SpeechCostMetric::PerToken => Ok(cost_per_token(self, request)?), - } - } - - pub fn transcription_cost( - &self, - request: ModelCostRequest<'_>, - duration_seconds: f64, - ) -> Result<(f64, f64), CostError> { - let key = self - .select_model_key(request.model, request.provider, request.region) - .ok_or(CostError::ModelNotFound)?; - let model_info = &self.entries[key]; - if transcription_usage_has_token_details(request.usage) { - return Ok(calculate_generic_cost_from_model_info_with_region( - request.usage, - model_info, - request.service_tier, - false, - request.data_residency, - request.vertex_location, - request.at, - )); - } - Ok(cost_per_second(model_info, duration_seconds)) - } - - pub fn handle_realtime_transcription_cost_calculation( - &self, - results: &[Value], - provider: &str, - requested_model: &str, - ) -> f64 { - let model = get_transcription_model_name_from_results(results).unwrap_or(requested_model); - let model_info = self - .select_model_key(model, Some(provider), None) - .and_then(|key| self.entries.get(key)); - results - .iter() - .filter(|event| { - event.get("type").and_then(Value::as_str) - == Some("conversation.item.input_audio_transcription.completed") - }) - .map(|event| transcription_usage_cost(&event["usage"], model_info)) - .sum() - } - - pub fn handle_realtime_stream_cost_calculation( - &self, - results: &[Value], - combined_usage: &ChatUsage, - provider: &str, - requested_model: &str, - data_residency: Option<&str>, - at: Timestamp, - ) -> f64 { - let models = results - .iter() - .filter(|event| event.get("type").and_then(Value::as_str) == Some("session.created")) - .filter_map(|event| event.pointer("/session/model").and_then(Value::as_str)) - .chain(std::iter::once(requested_model)); - let token_cost = models - .filter_map(|model| { - let key = self.select_model_key(model, Some(provider), None)?; - let cost = calculate_generic_cost_from_model_info_with_region( - combined_usage, - &self.entries[key], - None, - false, - data_residency, - None, - at, - ); - let declares_pricing = [ - self.entries.get(model), - self.entries.get(&format!("{provider}/{model}")), - ] - .into_iter() - .flatten() - .any(|entry| { - entry.as_object().is_some_and(|fields| { - fields - .iter() - .any(|(field, value)| field.contains("cost_per") && !value.is_null()) - }) - }); - (cost.0 + cost.1 > 0.0 || declares_pricing).then_some(cost) - }) - .next() - .unwrap_or((0.0, 0.0)); - token_cost.0 - + token_cost.1 - + self.handle_realtime_transcription_cost_calculation( - results, - provider, - requested_model, - ) - } - - pub fn responses_ws_token_cost_by_tier( - &self, - results: &[Value], - model: &str, - provider: Option<&str>, - region: Option<&str>, - data_residency: Option<&str>, - at: Timestamp, - ) -> Result { - partition_results_by_service_tier(results) - .into_iter() - .map(|(service_tier, events)| { - let usage = combine_usage_objects( - events - .into_iter() - .map(event_usage) - .collect::, _>>()?, - )?; - let (prompt, completion) = cost_per_token( - self, - ModelCostRequest { - model, - provider, - region, - usage: &usage, - service_tier, - data_residency, - vertex_location: None, - at, - response_time_ms: None, - }, - )?; - Ok(prompt + completion) - }) - .sum() - } - - pub fn get_billed_token_rates( - &self, - request: ModelCostRequest<'_>, - custom_cost_per_token: Option, - ) -> Option { - let model_info = self - .select_model_key(request.model, request.provider, request.region) - .and_then(|key| self.entries.get(key)); - calculate_billed_token_rates(BilledRatesRequest { - model_info, - usage: request.usage, - provider: request.provider, - service_tier: request.service_tier, - data_residency: request.data_residency, - vertex_location: request.vertex_location, - at: request.at, - custom_cost_per_token, - }) - } - - pub fn get_token_type_cost_breakdown( - &self, - request: ModelCostRequest<'_>, - custom_cost_per_token: Option, - ) -> TokenTypeCostBreakdown { - let model_info = self - .select_model_key(request.model, request.provider, request.region) - .and_then(|key| self.entries.get(key)); - calculate_token_type_cost_breakdown(BilledRatesRequest { - model_info, - usage: request.usage, - provider: request.provider, - service_tier: request.service_tier, - data_residency: request.data_residency, - vertex_location: request.vertex_location, - at: request.at, - custom_cost_per_token, - }) - } - - pub fn calculate_prompt_caching_savings(&self, request: ModelCostRequest<'_>) -> Option { - let model_info = self - .select_model_key(request.model, request.provider, request.region) - .and_then(|key| self.entries.get(key))?; - Some(calculate_prompt_caching_savings( - PromptCachingSavingsRequest { - model_info, - usage: request.usage, - provider: request.provider, - service_tier: request.service_tier, - data_residency: request.data_residency, - vertex_location: request.vertex_location, - at: request.at, - }, - )) - } - - pub fn calculate_image_response_cost_from_usage( - &self, - model: &str, - provider: Option<&str>, - region: Option<&str>, - image_response: &Value, - at: Timestamp, - ) -> Option { - let model_info = self - .select_model_key(model, provider, region) - .and_then(|key| self.entries.get(key))?; - calculate_image_response_cost_from_usage(image_response, model_info, provider, at) - } - - pub fn google_image_generation_cost( - &self, - model: &str, - provider: &str, - image_response: &Value, - supplied_model_info: Option<&Value>, - at: Timestamp, - ) -> Result { - let shared = self - .select_model_key(model, Some(provider), None) - .and_then(|key| self.entries.get(key)); - let model_info = resolve_image_model_info(shared, supplied_model_info) - .ok_or(CostError::ModelNotFound)?; - match provider { - "gemini" => Ok(gemini_image_generation_cost( - image_response, - &model_info, - at, - )), - "vertex_ai" => Ok(vertex_image_generation_cost( - image_response, - &model_info, - at, - )), - _ => Err(CostError::ModelNotFound), - } - } - - pub fn google_image_edit_cost( - &self, - model: &str, - provider: &str, - image_response: &Value, - supplied_model_info: Option<&Value>, - at: Timestamp, - ) -> Result { - let shared = self - .select_model_key(model, Some(provider), None) - .and_then(|key| self.entries.get(key)); - match provider { - "gemini" => { - let info = resolve_image_model_info(shared, supplied_model_info) - .ok_or(CostError::ModelNotFound)?; - Ok(gemini_image_edit_cost(image_response, &info, at)) - } - "vertex_ai" => shared - .map(|info| vertex_image_edit_cost(image_response, info)) - .ok_or(CostError::ModelNotFound), - _ => Err(CostError::ModelNotFound), - } - } - - pub fn azure_ai_image_generation_cost( - &self, - request: AzureAiImageCatalogRequest<'_>, - ) -> Result { - let shared = self - .select_model_key(request.model, Some("azure_ai"), None) - .and_then(|key| self.entries.get(key)); - let model_info = resolve_image_model_info(shared, request.supplied_model_info) - .ok_or(CostError::ModelNotFound)?; - let shared_pricing = model_info - .get("key") - .and_then(Value::as_str) - .and_then(|key| self.entries.get(key)) - .or(shared); - azure_ai_image_cost_calculator(AzureAiImageRequest { - image_response: request.image_response, - model_info: &model_info, - supplied_model_info: request.supplied_model_info, - shared_pricing, - size: request.size, - n: request.n, - optional_params: request.optional_params, - at: request.at, - }) - } - - pub fn flat_image_generation_cost( - &self, - model: &str, - provider: &str, - image_response: &Value, - supplied_model_info: Option<&Value>, - ) -> Result { - let shared = self - .select_model_key(model, Some(provider), None) - .and_then(|key| self.entries.get(key)); - let model_info = resolve_image_model_info(shared, supplied_model_info) - .ok_or(CostError::ModelNotFound)?; - Ok(flat_image_cost(image_response, &model_info)) - } - - pub fn openai_image_generation_cost( - &self, - model: &str, - provider: &str, - image_response: &Value, - supplied_model_info: Option<&Value>, - at: Timestamp, - ) -> Result { - let shared = self - .select_model_key(model, Some(provider), None) - .and_then(|key| self.entries.get(key)); - let model_info = resolve_image_model_info(shared, supplied_model_info) - .ok_or(CostError::ModelNotFound)?; - openai_image_cost_calculator(image_response, &model_info, provider, at) - } - - pub fn fal_ai_image_generation_cost( - &self, - model: &str, - image_response: &Value, - optional_params: &Value, - deployment_prices: Option<&Value>, - ) -> Result { - fal_ai_image_cost_calculator( - model, - image_response, - optional_params, - deployment_prices, - &self.entries, - ) - .ok_or(CostError::ModelNotFound) - } - - pub fn fal_ai_passthrough_cost(&self, model: &str, request_body: &Value) -> Option { - let normalized = model.strip_prefix("fal_ai/").unwrap_or(model); - calculate_fal_ai_passthrough_cost( - self.entries.get(&format!("fal_ai/{normalized}")), - request_body, - ) - } - - pub fn bedrock_image_generation_cost( - &self, - model: &str, - image_response: &Value, - size: Option<&str>, - optional_params: &Value, - ) -> Result { - bedrock_image_cost_calculator(model, image_response, size, optional_params, &self.entries) - .ok_or(CostError::ModelNotFound) - } - - pub fn video_generation_cost( - &self, - model: &str, - provider: Option<&str>, - deployment_info: Option<&Value>, - duration_seconds: f64, - video_resolution: Option<&str>, - ) -> Result { - let model_info = deployment_info - .or_else(|| { - self.select_model_key(model, provider, None) - .and_then(|key| self.entries.get(key)) - }) - .ok_or(CostError::ModelNotFound)?; - calculate_video_generation_cost(model_info, duration_seconds, video_resolution) - } - - pub fn default_image_cost_calculator( - &self, - request: DefaultImageCostRequest<'_>, - ) -> Result { - let raw_size = request.size.unwrap_or("1024-x-1024"); - let size = if raw_size.contains("-x-") { - raw_size.to_owned() - } else { - raw_size.replace('x', "-x-") - }; - let (height, width) = size.split_once("-x-").ok_or(CostError::InvalidQuantity)?; - let height = height - .parse::() - .map_err(|_| CostError::InvalidQuantity)?; - let width = width - .parse::() - .map_err(|_| CostError::InvalidQuantity)?; - let provider_prefix = request.provider.map(|provider| format!("{provider}/")); - let without_provider = provider_prefix - .as_deref() - .and_then(|prefix| request.model.strip_prefix(prefix)); - let base = match (request.provider, without_provider) { - (Some(provider), Some(model)) => format!("{provider}/{size}/{model}"), - _ => format!("{size}/{}", request.model), - }; - let model_tail = request.model.rsplit('/').next().unwrap_or(request.model); - let without_prefix = format!("{size}/{model_tail}"); - let candidates = [ - request.quality.map(|quality| format!("{quality}/{base}")), - match (request.provider, request.quality) { - (Some(provider), Some(quality)) => Some(format!( - "{provider}/{quality}/{size}/{}", - without_provider.unwrap_or(request.model) - )), - _ => None, - }, - Some(base.clone()), - Some(format!("high/{base}")), - request - .quality - .map(|quality| format!("{quality}/{without_prefix}")), - Some(without_prefix), - Some(request.model.to_owned()), - without_provider.map(str::to_owned), - ]; - let shared = candidates - .iter() - .flatten() - .find_map(|candidate| self.entries.get(candidate)); - if shared.is_none() && request.supplied_model_info.is_none() { - return Err(CostError::ModelNotFound); - } - let rate = |info: &Value, key: &str| { - crate::generic_input::get_cost_per_unit(info, key, None) - .map_or(crate::pricing::Rate::Missing, crate::pricing::Rate::Value) - }; - let tables = [request.supplied_model_info, shared] - .into_iter() - .flatten() - .map(|info| ImageRates { - input_per_image: rate(info, "input_cost_per_image"), - output_per_image: rate(info, "output_cost_per_image"), - input_per_pixel: rate(info, "input_cost_per_pixel"), - }) - .collect::>(); - Ok(calculate_image( - &tables, - ImageUsage { - count: request.n.unwrap_or(1), - width, - height, - }, - )? - .total) - } - - pub fn rerank_cost( - &self, - model: &str, - provider: &str, - region: Option<&str>, - billed_units: Option<&Value>, - ) -> (f64, f64) { - let model_info = self - .select_model_key(model, Some(provider), region) - .and_then(|key| self.entries.get(key)); - rerank_cost(provider, model_info, billed_units) - } - - pub fn vector_store_search_cost(&self, provider: &str, api_type: Option<&str>) -> (f64, f64) { - vector_store_search_cost(provider, api_type, self.entries.get("vertex_ai/search_api")) - } - - pub fn search_provider_cost_per_query( - &self, - model: &str, - provider: Option<&str>, - number_of_queries: u64, - optional_params: &Value, - ) -> Result<(f64, f64), CostError> { - if provider == Some("parallel_ai") { - let pricing_model = match effective_mode(optional_params) { - "fast" => "parallel_ai/search-fast", - "turbo" => "parallel_ai/search-turbo", - _ => "parallel_ai/search", - }; - let model_info = self - .entries - .get(pricing_model) - .ok_or(CostError::ModelNotFound)?; - let request_cost = model_info - .get("input_cost_per_query") - .and_then(|value| match value { - Value::Number(value) => value.as_f64(), - Value::String(value) => value.parse().ok(), - _ => None, - }) - .unwrap_or(0.0); - return Ok(( - parallel_ai_search_cost( - optional_params, - provider_usage(optional_params), - ParallelAiPricing { - request_cost, - default_results: 10, - additional_result_cost: 0.001, - }, - ), - 0.0, - )); - } - let model_info = self - .select_model_key(model, provider, None) - .and_then(|key| self.entries.get(key)) - .ok_or(CostError::ModelNotFound)?; - Ok(search_provider_cost_per_query( - model_info, - number_of_queries, - optional_params, - )) - } - - pub fn built_in_tool_cost( - &self, - model: &str, - provider: Option<&str>, - region: Option<&str>, - request: BuiltInToolCostRequest<'_>, - ) -> f64 { - let model_info = self - .select_model_key(model, provider, region) - .and_then(|key| self.entries.get(key)); - get_cost_for_built_in_tools(request, model_info) - } - - pub fn completion_cost( - &self, - request: CompletionCostRequest<'_>, - ) -> Result { - let (prompt, output) = cost_per_token(self, request.token)?; - let built_in_tools = match request.built_in_tools { - BuiltInToolCharge::Provided(cost) => cost, - BuiltInToolCharge::FromResponse(tool_request) => self.built_in_tool_cost( - request.token.model, - request.token.provider, - request.token.region, - tool_request, - ), - }; - Ok(completion_cost( - prompt, - output, - built_in_tools, - request.additional_costs, - request.token.provider, - request.discount_config, - request.margin_config, - )) - } - - pub fn response_cost_calculator( - &self, - request: ResponseCostRequest<'_>, - ) -> Result { - if request.cache_hit { - return Ok(0.0); - } - if let Some(reported) = get_response_cost_from_hidden_params(request.hidden_params)? { - return Ok(reported); - } - Ok(self.completion_cost(request.completion)?.total) - } } diff --git a/litellm-rust/crates/cost/src/completion_response.rs b/litellm-rust/crates/cost/src/completion_response.rs index e37c4a41025..669ac0c78fa 100644 --- a/litellm-rust/crates/cost/src/completion_response.rs +++ b/litellm-rust/crates/cost/src/completion_response.rs @@ -6,13 +6,16 @@ use serde_json::Value; use crate::a2a_cost::calculate_a2a_cost; use crate::azure_ai_cost::{azure_ai_router_fee, is_azure_model_router}; -use crate::billed_token_rates::TokenTypeCostBreakdown; +use crate::billed_token_rates::{TokenTypeCostBreakdown, get_token_type_cost_breakdown}; use crate::catalog::{CostCall, ModelCostRequest, ModelInfoCatalog}; use crate::completion_cost::{ CompletionCost, completion_cost, get_response_cost_from_hidden_params, }; use crate::completion_input::{CompletionInputRequest, PreparedCompletionInput, ResponseKind}; -use crate::cost_calculator::{cost_per_token, cost_per_token_for_call}; +use crate::cost_calculator::{ + built_in_tool_cost, cost_per_token, cost_per_token_for_call, default_video_cost_calculator, + handle_realtime_stream_cost_calculation, +}; use crate::custom_pricing::{CustomPricing, cost_from_chat_usage}; use crate::error::CostError; use crate::image_cost_router::{ @@ -382,15 +385,15 @@ fn price_video_response( .and_then(Value::as_str) .map(|value| value.trim().to_ascii_lowercase()); let (model, total) = price_candidates(&prepared.model_candidates, |model| { - catalog - .video_generation_cost( - model, - provider, - deployment_info, - duration, - resolution.as_deref(), - ) - .map(|cost| cost * count as f64) + default_video_cost_calculator( + catalog, + model, + duration, + provider, + deployment_info, + resolution.as_deref(), + ) + .map(|cost| cost * count as f64) }) .map_err(|error| match error { CandidatePriceError::MissingModel => CostError::MissingModel, @@ -460,7 +463,8 @@ fn price_realtime_response( .next() .ok_or(CostError::MissingModel)? .clone(); - let total = catalog.handle_realtime_stream_cost_calculation( + let total = handle_realtime_stream_cost_calculation( + catalog, results, &combined, provider, @@ -502,7 +506,8 @@ fn config_driven_tool_cost( let Some(response) = request.input.model_selection.response else { return 0.0; }; - catalog.built_in_tool_cost( + built_in_tool_cost( + catalog, model, provider, region, @@ -842,7 +847,8 @@ pub fn completion_cost_from_response( request.margin_config, ); let token_breakdown = prepared.usage.as_ref().map(|response_usage| { - catalog.get_token_type_cost_breakdown( + get_token_type_cost_breakdown( + catalog, ModelCostRequest { model: &model, provider, diff --git a/litellm-rust/crates/cost/src/cost_calculator.rs b/litellm-rust/crates/cost/src/cost_calculator.rs index 89892f4124c..59872fe6c37 100644 --- a/litellm-rust/crates/cost/src/cost_calculator.rs +++ b/litellm-rust/crates/cost/src/cost_calculator.rs @@ -1,5 +1,6 @@ //! The public cost entry points mirroring litellm/cost_calculator.py. +use jiff::Timestamp; use serde_json::Value; use crate::anthropic_cost::fast_speed_multiplier; @@ -8,6 +9,10 @@ use crate::azure_cost::output_per_second_cost; use crate::base_rate_selection::uses_inclusive_token_thresholds; use crate::batch::batch_cost_from_model_info; use crate::catalog::{CostCall, ModelCostRequest, ModelInfoCatalog}; +use crate::completion_cost::{ + CompletionCost, completion_cost as apply_completion_cost_adjustments, + get_response_cost_from_hidden_params, +}; use crate::custom_pricing::{CustomPricing, cost_from_chat_usage}; use crate::dashscope_cost::cost_per_token as dashscope_cost_per_token; use crate::databricks_cost::databricks_cost_per_token; @@ -18,18 +23,53 @@ use crate::fireworks_cost::{ use crate::generic_cost::calculate_generic_cost_from_model_info_with_region; use crate::lemonade_cost::lemonade_cost_per_token; use crate::ocr_cost::ocr_cost; +use crate::openai_cost::video_generation_cost; use crate::per_second::{has_token_or_tiered_pricing, per_second_pricing_cost}; use crate::perplexity_cost::cost_per_token as perplexity_cost_per_token; use crate::provider::LlmProviders; use crate::provider_cache::apply_provider_cache_read_default; +use crate::realtime_cost::{ + combine_usage_objects, event_usage, get_transcription_model_name_from_results, + partition_results_by_service_tier, transcription_usage_cost, +}; use crate::regional_uplift::get_provider_specific_geo_multiplier; +use crate::responses_usage::ChatUsage; +use crate::retrieval_cost::{rerank_cost, vector_store_search_cost}; +use crate::search_cost::search_provider_cost_per_query; +use crate::speech_cost::{ + SpeechCostMetric, cost_per_second, generic_cost_per_character, lyria_generation_cost, + select_cost_metric_for_model, transcription_usage_has_token_details, +}; use crate::together_cost::{ TogetherThresholds, get_model_params_and_category, has_together_registry_pricing, together_ai_cost_per_token, }; +use crate::tool_cost_dispatch::{BuiltInToolCostRequest, get_cost_for_built_in_tools}; use crate::vertex_cost::{cost_per_token as vertex_cost_per_token, vertex_cost}; use crate::xai_cost::{cost_per_token as xai_cost_per_token, reported_cost as xai_reported_cost}; +#[derive(Clone, Copy, Debug)] +pub struct CompletionCostRequest<'a> { + pub token: ModelCostRequest<'a>, + pub built_in_tools: BuiltInToolCharge<'a>, + pub additional_costs: &'a [f64], + pub discount_config: &'a Value, + pub margin_config: &'a Value, +} + +#[derive(Clone, Copy, Debug)] +pub enum BuiltInToolCharge<'a> { + Provided(f64), + FromResponse(BuiltInToolCostRequest<'a>), +} + +#[derive(Clone, Copy, Debug)] +pub struct ResponseCostRequest<'a> { + pub completion: CompletionCostRequest<'a>, + pub cache_hit: bool, + pub hidden_params: &'a Value, +} + pub fn cost_per_token_with_custom( catalog: &ModelInfoCatalog, request: ModelCostRequest<'_>, @@ -89,29 +129,34 @@ pub fn cost_per_token_for_call( } Ok(cost_per_token(catalog, request)?) } - CostCall::Speech { prompt_characters } => { - Ok(catalog.speech_cost(request, prompt_characters)?) - } + CostCall::Speech { prompt_characters } => speech_cost(catalog, request, prompt_characters), CostCall::Transcription { duration_seconds } => { - Ok(catalog.transcription_cost(request, duration_seconds)?) + transcription_cost(catalog, request, duration_seconds) } CostCall::Rerank { billed_units } => { let provider = request.provider.ok_or(CostError::MissingProvider)?; - Ok(catalog.rerank_cost(request.model, provider, request.region, billed_units)) + Ok(rerank_cost( + catalog, + request.model, + provider, + request.region, + billed_units, + )) } CostCall::VectorStoreSearch { api_type } => { let provider = request.provider.ok_or(CostError::MissingProvider)?; - Ok(catalog.vector_store_search_cost(provider, api_type)) + Ok(vector_store_search_cost(catalog, provider, api_type)) } CostCall::Search { number_of_queries, optional_params, - } => Ok(catalog.search_provider_cost_per_query( + } => search_provider_cost_per_query( + catalog, request.model, request.provider, number_of_queries.filter(|count| *count > 0).unwrap_or(1), optional_params, - )?), + ), CostCall::Ocr { response, deployment_info, @@ -296,3 +341,253 @@ pub fn cost_per_token( }; Ok((cost.0 * speed, cost.1 * speed)) } + +pub fn speech_cost( + catalog: &ModelInfoCatalog, + request: ModelCostRequest<'_>, + prompt_characters: Option, +) -> Result<(f64, f64), CostError> { + let model_info = catalog + .entry(request.model, request.provider, request.region) + .ok_or(CostError::ModelNotFound)?; + if matches!(request.provider, Some("vertex_ai" | "vertex_ai_beta")) + && let Some(cost) = lyria_generation_cost(model_info) + { + return Ok((0.0, cost)); + } + match select_cost_metric_for_model(model_info)? { + SpeechCostMetric::PerCharacter => { + let characters = prompt_characters.ok_or(CostError::MissingPromptCharacters)?; + let (prompt, completion) = + generic_cost_per_character(model_info, characters, 0.0, None, Some(0.0)); + Ok(( + prompt.ok_or(CostError::MissingInputCharacterRate)?, + completion.unwrap_or(0.0), + )) + } + SpeechCostMetric::PerToken => cost_per_token(catalog, request), + } +} + +pub fn transcription_cost( + catalog: &ModelInfoCatalog, + request: ModelCostRequest<'_>, + duration_seconds: f64, +) -> Result<(f64, f64), CostError> { + let model_info = catalog + .entry(request.model, request.provider, request.region) + .ok_or(CostError::ModelNotFound)?; + if transcription_usage_has_token_details(request.usage) { + return Ok(calculate_generic_cost_from_model_info_with_region( + request.usage, + model_info, + request.service_tier, + false, + request.data_residency, + request.vertex_location, + request.at, + )); + } + Ok(cost_per_second(model_info, duration_seconds)) +} + +pub fn handle_realtime_transcription_cost_calculation( + catalog: &ModelInfoCatalog, + results: &[Value], + provider: &str, + requested_model: &str, +) -> f64 { + let model = get_transcription_model_name_from_results(results).unwrap_or(requested_model); + let model_info = catalog.entry(model, Some(provider), None); + results + .iter() + .filter(|event| { + event.get("type").and_then(Value::as_str) + == Some("conversation.item.input_audio_transcription.completed") + }) + .map(|event| transcription_usage_cost(&event["usage"], model_info)) + .sum() +} + +fn cost_map_entry_declares_pricing( + catalog: &ModelInfoCatalog, + model: &str, + provider: &str, +) -> bool { + [ + catalog.entries().get(model), + catalog.entries().get(&format!("{provider}/{model}")), + ] + .into_iter() + .flatten() + .filter_map(Value::as_object) + .any(|fields| { + fields + .iter() + .any(|(field, value)| field.contains("cost_per") && !value.is_null()) + }) +} + +pub fn handle_realtime_stream_cost_calculation( + catalog: &ModelInfoCatalog, + results: &[Value], + combined_usage: &ChatUsage, + provider: &str, + requested_model: &str, + data_residency: Option<&str>, + at: Timestamp, +) -> f64 { + let (prompt, completion) = results + .iter() + .filter(|event| event.get("type").and_then(Value::as_str) == Some("session.created")) + .filter_map(|event| event.pointer("/session/model").and_then(Value::as_str)) + .chain(std::iter::once(requested_model)) + .find_map(|model| { + let model_info = catalog.entry(model, Some(provider), None)?; + let cost = calculate_generic_cost_from_model_info_with_region( + combined_usage, + model_info, + None, + false, + data_residency, + None, + at, + ); + (cost.0 + cost.1 > 0.0 || cost_map_entry_declares_pricing(catalog, model, provider)) + .then_some(cost) + }) + .unwrap_or((0.0, 0.0)); + prompt + + completion + + handle_realtime_transcription_cost_calculation( + catalog, + results, + provider, + requested_model, + ) +} + +pub fn responses_ws_token_cost_by_tier( + catalog: &ModelInfoCatalog, + results: &[Value], + model: &str, + provider: Option<&str>, + region: Option<&str>, + data_residency: Option<&str>, + at: Timestamp, +) -> Result { + partition_results_by_service_tier(results) + .into_iter() + .map(|(service_tier, events)| { + let usage = combine_usage_objects( + events + .into_iter() + .map(event_usage) + .collect::, _>>()?, + )?; + let (prompt, completion) = cost_per_token( + catalog, + ModelCostRequest { + model, + provider, + region, + usage: &usage, + service_tier, + data_residency, + vertex_location: None, + at, + response_time_ms: None, + }, + )?; + Ok(prompt + completion) + }) + .sum() +} + +pub fn built_in_tool_cost( + catalog: &ModelInfoCatalog, + model: &str, + provider: Option<&str>, + region: Option<&str>, + request: BuiltInToolCostRequest<'_>, +) -> f64 { + get_cost_for_built_in_tools(request, catalog.entry(model, provider, region)) +} + +pub fn completion_cost( + catalog: &ModelInfoCatalog, + request: CompletionCostRequest<'_>, +) -> Result { + let (prompt, output) = cost_per_token(catalog, request.token)?; + let built_in_tools = match request.built_in_tools { + BuiltInToolCharge::Provided(cost) => cost, + BuiltInToolCharge::FromResponse(tool_request) => built_in_tool_cost( + catalog, + request.token.model, + request.token.provider, + request.token.region, + tool_request, + ), + }; + Ok(apply_completion_cost_adjustments( + prompt, + output, + built_in_tools, + request.additional_costs, + request.token.provider, + request.discount_config, + request.margin_config, + )) +} + +pub fn response_cost_calculator( + catalog: &ModelInfoCatalog, + request: ResponseCostRequest<'_>, +) -> Result { + if request.cache_hit { + return Ok(0.0); + } + if let Some(reported) = get_response_cost_from_hidden_params(request.hidden_params)? { + return Ok(reported); + } + Ok(completion_cost(catalog, request.completion)?.total) +} + +pub fn default_video_cost_calculator( + catalog: &ModelInfoCatalog, + model: &str, + duration_seconds: f64, + provider: Option<&str>, + deployment_info: Option<&Value>, + video_resolution: Option<&str>, +) -> Result { + let model_info = match deployment_info { + Some(info) => info, + None => { + let without_provider = provider.and_then(|provider| { + model + .strip_prefix(provider) + .and_then(|rest| rest.strip_prefix('/')) + }); + let base_model = match (provider, without_provider) { + (Some(provider), Some(bare)) => format!("{provider}/{bare}"), + _ => model.to_owned(), + }; + let model_tail = model.rsplit('/').next().unwrap_or(model); + [ + Some(base_model.as_str()), + Some(model), + Some(model_tail), + without_provider, + ] + .into_iter() + .flatten() + .find_map(|candidate| catalog.entries().get(candidate)) + .or_else(|| { + provider.and_then(|provider| catalog.entries().get(&format!("{provider}/{model}"))) + }) + .ok_or(CostError::ModelNotFound)? + } + }; + video_generation_cost(model_info, duration_seconds, video_resolution) +} diff --git a/litellm-rust/crates/cost/src/fal_ai_image_cost.rs b/litellm-rust/crates/cost/src/fal_ai_image_cost.rs index ca7383f3950..152cdc2449f 100644 --- a/litellm-rust/crates/cost/src/fal_ai_image_cost.rs +++ b/litellm-rust/crates/cost/src/fal_ai_image_cost.rs @@ -2,6 +2,7 @@ use std::collections::HashMap; use serde_json::Value; +use crate::catalog::ModelInfoCatalog; use crate::image_response_cost::resolve_image_model_info; pub const PIXELS_PER_MEGAPIXEL: u64 = 1_048_576; @@ -102,7 +103,21 @@ pub fn flat_cost_per_image( pixel_rate * PIXELS_PER_MEGAPIXEL as f64 * megapixels as f64 } -pub fn fal_ai_passthrough_cost(model_info: Option<&Value>, request_body: &Value) -> Option { +pub fn fal_ai_passthrough_cost( + catalog: &ModelInfoCatalog, + model: &str, + request_body: &Value, +) -> Option { + fal_ai_passthrough_cost_from_model_info( + catalog.entries().get(&format!("fal_ai/{model}")), + request_body, + ) +} + +pub fn fal_ai_passthrough_cost_from_model_info( + model_info: Option<&Value>, + request_body: &Value, +) -> Option { let info = model_info?; let resolution = request_body .get("resolution") diff --git a/litellm-rust/crates/cost/src/image_cost_router.rs b/litellm-rust/crates/cost/src/image_cost_router.rs index 513ea77aa97..901785fc592 100644 --- a/litellm-rust/crates/cost/src/image_cost_router.rs +++ b/litellm-rust/crates/cost/src/image_cost_router.rs @@ -1,9 +1,21 @@ -use crate::error::CostError; use jiff::Timestamp; use serde_json::Value; -use crate::catalog::{AzureAiImageCatalogRequest, DefaultImageCostRequest, ModelInfoCatalog}; +use crate::azure_ai_image_cost::{ + AzureAiImageRequest, cost_calculator as azure_ai_image_cost_calculator, +}; +use crate::bedrock_image_cost::cost_calculator as bedrock_image_cost_calculator; +use crate::catalog::ModelInfoCatalog; +use crate::error::CostError; +use crate::fal_ai_image_cost::cost_calculator as fal_ai_image_cost_calculator; use crate::generic_input::get_cost_per_unit; +use crate::image_response_cost::{ + flat_image_cost, gemini_image_edit_cost, gemini_image_generation_cost, + resolve_image_model_info, vertex_image_edit_cost, vertex_image_generation_cost, +}; +use crate::non_token::{ImageRates, ImageUsage, calculate_image}; +use crate::openai_image_cost::cost_calculator as openai_image_cost_calculator; +use crate::pricing::Rate; #[derive(Clone, Copy, Debug)] pub struct ImageCostRouteRequest<'a> { @@ -19,6 +31,27 @@ pub struct ImageCostRouteRequest<'a> { pub at: Timestamp, } +#[derive(Clone, Copy, Debug)] +pub struct AzureAiImageCatalogRequest<'a> { + pub model: &'a str, + pub image_response: &'a Value, + pub size: Option<&'a str>, + pub n: Option, + pub optional_params: &'a Value, + pub supplied_model_info: Option<&'a Value>, + pub at: Timestamp, +} + +#[derive(Clone, Copy, Debug)] +pub struct DefaultImageCostRequest<'a> { + pub model: &'a str, + pub provider: Option<&'a str>, + pub quality: Option<&'a str>, + pub n: Option, + pub size: Option<&'a str>, + pub supplied_model_info: Option<&'a Value>, +} + pub fn call_type_has_image_response(call_type: &str) -> bool { matches!( call_type, @@ -197,30 +230,34 @@ pub fn route_image_generation_cost_calculator( .map(|data| data.len() as u64) }); match request.provider { - Some("vertex_ai") => Ok(catalog.google_image_generation_cost( + Some("vertex_ai") => google_image_generation_cost( + catalog, request.model, "vertex_ai", request.image_response, pricing.as_ref(), request.at, - )?), + ), Some("gemini") if matches!(request.call_type, Some("image_edit" | "aimage_edit")) => { - Ok(catalog.google_image_edit_cost( + google_image_edit_cost( + catalog, request.model, "gemini", request.image_response, pricing.as_ref(), request.at, - )?) + ) } - Some("gemini") => Ok(catalog.google_image_generation_cost( + Some("gemini") => google_image_generation_cost( + catalog, request.model, "gemini", request.image_response, pricing.as_ref(), request.at, - )?), - Some("azure_ai") => Ok(catalog.azure_ai_image_generation_cost( + ), + Some("azure_ai") => azure_ai_image_generation_cost( + catalog, AzureAiImageCatalogRequest { model: request.model, image_response: request.image_response, @@ -230,46 +267,236 @@ pub fn route_image_generation_cost_calculator( supplied_model_info: pricing.as_ref(), at: request.at, }, - )?), - Some(provider @ ("recraft" | "aiml" | "cometapi" | "runwayml")) => Ok(catalog - .flat_image_generation_cost( + ), + Some(provider @ ("recraft" | "aiml" | "cometapi" | "runwayml")) => { + flat_image_generation_cost( + catalog, request.model, provider, request.image_response, pricing.as_ref(), - )?), - Some("fal_ai") => Ok(catalog.fal_ai_image_generation_cost( + ) + } + Some("fal_ai") => fal_ai_image_cost_calculator( request.model, request.image_response, request.optional_params, pricing.as_ref(), - )?), - Some("bedrock") => Ok(catalog.bedrock_image_generation_cost( + catalog.entries(), + ) + .ok_or(CostError::ModelNotFound), + Some("bedrock") => bedrock_image_cost_calculator( request.model, request.image_response, resolved_size, request.optional_params, - )?), + catalog.entries(), + ) + .ok_or(CostError::ModelNotFound), Some(provider @ ("openai" | "azure")) if request.model.to_ascii_lowercase().contains("gpt-image") => { - Ok(catalog.openai_image_generation_cost( + openai_image_generation_cost( + catalog, request.model, provider, request.image_response, pricing.as_ref(), request.at, - )?) + ) } - _ => Ok( - catalog.default_image_cost_calculator(DefaultImageCostRequest { + _ => default_image_cost_calculator( + catalog, + DefaultImageCostRequest { model: request.model, provider: request.provider, quality: resolved_quality, n: resolved_n, size: resolved_size, supplied_model_info: pricing.as_ref(), - })?, + }, ), } } + +fn resolved_model_info( + catalog: &ModelInfoCatalog, + model: &str, + provider: &str, + supplied_model_info: Option<&Value>, +) -> Result { + resolve_image_model_info( + catalog.entry(model, Some(provider), None), + supplied_model_info, + ) + .ok_or(CostError::ModelNotFound) +} + +pub fn google_image_generation_cost( + catalog: &ModelInfoCatalog, + model: &str, + provider: &str, + image_response: &Value, + supplied_model_info: Option<&Value>, + at: Timestamp, +) -> Result { + let model_info = resolved_model_info(catalog, model, provider, supplied_model_info)?; + match provider { + "gemini" => Ok(gemini_image_generation_cost( + image_response, + &model_info, + at, + )), + "vertex_ai" => Ok(vertex_image_generation_cost( + image_response, + &model_info, + at, + )), + _ => Err(CostError::ModelNotFound), + } +} + +pub fn google_image_edit_cost( + catalog: &ModelInfoCatalog, + model: &str, + provider: &str, + image_response: &Value, + supplied_model_info: Option<&Value>, + at: Timestamp, +) -> Result { + match provider { + "gemini" => { + let model_info = resolved_model_info(catalog, model, provider, supplied_model_info)?; + Ok(gemini_image_edit_cost(image_response, &model_info, at)) + } + "vertex_ai" => catalog + .entry(model, Some(provider), None) + .map(|model_info| vertex_image_edit_cost(image_response, model_info)) + .ok_or(CostError::ModelNotFound), + _ => Err(CostError::ModelNotFound), + } +} + +pub fn azure_ai_image_generation_cost( + catalog: &ModelInfoCatalog, + request: AzureAiImageCatalogRequest<'_>, +) -> Result { + let shared = catalog.entry(request.model, Some("azure_ai"), None); + let model_info = resolve_image_model_info(shared, request.supplied_model_info) + .ok_or(CostError::ModelNotFound)?; + let shared_pricing = model_info + .get("key") + .and_then(Value::as_str) + .and_then(|key| catalog.entries().get(key)) + .or(shared); + azure_ai_image_cost_calculator(AzureAiImageRequest { + image_response: request.image_response, + model_info: &model_info, + supplied_model_info: request.supplied_model_info, + shared_pricing, + size: request.size, + n: request.n, + optional_params: request.optional_params, + at: request.at, + }) +} + +pub fn flat_image_generation_cost( + catalog: &ModelInfoCatalog, + model: &str, + provider: &str, + image_response: &Value, + supplied_model_info: Option<&Value>, +) -> Result { + let model_info = resolved_model_info(catalog, model, provider, supplied_model_info)?; + Ok(flat_image_cost(image_response, &model_info)) +} + +pub fn openai_image_generation_cost( + catalog: &ModelInfoCatalog, + model: &str, + provider: &str, + image_response: &Value, + supplied_model_info: Option<&Value>, + at: Timestamp, +) -> Result { + let model_info = resolved_model_info(catalog, model, provider, supplied_model_info)?; + openai_image_cost_calculator(image_response, &model_info, provider, at) +} + +fn image_dimensions(size: &str) -> Result<(u32, u32), CostError> { + let (height, width) = size.split_once("-x-").ok_or(CostError::InvalidQuantity)?; + let parse = |value: &str| value.parse::().map_err(|_| CostError::InvalidQuantity); + Ok((parse(height)?, parse(width)?)) +} + +pub fn default_image_cost_calculator( + catalog: &ModelInfoCatalog, + request: DefaultImageCostRequest<'_>, +) -> Result { + let raw_size = request.size.unwrap_or("1024-x-1024"); + let size = if raw_size.contains("-x-") { + raw_size.to_owned() + } else { + raw_size.replace('x', "-x-") + }; + let (height, width) = image_dimensions(&size)?; + let provider_prefix = request.provider.map(|provider| format!("{provider}/")); + let without_provider = provider_prefix + .as_deref() + .and_then(|prefix| request.model.strip_prefix(prefix)); + let base = match (request.provider, without_provider) { + (Some(provider), Some(model)) => format!("{provider}/{size}/{model}"), + _ => format!("{size}/{}", request.model), + }; + let model_tail = request.model.rsplit('/').next().unwrap_or(request.model); + let without_prefix = format!("{size}/{model_tail}"); + let candidates = [ + request.quality.map(|quality| format!("{quality}/{base}")), + request + .provider + .zip(request.quality) + .map(|(provider, quality)| { + format!( + "{provider}/{quality}/{size}/{}", + without_provider.unwrap_or(request.model) + ) + }), + Some(base.clone()), + Some(format!("high/{base}")), + request + .quality + .map(|quality| format!("{quality}/{without_prefix}")), + Some(without_prefix), + Some(request.model.to_owned()), + without_provider.map(str::to_owned), + ]; + let shared = candidates + .iter() + .flatten() + .find_map(|candidate| catalog.entries().get(candidate)); + if shared.is_none() && request.supplied_model_info.is_none() { + return Err(CostError::ModelNotFound); + } + let rate = |info: &Value, key: &str| { + get_cost_per_unit(info, key, None).map_or(Rate::Missing, Rate::Value) + }; + let tables = [request.supplied_model_info, shared] + .into_iter() + .flatten() + .map(|info| ImageRates { + input_per_image: rate(info, "input_cost_per_image"), + output_per_image: rate(info, "output_cost_per_image"), + input_per_pixel: rate(info, "input_cost_per_pixel"), + }) + .collect::>(); + Ok(calculate_image( + &tables, + ImageUsage { + count: request.n.unwrap_or(1), + width, + height, + }, + )? + .total) +} diff --git a/litellm-rust/crates/cost/src/prompt_caching_savings.rs b/litellm-rust/crates/cost/src/prompt_caching_savings.rs index 64e97265522..d694e600fdd 100644 --- a/litellm-rust/crates/cost/src/prompt_caching_savings.rs +++ b/litellm-rust/crates/cost/src/prompt_caching_savings.rs @@ -3,6 +3,7 @@ use serde_json::Value; use crate::base_rate_selection::get_token_base_cost; use crate::base_rate_selection::uses_inclusive_token_thresholds; +use crate::catalog::{ModelCostRequest, ModelInfoCatalog}; use crate::generic_input::calculate_cache_writing_cost; use crate::generic_usage::parse_prompt_tokens_details; use crate::provider_cache::apply_provider_cache_read_default; @@ -51,3 +52,21 @@ pub fn calculate_prompt_caching_savings(request: PromptCachingSavingsRequest<'_> * get_vertex_regional_endpoint_uplift(request.model_info, request.vertex_location); (read_discount - write_premium) * uplift } + +pub fn prompt_caching_savings_for_model( + catalog: &ModelInfoCatalog, + request: ModelCostRequest<'_>, +) -> Option { + let model_info = catalog.entry(request.model, request.provider, request.region)?; + Some(calculate_prompt_caching_savings( + PromptCachingSavingsRequest { + model_info, + usage: request.usage, + provider: request.provider, + service_tier: request.service_tier, + data_residency: request.data_residency, + vertex_location: request.vertex_location, + at: request.at, + }, + )) +} diff --git a/litellm-rust/crates/cost/src/retrieval_cost.rs b/litellm-rust/crates/cost/src/retrieval_cost.rs index 0e1d474e219..1b9462b7ec9 100644 --- a/litellm-rust/crates/cost/src/retrieval_cost.rs +++ b/litellm-rust/crates/cost/src/retrieval_cost.rs @@ -1,6 +1,8 @@ use serde_json::Value; -pub fn rerank_cost( +use crate::catalog::ModelInfoCatalog; + +pub fn rerank_cost_from_model_info( provider: &str, model_info: Option<&Value>, billed_units: Option<&Value>, @@ -21,7 +23,7 @@ pub fn rerank_cost( (rate * units as f64, 0.0) } -pub fn vector_store_search_cost( +pub fn vector_store_search_cost_from_model_info( provider: &str, api_type: Option<&str>, search_api_model_info: Option<&Value>, @@ -35,3 +37,29 @@ pub fn vector_store_search_cost( .unwrap_or(0.0); (rate, 0.0) } + +pub fn rerank_cost( + catalog: &ModelInfoCatalog, + model: &str, + provider: &str, + region: Option<&str>, + billed_units: Option<&Value>, +) -> (f64, f64) { + rerank_cost_from_model_info( + provider, + catalog.entry(model, Some(provider), region), + billed_units, + ) +} + +pub fn vector_store_search_cost( + catalog: &ModelInfoCatalog, + provider: &str, + api_type: Option<&str>, +) -> (f64, f64) { + vector_store_search_cost_from_model_info( + provider, + api_type, + catalog.entries().get("vertex_ai/search_api"), + ) +} diff --git a/litellm-rust/crates/cost/src/search_cost.rs b/litellm-rust/crates/cost/src/search_cost.rs index b7897956672..f8582f41085 100644 --- a/litellm-rust/crates/cost/src/search_cost.rs +++ b/litellm-rust/crates/cost/src/search_cost.rs @@ -1,5 +1,8 @@ use serde_json::Value; +use crate::catalog::ModelInfoCatalog; +use crate::error::CostError; + #[derive(Clone, Copy, Debug, PartialEq)] pub struct ParallelAiPricing { pub request_cost: f64, @@ -68,7 +71,7 @@ fn rate(value: Option<&Value>) -> f64 { } } -pub fn search_provider_cost_per_query( +pub fn search_provider_cost_from_model_info( model_info: &Value, number_of_queries: u64, optional_params: &Value, @@ -107,3 +110,40 @@ pub fn provider_usage(optional_params: &Value) -> Option<&[Value]> { .filter(|usage| usage.iter().all(Value::is_object)) .map(Vec::as_slice) } + +pub fn search_provider_cost_per_query( + catalog: &ModelInfoCatalog, + model: &str, + provider: Option<&str>, + number_of_queries: u64, + optional_params: &Value, +) -> Result<(f64, f64), CostError> { + if provider == Some("parallel_ai") { + let pricing_model = match effective_mode(optional_params) { + "fast" => "parallel_ai/search-fast", + "turbo" => "parallel_ai/search-turbo", + _ => "parallel_ai/search", + }; + let model_info = catalog + .entries() + .get(pricing_model) + .ok_or(CostError::ModelNotFound)?; + let pricing = ParallelAiPricing { + request_cost: rate(model_info.get("input_cost_per_query")), + default_results: 10, + additional_result_cost: 0.001, + }; + return Ok(( + parallel_ai_search_cost(optional_params, provider_usage(optional_params), pricing), + 0.0, + )); + } + let model_info = catalog + .entry(model, provider, None) + .ok_or(CostError::ModelNotFound)?; + Ok(search_provider_cost_from_model_info( + model_info, + number_of_queries, + optional_params, + )) +} diff --git a/litellm-rust/crates/cost/tests/python_azure_ai_image_cost.rs b/litellm-rust/crates/cost/tests/python_azure_ai_image_cost.rs index d4e5bd860d6..e3848919eb6 100644 --- a/litellm-rust/crates/cost/tests/python_azure_ai_image_cost.rs +++ b/litellm-rust/crates/cost/tests/python_azure_ai_image_cost.rs @@ -9,7 +9,8 @@ use jiff::Timestamp; use litellm_cost::azure_ai_image_cost::{ AzureAiImageRequest, cost_calculator, input_cost_per_pixel, }; -use litellm_cost::catalog::{AzureAiImageCatalogRequest, ModelInfoCatalog}; +use litellm_cost::catalog::ModelInfoCatalog; +use litellm_cost::image_cost_router::AzureAiImageCatalogRequest; use rstest::rstest; use serde_json::{Value, json}; @@ -107,8 +108,9 @@ fn catalog_azure_image_cost_uses_deployment_pixel_rate_for_unlisted_model() { let catalog = ModelInfoCatalog::default(); let response = json!({"data": [{}, {}], "size": "3x4"}); let supplied = json!({"input_cost_per_pixel": 0.02}); - let cost = catalog - .azure_ai_image_generation_cost(AzureAiImageCatalogRequest { + let cost = litellm_cost::image_cost_router::azure_ai_image_generation_cost( + &catalog, + AzureAiImageCatalogRequest { model: "unlisted", image_response: &response, size: None, @@ -116,8 +118,9 @@ fn catalog_azure_image_cost_uses_deployment_pixel_rate_for_unlisted_model() { optional_params: &json!({"width": 5, "height": 6}), supplied_model_info: Some(&supplied), at: at(), - }) - .unwrap(); + }, + ) + .unwrap(); assert!((cost - 2.0 * 5.0 * 6.0 * 0.02).abs() < 1e-12); } @@ -144,15 +147,19 @@ fn catalog_azure_image_cost_uses_shared_flat_rate_and_zero_when_unpriced() { at: at(), }; assert_eq!( - catalog - .azure_ai_image_generation_cost(for_model("flat")) - .unwrap(), + litellm_cost::image_cost_router::azure_ai_image_generation_cost( + &catalog, + for_model("flat") + ) + .unwrap(), 0.6 ); assert_eq!( - catalog - .azure_ai_image_generation_cost(for_model("free")) - .unwrap(), + litellm_cost::image_cost_router::azure_ai_image_generation_cost( + &catalog, + for_model("free") + ) + .unwrap(), 0.0 ); } diff --git a/litellm-rust/crates/cost/tests/python_billed_token_rates.rs b/litellm-rust/crates/cost/tests/python_billed_token_rates.rs index 55c28eda516..da82819976d 100644 --- a/litellm-rust/crates/cost/tests/python_billed_token_rates.rs +++ b/litellm-rust/crates/cost/tests/python_billed_token_rates.rs @@ -5,7 +5,10 @@ use std::collections::HashMap; use jiff::Timestamp; -use litellm_cost::billed_token_rates::{BilledRatesRequest, get_token_type_cost_breakdown}; +use litellm_cost::billed_token_rates::{ + BilledRatesRequest, calculate_token_type_cost_breakdown, get_billed_token_rates, + get_token_type_cost_breakdown, +}; use litellm_cost::catalog::{ModelCostRequest, ModelInfoCatalog}; use litellm_cost::custom_pricing::CustomTokenRates; use litellm_cost::usage_dispatch::get_usage_object; @@ -45,7 +48,7 @@ fn get_token_type_cost_breakdown_uses_off_peak_reasoning_and_cache_write_rates( }})) .unwrap() .unwrap(); - let breakdown = get_token_type_cost_breakdown(BilledRatesRequest { + let breakdown = calculate_token_type_cost_breakdown(BilledRatesRequest { model_info: Some(&model_info), usage: &usage, provider: Some("openai"), @@ -93,15 +96,15 @@ fn get_token_type_cost_breakdown_uses_flat_custom_rates_without_model_metadata() cache_read: Some(1e-7), cache_creation: None, }; - let breakdown = catalog.get_token_type_cost_breakdown(request, Some(custom)); + let breakdown = get_token_type_cost_breakdown(&catalog, request, Some(custom)); assert!((breakdown.cache_read_cost - 800.0 * 1e-7).abs() < 1e-12); assert!((breakdown.cache_creation_cost - 100.0 * 1e-6).abs() < 1e-12); assert!((breakdown.reasoning_cost - 200.0 * 2e-6).abs() < 1e-12); assert_eq!( - catalog.get_billed_token_rates(request, Some(custom)), + get_billed_token_rates(&catalog, request, Some(custom)), breakdown.rates ); - assert_eq!(catalog.get_billed_token_rates(request, None), None); + assert_eq!(get_billed_token_rates(&catalog, request, None), None); } #[rstest] @@ -139,8 +142,8 @@ fn get_billed_token_rates_follows_selected_threshold_tier() { at: at("2026-01-01T18:00Z"), response_time_ms: None, }; - let rates = catalog.get_billed_token_rates(request, None).unwrap(); - let breakdown = catalog.get_token_type_cost_breakdown(request, None); + let rates = get_billed_token_rates(&catalog, request, None).unwrap(); + let breakdown = get_token_type_cost_breakdown(&catalog, request, None); assert_eq!(rates.input_cost_per_token, 6e-6); assert_eq!(rates.output_cost_per_token, 3e-5); assert_eq!(rates.cache_read_input_token_cost, 6e-7); @@ -180,7 +183,7 @@ fn get_token_type_cost_breakdown_scales_cache_audio_and_ttl_with_region() { }})) .unwrap() .unwrap(); - let breakdown = get_token_type_cost_breakdown(BilledRatesRequest { + let breakdown = calculate_token_type_cost_breakdown(BilledRatesRequest { model_info: Some(&model_info), usage: &usage, provider: Some("openai"), @@ -216,7 +219,8 @@ fn token_type_cost_breakdown_is_provider_agnostic_for_perplexity() { }})) .unwrap() .unwrap(); - let breakdown = catalog.get_token_type_cost_breakdown( + let breakdown = get_token_type_cost_breakdown( + &catalog, ModelCostRequest { model: "perplexity/sonar-reasoning", provider: Some("perplexity"), diff --git a/litellm-rust/crates/cost/tests/python_completion_cost.rs b/litellm-rust/crates/cost/tests/python_completion_cost.rs index f853a4b1b35..ea1e083e5be 100644 --- a/litellm-rust/crates/cost/tests/python_completion_cost.rs +++ b/litellm-rust/crates/cost/tests/python_completion_cost.rs @@ -5,14 +5,14 @@ use litellm_cost::error::CostError; use std::collections::HashMap; use jiff::Timestamp; -use litellm_cost::catalog::{ - BuiltInToolCharge, CompletionCostRequest, ModelCostRequest, ModelInfoCatalog, - ResponseCostRequest, -}; +use litellm_cost::catalog::{ModelCostRequest, ModelInfoCatalog}; use litellm_cost::completion_cost::{ apply_cost_discount, apply_cost_margin, completion_cost, get_response_cost_from_hidden_params, response_cost_calculator, }; +use litellm_cost::cost_calculator::{ + BuiltInToolCharge, CompletionCostRequest, ResponseCostRequest, +}; use litellm_cost::usage_dispatch::get_usage_object; use rstest::rstest; use serde_json::json; @@ -123,26 +123,32 @@ fn model_info_catalog_completion_and_response_cost_use_selected_metadata() { discount_config: &discount, margin_config: &margin, }; - let calculated = catalog.completion_cost(completion).unwrap(); + let calculated = litellm_cost::cost_calculator::completion_cost(&catalog, completion).unwrap(); assert_eq!(calculated.built_in_tools, 0.01); assert_eq!(calculated.additional, 0.02); assert!((calculated.input + calculated.output - 0.0004).abs() < 1e-12); assert!((calculated.original - 0.0304).abs() < 1e-12); assert!((calculated.total - 0.033832).abs() < 1e-12); assert_eq!( - catalog.response_cost_calculator(ResponseCostRequest { - completion, - cache_hit: false, - hidden_params: &json!({}), - }), + litellm_cost::cost_calculator::response_cost_calculator( + &catalog, + ResponseCostRequest { + completion, + cache_hit: false, + hidden_params: &json!({}), + } + ), Ok(calculated.total) ); assert_eq!( - catalog.response_cost_calculator(ResponseCostRequest { - completion, - cache_hit: true, - hidden_params: &json!({}), - }), + litellm_cost::cost_calculator::response_cost_calculator( + &catalog, + ResponseCostRequest { + completion, + cache_hit: true, + hidden_params: &json!({}), + } + ), Ok(0.0) ); } diff --git a/litellm-rust/crates/cost/tests/python_cost_calculator.rs b/litellm-rust/crates/cost/tests/python_cost_calculator.rs index 0e7a96b505b..054e810553b 100644 --- a/litellm-rust/crates/cost/tests/python_cost_calculator.rs +++ b/litellm-rust/crates/cost/tests/python_cost_calculator.rs @@ -2,6 +2,7 @@ // mirrors: test_litellm/test_cost_calculator.py::test_default_image_cost_calculator // mirrors: integration/pricing/test_ocr_page_pricing.py +// mirrors: test_litellm/test_video_generation.py::TestVideoGeneration::test_default_video_cost_calculator use std::collections::HashMap; @@ -11,7 +12,10 @@ use litellm_cost::batch::{ BatchCostRates, BatchPricing, BatchTier, BatchUsage, ModalityRates, batch_cost_calculator, batch_cost_from_model_info, get_batch_cost_rates, }; -use litellm_cost::catalog::{DefaultImageCostRequest, ModelInfoCatalog}; +use litellm_cost::catalog::ModelInfoCatalog; +use litellm_cost::cost_calculator::default_video_cost_calculator; +use litellm_cost::error::CostError; +use litellm_cost::image_cost_router::DefaultImageCostRequest; use litellm_cost::non_token::{ ImageRates, ImageUsage, OcrBatchRates, OcrRates, OcrUsage, Unit, VideoRates, calculate_image, calculate_ocr_batch, calculate_ocr_with_tables, calculate_video, @@ -584,16 +588,18 @@ fn default_image_cost_calculator_selects_quality_and_normalized_size( json!({"input_cost_per_image": 0.08}), ), ])); - let cost = catalog - .default_image_cost_calculator(DefaultImageCostRequest { + let cost = litellm_cost::image_cost_router::default_image_cost_calculator( + &catalog, + DefaultImageCostRequest { model: "xai/model", provider: Some("xai"), quality, n: Some(1), size, supplied_model_info: None, - }) - .unwrap(); + }, + ) + .unwrap(); assert!((cost - expected).abs() < 1e-12); } @@ -605,16 +611,18 @@ fn default_image_cost_calculator_prefers_deployment_rate_and_explicit_zero() { )])); for (rate, expected) in [(0.07, 0.14), (0.0, 0.0)] { let supplied = json!({"input_cost_per_image": rate}); - let cost = catalog - .default_image_cost_calculator(DefaultImageCostRequest { + let cost = litellm_cost::image_cost_router::default_image_cost_calculator( + &catalog, + DefaultImageCostRequest { model: "dall-e", provider: Some("openai"), quality: Some("standard"), n: Some(2), size: Some("1024x1024"), supplied_model_info: Some(&supplied), - }) - .unwrap(); + }, + ) + .unwrap(); assert_eq!(cost, expected); } } @@ -625,16 +633,18 @@ fn default_image_cost_calculator_prices_pixels_after_image_rates() { "provider/model".to_owned(), json!({"input_cost_per_pixel": 0.0001}), )])); - let cost = catalog - .default_image_cost_calculator(DefaultImageCostRequest { + let cost = litellm_cost::image_cost_router::default_image_cost_calculator( + &catalog, + DefaultImageCostRequest { model: "provider/model", provider: Some("provider"), quality: None, n: Some(2), size: Some("20x10"), supplied_model_info: None, - }) - .unwrap(); + }, + ) + .unwrap(); assert!((cost - 2.0 * 20.0 * 10.0 * 0.0001).abs() < 1e-12); } @@ -678,16 +688,18 @@ fn default_image_cost_calculator_tries_provider_first_quality_candidate() { "openai/high/1024-x-1024/gpt-image-1".to_owned(), json!({"input_cost_per_image": 0.07}), )])); - let cost = catalog - .default_image_cost_calculator(DefaultImageCostRequest { + let cost = litellm_cost::image_cost_router::default_image_cost_calculator( + &catalog, + DefaultImageCostRequest { model: "gpt-image-1", provider: Some("openai"), quality: Some("high"), n: Some(1), size: Some("1024x1024"), supplied_model_info: None, - }) - .unwrap(); + }, + ) + .unwrap(); assert!((cost - 0.07).abs() < 1e-12); let both = ModelInfoCatalog::new(HashMap::from([ ( @@ -699,18 +711,70 @@ fn default_image_cost_calculator_tries_provider_first_quality_candidate() { json!({"input_cost_per_image": 0.07}), ), ])); - let cost = both - .default_image_cost_calculator(DefaultImageCostRequest { + let cost = litellm_cost::image_cost_router::default_image_cost_calculator( + &both, + DefaultImageCostRequest { model: "gpt-image-1", provider: Some("openai"), quality: Some("high"), n: Some(1), size: Some("1024x1024"), supplied_model_info: None, - }) - .unwrap(); + }, + ) + .unwrap(); assert!( (cost - 0.04).abs() < 1e-12, "quality-prefixed candidate is checked before the provider-first one" ); } + +fn video_catalog() -> ModelInfoCatalog { + ModelInfoCatalog::new(HashMap::from([ + ( + "model".to_owned(), + json!({"output_cost_per_video_per_second": 0.1}), + ), + ( + "prov/model".to_owned(), + json!({"output_cost_per_video_per_second": 0.2}), + ), + ( + "prov/other".to_owned(), + json!({"output_cost_per_video_per_second": 0.3}), + ), + ])) +} + +#[rstest] +#[case::bare_model_before_provider_prefix("model", Some("prov"), Ok(0.2))] +#[case::already_prefixed_model("prov/model", Some("prov"), Ok(0.4))] +#[case::last_path_segment("a/b/model", None, Ok(0.2))] +#[case::provider_prefix_is_the_last_resort("other", Some("prov"), Ok(0.6))] +#[case::unknown_model("missing", Some("prov"), Err(CostError::ModelNotFound))] +fn default_video_cost_calculator_follows_python_lookup_order( + #[case] model: &str, + #[case] provider: Option<&str>, + #[case] expected: Result, +) { + let cost = default_video_cost_calculator(&video_catalog(), model, 2.0, provider, None, None); + match (cost, expected) { + (Ok(cost), Ok(expected)) => assert!((cost - expected).abs() < 1e-12), + (cost, expected) => assert_eq!(cost, expected), + } +} + +#[rstest] +fn default_video_cost_calculator_prefers_deployment_pricing_over_the_cost_map() { + let deployment = json!({"output_cost_per_second": 0.5}); + let cost = default_video_cost_calculator( + &video_catalog(), + "model", + 2.0, + Some("prov"), + Some(&deployment), + None, + ) + .unwrap(); + assert!((cost - 1.0).abs() < 1e-12); +} diff --git a/litellm-rust/crates/cost/tests/python_fal_ai_image_cost.rs b/litellm-rust/crates/cost/tests/python_fal_ai_image_cost.rs index 694be457ff0..21399130a1d 100644 --- a/litellm-rust/crates/cost/tests/python_fal_ai_image_cost.rs +++ b/litellm-rust/crates/cost/tests/python_fal_ai_image_cost.rs @@ -7,9 +7,9 @@ use std::collections::HashMap; use jiff::Timestamp; use litellm_cost::catalog::ModelInfoCatalog; use litellm_cost::fal_ai_image_cost::{ - PIXELS_PER_MEGAPIXEL, cost_calculator, fal_ai_passthrough_cost, flat_cost_per_image, - image_dimensions, keyed_cost_per_image, keyed_quality, keyed_rows, keyed_size, - parse_keyed_dimensions, + PIXELS_PER_MEGAPIXEL, cost_calculator, fal_ai_passthrough_cost, + fal_ai_passthrough_cost_from_model_info, flat_cost_per_image, image_dimensions, + keyed_cost_per_image, keyed_quality, keyed_rows, keyed_size, parse_keyed_dimensions, }; use litellm_cost::image_cost_router::{ ImageCostRouteRequest, route_image_generation_cost_calculator, @@ -179,25 +179,33 @@ fn catalog_fal_image_route_uses_keyed_row_for_image_edit() { fn fal_passthrough_selects_resolution_rate_and_falls_back_to_base() { let info = json!({"output_cost_per_image": 0.3, "output_cost_per_image_512": 0.25}); assert_eq!( - fal_ai_passthrough_cost(Some(&info), &json!({"resolution": 512})), + fal_ai_passthrough_cost_from_model_info(Some(&info), &json!({"resolution": 512})), Some(0.25) ); assert_eq!( - fal_ai_passthrough_cost(Some(&info), &json!({"resolution": "512"})), + fal_ai_passthrough_cost_from_model_info(Some(&info), &json!({"resolution": "512"})), Some(0.25) ); assert_eq!( - fal_ai_passthrough_cost(Some(&info), &json!({"resolution": true})), + fal_ai_passthrough_cost_from_model_info(Some(&info), &json!({"resolution": true})), Some(0.3) ); assert_eq!( - fal_ai_passthrough_cost(Some(&info), &json!({"resolution": 512.0})), + fal_ai_passthrough_cost_from_model_info(Some(&info), &json!({"resolution": 512.0})), Some(0.3) ); let catalog = ModelInfoCatalog::new(HashMap::from([("fal_ai/fal-ai/trellis".to_owned(), info)])); assert_eq!( - catalog.fal_ai_passthrough_cost("fal-ai/trellis", &json!({"resolution": 512})), + fal_ai_passthrough_cost(&catalog, "fal-ai/trellis", &json!({"resolution": 512})), Some(0.25) ); + assert_eq!( + fal_ai_passthrough_cost( + &catalog, + "fal_ai/fal-ai/trellis", + &json!({"resolution": 512}) + ), + None + ); } diff --git a/litellm-rust/crates/cost/tests/python_fixture_replay.rs b/litellm-rust/crates/cost/tests/python_fixture_replay.rs index bc54042e611..426b8323537 100644 --- a/litellm-rust/crates/cost/tests/python_fixture_replay.rs +++ b/litellm-rust/crates/cost/tests/python_fixture_replay.rs @@ -9,7 +9,7 @@ use litellm_cost::Rate; use litellm_cost::anthropic_usage::transform_anthropic_usage_to_chat_usage; use litellm_cost::base_rate_selection::uses_inclusive_token_thresholds; use litellm_cost::batch::batch_cost_rates_from_model_info; -use litellm_cost::billed_token_rates::{BilledRatesRequest, get_billed_token_rates}; +use litellm_cost::billed_token_rates::{BilledRatesRequest, calculate_billed_token_rates}; use litellm_cost::generic_cost::calculate_generic_cost_from_model_info_with_region; use litellm_cost::off_peak::is_off_peak; use litellm_cost::prompt_caching_savings::{ @@ -93,7 +93,7 @@ fn billed_token_rates_match_executed_python_fixtures() { let provider = opt_str(row, "custom_llm_provider"); let at = opt_str(row, "current_time") .map_or_else(|| "2026-03-12T00:00:00Z".parse().unwrap(), instant); - let rates = get_billed_token_rates(BilledRatesRequest { + let rates = calculate_billed_token_rates(BilledRatesRequest { model_info: Some(row.get("model_info").unwrap()), usage: &usage, provider, diff --git a/litellm-rust/crates/cost/tests/python_image_response_cost.rs b/litellm-rust/crates/cost/tests/python_image_response_cost.rs index 8366d6c700d..7d91509cb7b 100644 --- a/litellm-rust/crates/cost/tests/python_image_response_cost.rs +++ b/litellm-rust/crates/cost/tests/python_image_response_cost.rs @@ -61,23 +61,9 @@ fn calculate_image_response_cost_from_usage_prices_image_input_and_output_detail "input_tokens_details": {"text_tokens": 19, "image_tokens": 512}, "output_tokens_details": output_details }}); - let direct = + let cost = calculate_image_response_cost_from_usage(&response, &model_info(), Some("openai"), at()); - assert!((direct.unwrap() - expected).abs() < 1e-12); - let catalog = ModelInfoCatalog::new(HashMap::from([( - "openai/image-model".to_owned(), - model_info(), - )])); - assert_eq!( - catalog.calculate_image_response_cost_from_usage( - "image-model", - Some("openai"), - None, - &response, - at(), - ), - direct - ); + assert!((cost.unwrap() - expected).abs() < 1e-12); } #[rstest] @@ -171,9 +157,15 @@ fn google_image_generation_prefers_token_usage_and_adds_web_search(#[case] provi assert!((direct - (3.0 * 0.001 + 2.0 * 0.003 + 2.0 * 0.02)).abs() < 1e-12); let catalog = ModelInfoCatalog::new(HashMap::from([(format!("{provider}/image-model"), info)])); assert_eq!( - catalog - .google_image_generation_cost("image-model", provider, &response, None, at()) - .unwrap(), + litellm_cost::image_cost_router::google_image_generation_cost( + &catalog, + "image-model", + provider, + &response, + None, + at() + ) + .unwrap(), direct ); } @@ -194,9 +186,15 @@ fn google_image_generation_falls_back_to_images_and_preserves_supplied_prices() "vertex_ai/image-model".to_owned(), shared, )])); - let cost = catalog - .google_image_generation_cost("image-model", "vertex_ai", &response, Some(&supplied), at()) - .unwrap(); + let cost = litellm_cost::image_cost_router::google_image_generation_cost( + &catalog, + "image-model", + "vertex_ai", + &response, + Some(&supplied), + at(), + ) + .unwrap(); assert!((cost - 0.52).abs() < 1e-12); } @@ -217,9 +215,15 @@ fn gemini_image_edit_shares_generation_token_and_search_billing() { assert_eq!(gemini_image_edit_cost(&response, &info, at()), generation); let catalog = ModelInfoCatalog::new(HashMap::from([("gemini/image-model".to_owned(), info)])); assert_eq!( - catalog - .google_image_edit_cost("image-model", "gemini", &response, None, at()) - .unwrap(), + litellm_cost::image_cost_router::google_image_edit_cost( + &catalog, + "image-model", + "gemini", + &response, + None, + at() + ) + .unwrap(), generation ); } @@ -234,15 +238,15 @@ fn vertex_image_edit_uses_flat_shared_rate_even_with_token_usage() { let catalog = ModelInfoCatalog::new(HashMap::from([("vertex_ai/image-model".to_owned(), info)])); assert_eq!( - catalog - .google_image_edit_cost( - "image-model", - "vertex_ai", - &response, - Some(&json!({"output_cost_per_image": 10.0})), - at() - ) - .unwrap(), + litellm_cost::image_cost_router::google_image_edit_cost( + &catalog, + "image-model", + "vertex_ai", + &response, + Some(&json!({"output_cost_per_image": 10.0})), + at() + ) + .unwrap(), 0.5 ); } diff --git a/litellm-rust/crates/cost/tests/python_prompt_caching_savings.rs b/litellm-rust/crates/cost/tests/python_prompt_caching_savings.rs index 6c6d2a09177..e5a1eea8753 100644 --- a/litellm-rust/crates/cost/tests/python_prompt_caching_savings.rs +++ b/litellm-rust/crates/cost/tests/python_prompt_caching_savings.rs @@ -108,17 +108,20 @@ fn calculate_prompt_caching_savings_matches_uncached_bill_minus_cached_bill( assert!((savings - expected).abs() < 1e-12); let catalog = ModelInfoCatalog::new(HashMap::from([("openai/model".to_owned(), model_info)])); assert_eq!( - catalog.calculate_prompt_caching_savings(ModelCostRequest { - model: "model", - provider: Some("openai"), - region: None, - usage: &cached, - service_tier, - data_residency, - vertex_location, - at: billed_at, - response_time_ms: None, - }), + litellm_cost::prompt_caching_savings::prompt_caching_savings_for_model( + &catalog, + ModelCostRequest { + model: "model", + provider: Some("openai"), + region: None, + usage: &cached, + service_tier, + data_residency, + vertex_location, + at: billed_at, + response_time_ms: None, + } + ), Some(savings) ); } diff --git a/litellm-rust/crates/cost/tests/python_realtime_cost.rs b/litellm-rust/crates/cost/tests/python_realtime_cost.rs index 422fb360f40..4db92852459 100644 --- a/litellm-rust/crates/cost/tests/python_realtime_cost.rs +++ b/litellm-rust/crates/cost/tests/python_realtime_cost.rs @@ -75,11 +75,21 @@ fn handle_realtime_transcription_cost_calculation_uses_session_model_and_complet json!({"type": "conversation.item.input_audio_transcription.completed", "usage": {"type": "duration", "seconds": 3.0}}), ]; assert_eq!( - catalog.handle_realtime_transcription_cost_calculation(&events, "openai", "requested"), + litellm_cost::cost_calculator::handle_realtime_transcription_cost_calculation( + &catalog, + &events, + "openai", + "requested" + ), 5.0 * 0.02 ); assert_eq!( - catalog.handle_realtime_transcription_cost_calculation(&events[..1], "openai", "requested"), + litellm_cost::cost_calculator::handle_realtime_transcription_cost_calculation( + &catalog, + &events[..1], + "openai", + "requested" + ), 0.0 ); } @@ -154,9 +164,16 @@ fn responses_ws_token_cost_prices_each_tier_at_its_returned_rate() { json!({"type": "response.failed", "response": {"service_tier": "priority", "usage": {"input_tokens": 1000, "output_tokens": 1000}}}), ]; let at: Timestamp = "2026-09-22T12:00:00Z".parse().unwrap(); - let actual = catalog - .responses_ws_token_cost_by_tier(&events, "model", Some("openai"), None, None, at) - .unwrap(); + let actual = litellm_cost::cost_calculator::responses_ws_token_cost_by_tier( + &catalog, + &events, + "model", + Some("openai"), + None, + None, + at, + ) + .unwrap(); let expected = 100.0 * 0.002 + 40.0 * 0.003 + 60.0 * 0.005 + 10.0 * 0.007; assert!((actual - expected).abs() < 1e-12); assert!((actual - (160.0 * 0.002 + 50.0 * 0.003)).abs() > 1e-9); @@ -182,7 +199,8 @@ fn handle_realtime_stream_cost_calculation_falls_through_only_for_priceless_sess ]; let usage = collect_and_combine_usage_from_realtime_stream_results(&events).unwrap(); let at: Timestamp = "2026-09-22T12:00:00Z".parse().unwrap(); - let actual = catalog.handle_realtime_stream_cost_calculation( + let actual = litellm_cost::cost_calculator::handle_realtime_stream_cost_calculation( + &catalog, &events, &usage, "openai", @@ -212,7 +230,8 @@ fn handle_realtime_stream_cost_calculation_adds_transcription_events() { ]; let usage = collect_and_combine_usage_from_realtime_stream_results(&events).unwrap(); let at: Timestamp = "2026-09-22T12:00:00Z".parse().unwrap(); - let actual = catalog.handle_realtime_stream_cost_calculation( + let actual = litellm_cost::cost_calculator::handle_realtime_stream_cost_calculation( + &catalog, &events, &usage, "openai", diff --git a/litellm-rust/crates/cost/tests/python_retrieval_cost.rs b/litellm-rust/crates/cost/tests/python_retrieval_cost.rs index a7142ca7f16..a65784aa830 100644 --- a/litellm-rust/crates/cost/tests/python_retrieval_cost.rs +++ b/litellm-rust/crates/cost/tests/python_retrieval_cost.rs @@ -2,9 +2,12 @@ // mirrors: test_litellm/rag/test_main.py::test_aquery_billed_cost_includes_priced_vector_store_search -use litellm_cost::retrieval_cost::{rerank_cost, vector_store_search_cost}; use rstest::rstest; use serde_json::json; +use litellm_cost::retrieval_cost::{ + rerank_cost, rerank_cost_from_model_info, vector_store_search_cost, + vector_store_search_cost_from_model_info, +}; #[rstest] #[case("vertex_ai", json!({"input_cost_per_query": 0.25}), json!({"search_units": 3}), (0.75, 0.0))] @@ -19,7 +22,7 @@ fn rerank_cost_uses_provider_billing_units( #[case] billed_units: serde_json::Value, #[case] expected: (f64, f64), ) { - let actual = rerank_cost(provider, Some(&model_info), Some(&billed_units)); + let actual = rerank_cost_from_model_info(provider, Some(&model_info), Some(&billed_units)); assert!((actual.0 - expected.0).abs() < 1e-12); assert_eq!(actual.1, expected.1); } @@ -27,15 +30,15 @@ fn rerank_cost_uses_provider_billing_units( #[rstest] fn rerank_cost_returns_zero_when_pricing_or_units_are_missing() { assert_eq!( - rerank_cost("cohere", None, Some(&json!({"search_units": 2}))), + rerank_cost_from_model_info("cohere", None, Some(&json!({"search_units": 2}))), (0.0, 0.0) ); assert_eq!( - rerank_cost("cohere", Some(&json!({"input_cost_per_query": 0.25})), None), + rerank_cost_from_model_info("cohere", Some(&json!({"input_cost_per_query": 0.25})), None), (0.0, 0.0) ); assert_eq!( - rerank_cost( + rerank_cost_from_model_info( "cohere", Some(&json!({"input_cost_per_query": 0.25})), Some(&json!({})) @@ -55,7 +58,7 @@ fn vector_store_search_cost_only_prices_vertex_search_api( #[case] expected: f64, ) { assert_eq!( - vector_store_search_cost( + vector_store_search_cost_from_model_info( provider, api_type, Some(&json!({"input_cost_per_query": 0.25})) @@ -81,7 +84,8 @@ fn model_info_catalog_routes_retrieval_costs_from_selected_metadata() { ), ])); assert_eq!( - catalog.rerank_cost( + rerank_cost( + &catalog, "model", "vertex_ai", None, @@ -90,15 +94,22 @@ fn model_info_catalog_routes_retrieval_costs_from_selected_metadata() { (0.75, 0.0) ); assert_eq!( - catalog.rerank_cost("model", "jina_ai", None, Some(&json!({"total_tokens": 30}))), + rerank_cost( + &catalog, + "model", + "jina_ai", + None, + Some(&json!({"total_tokens": 30})) + ), (0.3, 0.0) ); assert_eq!( - catalog.vector_store_search_cost("vertex_ai", Some("search_api")), + vector_store_search_cost(&catalog, "vertex_ai", Some("search_api")), (0.4, 0.0) ); assert_eq!( - catalog.rerank_cost( + rerank_cost( + &catalog, "missing", "vertex_ai", None, diff --git a/litellm-rust/crates/cost/tests/python_search_cost.rs b/litellm-rust/crates/cost/tests/python_search_cost.rs index cef859ca828..f015ad7f12f 100644 --- a/litellm-rust/crates/cost/tests/python_search_cost.rs +++ b/litellm-rust/crates/cost/tests/python_search_cost.rs @@ -8,7 +8,8 @@ use std::collections::HashMap; use litellm_cost::catalog::ModelInfoCatalog; use litellm_cost::search_cost::{ ParallelAiPricing, effective_max_results, effective_mode, parallel_ai_search_cost, - provider_usage, search_provider_cost_per_query, usage_count, + provider_usage, search_provider_cost_from_model_info, search_provider_cost_per_query, + usage_count, }; use rstest::rstest; use serde_json::{Value, json}; @@ -29,7 +30,7 @@ fn search_provider_cost_per_query_selects_inclusive_result_tier_and_last_fallbac ] }); assert_eq!( - search_provider_cost_per_query(&model_info, 3, &json!({"max_results": max_results})), + search_provider_cost_from_model_info(&model_info, 3, &json!({"max_results": max_results})), (3.0 * expected_rate, 0.0) ); } @@ -41,9 +42,7 @@ fn search_provider_cost_per_query_uses_flat_rate_without_tiers() { json!({"input_cost_per_query": 0.003}), )])); assert_eq!( - catalog - .search_provider_cost_per_query("search", Some("exa_ai"), 4, &json!({})) - .unwrap(), + search_provider_cost_per_query(&catalog, "search", Some("exa_ai"), 4, &json!({})).unwrap(), (0.012, 0.0) ); } @@ -122,8 +121,7 @@ fn parallel_ai_catalog_selects_mode_price_and_zero_additional_results_for_usage_ "_parallel_ai_usage": [{"name": "unrelated_sku", "count": 5}] }); assert_eq!( - catalog - .search_provider_cost_per_query("ignored", Some("parallel_ai"), 9, ¶ms) + search_provider_cost_per_query(&catalog, "ignored", Some("parallel_ai"), 9, ¶ms) .unwrap(), (0.03, 0.0) ); diff --git a/litellm-rust/crates/cost/tests/python_speech_cost.rs b/litellm-rust/crates/cost/tests/python_speech_cost.rs index 265fff3cd4d..553313c2354 100644 --- a/litellm-rust/crates/cost/tests/python_speech_cost.rs +++ b/litellm-rust/crates/cost/tests/python_speech_cost.rs @@ -87,19 +87,29 @@ fn speech_cost_selects_character_or_token_pricing_and_requires_characters() { .unwrap() .unwrap(); assert_eq!( - catalog - .speech_cost(request("character", "openai", &usage), Some(5.0)) - .unwrap(), + litellm_cost::cost_calculator::speech_cost( + &catalog, + request("character", "openai", &usage), + Some(5.0) + ) + .unwrap(), (0.01, 0.0) ); assert_eq!( - catalog - .speech_cost(request("token", "openai", &usage), None) - .unwrap(), + litellm_cost::cost_calculator::speech_cost( + &catalog, + request("token", "openai", &usage), + None + ) + .unwrap(), (0.03, 0.008) ); assert_eq!( - catalog.speech_cost(request("character", "openai", &usage), None), + litellm_cost::cost_calculator::speech_cost( + &catalog, + request("character", "openai", &usage), + None + ), Err(CostError::MissingPromptCharacters) ); } @@ -113,9 +123,12 @@ fn speech_cost_uses_lyria_generation_price_before_character_or_token_rates() { )])); assert_eq!(lyria_generation_cost(&info), Some(0.08)); assert_eq!( - catalog - .speech_cost(request("lyria", "vertex_ai", &ChatUsage::default()), None) - .unwrap(), + litellm_cost::cost_calculator::speech_cost( + &catalog, + request("lyria", "vertex_ai", &ChatUsage::default()), + None + ) + .unwrap(), (0.0, 0.08) ); assert_eq!( @@ -143,15 +156,21 @@ fn transcription_cost_selects_token_usage_or_duration_pricing() { assert!(transcription_usage_has_token_details(&audio_detail_usage)); assert!(!transcription_usage_has_token_details(&duration_usage)); assert_eq!( - catalog - .transcription_cost(request("transcribe", "openai", &token_usage), 5.0) - .unwrap(), + litellm_cost::cost_calculator::transcription_cost( + &catalog, + request("transcribe", "openai", &token_usage), + 5.0 + ) + .unwrap(), (0.02, 0.006) ); assert_eq!( - catalog - .transcription_cost(request("transcribe", "openai", &duration_usage), 5.0) - .unwrap(), + litellm_cost::cost_calculator::transcription_cost( + &catalog, + request("transcribe", "openai", &duration_usage), + 5.0 + ) + .unwrap(), (0.05, 0.0) ); } diff --git a/litellm-rust/crates/cost/tests/python_tool_cost_dispatch.rs b/litellm-rust/crates/cost/tests/python_tool_cost_dispatch.rs index 21813869ce7..f5178a140c3 100644 --- a/litellm-rust/crates/cost/tests/python_tool_cost_dispatch.rs +++ b/litellm-rust/crates/cost/tests/python_tool_cost_dispatch.rs @@ -196,8 +196,9 @@ fn model_info_catalog_completion_cost_dispatches_built_in_tools() { let discount = json!({}); let margin = json!({}); let at: Timestamp = "2026-01-01T18:00Z".parse().unwrap(); - let actual = catalog - .completion_cost(CompletionCostRequest { + let actual = litellm_cost::cost_calculator::completion_cost( + &catalog, + CompletionCostRequest { token: ModelCostRequest { model: "model", provider: Some("openai"), @@ -220,13 +221,13 @@ fn model_info_catalog_completion_cost_dispatches_built_in_tools() { additional_costs: &[], discount_config: &discount, margin_config: &margin, - }) - .unwrap(); + }, + ) + .unwrap(); assert!((actual.total - (100.0 * 2e-6 + 50.0 * 4e-6 + 2.0 * 0.01)).abs() < 1e-12); } use std::collections::HashMap; use jiff::Timestamp; -use litellm_cost::catalog::{ - BuiltInToolCharge, CompletionCostRequest, ModelCostRequest, ModelInfoCatalog, -}; +use litellm_cost::catalog::{ModelCostRequest, ModelInfoCatalog}; +use litellm_cost::cost_calculator::{BuiltInToolCharge, CompletionCostRequest};