diff --git a/litellm-rust/crates/cost/src/anthropic_cost.rs b/litellm-rust/crates/cost/src/anthropic_cost.rs index 4c580c37c4c..5754aba729a 100644 --- a/litellm-rust/crates/cost/src/anthropic_cost.rs +++ b/litellm-rust/crates/cost/src/anthropic_cost.rs @@ -1,6 +1,7 @@ use serde_json::Value; use crate::responses_usage::ChatUsage; +use crate::wire::lax_int; pub fn fast_speed_multiplier(model_info: &Value, usage: &ChatUsage) -> f64 { if usage.extra.get("speed").and_then(Value::as_str) != Some("fast") { @@ -20,7 +21,7 @@ pub fn get_anthropic_web_search_requests_from_response(response: &Value) -> Opti pub fn get_web_search_requests(server_tool_use: Option<&Value>) -> Option { server_tool_use? .get("web_search_requests") - .and_then(Value::as_i64) + .and_then(lax_int) } pub fn get_web_search_requests_from_usage(usage: &ChatUsage) -> Option { diff --git a/litellm-rust/crates/cost/src/catalog.rs b/litellm-rust/crates/cost/src/catalog.rs index 7078f7b0970..10792c5e062 100644 --- a/litellm-rust/crates/cost/src/catalog.rs +++ b/litellm-rust/crates/cost/src/catalog.rs @@ -63,12 +63,32 @@ pub enum CostCall<'a> { }, } -fn select_model_key<'a, T>( - entries: &'a HashMap, +pub fn check_provider_match(model_info: &Value, provider: Option<&str>) -> bool { + let Some(provider) = provider.filter(|provider| !provider.is_empty()) else { + return true; + }; + let Some(entry_provider) = model_info.get("litellm_provider").and_then(Value::as_str) else { + return true; + }; + entry_provider == provider + || (provider == "vertex_ai" && entry_provider.starts_with("vertex_ai")) + || (provider == "fireworks_ai" && entry_provider.starts_with("fireworks_ai")) + || (provider.starts_with("bedrock") && entry_provider.starts_with("bedrock")) + || provider == "litellm_proxy" + || (provider == "azure_ai" && matches!(entry_provider, "azure" | "openai")) + || provider == "github" +} + +fn select_model_key<'a>( + entries: &'a HashMap, model: &str, provider: Option<&str>, region: Option<&str>, ) -> Option<&'a str> { + let match_provider = provider.map(|provider| match provider { + "vertex_ai_beta" => "vertex_ai", + provider => provider, + }); let normalized = match provider { Some(provider) => { let prefix = format!("{provider}/"); @@ -106,6 +126,7 @@ fn select_model_key<'a, T>( .find_map(|candidate| { entries .get_key_value(candidate) + .filter(|(_, model_info)| check_provider_match(model_info, match_provider)) .map(|(key, _)| key.as_str()) }) } diff --git a/litellm-rust/crates/cost/src/completion_response.rs b/litellm-rust/crates/cost/src/completion_response.rs index 6e3698cd6d2..50442d008a6 100644 --- a/litellm-rust/crates/cost/src/completion_response.rs +++ b/litellm-rust/crates/cost/src/completion_response.rs @@ -14,7 +14,7 @@ use crate::completion_cost::{ }; use crate::completion_input::{CompletionInputRequest, PreparedCompletionInput, ResponseKind}; use crate::cost_calculator::{ - built_in_tool_cost, cost_per_token, cost_per_token_for_call, default_video_cost_calculator, + cost_per_token, cost_per_token_for_call, default_video_cost_calculator, handle_realtime_stream_cost_calculation, }; use crate::custom_pricing::{CustomPricing, cost_from_chat_usage}; @@ -32,7 +32,7 @@ use crate::realtime_cost::{ use crate::responses_usage::ChatUsage; use crate::speech_cost::count_characters; use crate::tool_call_cost_tracking::{DefaultToolRates, ResponseKind as ToolResponseKind}; -use crate::tool_cost_dispatch::BuiltInToolCostRequest; +use crate::tool_cost_dispatch::{BuiltInToolCostRequest, get_cost_for_built_in_tools}; #[derive(Clone, Copy, Debug)] pub struct BuiltInToolCostConfig<'a> { @@ -508,10 +508,9 @@ fn config_driven_tool_cost( let Some(response) = request.input.model_selection.response else { return 0.0; }; - built_in_tool_cost( + get_cost_for_built_in_tools( catalog, model, - provider, region, BuiltInToolCostRequest { response, diff --git a/litellm-rust/crates/cost/src/cost_calculator.rs b/litellm-rust/crates/cost/src/cost_calculator.rs index fa73e47eb9b..033c1dfe967 100644 --- a/litellm-rust/crates/cost/src/cost_calculator.rs +++ b/litellm-rust/crates/cost/src/cost_calculator.rs @@ -513,16 +513,6 @@ pub fn responses_ws_token_cost_by_tier( .sum() } -pub fn built_in_tool_cost( - catalog: &ModelInfoCatalog, - model: &str, - provider: Option<&str>, - region: Option<&str>, - request: BuiltInToolCostRequest<'_>, -) -> f64 { - get_cost_for_built_in_tools(request, catalog.entry(model, provider, region)) -} - pub fn completion_cost( catalog: &ModelInfoCatalog, request: CompletionCostRequest<'_>, @@ -530,10 +520,9 @@ pub fn completion_cost( let (prompt, output) = cost_per_token(catalog, request.token)?; let built_in_tools = match request.built_in_tools { BuiltInToolCharge::Provided(cost) => cost, - BuiltInToolCharge::FromResponse(tool_request) => built_in_tool_cost( + BuiltInToolCharge::FromResponse(tool_request) => get_cost_for_built_in_tools( catalog, request.token.model, - request.token.provider, request.token.region, tool_request, ), diff --git a/litellm-rust/crates/cost/src/gemini_cost.rs b/litellm-rust/crates/cost/src/gemini_cost.rs index 72b4d392f97..83acef355d2 100644 --- a/litellm-rust/crates/cost/src/gemini_cost.rs +++ b/litellm-rust/crates/cost/src/gemini_cost.rs @@ -1,40 +1,39 @@ use serde_json::Value; +use crate::anthropic_cost::get_web_search_requests_from_usage; use crate::responses_usage::ChatUsage; +use crate::wire::is_truthy; const DEFAULT_WEB_SEARCH_COST: f64 = 35e-3; const DEFAULT_MAPS_QUERY_COST: f64 = 14e-3; const DEFAULT_MAPS_PROMPT_COST: f64 = 25e-3; -fn per_query(model_info: &Value) -> bool { - model_info - .get("web_search_billing_unit") - .and_then(Value::as_str) - == Some("per_query") +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum WebSearchBillingUnit { + PerPrompt, + PerQuery, + Unrecognized, } -fn billed_requests(requests: u64, per_query: bool) -> u64 { - if per_query { - requests - } else { - u64::from(requests > 0) +pub fn web_search_billing_unit(model_info: &Value) -> WebSearchBillingUnit { + match model_info.get("web_search_billing_unit") { + Some(unit) if is_truthy(unit) => match unit.as_str() { + Some("per_prompt") => WebSearchBillingUnit::PerPrompt, + Some("per_query") => WebSearchBillingUnit::PerQuery, + _ => WebSearchBillingUnit::Unrecognized, + }, + _ => WebSearchBillingUnit::PerPrompt, } } -fn web_search_requests(usage: &ChatUsage) -> u64 { - let from_details = usage +fn web_search_requests(usage: &ChatUsage) -> i64 { + usage .prompt_tokens_details .as_ref() .and_then(|details| details.web_search_requests) - .unwrap_or(0); - if from_details > 0 { - return from_details; - } - usage - .extra - .get("server_tool_use") - .and_then(|value| value.get("web_search_requests")) - .and_then(Value::as_u64) + .and_then(|requests| i64::try_from(requests).ok()) + .filter(|requests| *requests != 0) + .or_else(|| get_web_search_requests_from_usage(usage)) .unwrap_or(0) } @@ -44,7 +43,14 @@ pub fn cost_per_web_search_request(usage: &ChatUsage, model_info: &Value) -> f64 .and_then(|rates| rates.get("search_context_size_medium")) .and_then(Value::as_f64) .unwrap_or(DEFAULT_WEB_SEARCH_COST); - rate * billed_requests(web_search_requests(usage), per_query(model_info)) as f64 + let requests = web_search_requests(usage); + let billed = + if requests > 0 && web_search_billing_unit(model_info) == WebSearchBillingUnit::PerPrompt { + 1 + } else { + requests + }; + rate * billed as f64 } pub fn google_maps_grounding_requests(usage: Option<&ChatUsage>) -> Option { @@ -59,7 +65,7 @@ pub fn cost_per_google_maps_grounding_request(usage: &ChatUsage, model_info: &Va else { return 0.0; }; - let per_query = per_query(model_info); + let per_query = web_search_billing_unit(model_info) == WebSearchBillingUnit::PerQuery; let default_rate = if per_query { DEFAULT_MAPS_QUERY_COST } else { @@ -69,5 +75,6 @@ pub fn cost_per_google_maps_grounding_request(usage: &ChatUsage, model_info: &Va .get("google_maps_grounding_cost_per_query") .and_then(Value::as_f64) .unwrap_or(default_rate); - rate * billed_requests(requests, per_query) as f64 + let billed = if per_query { requests } else { 1 }; + rate * billed as f64 } diff --git a/litellm-rust/crates/cost/src/ptu_pricing.rs b/litellm-rust/crates/cost/src/ptu_pricing.rs index 6696970f52e..a9e83154151 100644 --- a/litellm-rust/crates/cost/src/ptu_pricing.rs +++ b/litellm-rust/crates/cost/src/ptu_pricing.rs @@ -2,7 +2,7 @@ use jiff::Timestamp; use jiff::civil::{Date, DateTime}; use serde_json::{Map, Value, json}; -use crate::wire::is_truthy; +use crate::wire::{is_truthy, py_float, py_int}; pub const PTU_COST_ATTRIBUTION_ENV_VAR: &str = "LITELLM_ENABLE_PTU_COST_ATTRIBUTION"; pub const AZURE_SPILLOVER_HEADER: &str = "x-ms-is-spilled-over"; @@ -176,24 +176,6 @@ fn py_str(value: &Value) -> String { } } -fn py_int(value: &Value) -> Option { - match value { - Value::Number(number) => Some(number.as_f64()?.trunc() as i64), - Value::Bool(flag) => Some(i64::from(*flag)), - Value::String(text) => text.trim().parse::().ok(), - _ => None, - } -} - -fn py_float(value: &Value) -> Option { - match value { - Value::Number(number) => number.as_f64(), - Value::Bool(flag) => Some(f64::from(u8::from(*flag))), - Value::String(text) => text.trim().parse::().ok(), - _ => None, - } -} - fn as_utc(value: &Value) -> Option { let Value::String(text) = value else { return None; diff --git a/litellm-rust/crates/cost/src/tool_call_cost_tracking.rs b/litellm-rust/crates/cost/src/tool_call_cost_tracking.rs index 199ca59692a..00f7ae2e1ca 100644 --- a/litellm-rust/crates/cost/src/tool_call_cost_tracking.rs +++ b/litellm-rust/crates/cost/src/tool_call_cost_tracking.rs @@ -1,6 +1,10 @@ use serde_json::Value; +use crate::anthropic_cost::{ + get_anthropic_web_search_requests_from_response, get_web_search_requests, +}; use crate::provider::LlmProviders; +use crate::wire::{lax_int, py_float, py_int}; #[derive(Clone, Copy, Debug, PartialEq)] pub struct DefaultToolRates { @@ -23,23 +27,11 @@ pub enum ResponseKind { } fn number(value: Option<&Value>) -> Option { - match value? { - Value::Number(value) => value.as_f64(), - Value::String(value) => value.parse().ok(), - _ => None, - } + value.and_then(py_float) } pub fn safe_convert_to_int(value: Option<&Value>) -> Option { - match value? { - Value::Number(value) => value.as_i64().or_else(|| { - value - .as_f64() - .and_then(|value| value.is_finite().then_some(value as i64)) - }), - Value::String(value) => value.parse::().ok(), - _ => None, - } + value.and_then(py_int) } pub fn extract_file_search_params(file_search: &Value) -> (Option, Option) { @@ -115,11 +107,7 @@ pub fn response_object_includes_web_search_call( usage: Option<&Value>, ) -> bool { if kind == ResponseKind::Anthropic - && response - .get("usage") - .and_then(|usage| usage.get("server_tool_use")) - .and_then(|tools| tools.get("web_search_requests")) - .is_some() + && get_anthropic_web_search_requests_from_response(response).is_some() { return true; } @@ -133,23 +121,24 @@ pub fn response_object_includes_web_search_call( } } +pub fn usage_reports_server_side_web_search_calls(details: Option<&Value>) -> bool { + match details.and_then(|details| details.get("web_search_calls")) { + Some(Value::Bool(flag)) => *flag, + Some(Value::Number(calls)) => calls.as_i64().is_some_and(|calls| calls > 0), + _ => false, + } +} + fn usage_reports_web_search(usage: Option<&Value>) -> bool { let Some(usage) = usage else { return false; }; - usage - .get("server_tool_use") - .and_then(|tools| tools.get("web_search_requests")) - .is_some() + get_web_search_requests(usage.get("server_tool_use")).is_some() || usage .get("prompt_tokens_details") .and_then(|details| details.get("web_search_requests")) - .is_some() - || usage - .get("server_side_tool_usage_details") - .and_then(|details| details.get("web_search_calls")) - .and_then(Value::as_i64) - .is_some_and(|calls| calls > 0) + .is_some_and(|requests| !requests.is_null()) + || usage_reports_server_side_web_search_calls(usage.get("server_side_tool_usage_details")) } pub fn count_web_search_calls(response: &Value, kind: ResponseKind) -> u64 { @@ -160,7 +149,8 @@ pub fn count_web_search_calls(response: &Value, kind: ResponseKind) -> u64 { .get("tool_usage") .and_then(|usage| usage.get("web_search")) .and_then(|search| search.get("num_requests")) - .and_then(Value::as_u64) + .and_then(lax_int) + .and_then(|requests| u64::try_from(requests).ok()) { return reported; } diff --git a/litellm-rust/crates/cost/src/tool_cost_dispatch.rs b/litellm-rust/crates/cost/src/tool_cost_dispatch.rs index 0d44eee0b19..ca7501e65e0 100644 --- a/litellm-rust/crates/cost/src/tool_cost_dispatch.rs +++ b/litellm-rust/crates/cost/src/tool_cost_dispatch.rs @@ -1,10 +1,16 @@ -use serde_json::Value; +use std::borrow::Cow; + +use serde_json::{Value, json}; use crate::anthropic_cost::{ get_anthropic_web_search_requests_from_response, get_cost_for_anthropic_web_search, get_web_search_requests_from_usage, }; -use crate::gemini_cost::{cost_per_google_maps_grounding_request, cost_per_web_search_request}; +use crate::catalog::ModelInfoCatalog; +use crate::gemini_cost::{ + cost_per_google_maps_grounding_request, cost_per_web_search_request, + google_maps_grounding_requests, +}; use crate::groq_cost::cost_per_web_search_request as groq_web_search_cost; use crate::provider::LlmProviders; use crate::responses_usage::ChatUsage; @@ -13,7 +19,7 @@ use crate::tool_call_cost_tracking::{ extract_token_counts, get_cost_for_code_interpreter, get_cost_for_computer_use, get_cost_for_file_search, get_cost_for_vector_store, get_cost_for_web_search, response_object_includes_file_search_call, response_object_includes_web_search_call, - safe_convert_to_int, + safe_convert_to_int, usage_reports_server_side_web_search_calls, }; use crate::wire::is_truthy; @@ -27,110 +33,162 @@ pub struct BuiltInToolCostRequest<'a> { pub defaults: DefaultToolRates, } -fn server_tool_count(usage: &ChatUsage, key: &str) -> Option { - usage - .extra - .get("server_tool_use") - .and_then(|tools| tools.get(key)) - .and_then(Value::as_u64) -} - fn usage_reports_web_search(usage: &ChatUsage) -> bool { usage .prompt_tokens_details .as_ref() .and_then(|details| details.web_search_requests) .is_some() - || server_tool_count(usage, "web_search_requests").is_some() - || usage - .extra - .get("server_side_tool_usage_details") - .and_then(|details| details.get("web_search_calls")) - .and_then(Value::as_u64) - .is_some_and(|calls| calls > 0) + || get_web_search_requests_from_usage(usage).is_some() + || usage_reports_server_side_web_search_calls( + usage.extra.get("server_side_tool_usage_details"), + ) } -fn web_search_requests(request: BuiltInToolCostRequest<'_>) -> Option { - request - .usage - .and_then(|usage| server_tool_count(usage, "web_search_requests")) - .or_else(|| { - request - .response - .get("usage") - .and_then(|usage| usage.get("server_tool_use")) - .and_then(|tools| tools.get("web_search_requests")) - .and_then(Value::as_u64) - }) +#[derive(Clone, Copy, Debug, Default)] +pub struct ResolvedModelInfo<'a> { + pub model_info: Option<&'a Value>, + pub provider: Option<&'a str>, } -fn context_rate(model_info: &Value) -> f64 { - model_info - .get("search_context_cost_per_query") - .and_then(|rates| rates.get("search_context_size_medium")) - .and_then(Value::as_f64) - .unwrap_or(0.0) -} - -fn provider_web_search_cost( - request: BuiltInToolCostRequest<'_>, - model_info: &Value, -) -> Option { - let provider = request.provider?; - let usage = request.usage; - match provider { - "gemini" => usage.map(|usage| cost_per_web_search_request(usage, model_info)), - "anthropic" => match usage.and_then(get_web_search_requests_from_usage) { - Some(_) => Some(get_cost_for_anthropic_web_search(Some(model_info), usage)), - None => get_anthropic_web_search_requests_from_response(request.response) - .map(|count| context_rate(model_info) * count as f64), +pub fn resolve_model_info<'a>( + catalog: &'a ModelInfoCatalog, + model: &str, + provider: Option<&'a str>, + region: Option<&str>, +) -> ResolvedModelInfo<'a> { + let litellm_provider = + |model_info: &'a Value| model_info.get("litellm_provider").and_then(Value::as_str); + if let Some(direct) = catalog.entry(model, provider, region) { + return ResolvedModelInfo { + model_info: Some(direct), + provider: provider + .filter(|provider| !provider.is_empty()) + .or_else(|| litellm_provider(direct)), + }; + } + let by_prefix = model + .contains('/') + .then(|| catalog.entry(model, None, None)) + .flatten(); + match by_prefix { + Some(by_prefix) => ResolvedModelInfo { + model_info: Some(by_prefix), + provider: litellm_provider(by_prefix), }, - "perplexity" => Some(0.0), - "xai" => usage.map(|usage| { - crate::xai_cost::cost_per_web_search_request( - usage, - model_info, - request.defaults.xai_web_search_per_call, - ) - }), - "groq" => usage.map(|usage| { - groq_web_search_cost( - usage, - model_info, - request.defaults.groq_browser_open_per_call, - ) - }), - provider if provider.starts_with("vertex_ai") => { + None => ResolvedModelInfo { + model_info: None, + provider, + }, + } +} + +pub fn get_cost_for_web_search_request( + provider: &str, + usage: &ChatUsage, + model_info: &Value, + defaults: DefaultToolRates, +) -> Option { + match provider.parse::().ok() { + Some(LlmProviders::GEMINI) => Some(cost_per_web_search_request(usage, model_info)), + Some(LlmProviders::ANTHROPIC) => Some(get_cost_for_anthropic_web_search( + Some(model_info), + Some(usage), + )), + _ if provider.starts_with(LlmProviders::VERTEX_AI.as_str()) => { let is_claude = model_info .get("key") .and_then(Value::as_str) .is_some_and(|key| key.to_ascii_lowercase().contains("claude")); - if is_claude { - web_search_requests(request).map(|count| context_rate(model_info) * count as f64) + Some(if is_claude { + get_cost_for_anthropic_web_search(Some(model_info), Some(usage)) } else { - usage.map(|usage| cost_per_web_search_request(usage, model_info)) - } + cost_per_web_search_request(usage, model_info) + }) } + Some(LlmProviders::PERPLEXITY) => Some(0.0), + Some(LlmProviders::XAI) => Some(crate::xai_cost::cost_per_web_search_request( + usage, + model_info, + defaults.xai_web_search_per_call, + )), + Some(LlmProviders::GROQ) => Some(groq_web_search_cost( + usage, + model_info, + defaults.groq_browser_open_per_call, + )), _ => None, } } -fn maps_cost(request: BuiltInToolCostRequest<'_>, model_info: Option<&Value>) -> f64 { - let supports_maps = LlmProviders::GEMINI.matches(request.provider) - || request - .provider - .is_some_and(|provider| provider.starts_with("vertex_ai")); - if !supports_maps { - return 0.0; +pub fn get_cost_for_google_maps_grounding_request( + provider: &str, + usage: &ChatUsage, + model_info: &Value, +) -> Option { + (LlmProviders::GEMINI.matches(Some(provider)) + || provider.starts_with(LlmProviders::VERTEX_AI.as_str())) + .then(|| cost_per_google_maps_grounding_request(usage, model_info)) +} + +fn usage_with_anthropic_web_search<'a>( + usage: Option<&'a ChatUsage>, + response: &Value, + kind: ResponseKind, +) -> Option> { + if usage.is_some_and(|usage| get_web_search_requests_from_usage(usage).is_some()) { + return usage.map(Cow::Borrowed); } - match (request.usage, model_info) { - (Some(usage), Some(model_info)) => { - cost_per_google_maps_grounding_request(usage, model_info) - } + let from_response = (kind == ResponseKind::Anthropic) + .then(|| get_anthropic_web_search_requests_from_response(response)) + .flatten(); + let Some(web_search_requests) = from_response else { + return usage.map(Cow::Borrowed); + }; + let base = usage.cloned().unwrap_or_default(); + let extra = base + .extra + .into_iter() + .filter(|(key, _)| key != "server_tool_use") + .chain([( + "server_tool_use".to_owned(), + json!({"web_search_requests": web_search_requests}), + )]) + .collect(); + Some(Cow::Owned(ChatUsage { extra, ..base })) +} + +fn maps_cost(usage: Option<&ChatUsage>, resolved: ResolvedModelInfo<'_>) -> f64 { + let Some(usage) = usage.filter(|usage| google_maps_grounding_requests(Some(usage)).is_some()) + else { + return 0.0; + }; + match resolved { + ResolvedModelInfo { + model_info: Some(model_info), + provider: Some(provider), + } => get_cost_for_google_maps_grounding_request(provider, usage, model_info).unwrap_or(0.0), _ => 0.0, } } +fn web_search_cost(request: BuiltInToolCostRequest<'_>, resolved: ResolvedModelInfo<'_>) -> f64 { + let usage = + usage_with_anthropic_web_search(request.usage, request.response, request.response_kind); + let routed = match (resolved.model_info, usage.as_deref(), resolved.provider) { + (Some(model_info), Some(usage), Some(provider)) => { + get_cost_for_web_search_request(provider, usage, model_info, request.defaults) + } + _ => None, + }; + routed.unwrap_or_else(|| { + get_cost_for_web_search( + request.params.get("web_search_options"), + resolved.model_info, + ) * count_web_search_calls(request.response, request.response_kind) as f64 + }) +} + fn file_search_cost(request: BuiltInToolCostRequest<'_>, model_info: Option<&Value>) -> f64 { let file_search = request .params @@ -179,24 +237,23 @@ fn azure_assistant_cost(request: BuiltInToolCostRequest<'_>, model_info: Option< } pub fn get_cost_for_built_in_tools( + catalog: &ModelInfoCatalog, + model: &str, + region: Option<&str>, request: BuiltInToolCostRequest<'_>, - model_info: Option<&Value>, ) -> f64 { - let maps = maps_cost(request, model_info); + let resolved = resolve_model_info(catalog, model, request.provider, region); + let maps = maps_cost(request.usage, resolved); let web_search = response_object_includes_web_search_call(request.response, request.response_kind, None) || (request.response_kind != ResponseKind::Responses && request.usage.is_some_and(usage_reports_web_search)); if web_search { - let routed = - model_info.and_then(|model_info| provider_web_search_cost(request, model_info)); - let fallback = - get_cost_for_web_search(request.params.get("web_search_options"), model_info) - * count_web_search_calls(request.response, request.response_kind) as f64; - return maps + routed.unwrap_or(fallback); + return maps + web_search_cost(request, resolved); } + let direct = catalog.entry(model, request.provider, region); if response_object_includes_file_search_call(request.response, request.response_kind) { - return maps + file_search_cost(request, model_info); + return maps + file_search_cost(request, direct); } - maps + azure_assistant_cost(request, model_info) + maps + azure_assistant_cost(request, direct) } diff --git a/litellm-rust/crates/cost/src/wire.rs b/litellm-rust/crates/cost/src/wire.rs index 35ad3f6e6d6..6e85175bbb6 100644 --- a/litellm-rust/crates/cost/src/wire.rs +++ b/litellm-rust/crates/cost/src/wire.rs @@ -83,6 +83,55 @@ pub fn is_truthy(value: &Value) -> bool { } } +fn python_int_literal(text: &str) -> Option { + let digits = text.strip_prefix(['+', '-']).unwrap_or(text); + let well_formed = digits + .split('_') + .all(|group| !group.is_empty() && group.bytes().all(|byte| byte.is_ascii_digit())); + if !well_formed { + return None; + } + text.replace('_', "").parse().ok() +} + +pub fn py_int(value: &Value) -> Option { + match value { + Value::Number(number) => number.as_i64().or_else(|| { + number + .as_f64() + .map(f64::trunc) + .filter(|float| float.abs() < i64::MAX as f64) + .map(|float| float as i64) + }), + Value::Bool(flag) => Some(i64::from(*flag)), + Value::String(text) => python_int_literal(text.trim()), + _ => None, + } +} + +pub fn py_float(value: &Value) -> Option { + match value { + Value::Number(number) => number.as_f64(), + Value::Bool(flag) => Some(f64::from(u8::from(*flag))), + Value::String(text) => text.trim().parse().ok(), + _ => None, + } +} + +pub fn lax_int(value: &Value) -> Option { + match value { + Value::Number(number) => number.as_i64().or_else(|| { + number + .as_f64() + .filter(|float| float.fract() == 0.0 && float.abs() < i64::MAX as f64) + .map(|float| float as i64) + }), + Value::Bool(flag) => Some(i64::from(*flag)), + Value::String(text) => python_int_literal(text.trim()), + _ => None, + } +} + pub fn rate(value: Option<&Value>) -> Rate { match value { None => Rate::Missing, @@ -838,6 +887,52 @@ mod tests { )) } + #[rstest] + #[case(serde_json::json!(3), Some(3))] + #[case(serde_json::json!(3.9), Some(3))] + #[case(serde_json::json!(-3.9), Some(-3))] + #[case(serde_json::json!(true), Some(1))] + #[case(serde_json::json!(" 12 "), Some(12))] + #[case(serde_json::json!("1_0"), Some(10))] + #[case(serde_json::json!("3.5"), None)] + #[case(serde_json::json!(null), None)] + #[case(serde_json::json!({}), None)] + fn py_int_follows_python_int(#[case] value: Value, #[case] expected: Option) { + assert_eq!(py_int(&value), expected); + } + + #[rstest] + #[case(serde_json::json!(2), Some(2.0))] + #[case(serde_json::json!(0.25), Some(0.25))] + #[case(serde_json::json!(false), Some(0.0))] + #[case(serde_json::json!(" 0.5 "), Some(0.5))] + #[case(serde_json::json!("abc"), None)] + #[case(serde_json::json!(null), None)] + #[case(serde_json::json!([1]), None)] + fn py_float_follows_python_float(#[case] value: Value, #[case] expected: Option) { + assert_eq!(py_float(&value), expected); + } + + #[rstest] + #[case(serde_json::json!(3), Some(3))] + #[case(serde_json::json!(-1), Some(-1))] + #[case(serde_json::json!(2.0), Some(2))] + #[case(serde_json::json!(2.5), None)] + #[case(serde_json::json!(true), Some(1))] + #[case(serde_json::json!("3"), Some(3))] + #[case(serde_json::json!(" 3 "), Some(3))] + #[case(serde_json::json!("-4"), Some(-4))] + #[case(serde_json::json!("1_000"), Some(1000))] + #[case(serde_json::json!("1__000"), None)] + #[case(serde_json::json!("_1"), None)] + #[case(serde_json::json!("3.0"), None)] + #[case(serde_json::json!("x"), None)] + #[case(serde_json::json!(null), None)] + #[case(serde_json::json!([3]), None)] + fn lax_int_follows_pydantic_lax_int(#[case] value: Value, #[case] expected: Option) { + assert_eq!(lax_int(&value), expected); + } + #[rstest] #[case(serde_json::json!(null), false)] #[case(serde_json::json!(false), false)] diff --git a/litellm-rust/crates/cost/tests/python_anthropic_cost.rs b/litellm-rust/crates/cost/tests/python_anthropic_cost.rs index d77f43e789a..1786249a72e 100644 --- a/litellm-rust/crates/cost/tests/python_anthropic_cost.rs +++ b/litellm-rust/crates/cost/tests/python_anthropic_cost.rs @@ -17,6 +17,17 @@ use litellm_cost::usage_dispatch::get_usage_object; use rstest::rstest; use serde_json::{Value, json}; +fn priced(request: BuiltInToolCostRequest<'_>, model_info: Option<&Value>) -> f64 { + let provider = request.provider.expect("tests price a provider"); + let catalog = ModelInfoCatalog::new( + model_info + .map(|info| (format!("{provider}/model"), info.clone())) + .into_iter() + .collect(), + ); + get_cost_for_built_in_tools(&catalog, "model", None, request) +} + fn usage(value: &Value) -> litellm_cost::responses_usage::ChatUsage { get_usage_object(&json!({"usage": value})).unwrap().unwrap() } @@ -91,7 +102,10 @@ fn get_anthropic_web_search_requests_from_response_reads_nested_usage( #[case(json!({"web_search_requests": 0}), Some(0))] #[case(json!({"web_search_requests": 3}), Some(3))] #[case(json!({}), None)] -#[case(json!({"web_search_requests": "3"}), None)] +#[case(json!({"web_search_requests": "3"}), Some(3))] +#[case(json!({"web_search_requests": 2.0}), Some(2))] +#[case(json!({"web_search_requests": 2.5}), None)] +#[case(json!({"web_search_requests": null}), None)] fn get_web_search_requests_reads_serialized_server_tool_use( #[case] server_tool_use: Value, #[case] expected: Option, @@ -114,7 +128,7 @@ fn get_cost_for_anthropic_web_search_prices_server_tool_requests() { get_cost_for_anthropic_web_search(Some(&model_info), None), 0.0 ); - let dispatched = get_cost_for_built_in_tools( + let dispatched = priced( BuiltInToolCostRequest { response: &json!({}), response_kind: ResponseKind::Anthropic, diff --git a/litellm-rust/crates/cost/tests/python_model_lookup.rs b/litellm-rust/crates/cost/tests/python_model_lookup.rs index dcd54d047a9..d53e6314e68 100644 --- a/litellm-rust/crates/cost/tests/python_model_lookup.rs +++ b/litellm-rust/crates/cost/tests/python_model_lookup.rs @@ -1,12 +1,13 @@ #![allow(clippy::disallowed_types)] // mirrors: test_litellm/test_cost_calculator.py::test_cost_per_token_duplicate_openai_prefix_matches_model_cost +// mirrors: test_litellm/test_utils.py::test_check_provider_match_azure_ai_allows_openai_and_azure use litellm_cost::cost_calculator::{cost_per_token, cost_per_token_for_call}; use litellm_cost::error::CostError; use std::collections::HashMap; use jiff::Timestamp; -use litellm_cost::catalog::{ModelCostRequest, ModelInfoCatalog}; +use litellm_cost::catalog::{ModelCostRequest, ModelInfoCatalog, check_provider_match}; use litellm_cost::usage_dispatch::get_usage_object; use rstest::rstest; use serde_json::json; @@ -209,3 +210,57 @@ fn model_info_catalog_applies_xai_inclusive_threshold_policy() { assert!((actual.0 - 128_000.0 * 5e-6).abs() < 1e-12); assert!((actual.1 - 10.0 * 7e-6).abs() < 1e-12); } + +#[rstest] +#[case::no_requested_provider(Some("gemini"), None, true)] +#[case::empty_requested_provider(Some("gemini"), Some(""), true)] +#[case::entry_without_provider(None, Some("openai"), true)] +#[case::same_provider(Some("openai"), Some("openai"), true)] +#[case::different_provider(Some("gemini"), Some("openai"), false)] +#[case::vertex_family(Some("vertex_ai-language-models"), Some("vertex_ai"), true)] +#[case::vertex_family_is_one_way(Some("vertex_ai"), Some("vertex_ai-language-models"), false)] +#[case::fireworks_family(Some("fireworks_ai-embedding-models"), Some("fireworks_ai"), true)] +#[case::bedrock_family(Some("bedrock_converse"), Some("bedrock"), true)] +#[case::litellm_proxy_matches_anything(Some("anthropic"), Some("litellm_proxy"), true)] +#[case::azure_ai_falls_back_to_azure(Some("azure"), Some("azure_ai"), true)] +#[case::azure_ai_falls_back_to_openai(Some("openai"), Some("azure_ai"), true)] +#[case::azure_does_not_fall_back_to_azure_ai(Some("azure_ai"), Some("azure"), false)] +#[case::github_reuses_any_entry(Some("openai"), Some("github"), true)] +fn check_provider_match_follows_python( + #[case] entry_provider: Option<&str>, + #[case] requested: Option<&str>, + #[case] expected: bool, +) { + let model_info = json!({"litellm_provider": entry_provider}); + assert_eq!(check_provider_match(&model_info, requested), expected); +} + +#[rstest] +#[case::matching_provider(Some("gemini"), Some("gemini/gemini-x"))] +#[case::no_provider(None, Some("gemini/gemini-x"))] +#[case::mismatched_provider(Some("openai"), None)] +fn select_model_key_skips_entries_owned_by_another_provider( + #[case] provider: Option<&str>, + #[case] expected: Option<&str>, +) { + let catalog = ModelInfoCatalog::new(HashMap::from([( + "gemini/gemini-x".to_owned(), + json!({"litellm_provider": "gemini", "input_cost_per_token": 1e-6}), + )])); + assert_eq!( + catalog.select_model_key("gemini/gemini-x", provider, None), + expected + ); +} + +#[rstest] +fn select_model_key_lets_vertex_ai_beta_use_vertex_entries() { + let catalog = ModelInfoCatalog::new(HashMap::from([( + "vertex_ai/gemini-x".to_owned(), + json!({"litellm_provider": "vertex_ai-language-models"}), + )])); + assert_eq!( + catalog.select_model_key("vertex_ai/gemini-x", Some("vertex_ai_beta"), None), + Some("vertex_ai/gemini-x") + ); +} diff --git a/litellm-rust/crates/cost/tests/python_tool_cost_dispatch.rs b/litellm-rust/crates/cost/tests/python_tool_cost_dispatch.rs index f5178a140c3..553793f71c0 100644 --- a/litellm-rust/crates/cost/tests/python_tool_cost_dispatch.rs +++ b/litellm-rust/crates/cost/tests/python_tool_cost_dispatch.rs @@ -2,12 +2,28 @@ // mirrors: test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py::test_get_cost_for_built_in_tools_web_search +use std::collections::HashMap; + +use jiff::Timestamp; +use litellm_cost::catalog::{ModelCostRequest, ModelInfoCatalog}; +use litellm_cost::cost_calculator::{BuiltInToolCharge, CompletionCostRequest}; use litellm_cost::tool_call_cost_tracking::{DefaultToolRates, ResponseKind}; use litellm_cost::tool_cost_dispatch::{BuiltInToolCostRequest, get_cost_for_built_in_tools}; use litellm_cost::usage_dispatch::get_usage_object; use rstest::rstest; use serde_json::{Value, json}; +fn priced(request: BuiltInToolCostRequest<'_>, model_info: Option<&Value>) -> f64 { + let provider = request.provider.expect("tests price a provider"); + let catalog = ModelInfoCatalog::new( + model_info + .map(|info| (format!("{provider}/model"), info.clone())) + .into_iter() + .collect(), + ); + get_cost_for_built_in_tools(&catalog, "model", None, request) +} + fn defaults() -> DefaultToolRates { DefaultToolRates { file_search_per_call: 0.25, @@ -42,7 +58,7 @@ fn get_cost_for_built_in_tools_adds_gemini_search_and_maps( "search_context_cost_per_query": {"search_context_size_medium": 0.01}, "google_maps_grounding_cost_per_query": 0.02 }); - let actual = get_cost_for_built_in_tools( + let actual = priced( BuiltInToolCostRequest { response: &json!({"choices": []}), response_kind: ResponseKind::Chat, @@ -62,7 +78,7 @@ fn get_cost_for_built_in_tools_prefers_reported_responses_search_count() { "output": [{"type": "web_search_call"}, {"type": "web_search_call"}], "tool_usage": {"web_search": {"num_requests": 1}} }); - let actual = get_cost_for_built_in_tools( + let actual = priced( BuiltInToolCostRequest { response: &response, response_kind: ResponseKind::Responses, @@ -79,7 +95,7 @@ fn get_cost_for_built_in_tools_prefers_reported_responses_search_count() { #[rstest] fn get_cost_for_built_in_tools_reads_anthropic_raw_web_search_usage() { let response = json!({"usage": {"server_tool_use": {"web_search_requests": 2}}}); - let actual = get_cost_for_built_in_tools( + let actual = priced( BuiltInToolCostRequest { response: &response, response_kind: ResponseKind::Anthropic, @@ -106,7 +122,7 @@ fn get_cost_for_built_in_tools_uses_xai_reported_cost_or_search_calls( "cost": reported, "server_side_tool_usage_details": {"web_search_calls": 2} })); - let actual = get_cost_for_built_in_tools( + let actual = priced( BuiltInToolCostRequest { response: &json!({"choices": []}), response_kind: ResponseKind::Chat, @@ -127,7 +143,7 @@ fn get_cost_for_built_in_tools_prices_groq_search_and_browser_open() { "completion_tokens": 2, "server_tool_use": {"web_search_requests": 2, "browser_open_requests": 3} })); - let actual = get_cost_for_built_in_tools( + let actual = priced( BuiltInToolCostRequest { response: &json!({"choices": []}), response_kind: ResponseKind::Chat, @@ -143,7 +159,7 @@ fn get_cost_for_built_in_tools_prices_groq_search_and_browser_open() { #[rstest] fn get_cost_for_built_in_tools_prices_file_search_before_assistant_features() { - let actual = get_cost_for_built_in_tools( + let actual = priced( BuiltInToolCostRequest { response: &json!({"output": [{"type": "file_search_call"}]}), response_kind: ResponseKind::Responses, @@ -162,7 +178,7 @@ fn get_cost_for_built_in_tools_prices_file_search_before_assistant_features() { #[rstest] fn get_cost_for_built_in_tools_adds_azure_assistant_features() { - let actual = get_cost_for_built_in_tools( + let actual = priced( BuiltInToolCostRequest { response: &json!({"output": []}), response_kind: ResponseKind::Responses, @@ -226,8 +242,167 @@ fn model_info_catalog_completion_cost_dispatches_built_in_tools() { .unwrap(); assert!((actual.total - (100.0 * 2e-6 + 50.0 * 4e-6 + 2.0 * 0.01)).abs() < 1e-12); } -use std::collections::HashMap; -use jiff::Timestamp; -use litellm_cost::catalog::{ModelCostRequest, ModelInfoCatalog}; -use litellm_cost::cost_calculator::{BuiltInToolCharge, CompletionCostRequest}; +fn gemini_catalog() -> ModelInfoCatalog { + ModelInfoCatalog::new(HashMap::from([( + "gemini/gemini-x".to_owned(), + json!({ + "litellm_provider": "gemini", + "web_search_billing_unit": "per_query", + "search_context_cost_per_query": {"search_context_size_medium": 0.014}, + "google_maps_grounding_cost_per_query": 0.0125 + }), + )])) +} + +#[rstest] +#[case::no_provider(None)] +#[case::provider_the_entry_does_not_belong_to(Some("openai"))] +fn get_cost_for_built_in_tools_adopts_the_provider_of_the_resolved_entry( + #[case] provider: Option<&str>, +) { + let usage = usage(&json!({ + "prompt_tokens": 10, + "completion_tokens": 2, + "prompt_tokens_details": {"web_search_requests": 3, "google_maps_grounding_requests": 2} + })); + let actual = get_cost_for_built_in_tools( + &gemini_catalog(), + "gemini/gemini-x", + None, + BuiltInToolCostRequest { + response: &json!({"choices": []}), + response_kind: ResponseKind::Chat, + usage: Some(&usage), + provider, + params: &json!({}), + defaults: defaults(), + }, + ); + assert!((actual - (3.0 * 0.014 + 2.0 * 0.0125)).abs() < 1e-12); +} + +fn cited_chat_response() -> Value { + json!({"choices": [{"message": {"annotations": [{"type": "url_citation"}]}}]}) +} + +#[rstest] +fn anthropic_usage_without_a_search_count_bills_nothing_instead_of_the_per_call_price() { + let usage = usage(&json!({"prompt_tokens": 10, "completion_tokens": 2})); + let actual = priced( + BuiltInToolCostRequest { + response: &cited_chat_response(), + response_kind: ResponseKind::Chat, + usage: Some(&usage), + provider: Some("anthropic"), + params: &json!({}), + defaults: defaults(), + }, + Some(&json!({"search_context_cost_per_query": {"search_context_size_medium": 0.01}})), + ); + assert_eq!(actual, 0.0); +} + +#[rstest] +#[case::with_usage(true, 0.0)] +#[case::without_usage(false, 0.01)] +fn perplexity_search_is_free_only_when_usage_reaches_the_router( + #[case] with_usage: bool, + #[case] expected: f64, +) { + let usage = usage(&json!({"prompt_tokens": 10, "completion_tokens": 2})); + let actual = priced( + BuiltInToolCostRequest { + response: &cited_chat_response(), + response_kind: ResponseKind::Chat, + usage: with_usage.then_some(&usage), + provider: Some("perplexity"), + params: &json!({}), + defaults: defaults(), + }, + Some(&json!({"search_context_cost_per_query": {"search_context_size_medium": 0.01}})), + ); + assert_eq!(actual, expected); +} + +#[rstest] +#[case::per_prompt(json!("per_prompt"), 0.01)] +#[case::unset(json!(null), 0.01)] +#[case::empty(json!(""), 0.01)] +#[case::per_query(json!("per_query"), 0.03)] +#[case::unrecognized(json!("per_request"), 0.03)] +fn gemini_bills_one_search_only_for_per_prompt_units( + #[case] billing_unit: Value, + #[case] expected: f64, +) { + let usage = usage(&json!({ + "prompt_tokens": 10, + "completion_tokens": 2, + "prompt_tokens_details": {"web_search_requests": 3} + })); + let actual = priced( + BuiltInToolCostRequest { + response: &json!({"choices": []}), + response_kind: ResponseKind::Chat, + usage: Some(&usage), + provider: Some("gemini"), + params: &json!({}), + defaults: defaults(), + }, + Some(&json!({ + "web_search_billing_unit": billing_unit, + "search_context_cost_per_query": {"search_context_size_medium": 0.01} + })), + ); + assert!((actual - expected).abs() < 1e-12); +} + +#[rstest] +#[case::string_count(json!("3"), 0.03)] +#[case::integral_float_count(json!(2.0), 0.02)] +#[case::null_count_is_not_a_search(json!(null), 0.0)] +fn raw_anthropic_search_counts_are_coerced_like_pydantic( + #[case] requests: Value, + #[case] expected: f64, +) { + let response = json!({"usage": {"server_tool_use": {"web_search_requests": requests}}}); + let actual = priced( + BuiltInToolCostRequest { + response: &response, + response_kind: ResponseKind::Anthropic, + usage: None, + provider: Some("anthropic"), + params: &json!({}), + defaults: defaults(), + }, + Some(&json!({"search_context_cost_per_query": {"search_context_size_medium": 0.01}})), + ); + assert!((actual - expected).abs() < 1e-12); +} + +#[rstest] +#[case::integer(json!(1), 0.015)] +#[case::boolean_counts_as_one_call(json!(true), 0.015)] +#[case::float_is_not_an_int(json!(1.0), 0.0)] +fn xai_server_side_search_calls_follow_python_isinstance_int( + #[case] calls: Value, + #[case] expected: f64, +) { + let usage = usage(&json!({ + "prompt_tokens": 10, + "completion_tokens": 2, + "server_side_tool_usage_details": {"web_search_calls": calls} + })); + let actual = priced( + BuiltInToolCostRequest { + response: &json!({"choices": []}), + response_kind: ResponseKind::Chat, + usage: Some(&usage), + provider: Some("xai"), + params: &json!({}), + defaults: defaults(), + }, + Some(&json!({"search_context_cost_per_query": {"search_context_size_medium": 0.015}})), + ); + assert!((actual - expected).abs() < 1e-12); +}