fix(cost): resolve built-in tool pricing and model keys the way Python does

get_cost_for_built_in_tools now takes the catalog and model like Python,
adopts the resolved entry's litellm_provider, returns zero for Anthropic usage
without a search count, keeps Perplexity's per-call fallback when no usage
reaches the router, and bills Gemini per query for any unit other than
per_prompt. Search counts are read with pydantic's lax int rules, and
select_model_key applies _check_provider_match. Python int(), float(), and
lax-int coercion now live once in wire.rs

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Yujong Lee 2026-09-23 19:00:37 -07:00
parent a323ad73bc
commit f74ee2b856
12 changed files with 582 additions and 197 deletions

View file

@ -1,6 +1,7 @@
use serde_json::Value;
use crate::responses_usage::ChatUsage;
use crate::wire::lax_int;
pub fn fast_speed_multiplier(model_info: &Value, usage: &ChatUsage) -> f64 {
if usage.extra.get("speed").and_then(Value::as_str) != Some("fast") {
@ -20,7 +21,7 @@ pub fn get_anthropic_web_search_requests_from_response(response: &Value) -> Opti
pub fn get_web_search_requests(server_tool_use: Option<&Value>) -> Option<i64> {
server_tool_use?
.get("web_search_requests")
.and_then(Value::as_i64)
.and_then(lax_int)
}
pub fn get_web_search_requests_from_usage(usage: &ChatUsage) -> Option<i64> {

View file

@ -63,12 +63,32 @@ pub enum CostCall<'a> {
},
}
fn select_model_key<'a, T>(
entries: &'a HashMap<String, T>,
pub fn check_provider_match(model_info: &Value, provider: Option<&str>) -> bool {
let Some(provider) = provider.filter(|provider| !provider.is_empty()) else {
return true;
};
let Some(entry_provider) = model_info.get("litellm_provider").and_then(Value::as_str) else {
return true;
};
entry_provider == provider
|| (provider == "vertex_ai" && entry_provider.starts_with("vertex_ai"))
|| (provider == "fireworks_ai" && entry_provider.starts_with("fireworks_ai"))
|| (provider.starts_with("bedrock") && entry_provider.starts_with("bedrock"))
|| provider == "litellm_proxy"
|| (provider == "azure_ai" && matches!(entry_provider, "azure" | "openai"))
|| provider == "github"
}
fn select_model_key<'a>(
entries: &'a HashMap<String, Value>,
model: &str,
provider: Option<&str>,
region: Option<&str>,
) -> Option<&'a str> {
let match_provider = provider.map(|provider| match provider {
"vertex_ai_beta" => "vertex_ai",
provider => provider,
});
let normalized = match provider {
Some(provider) => {
let prefix = format!("{provider}/");
@ -106,6 +126,7 @@ fn select_model_key<'a, T>(
.find_map(|candidate| {
entries
.get_key_value(candidate)
.filter(|(_, model_info)| check_provider_match(model_info, match_provider))
.map(|(key, _)| key.as_str())
})
}

View file

@ -14,7 +14,7 @@ use crate::completion_cost::{
};
use crate::completion_input::{CompletionInputRequest, PreparedCompletionInput, ResponseKind};
use crate::cost_calculator::{
built_in_tool_cost, cost_per_token, cost_per_token_for_call, default_video_cost_calculator,
cost_per_token, cost_per_token_for_call, default_video_cost_calculator,
handle_realtime_stream_cost_calculation,
};
use crate::custom_pricing::{CustomPricing, cost_from_chat_usage};
@ -32,7 +32,7 @@ use crate::realtime_cost::{
use crate::responses_usage::ChatUsage;
use crate::speech_cost::count_characters;
use crate::tool_call_cost_tracking::{DefaultToolRates, ResponseKind as ToolResponseKind};
use crate::tool_cost_dispatch::BuiltInToolCostRequest;
use crate::tool_cost_dispatch::{BuiltInToolCostRequest, get_cost_for_built_in_tools};
#[derive(Clone, Copy, Debug)]
pub struct BuiltInToolCostConfig<'a> {
@ -508,10 +508,9 @@ fn config_driven_tool_cost(
let Some(response) = request.input.model_selection.response else {
return 0.0;
};
built_in_tool_cost(
get_cost_for_built_in_tools(
catalog,
model,
provider,
region,
BuiltInToolCostRequest {
response,

View file

@ -513,16 +513,6 @@ pub fn responses_ws_token_cost_by_tier(
.sum()
}
pub fn built_in_tool_cost(
catalog: &ModelInfoCatalog,
model: &str,
provider: Option<&str>,
region: Option<&str>,
request: BuiltInToolCostRequest<'_>,
) -> f64 {
get_cost_for_built_in_tools(request, catalog.entry(model, provider, region))
}
pub fn completion_cost(
catalog: &ModelInfoCatalog,
request: CompletionCostRequest<'_>,
@ -530,10 +520,9 @@ pub fn completion_cost(
let (prompt, output) = cost_per_token(catalog, request.token)?;
let built_in_tools = match request.built_in_tools {
BuiltInToolCharge::Provided(cost) => cost,
BuiltInToolCharge::FromResponse(tool_request) => built_in_tool_cost(
BuiltInToolCharge::FromResponse(tool_request) => get_cost_for_built_in_tools(
catalog,
request.token.model,
request.token.provider,
request.token.region,
tool_request,
),

View file

@ -1,40 +1,39 @@
use serde_json::Value;
use crate::anthropic_cost::get_web_search_requests_from_usage;
use crate::responses_usage::ChatUsage;
use crate::wire::is_truthy;
const DEFAULT_WEB_SEARCH_COST: f64 = 35e-3;
const DEFAULT_MAPS_QUERY_COST: f64 = 14e-3;
const DEFAULT_MAPS_PROMPT_COST: f64 = 25e-3;
fn per_query(model_info: &Value) -> bool {
model_info
.get("web_search_billing_unit")
.and_then(Value::as_str)
== Some("per_query")
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum WebSearchBillingUnit {
PerPrompt,
PerQuery,
Unrecognized,
}
fn billed_requests(requests: u64, per_query: bool) -> u64 {
if per_query {
requests
} else {
u64::from(requests > 0)
pub fn web_search_billing_unit(model_info: &Value) -> WebSearchBillingUnit {
match model_info.get("web_search_billing_unit") {
Some(unit) if is_truthy(unit) => match unit.as_str() {
Some("per_prompt") => WebSearchBillingUnit::PerPrompt,
Some("per_query") => WebSearchBillingUnit::PerQuery,
_ => WebSearchBillingUnit::Unrecognized,
},
_ => WebSearchBillingUnit::PerPrompt,
}
}
fn web_search_requests(usage: &ChatUsage) -> u64 {
let from_details = usage
fn web_search_requests(usage: &ChatUsage) -> i64 {
usage
.prompt_tokens_details
.as_ref()
.and_then(|details| details.web_search_requests)
.unwrap_or(0);
if from_details > 0 {
return from_details;
}
usage
.extra
.get("server_tool_use")
.and_then(|value| value.get("web_search_requests"))
.and_then(Value::as_u64)
.and_then(|requests| i64::try_from(requests).ok())
.filter(|requests| *requests != 0)
.or_else(|| get_web_search_requests_from_usage(usage))
.unwrap_or(0)
}
@ -44,7 +43,14 @@ pub fn cost_per_web_search_request(usage: &ChatUsage, model_info: &Value) -> f64
.and_then(|rates| rates.get("search_context_size_medium"))
.and_then(Value::as_f64)
.unwrap_or(DEFAULT_WEB_SEARCH_COST);
rate * billed_requests(web_search_requests(usage), per_query(model_info)) as f64
let requests = web_search_requests(usage);
let billed =
if requests > 0 && web_search_billing_unit(model_info) == WebSearchBillingUnit::PerPrompt {
1
} else {
requests
};
rate * billed as f64
}
pub fn google_maps_grounding_requests(usage: Option<&ChatUsage>) -> Option<u64> {
@ -59,7 +65,7 @@ pub fn cost_per_google_maps_grounding_request(usage: &ChatUsage, model_info: &Va
else {
return 0.0;
};
let per_query = per_query(model_info);
let per_query = web_search_billing_unit(model_info) == WebSearchBillingUnit::PerQuery;
let default_rate = if per_query {
DEFAULT_MAPS_QUERY_COST
} else {
@ -69,5 +75,6 @@ pub fn cost_per_google_maps_grounding_request(usage: &ChatUsage, model_info: &Va
.get("google_maps_grounding_cost_per_query")
.and_then(Value::as_f64)
.unwrap_or(default_rate);
rate * billed_requests(requests, per_query) as f64
let billed = if per_query { requests } else { 1 };
rate * billed as f64
}

View file

@ -2,7 +2,7 @@ use jiff::Timestamp;
use jiff::civil::{Date, DateTime};
use serde_json::{Map, Value, json};
use crate::wire::is_truthy;
use crate::wire::{is_truthy, py_float, py_int};
pub const PTU_COST_ATTRIBUTION_ENV_VAR: &str = "LITELLM_ENABLE_PTU_COST_ATTRIBUTION";
pub const AZURE_SPILLOVER_HEADER: &str = "x-ms-is-spilled-over";
@ -176,24 +176,6 @@ fn py_str(value: &Value) -> String {
}
}
fn py_int(value: &Value) -> Option<i64> {
match value {
Value::Number(number) => Some(number.as_f64()?.trunc() as i64),
Value::Bool(flag) => Some(i64::from(*flag)),
Value::String(text) => text.trim().parse::<i64>().ok(),
_ => None,
}
}
fn py_float(value: &Value) -> Option<f64> {
match value {
Value::Number(number) => number.as_f64(),
Value::Bool(flag) => Some(f64::from(u8::from(*flag))),
Value::String(text) => text.trim().parse::<f64>().ok(),
_ => None,
}
}
fn as_utc(value: &Value) -> Option<Timestamp> {
let Value::String(text) = value else {
return None;

View file

@ -1,6 +1,10 @@
use serde_json::Value;
use crate::anthropic_cost::{
get_anthropic_web_search_requests_from_response, get_web_search_requests,
};
use crate::provider::LlmProviders;
use crate::wire::{lax_int, py_float, py_int};
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct DefaultToolRates {
@ -23,23 +27,11 @@ pub enum ResponseKind {
}
fn number(value: Option<&Value>) -> Option<f64> {
match value? {
Value::Number(value) => value.as_f64(),
Value::String(value) => value.parse().ok(),
_ => None,
}
value.and_then(py_float)
}
pub fn safe_convert_to_int(value: Option<&Value>) -> Option<i64> {
match value? {
Value::Number(value) => value.as_i64().or_else(|| {
value
.as_f64()
.and_then(|value| value.is_finite().then_some(value as i64))
}),
Value::String(value) => value.parse::<i64>().ok(),
_ => None,
}
value.and_then(py_int)
}
pub fn extract_file_search_params(file_search: &Value) -> (Option<f64>, Option<f64>) {
@ -115,11 +107,7 @@ pub fn response_object_includes_web_search_call(
usage: Option<&Value>,
) -> bool {
if kind == ResponseKind::Anthropic
&& response
.get("usage")
.and_then(|usage| usage.get("server_tool_use"))
.and_then(|tools| tools.get("web_search_requests"))
.is_some()
&& get_anthropic_web_search_requests_from_response(response).is_some()
{
return true;
}
@ -133,23 +121,24 @@ pub fn response_object_includes_web_search_call(
}
}
pub fn usage_reports_server_side_web_search_calls(details: Option<&Value>) -> bool {
match details.and_then(|details| details.get("web_search_calls")) {
Some(Value::Bool(flag)) => *flag,
Some(Value::Number(calls)) => calls.as_i64().is_some_and(|calls| calls > 0),
_ => false,
}
}
fn usage_reports_web_search(usage: Option<&Value>) -> bool {
let Some(usage) = usage else {
return false;
};
usage
.get("server_tool_use")
.and_then(|tools| tools.get("web_search_requests"))
.is_some()
get_web_search_requests(usage.get("server_tool_use")).is_some()
|| usage
.get("prompt_tokens_details")
.and_then(|details| details.get("web_search_requests"))
.is_some()
|| usage
.get("server_side_tool_usage_details")
.and_then(|details| details.get("web_search_calls"))
.and_then(Value::as_i64)
.is_some_and(|calls| calls > 0)
.is_some_and(|requests| !requests.is_null())
|| usage_reports_server_side_web_search_calls(usage.get("server_side_tool_usage_details"))
}
pub fn count_web_search_calls(response: &Value, kind: ResponseKind) -> u64 {
@ -160,7 +149,8 @@ pub fn count_web_search_calls(response: &Value, kind: ResponseKind) -> u64 {
.get("tool_usage")
.and_then(|usage| usage.get("web_search"))
.and_then(|search| search.get("num_requests"))
.and_then(Value::as_u64)
.and_then(lax_int)
.and_then(|requests| u64::try_from(requests).ok())
{
return reported;
}

View file

@ -1,10 +1,16 @@
use serde_json::Value;
use std::borrow::Cow;
use serde_json::{Value, json};
use crate::anthropic_cost::{
get_anthropic_web_search_requests_from_response, get_cost_for_anthropic_web_search,
get_web_search_requests_from_usage,
};
use crate::gemini_cost::{cost_per_google_maps_grounding_request, cost_per_web_search_request};
use crate::catalog::ModelInfoCatalog;
use crate::gemini_cost::{
cost_per_google_maps_grounding_request, cost_per_web_search_request,
google_maps_grounding_requests,
};
use crate::groq_cost::cost_per_web_search_request as groq_web_search_cost;
use crate::provider::LlmProviders;
use crate::responses_usage::ChatUsage;
@ -13,7 +19,7 @@ use crate::tool_call_cost_tracking::{
extract_token_counts, get_cost_for_code_interpreter, get_cost_for_computer_use,
get_cost_for_file_search, get_cost_for_vector_store, get_cost_for_web_search,
response_object_includes_file_search_call, response_object_includes_web_search_call,
safe_convert_to_int,
safe_convert_to_int, usage_reports_server_side_web_search_calls,
};
use crate::wire::is_truthy;
@ -27,110 +33,162 @@ pub struct BuiltInToolCostRequest<'a> {
pub defaults: DefaultToolRates,
}
fn server_tool_count(usage: &ChatUsage, key: &str) -> Option<u64> {
usage
.extra
.get("server_tool_use")
.and_then(|tools| tools.get(key))
.and_then(Value::as_u64)
}
fn usage_reports_web_search(usage: &ChatUsage) -> bool {
usage
.prompt_tokens_details
.as_ref()
.and_then(|details| details.web_search_requests)
.is_some()
|| server_tool_count(usage, "web_search_requests").is_some()
|| usage
.extra
.get("server_side_tool_usage_details")
.and_then(|details| details.get("web_search_calls"))
.and_then(Value::as_u64)
.is_some_and(|calls| calls > 0)
|| get_web_search_requests_from_usage(usage).is_some()
|| usage_reports_server_side_web_search_calls(
usage.extra.get("server_side_tool_usage_details"),
)
}
fn web_search_requests(request: BuiltInToolCostRequest<'_>) -> Option<u64> {
request
.usage
.and_then(|usage| server_tool_count(usage, "web_search_requests"))
.or_else(|| {
request
.response
.get("usage")
.and_then(|usage| usage.get("server_tool_use"))
.and_then(|tools| tools.get("web_search_requests"))
.and_then(Value::as_u64)
})
#[derive(Clone, Copy, Debug, Default)]
pub struct ResolvedModelInfo<'a> {
pub model_info: Option<&'a Value>,
pub provider: Option<&'a str>,
}
fn context_rate(model_info: &Value) -> f64 {
model_info
.get("search_context_cost_per_query")
.and_then(|rates| rates.get("search_context_size_medium"))
.and_then(Value::as_f64)
.unwrap_or(0.0)
}
fn provider_web_search_cost(
request: BuiltInToolCostRequest<'_>,
model_info: &Value,
) -> Option<f64> {
let provider = request.provider?;
let usage = request.usage;
match provider {
"gemini" => usage.map(|usage| cost_per_web_search_request(usage, model_info)),
"anthropic" => match usage.and_then(get_web_search_requests_from_usage) {
Some(_) => Some(get_cost_for_anthropic_web_search(Some(model_info), usage)),
None => get_anthropic_web_search_requests_from_response(request.response)
.map(|count| context_rate(model_info) * count as f64),
pub fn resolve_model_info<'a>(
catalog: &'a ModelInfoCatalog,
model: &str,
provider: Option<&'a str>,
region: Option<&str>,
) -> ResolvedModelInfo<'a> {
let litellm_provider =
|model_info: &'a Value| model_info.get("litellm_provider").and_then(Value::as_str);
if let Some(direct) = catalog.entry(model, provider, region) {
return ResolvedModelInfo {
model_info: Some(direct),
provider: provider
.filter(|provider| !provider.is_empty())
.or_else(|| litellm_provider(direct)),
};
}
let by_prefix = model
.contains('/')
.then(|| catalog.entry(model, None, None))
.flatten();
match by_prefix {
Some(by_prefix) => ResolvedModelInfo {
model_info: Some(by_prefix),
provider: litellm_provider(by_prefix),
},
"perplexity" => Some(0.0),
"xai" => usage.map(|usage| {
crate::xai_cost::cost_per_web_search_request(
usage,
model_info,
request.defaults.xai_web_search_per_call,
)
}),
"groq" => usage.map(|usage| {
groq_web_search_cost(
usage,
model_info,
request.defaults.groq_browser_open_per_call,
)
}),
provider if provider.starts_with("vertex_ai") => {
None => ResolvedModelInfo {
model_info: None,
provider,
},
}
}
pub fn get_cost_for_web_search_request(
provider: &str,
usage: &ChatUsage,
model_info: &Value,
defaults: DefaultToolRates,
) -> Option<f64> {
match provider.parse::<LlmProviders>().ok() {
Some(LlmProviders::GEMINI) => Some(cost_per_web_search_request(usage, model_info)),
Some(LlmProviders::ANTHROPIC) => Some(get_cost_for_anthropic_web_search(
Some(model_info),
Some(usage),
)),
_ if provider.starts_with(LlmProviders::VERTEX_AI.as_str()) => {
let is_claude = model_info
.get("key")
.and_then(Value::as_str)
.is_some_and(|key| key.to_ascii_lowercase().contains("claude"));
if is_claude {
web_search_requests(request).map(|count| context_rate(model_info) * count as f64)
Some(if is_claude {
get_cost_for_anthropic_web_search(Some(model_info), Some(usage))
} else {
usage.map(|usage| cost_per_web_search_request(usage, model_info))
}
cost_per_web_search_request(usage, model_info)
})
}
Some(LlmProviders::PERPLEXITY) => Some(0.0),
Some(LlmProviders::XAI) => Some(crate::xai_cost::cost_per_web_search_request(
usage,
model_info,
defaults.xai_web_search_per_call,
)),
Some(LlmProviders::GROQ) => Some(groq_web_search_cost(
usage,
model_info,
defaults.groq_browser_open_per_call,
)),
_ => None,
}
}
fn maps_cost(request: BuiltInToolCostRequest<'_>, model_info: Option<&Value>) -> f64 {
let supports_maps = LlmProviders::GEMINI.matches(request.provider)
|| request
.provider
.is_some_and(|provider| provider.starts_with("vertex_ai"));
if !supports_maps {
return 0.0;
pub fn get_cost_for_google_maps_grounding_request(
provider: &str,
usage: &ChatUsage,
model_info: &Value,
) -> Option<f64> {
(LlmProviders::GEMINI.matches(Some(provider))
|| provider.starts_with(LlmProviders::VERTEX_AI.as_str()))
.then(|| cost_per_google_maps_grounding_request(usage, model_info))
}
fn usage_with_anthropic_web_search<'a>(
usage: Option<&'a ChatUsage>,
response: &Value,
kind: ResponseKind,
) -> Option<Cow<'a, ChatUsage>> {
if usage.is_some_and(|usage| get_web_search_requests_from_usage(usage).is_some()) {
return usage.map(Cow::Borrowed);
}
match (request.usage, model_info) {
(Some(usage), Some(model_info)) => {
cost_per_google_maps_grounding_request(usage, model_info)
}
let from_response = (kind == ResponseKind::Anthropic)
.then(|| get_anthropic_web_search_requests_from_response(response))
.flatten();
let Some(web_search_requests) = from_response else {
return usage.map(Cow::Borrowed);
};
let base = usage.cloned().unwrap_or_default();
let extra = base
.extra
.into_iter()
.filter(|(key, _)| key != "server_tool_use")
.chain([(
"server_tool_use".to_owned(),
json!({"web_search_requests": web_search_requests}),
)])
.collect();
Some(Cow::Owned(ChatUsage { extra, ..base }))
}
fn maps_cost(usage: Option<&ChatUsage>, resolved: ResolvedModelInfo<'_>) -> f64 {
let Some(usage) = usage.filter(|usage| google_maps_grounding_requests(Some(usage)).is_some())
else {
return 0.0;
};
match resolved {
ResolvedModelInfo {
model_info: Some(model_info),
provider: Some(provider),
} => get_cost_for_google_maps_grounding_request(provider, usage, model_info).unwrap_or(0.0),
_ => 0.0,
}
}
fn web_search_cost(request: BuiltInToolCostRequest<'_>, resolved: ResolvedModelInfo<'_>) -> f64 {
let usage =
usage_with_anthropic_web_search(request.usage, request.response, request.response_kind);
let routed = match (resolved.model_info, usage.as_deref(), resolved.provider) {
(Some(model_info), Some(usage), Some(provider)) => {
get_cost_for_web_search_request(provider, usage, model_info, request.defaults)
}
_ => None,
};
routed.unwrap_or_else(|| {
get_cost_for_web_search(
request.params.get("web_search_options"),
resolved.model_info,
) * count_web_search_calls(request.response, request.response_kind) as f64
})
}
fn file_search_cost(request: BuiltInToolCostRequest<'_>, model_info: Option<&Value>) -> f64 {
let file_search = request
.params
@ -179,24 +237,23 @@ fn azure_assistant_cost(request: BuiltInToolCostRequest<'_>, model_info: Option<
}
pub fn get_cost_for_built_in_tools(
catalog: &ModelInfoCatalog,
model: &str,
region: Option<&str>,
request: BuiltInToolCostRequest<'_>,
model_info: Option<&Value>,
) -> f64 {
let maps = maps_cost(request, model_info);
let resolved = resolve_model_info(catalog, model, request.provider, region);
let maps = maps_cost(request.usage, resolved);
let web_search =
response_object_includes_web_search_call(request.response, request.response_kind, None)
|| (request.response_kind != ResponseKind::Responses
&& request.usage.is_some_and(usage_reports_web_search));
if web_search {
let routed =
model_info.and_then(|model_info| provider_web_search_cost(request, model_info));
let fallback =
get_cost_for_web_search(request.params.get("web_search_options"), model_info)
* count_web_search_calls(request.response, request.response_kind) as f64;
return maps + routed.unwrap_or(fallback);
return maps + web_search_cost(request, resolved);
}
let direct = catalog.entry(model, request.provider, region);
if response_object_includes_file_search_call(request.response, request.response_kind) {
return maps + file_search_cost(request, model_info);
return maps + file_search_cost(request, direct);
}
maps + azure_assistant_cost(request, model_info)
maps + azure_assistant_cost(request, direct)
}

View file

@ -83,6 +83,55 @@ pub fn is_truthy(value: &Value) -> bool {
}
}
fn python_int_literal(text: &str) -> Option<i64> {
let digits = text.strip_prefix(['+', '-']).unwrap_or(text);
let well_formed = digits
.split('_')
.all(|group| !group.is_empty() && group.bytes().all(|byte| byte.is_ascii_digit()));
if !well_formed {
return None;
}
text.replace('_', "").parse().ok()
}
pub fn py_int(value: &Value) -> Option<i64> {
match value {
Value::Number(number) => number.as_i64().or_else(|| {
number
.as_f64()
.map(f64::trunc)
.filter(|float| float.abs() < i64::MAX as f64)
.map(|float| float as i64)
}),
Value::Bool(flag) => Some(i64::from(*flag)),
Value::String(text) => python_int_literal(text.trim()),
_ => None,
}
}
pub fn py_float(value: &Value) -> Option<f64> {
match value {
Value::Number(number) => number.as_f64(),
Value::Bool(flag) => Some(f64::from(u8::from(*flag))),
Value::String(text) => text.trim().parse().ok(),
_ => None,
}
}
pub fn lax_int(value: &Value) -> Option<i64> {
match value {
Value::Number(number) => number.as_i64().or_else(|| {
number
.as_f64()
.filter(|float| float.fract() == 0.0 && float.abs() < i64::MAX as f64)
.map(|float| float as i64)
}),
Value::Bool(flag) => Some(i64::from(*flag)),
Value::String(text) => python_int_literal(text.trim()),
_ => None,
}
}
pub fn rate(value: Option<&Value>) -> Rate {
match value {
None => Rate::Missing,
@ -838,6 +887,52 @@ mod tests {
))
}
#[rstest]
#[case(serde_json::json!(3), Some(3))]
#[case(serde_json::json!(3.9), Some(3))]
#[case(serde_json::json!(-3.9), Some(-3))]
#[case(serde_json::json!(true), Some(1))]
#[case(serde_json::json!(" 12 "), Some(12))]
#[case(serde_json::json!("1_0"), Some(10))]
#[case(serde_json::json!("3.5"), None)]
#[case(serde_json::json!(null), None)]
#[case(serde_json::json!({}), None)]
fn py_int_follows_python_int(#[case] value: Value, #[case] expected: Option<i64>) {
assert_eq!(py_int(&value), expected);
}
#[rstest]
#[case(serde_json::json!(2), Some(2.0))]
#[case(serde_json::json!(0.25), Some(0.25))]
#[case(serde_json::json!(false), Some(0.0))]
#[case(serde_json::json!(" 0.5 "), Some(0.5))]
#[case(serde_json::json!("abc"), None)]
#[case(serde_json::json!(null), None)]
#[case(serde_json::json!([1]), None)]
fn py_float_follows_python_float(#[case] value: Value, #[case] expected: Option<f64>) {
assert_eq!(py_float(&value), expected);
}
#[rstest]
#[case(serde_json::json!(3), Some(3))]
#[case(serde_json::json!(-1), Some(-1))]
#[case(serde_json::json!(2.0), Some(2))]
#[case(serde_json::json!(2.5), None)]
#[case(serde_json::json!(true), Some(1))]
#[case(serde_json::json!("3"), Some(3))]
#[case(serde_json::json!(" 3 "), Some(3))]
#[case(serde_json::json!("-4"), Some(-4))]
#[case(serde_json::json!("1_000"), Some(1000))]
#[case(serde_json::json!("1__000"), None)]
#[case(serde_json::json!("_1"), None)]
#[case(serde_json::json!("3.0"), None)]
#[case(serde_json::json!("x"), None)]
#[case(serde_json::json!(null), None)]
#[case(serde_json::json!([3]), None)]
fn lax_int_follows_pydantic_lax_int(#[case] value: Value, #[case] expected: Option<i64>) {
assert_eq!(lax_int(&value), expected);
}
#[rstest]
#[case(serde_json::json!(null), false)]
#[case(serde_json::json!(false), false)]

View file

@ -17,6 +17,17 @@ use litellm_cost::usage_dispatch::get_usage_object;
use rstest::rstest;
use serde_json::{Value, json};
fn priced(request: BuiltInToolCostRequest<'_>, model_info: Option<&Value>) -> f64 {
let provider = request.provider.expect("tests price a provider");
let catalog = ModelInfoCatalog::new(
model_info
.map(|info| (format!("{provider}/model"), info.clone()))
.into_iter()
.collect(),
);
get_cost_for_built_in_tools(&catalog, "model", None, request)
}
fn usage(value: &Value) -> litellm_cost::responses_usage::ChatUsage {
get_usage_object(&json!({"usage": value})).unwrap().unwrap()
}
@ -91,7 +102,10 @@ fn get_anthropic_web_search_requests_from_response_reads_nested_usage(
#[case(json!({"web_search_requests": 0}), Some(0))]
#[case(json!({"web_search_requests": 3}), Some(3))]
#[case(json!({}), None)]
#[case(json!({"web_search_requests": "3"}), None)]
#[case(json!({"web_search_requests": "3"}), Some(3))]
#[case(json!({"web_search_requests": 2.0}), Some(2))]
#[case(json!({"web_search_requests": 2.5}), None)]
#[case(json!({"web_search_requests": null}), None)]
fn get_web_search_requests_reads_serialized_server_tool_use(
#[case] server_tool_use: Value,
#[case] expected: Option<i64>,
@ -114,7 +128,7 @@ fn get_cost_for_anthropic_web_search_prices_server_tool_requests() {
get_cost_for_anthropic_web_search(Some(&model_info), None),
0.0
);
let dispatched = get_cost_for_built_in_tools(
let dispatched = priced(
BuiltInToolCostRequest {
response: &json!({}),
response_kind: ResponseKind::Anthropic,

View file

@ -1,12 +1,13 @@
#![allow(clippy::disallowed_types)]
// mirrors: test_litellm/test_cost_calculator.py::test_cost_per_token_duplicate_openai_prefix_matches_model_cost
// mirrors: test_litellm/test_utils.py::test_check_provider_match_azure_ai_allows_openai_and_azure
use litellm_cost::cost_calculator::{cost_per_token, cost_per_token_for_call};
use litellm_cost::error::CostError;
use std::collections::HashMap;
use jiff::Timestamp;
use litellm_cost::catalog::{ModelCostRequest, ModelInfoCatalog};
use litellm_cost::catalog::{ModelCostRequest, ModelInfoCatalog, check_provider_match};
use litellm_cost::usage_dispatch::get_usage_object;
use rstest::rstest;
use serde_json::json;
@ -209,3 +210,57 @@ fn model_info_catalog_applies_xai_inclusive_threshold_policy() {
assert!((actual.0 - 128_000.0 * 5e-6).abs() < 1e-12);
assert!((actual.1 - 10.0 * 7e-6).abs() < 1e-12);
}
#[rstest]
#[case::no_requested_provider(Some("gemini"), None, true)]
#[case::empty_requested_provider(Some("gemini"), Some(""), true)]
#[case::entry_without_provider(None, Some("openai"), true)]
#[case::same_provider(Some("openai"), Some("openai"), true)]
#[case::different_provider(Some("gemini"), Some("openai"), false)]
#[case::vertex_family(Some("vertex_ai-language-models"), Some("vertex_ai"), true)]
#[case::vertex_family_is_one_way(Some("vertex_ai"), Some("vertex_ai-language-models"), false)]
#[case::fireworks_family(Some("fireworks_ai-embedding-models"), Some("fireworks_ai"), true)]
#[case::bedrock_family(Some("bedrock_converse"), Some("bedrock"), true)]
#[case::litellm_proxy_matches_anything(Some("anthropic"), Some("litellm_proxy"), true)]
#[case::azure_ai_falls_back_to_azure(Some("azure"), Some("azure_ai"), true)]
#[case::azure_ai_falls_back_to_openai(Some("openai"), Some("azure_ai"), true)]
#[case::azure_does_not_fall_back_to_azure_ai(Some("azure_ai"), Some("azure"), false)]
#[case::github_reuses_any_entry(Some("openai"), Some("github"), true)]
fn check_provider_match_follows_python(
#[case] entry_provider: Option<&str>,
#[case] requested: Option<&str>,
#[case] expected: bool,
) {
let model_info = json!({"litellm_provider": entry_provider});
assert_eq!(check_provider_match(&model_info, requested), expected);
}
#[rstest]
#[case::matching_provider(Some("gemini"), Some("gemini/gemini-x"))]
#[case::no_provider(None, Some("gemini/gemini-x"))]
#[case::mismatched_provider(Some("openai"), None)]
fn select_model_key_skips_entries_owned_by_another_provider(
#[case] provider: Option<&str>,
#[case] expected: Option<&str>,
) {
let catalog = ModelInfoCatalog::new(HashMap::from([(
"gemini/gemini-x".to_owned(),
json!({"litellm_provider": "gemini", "input_cost_per_token": 1e-6}),
)]));
assert_eq!(
catalog.select_model_key("gemini/gemini-x", provider, None),
expected
);
}
#[rstest]
fn select_model_key_lets_vertex_ai_beta_use_vertex_entries() {
let catalog = ModelInfoCatalog::new(HashMap::from([(
"vertex_ai/gemini-x".to_owned(),
json!({"litellm_provider": "vertex_ai-language-models"}),
)]));
assert_eq!(
catalog.select_model_key("vertex_ai/gemini-x", Some("vertex_ai_beta"), None),
Some("vertex_ai/gemini-x")
);
}

View file

@ -2,12 +2,28 @@
// mirrors: test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py::test_get_cost_for_built_in_tools_web_search
use std::collections::HashMap;
use jiff::Timestamp;
use litellm_cost::catalog::{ModelCostRequest, ModelInfoCatalog};
use litellm_cost::cost_calculator::{BuiltInToolCharge, CompletionCostRequest};
use litellm_cost::tool_call_cost_tracking::{DefaultToolRates, ResponseKind};
use litellm_cost::tool_cost_dispatch::{BuiltInToolCostRequest, get_cost_for_built_in_tools};
use litellm_cost::usage_dispatch::get_usage_object;
use rstest::rstest;
use serde_json::{Value, json};
fn priced(request: BuiltInToolCostRequest<'_>, model_info: Option<&Value>) -> f64 {
let provider = request.provider.expect("tests price a provider");
let catalog = ModelInfoCatalog::new(
model_info
.map(|info| (format!("{provider}/model"), info.clone()))
.into_iter()
.collect(),
);
get_cost_for_built_in_tools(&catalog, "model", None, request)
}
fn defaults() -> DefaultToolRates {
DefaultToolRates {
file_search_per_call: 0.25,
@ -42,7 +58,7 @@ fn get_cost_for_built_in_tools_adds_gemini_search_and_maps(
"search_context_cost_per_query": {"search_context_size_medium": 0.01},
"google_maps_grounding_cost_per_query": 0.02
});
let actual = get_cost_for_built_in_tools(
let actual = priced(
BuiltInToolCostRequest {
response: &json!({"choices": []}),
response_kind: ResponseKind::Chat,
@ -62,7 +78,7 @@ fn get_cost_for_built_in_tools_prefers_reported_responses_search_count() {
"output": [{"type": "web_search_call"}, {"type": "web_search_call"}],
"tool_usage": {"web_search": {"num_requests": 1}}
});
let actual = get_cost_for_built_in_tools(
let actual = priced(
BuiltInToolCostRequest {
response: &response,
response_kind: ResponseKind::Responses,
@ -79,7 +95,7 @@ fn get_cost_for_built_in_tools_prefers_reported_responses_search_count() {
#[rstest]
fn get_cost_for_built_in_tools_reads_anthropic_raw_web_search_usage() {
let response = json!({"usage": {"server_tool_use": {"web_search_requests": 2}}});
let actual = get_cost_for_built_in_tools(
let actual = priced(
BuiltInToolCostRequest {
response: &response,
response_kind: ResponseKind::Anthropic,
@ -106,7 +122,7 @@ fn get_cost_for_built_in_tools_uses_xai_reported_cost_or_search_calls(
"cost": reported,
"server_side_tool_usage_details": {"web_search_calls": 2}
}));
let actual = get_cost_for_built_in_tools(
let actual = priced(
BuiltInToolCostRequest {
response: &json!({"choices": []}),
response_kind: ResponseKind::Chat,
@ -127,7 +143,7 @@ fn get_cost_for_built_in_tools_prices_groq_search_and_browser_open() {
"completion_tokens": 2,
"server_tool_use": {"web_search_requests": 2, "browser_open_requests": 3}
}));
let actual = get_cost_for_built_in_tools(
let actual = priced(
BuiltInToolCostRequest {
response: &json!({"choices": []}),
response_kind: ResponseKind::Chat,
@ -143,7 +159,7 @@ fn get_cost_for_built_in_tools_prices_groq_search_and_browser_open() {
#[rstest]
fn get_cost_for_built_in_tools_prices_file_search_before_assistant_features() {
let actual = get_cost_for_built_in_tools(
let actual = priced(
BuiltInToolCostRequest {
response: &json!({"output": [{"type": "file_search_call"}]}),
response_kind: ResponseKind::Responses,
@ -162,7 +178,7 @@ fn get_cost_for_built_in_tools_prices_file_search_before_assistant_features() {
#[rstest]
fn get_cost_for_built_in_tools_adds_azure_assistant_features() {
let actual = get_cost_for_built_in_tools(
let actual = priced(
BuiltInToolCostRequest {
response: &json!({"output": []}),
response_kind: ResponseKind::Responses,
@ -226,8 +242,167 @@ fn model_info_catalog_completion_cost_dispatches_built_in_tools() {
.unwrap();
assert!((actual.total - (100.0 * 2e-6 + 50.0 * 4e-6 + 2.0 * 0.01)).abs() < 1e-12);
}
use std::collections::HashMap;
use jiff::Timestamp;
use litellm_cost::catalog::{ModelCostRequest, ModelInfoCatalog};
use litellm_cost::cost_calculator::{BuiltInToolCharge, CompletionCostRequest};
fn gemini_catalog() -> ModelInfoCatalog {
ModelInfoCatalog::new(HashMap::from([(
"gemini/gemini-x".to_owned(),
json!({
"litellm_provider": "gemini",
"web_search_billing_unit": "per_query",
"search_context_cost_per_query": {"search_context_size_medium": 0.014},
"google_maps_grounding_cost_per_query": 0.0125
}),
)]))
}
#[rstest]
#[case::no_provider(None)]
#[case::provider_the_entry_does_not_belong_to(Some("openai"))]
fn get_cost_for_built_in_tools_adopts_the_provider_of_the_resolved_entry(
#[case] provider: Option<&str>,
) {
let usage = usage(&json!({
"prompt_tokens": 10,
"completion_tokens": 2,
"prompt_tokens_details": {"web_search_requests": 3, "google_maps_grounding_requests": 2}
}));
let actual = get_cost_for_built_in_tools(
&gemini_catalog(),
"gemini/gemini-x",
None,
BuiltInToolCostRequest {
response: &json!({"choices": []}),
response_kind: ResponseKind::Chat,
usage: Some(&usage),
provider,
params: &json!({}),
defaults: defaults(),
},
);
assert!((actual - (3.0 * 0.014 + 2.0 * 0.0125)).abs() < 1e-12);
}
fn cited_chat_response() -> Value {
json!({"choices": [{"message": {"annotations": [{"type": "url_citation"}]}}]})
}
#[rstest]
fn anthropic_usage_without_a_search_count_bills_nothing_instead_of_the_per_call_price() {
let usage = usage(&json!({"prompt_tokens": 10, "completion_tokens": 2}));
let actual = priced(
BuiltInToolCostRequest {
response: &cited_chat_response(),
response_kind: ResponseKind::Chat,
usage: Some(&usage),
provider: Some("anthropic"),
params: &json!({}),
defaults: defaults(),
},
Some(&json!({"search_context_cost_per_query": {"search_context_size_medium": 0.01}})),
);
assert_eq!(actual, 0.0);
}
#[rstest]
#[case::with_usage(true, 0.0)]
#[case::without_usage(false, 0.01)]
fn perplexity_search_is_free_only_when_usage_reaches_the_router(
#[case] with_usage: bool,
#[case] expected: f64,
) {
let usage = usage(&json!({"prompt_tokens": 10, "completion_tokens": 2}));
let actual = priced(
BuiltInToolCostRequest {
response: &cited_chat_response(),
response_kind: ResponseKind::Chat,
usage: with_usage.then_some(&usage),
provider: Some("perplexity"),
params: &json!({}),
defaults: defaults(),
},
Some(&json!({"search_context_cost_per_query": {"search_context_size_medium": 0.01}})),
);
assert_eq!(actual, expected);
}
#[rstest]
#[case::per_prompt(json!("per_prompt"), 0.01)]
#[case::unset(json!(null), 0.01)]
#[case::empty(json!(""), 0.01)]
#[case::per_query(json!("per_query"), 0.03)]
#[case::unrecognized(json!("per_request"), 0.03)]
fn gemini_bills_one_search_only_for_per_prompt_units(
#[case] billing_unit: Value,
#[case] expected: f64,
) {
let usage = usage(&json!({
"prompt_tokens": 10,
"completion_tokens": 2,
"prompt_tokens_details": {"web_search_requests": 3}
}));
let actual = priced(
BuiltInToolCostRequest {
response: &json!({"choices": []}),
response_kind: ResponseKind::Chat,
usage: Some(&usage),
provider: Some("gemini"),
params: &json!({}),
defaults: defaults(),
},
Some(&json!({
"web_search_billing_unit": billing_unit,
"search_context_cost_per_query": {"search_context_size_medium": 0.01}
})),
);
assert!((actual - expected).abs() < 1e-12);
}
#[rstest]
#[case::string_count(json!("3"), 0.03)]
#[case::integral_float_count(json!(2.0), 0.02)]
#[case::null_count_is_not_a_search(json!(null), 0.0)]
fn raw_anthropic_search_counts_are_coerced_like_pydantic(
#[case] requests: Value,
#[case] expected: f64,
) {
let response = json!({"usage": {"server_tool_use": {"web_search_requests": requests}}});
let actual = priced(
BuiltInToolCostRequest {
response: &response,
response_kind: ResponseKind::Anthropic,
usage: None,
provider: Some("anthropic"),
params: &json!({}),
defaults: defaults(),
},
Some(&json!({"search_context_cost_per_query": {"search_context_size_medium": 0.01}})),
);
assert!((actual - expected).abs() < 1e-12);
}
#[rstest]
#[case::integer(json!(1), 0.015)]
#[case::boolean_counts_as_one_call(json!(true), 0.015)]
#[case::float_is_not_an_int(json!(1.0), 0.0)]
fn xai_server_side_search_calls_follow_python_isinstance_int(
#[case] calls: Value,
#[case] expected: f64,
) {
let usage = usage(&json!({
"prompt_tokens": 10,
"completion_tokens": 2,
"server_side_tool_usage_details": {"web_search_calls": calls}
}));
let actual = priced(
BuiltInToolCostRequest {
response: &json!({"choices": []}),
response_kind: ResponseKind::Chat,
usage: Some(&usage),
provider: Some("xai"),
params: &json!({}),
defaults: defaults(),
},
Some(&json!({"search_context_cost_per_query": {"search_context_size_medium": 0.015}})),
);
assert!((actual - expected).abs() < 1e-12);
}