mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(cost): resolve built-in tool pricing and model keys the way Python does
get_cost_for_built_in_tools now takes the catalog and model like Python, adopts the resolved entry's litellm_provider, returns zero for Anthropic usage without a search count, keeps Perplexity's per-call fallback when no usage reaches the router, and bills Gemini per query for any unit other than per_prompt. Search counts are read with pydantic's lax int rules, and select_model_key applies _check_provider_match. Python int(), float(), and lax-int coercion now live once in wire.rs Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
a323ad73bc
commit
f74ee2b856
12 changed files with 582 additions and 197 deletions
|
|
@ -1,6 +1,7 @@
|
|||
use serde_json::Value;
|
||||
|
||||
use crate::responses_usage::ChatUsage;
|
||||
use crate::wire::lax_int;
|
||||
|
||||
pub fn fast_speed_multiplier(model_info: &Value, usage: &ChatUsage) -> f64 {
|
||||
if usage.extra.get("speed").and_then(Value::as_str) != Some("fast") {
|
||||
|
|
@ -20,7 +21,7 @@ pub fn get_anthropic_web_search_requests_from_response(response: &Value) -> Opti
|
|||
pub fn get_web_search_requests(server_tool_use: Option<&Value>) -> Option<i64> {
|
||||
server_tool_use?
|
||||
.get("web_search_requests")
|
||||
.and_then(Value::as_i64)
|
||||
.and_then(lax_int)
|
||||
}
|
||||
|
||||
pub fn get_web_search_requests_from_usage(usage: &ChatUsage) -> Option<i64> {
|
||||
|
|
|
|||
|
|
@ -63,12 +63,32 @@ pub enum CostCall<'a> {
|
|||
},
|
||||
}
|
||||
|
||||
fn select_model_key<'a, T>(
|
||||
entries: &'a HashMap<String, T>,
|
||||
pub fn check_provider_match(model_info: &Value, provider: Option<&str>) -> bool {
|
||||
let Some(provider) = provider.filter(|provider| !provider.is_empty()) else {
|
||||
return true;
|
||||
};
|
||||
let Some(entry_provider) = model_info.get("litellm_provider").and_then(Value::as_str) else {
|
||||
return true;
|
||||
};
|
||||
entry_provider == provider
|
||||
|| (provider == "vertex_ai" && entry_provider.starts_with("vertex_ai"))
|
||||
|| (provider == "fireworks_ai" && entry_provider.starts_with("fireworks_ai"))
|
||||
|| (provider.starts_with("bedrock") && entry_provider.starts_with("bedrock"))
|
||||
|| provider == "litellm_proxy"
|
||||
|| (provider == "azure_ai" && matches!(entry_provider, "azure" | "openai"))
|
||||
|| provider == "github"
|
||||
}
|
||||
|
||||
fn select_model_key<'a>(
|
||||
entries: &'a HashMap<String, Value>,
|
||||
model: &str,
|
||||
provider: Option<&str>,
|
||||
region: Option<&str>,
|
||||
) -> Option<&'a str> {
|
||||
let match_provider = provider.map(|provider| match provider {
|
||||
"vertex_ai_beta" => "vertex_ai",
|
||||
provider => provider,
|
||||
});
|
||||
let normalized = match provider {
|
||||
Some(provider) => {
|
||||
let prefix = format!("{provider}/");
|
||||
|
|
@ -106,6 +126,7 @@ fn select_model_key<'a, T>(
|
|||
.find_map(|candidate| {
|
||||
entries
|
||||
.get_key_value(candidate)
|
||||
.filter(|(_, model_info)| check_provider_match(model_info, match_provider))
|
||||
.map(|(key, _)| key.as_str())
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ use crate::completion_cost::{
|
|||
};
|
||||
use crate::completion_input::{CompletionInputRequest, PreparedCompletionInput, ResponseKind};
|
||||
use crate::cost_calculator::{
|
||||
built_in_tool_cost, cost_per_token, cost_per_token_for_call, default_video_cost_calculator,
|
||||
cost_per_token, cost_per_token_for_call, default_video_cost_calculator,
|
||||
handle_realtime_stream_cost_calculation,
|
||||
};
|
||||
use crate::custom_pricing::{CustomPricing, cost_from_chat_usage};
|
||||
|
|
@ -32,7 +32,7 @@ use crate::realtime_cost::{
|
|||
use crate::responses_usage::ChatUsage;
|
||||
use crate::speech_cost::count_characters;
|
||||
use crate::tool_call_cost_tracking::{DefaultToolRates, ResponseKind as ToolResponseKind};
|
||||
use crate::tool_cost_dispatch::BuiltInToolCostRequest;
|
||||
use crate::tool_cost_dispatch::{BuiltInToolCostRequest, get_cost_for_built_in_tools};
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct BuiltInToolCostConfig<'a> {
|
||||
|
|
@ -508,10 +508,9 @@ fn config_driven_tool_cost(
|
|||
let Some(response) = request.input.model_selection.response else {
|
||||
return 0.0;
|
||||
};
|
||||
built_in_tool_cost(
|
||||
get_cost_for_built_in_tools(
|
||||
catalog,
|
||||
model,
|
||||
provider,
|
||||
region,
|
||||
BuiltInToolCostRequest {
|
||||
response,
|
||||
|
|
|
|||
|
|
@ -513,16 +513,6 @@ pub fn responses_ws_token_cost_by_tier(
|
|||
.sum()
|
||||
}
|
||||
|
||||
pub fn built_in_tool_cost(
|
||||
catalog: &ModelInfoCatalog,
|
||||
model: &str,
|
||||
provider: Option<&str>,
|
||||
region: Option<&str>,
|
||||
request: BuiltInToolCostRequest<'_>,
|
||||
) -> f64 {
|
||||
get_cost_for_built_in_tools(request, catalog.entry(model, provider, region))
|
||||
}
|
||||
|
||||
pub fn completion_cost(
|
||||
catalog: &ModelInfoCatalog,
|
||||
request: CompletionCostRequest<'_>,
|
||||
|
|
@ -530,10 +520,9 @@ pub fn completion_cost(
|
|||
let (prompt, output) = cost_per_token(catalog, request.token)?;
|
||||
let built_in_tools = match request.built_in_tools {
|
||||
BuiltInToolCharge::Provided(cost) => cost,
|
||||
BuiltInToolCharge::FromResponse(tool_request) => built_in_tool_cost(
|
||||
BuiltInToolCharge::FromResponse(tool_request) => get_cost_for_built_in_tools(
|
||||
catalog,
|
||||
request.token.model,
|
||||
request.token.provider,
|
||||
request.token.region,
|
||||
tool_request,
|
||||
),
|
||||
|
|
|
|||
|
|
@ -1,40 +1,39 @@
|
|||
use serde_json::Value;
|
||||
|
||||
use crate::anthropic_cost::get_web_search_requests_from_usage;
|
||||
use crate::responses_usage::ChatUsage;
|
||||
use crate::wire::is_truthy;
|
||||
|
||||
const DEFAULT_WEB_SEARCH_COST: f64 = 35e-3;
|
||||
const DEFAULT_MAPS_QUERY_COST: f64 = 14e-3;
|
||||
const DEFAULT_MAPS_PROMPT_COST: f64 = 25e-3;
|
||||
|
||||
fn per_query(model_info: &Value) -> bool {
|
||||
model_info
|
||||
.get("web_search_billing_unit")
|
||||
.and_then(Value::as_str)
|
||||
== Some("per_query")
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum WebSearchBillingUnit {
|
||||
PerPrompt,
|
||||
PerQuery,
|
||||
Unrecognized,
|
||||
}
|
||||
|
||||
fn billed_requests(requests: u64, per_query: bool) -> u64 {
|
||||
if per_query {
|
||||
requests
|
||||
} else {
|
||||
u64::from(requests > 0)
|
||||
pub fn web_search_billing_unit(model_info: &Value) -> WebSearchBillingUnit {
|
||||
match model_info.get("web_search_billing_unit") {
|
||||
Some(unit) if is_truthy(unit) => match unit.as_str() {
|
||||
Some("per_prompt") => WebSearchBillingUnit::PerPrompt,
|
||||
Some("per_query") => WebSearchBillingUnit::PerQuery,
|
||||
_ => WebSearchBillingUnit::Unrecognized,
|
||||
},
|
||||
_ => WebSearchBillingUnit::PerPrompt,
|
||||
}
|
||||
}
|
||||
|
||||
fn web_search_requests(usage: &ChatUsage) -> u64 {
|
||||
let from_details = usage
|
||||
fn web_search_requests(usage: &ChatUsage) -> i64 {
|
||||
usage
|
||||
.prompt_tokens_details
|
||||
.as_ref()
|
||||
.and_then(|details| details.web_search_requests)
|
||||
.unwrap_or(0);
|
||||
if from_details > 0 {
|
||||
return from_details;
|
||||
}
|
||||
usage
|
||||
.extra
|
||||
.get("server_tool_use")
|
||||
.and_then(|value| value.get("web_search_requests"))
|
||||
.and_then(Value::as_u64)
|
||||
.and_then(|requests| i64::try_from(requests).ok())
|
||||
.filter(|requests| *requests != 0)
|
||||
.or_else(|| get_web_search_requests_from_usage(usage))
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
|
|
@ -44,7 +43,14 @@ pub fn cost_per_web_search_request(usage: &ChatUsage, model_info: &Value) -> f64
|
|||
.and_then(|rates| rates.get("search_context_size_medium"))
|
||||
.and_then(Value::as_f64)
|
||||
.unwrap_or(DEFAULT_WEB_SEARCH_COST);
|
||||
rate * billed_requests(web_search_requests(usage), per_query(model_info)) as f64
|
||||
let requests = web_search_requests(usage);
|
||||
let billed =
|
||||
if requests > 0 && web_search_billing_unit(model_info) == WebSearchBillingUnit::PerPrompt {
|
||||
1
|
||||
} else {
|
||||
requests
|
||||
};
|
||||
rate * billed as f64
|
||||
}
|
||||
|
||||
pub fn google_maps_grounding_requests(usage: Option<&ChatUsage>) -> Option<u64> {
|
||||
|
|
@ -59,7 +65,7 @@ pub fn cost_per_google_maps_grounding_request(usage: &ChatUsage, model_info: &Va
|
|||
else {
|
||||
return 0.0;
|
||||
};
|
||||
let per_query = per_query(model_info);
|
||||
let per_query = web_search_billing_unit(model_info) == WebSearchBillingUnit::PerQuery;
|
||||
let default_rate = if per_query {
|
||||
DEFAULT_MAPS_QUERY_COST
|
||||
} else {
|
||||
|
|
@ -69,5 +75,6 @@ pub fn cost_per_google_maps_grounding_request(usage: &ChatUsage, model_info: &Va
|
|||
.get("google_maps_grounding_cost_per_query")
|
||||
.and_then(Value::as_f64)
|
||||
.unwrap_or(default_rate);
|
||||
rate * billed_requests(requests, per_query) as f64
|
||||
let billed = if per_query { requests } else { 1 };
|
||||
rate * billed as f64
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ use jiff::Timestamp;
|
|||
use jiff::civil::{Date, DateTime};
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
use crate::wire::is_truthy;
|
||||
use crate::wire::{is_truthy, py_float, py_int};
|
||||
|
||||
pub const PTU_COST_ATTRIBUTION_ENV_VAR: &str = "LITELLM_ENABLE_PTU_COST_ATTRIBUTION";
|
||||
pub const AZURE_SPILLOVER_HEADER: &str = "x-ms-is-spilled-over";
|
||||
|
|
@ -176,24 +176,6 @@ fn py_str(value: &Value) -> String {
|
|||
}
|
||||
}
|
||||
|
||||
fn py_int(value: &Value) -> Option<i64> {
|
||||
match value {
|
||||
Value::Number(number) => Some(number.as_f64()?.trunc() as i64),
|
||||
Value::Bool(flag) => Some(i64::from(*flag)),
|
||||
Value::String(text) => text.trim().parse::<i64>().ok(),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn py_float(value: &Value) -> Option<f64> {
|
||||
match value {
|
||||
Value::Number(number) => number.as_f64(),
|
||||
Value::Bool(flag) => Some(f64::from(u8::from(*flag))),
|
||||
Value::String(text) => text.trim().parse::<f64>().ok(),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn as_utc(value: &Value) -> Option<Timestamp> {
|
||||
let Value::String(text) = value else {
|
||||
return None;
|
||||
|
|
|
|||
|
|
@ -1,6 +1,10 @@
|
|||
use serde_json::Value;
|
||||
|
||||
use crate::anthropic_cost::{
|
||||
get_anthropic_web_search_requests_from_response, get_web_search_requests,
|
||||
};
|
||||
use crate::provider::LlmProviders;
|
||||
use crate::wire::{lax_int, py_float, py_int};
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct DefaultToolRates {
|
||||
|
|
@ -23,23 +27,11 @@ pub enum ResponseKind {
|
|||
}
|
||||
|
||||
fn number(value: Option<&Value>) -> Option<f64> {
|
||||
match value? {
|
||||
Value::Number(value) => value.as_f64(),
|
||||
Value::String(value) => value.parse().ok(),
|
||||
_ => None,
|
||||
}
|
||||
value.and_then(py_float)
|
||||
}
|
||||
|
||||
pub fn safe_convert_to_int(value: Option<&Value>) -> Option<i64> {
|
||||
match value? {
|
||||
Value::Number(value) => value.as_i64().or_else(|| {
|
||||
value
|
||||
.as_f64()
|
||||
.and_then(|value| value.is_finite().then_some(value as i64))
|
||||
}),
|
||||
Value::String(value) => value.parse::<i64>().ok(),
|
||||
_ => None,
|
||||
}
|
||||
value.and_then(py_int)
|
||||
}
|
||||
|
||||
pub fn extract_file_search_params(file_search: &Value) -> (Option<f64>, Option<f64>) {
|
||||
|
|
@ -115,11 +107,7 @@ pub fn response_object_includes_web_search_call(
|
|||
usage: Option<&Value>,
|
||||
) -> bool {
|
||||
if kind == ResponseKind::Anthropic
|
||||
&& response
|
||||
.get("usage")
|
||||
.and_then(|usage| usage.get("server_tool_use"))
|
||||
.and_then(|tools| tools.get("web_search_requests"))
|
||||
.is_some()
|
||||
&& get_anthropic_web_search_requests_from_response(response).is_some()
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
|
@ -133,23 +121,24 @@ pub fn response_object_includes_web_search_call(
|
|||
}
|
||||
}
|
||||
|
||||
pub fn usage_reports_server_side_web_search_calls(details: Option<&Value>) -> bool {
|
||||
match details.and_then(|details| details.get("web_search_calls")) {
|
||||
Some(Value::Bool(flag)) => *flag,
|
||||
Some(Value::Number(calls)) => calls.as_i64().is_some_and(|calls| calls > 0),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn usage_reports_web_search(usage: Option<&Value>) -> bool {
|
||||
let Some(usage) = usage else {
|
||||
return false;
|
||||
};
|
||||
usage
|
||||
.get("server_tool_use")
|
||||
.and_then(|tools| tools.get("web_search_requests"))
|
||||
.is_some()
|
||||
get_web_search_requests(usage.get("server_tool_use")).is_some()
|
||||
|| usage
|
||||
.get("prompt_tokens_details")
|
||||
.and_then(|details| details.get("web_search_requests"))
|
||||
.is_some()
|
||||
|| usage
|
||||
.get("server_side_tool_usage_details")
|
||||
.and_then(|details| details.get("web_search_calls"))
|
||||
.and_then(Value::as_i64)
|
||||
.is_some_and(|calls| calls > 0)
|
||||
.is_some_and(|requests| !requests.is_null())
|
||||
|| usage_reports_server_side_web_search_calls(usage.get("server_side_tool_usage_details"))
|
||||
}
|
||||
|
||||
pub fn count_web_search_calls(response: &Value, kind: ResponseKind) -> u64 {
|
||||
|
|
@ -160,7 +149,8 @@ pub fn count_web_search_calls(response: &Value, kind: ResponseKind) -> u64 {
|
|||
.get("tool_usage")
|
||||
.and_then(|usage| usage.get("web_search"))
|
||||
.and_then(|search| search.get("num_requests"))
|
||||
.and_then(Value::as_u64)
|
||||
.and_then(lax_int)
|
||||
.and_then(|requests| u64::try_from(requests).ok())
|
||||
{
|
||||
return reported;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,10 +1,16 @@
|
|||
use serde_json::Value;
|
||||
use std::borrow::Cow;
|
||||
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use crate::anthropic_cost::{
|
||||
get_anthropic_web_search_requests_from_response, get_cost_for_anthropic_web_search,
|
||||
get_web_search_requests_from_usage,
|
||||
};
|
||||
use crate::gemini_cost::{cost_per_google_maps_grounding_request, cost_per_web_search_request};
|
||||
use crate::catalog::ModelInfoCatalog;
|
||||
use crate::gemini_cost::{
|
||||
cost_per_google_maps_grounding_request, cost_per_web_search_request,
|
||||
google_maps_grounding_requests,
|
||||
};
|
||||
use crate::groq_cost::cost_per_web_search_request as groq_web_search_cost;
|
||||
use crate::provider::LlmProviders;
|
||||
use crate::responses_usage::ChatUsage;
|
||||
|
|
@ -13,7 +19,7 @@ use crate::tool_call_cost_tracking::{
|
|||
extract_token_counts, get_cost_for_code_interpreter, get_cost_for_computer_use,
|
||||
get_cost_for_file_search, get_cost_for_vector_store, get_cost_for_web_search,
|
||||
response_object_includes_file_search_call, response_object_includes_web_search_call,
|
||||
safe_convert_to_int,
|
||||
safe_convert_to_int, usage_reports_server_side_web_search_calls,
|
||||
};
|
||||
use crate::wire::is_truthy;
|
||||
|
||||
|
|
@ -27,110 +33,162 @@ pub struct BuiltInToolCostRequest<'a> {
|
|||
pub defaults: DefaultToolRates,
|
||||
}
|
||||
|
||||
fn server_tool_count(usage: &ChatUsage, key: &str) -> Option<u64> {
|
||||
usage
|
||||
.extra
|
||||
.get("server_tool_use")
|
||||
.and_then(|tools| tools.get(key))
|
||||
.and_then(Value::as_u64)
|
||||
}
|
||||
|
||||
fn usage_reports_web_search(usage: &ChatUsage) -> bool {
|
||||
usage
|
||||
.prompt_tokens_details
|
||||
.as_ref()
|
||||
.and_then(|details| details.web_search_requests)
|
||||
.is_some()
|
||||
|| server_tool_count(usage, "web_search_requests").is_some()
|
||||
|| usage
|
||||
.extra
|
||||
.get("server_side_tool_usage_details")
|
||||
.and_then(|details| details.get("web_search_calls"))
|
||||
.and_then(Value::as_u64)
|
||||
.is_some_and(|calls| calls > 0)
|
||||
|| get_web_search_requests_from_usage(usage).is_some()
|
||||
|| usage_reports_server_side_web_search_calls(
|
||||
usage.extra.get("server_side_tool_usage_details"),
|
||||
)
|
||||
}
|
||||
|
||||
fn web_search_requests(request: BuiltInToolCostRequest<'_>) -> Option<u64> {
|
||||
request
|
||||
.usage
|
||||
.and_then(|usage| server_tool_count(usage, "web_search_requests"))
|
||||
.or_else(|| {
|
||||
request
|
||||
.response
|
||||
.get("usage")
|
||||
.and_then(|usage| usage.get("server_tool_use"))
|
||||
.and_then(|tools| tools.get("web_search_requests"))
|
||||
.and_then(Value::as_u64)
|
||||
})
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
pub struct ResolvedModelInfo<'a> {
|
||||
pub model_info: Option<&'a Value>,
|
||||
pub provider: Option<&'a str>,
|
||||
}
|
||||
|
||||
fn context_rate(model_info: &Value) -> f64 {
|
||||
model_info
|
||||
.get("search_context_cost_per_query")
|
||||
.and_then(|rates| rates.get("search_context_size_medium"))
|
||||
.and_then(Value::as_f64)
|
||||
.unwrap_or(0.0)
|
||||
}
|
||||
|
||||
fn provider_web_search_cost(
|
||||
request: BuiltInToolCostRequest<'_>,
|
||||
model_info: &Value,
|
||||
) -> Option<f64> {
|
||||
let provider = request.provider?;
|
||||
let usage = request.usage;
|
||||
match provider {
|
||||
"gemini" => usage.map(|usage| cost_per_web_search_request(usage, model_info)),
|
||||
"anthropic" => match usage.and_then(get_web_search_requests_from_usage) {
|
||||
Some(_) => Some(get_cost_for_anthropic_web_search(Some(model_info), usage)),
|
||||
None => get_anthropic_web_search_requests_from_response(request.response)
|
||||
.map(|count| context_rate(model_info) * count as f64),
|
||||
pub fn resolve_model_info<'a>(
|
||||
catalog: &'a ModelInfoCatalog,
|
||||
model: &str,
|
||||
provider: Option<&'a str>,
|
||||
region: Option<&str>,
|
||||
) -> ResolvedModelInfo<'a> {
|
||||
let litellm_provider =
|
||||
|model_info: &'a Value| model_info.get("litellm_provider").and_then(Value::as_str);
|
||||
if let Some(direct) = catalog.entry(model, provider, region) {
|
||||
return ResolvedModelInfo {
|
||||
model_info: Some(direct),
|
||||
provider: provider
|
||||
.filter(|provider| !provider.is_empty())
|
||||
.or_else(|| litellm_provider(direct)),
|
||||
};
|
||||
}
|
||||
let by_prefix = model
|
||||
.contains('/')
|
||||
.then(|| catalog.entry(model, None, None))
|
||||
.flatten();
|
||||
match by_prefix {
|
||||
Some(by_prefix) => ResolvedModelInfo {
|
||||
model_info: Some(by_prefix),
|
||||
provider: litellm_provider(by_prefix),
|
||||
},
|
||||
"perplexity" => Some(0.0),
|
||||
"xai" => usage.map(|usage| {
|
||||
crate::xai_cost::cost_per_web_search_request(
|
||||
usage,
|
||||
model_info,
|
||||
request.defaults.xai_web_search_per_call,
|
||||
)
|
||||
}),
|
||||
"groq" => usage.map(|usage| {
|
||||
groq_web_search_cost(
|
||||
usage,
|
||||
model_info,
|
||||
request.defaults.groq_browser_open_per_call,
|
||||
)
|
||||
}),
|
||||
provider if provider.starts_with("vertex_ai") => {
|
||||
None => ResolvedModelInfo {
|
||||
model_info: None,
|
||||
provider,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_cost_for_web_search_request(
|
||||
provider: &str,
|
||||
usage: &ChatUsage,
|
||||
model_info: &Value,
|
||||
defaults: DefaultToolRates,
|
||||
) -> Option<f64> {
|
||||
match provider.parse::<LlmProviders>().ok() {
|
||||
Some(LlmProviders::GEMINI) => Some(cost_per_web_search_request(usage, model_info)),
|
||||
Some(LlmProviders::ANTHROPIC) => Some(get_cost_for_anthropic_web_search(
|
||||
Some(model_info),
|
||||
Some(usage),
|
||||
)),
|
||||
_ if provider.starts_with(LlmProviders::VERTEX_AI.as_str()) => {
|
||||
let is_claude = model_info
|
||||
.get("key")
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(|key| key.to_ascii_lowercase().contains("claude"));
|
||||
if is_claude {
|
||||
web_search_requests(request).map(|count| context_rate(model_info) * count as f64)
|
||||
Some(if is_claude {
|
||||
get_cost_for_anthropic_web_search(Some(model_info), Some(usage))
|
||||
} else {
|
||||
usage.map(|usage| cost_per_web_search_request(usage, model_info))
|
||||
}
|
||||
cost_per_web_search_request(usage, model_info)
|
||||
})
|
||||
}
|
||||
Some(LlmProviders::PERPLEXITY) => Some(0.0),
|
||||
Some(LlmProviders::XAI) => Some(crate::xai_cost::cost_per_web_search_request(
|
||||
usage,
|
||||
model_info,
|
||||
defaults.xai_web_search_per_call,
|
||||
)),
|
||||
Some(LlmProviders::GROQ) => Some(groq_web_search_cost(
|
||||
usage,
|
||||
model_info,
|
||||
defaults.groq_browser_open_per_call,
|
||||
)),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn maps_cost(request: BuiltInToolCostRequest<'_>, model_info: Option<&Value>) -> f64 {
|
||||
let supports_maps = LlmProviders::GEMINI.matches(request.provider)
|
||||
|| request
|
||||
.provider
|
||||
.is_some_and(|provider| provider.starts_with("vertex_ai"));
|
||||
if !supports_maps {
|
||||
return 0.0;
|
||||
pub fn get_cost_for_google_maps_grounding_request(
|
||||
provider: &str,
|
||||
usage: &ChatUsage,
|
||||
model_info: &Value,
|
||||
) -> Option<f64> {
|
||||
(LlmProviders::GEMINI.matches(Some(provider))
|
||||
|| provider.starts_with(LlmProviders::VERTEX_AI.as_str()))
|
||||
.then(|| cost_per_google_maps_grounding_request(usage, model_info))
|
||||
}
|
||||
|
||||
fn usage_with_anthropic_web_search<'a>(
|
||||
usage: Option<&'a ChatUsage>,
|
||||
response: &Value,
|
||||
kind: ResponseKind,
|
||||
) -> Option<Cow<'a, ChatUsage>> {
|
||||
if usage.is_some_and(|usage| get_web_search_requests_from_usage(usage).is_some()) {
|
||||
return usage.map(Cow::Borrowed);
|
||||
}
|
||||
match (request.usage, model_info) {
|
||||
(Some(usage), Some(model_info)) => {
|
||||
cost_per_google_maps_grounding_request(usage, model_info)
|
||||
}
|
||||
let from_response = (kind == ResponseKind::Anthropic)
|
||||
.then(|| get_anthropic_web_search_requests_from_response(response))
|
||||
.flatten();
|
||||
let Some(web_search_requests) = from_response else {
|
||||
return usage.map(Cow::Borrowed);
|
||||
};
|
||||
let base = usage.cloned().unwrap_or_default();
|
||||
let extra = base
|
||||
.extra
|
||||
.into_iter()
|
||||
.filter(|(key, _)| key != "server_tool_use")
|
||||
.chain([(
|
||||
"server_tool_use".to_owned(),
|
||||
json!({"web_search_requests": web_search_requests}),
|
||||
)])
|
||||
.collect();
|
||||
Some(Cow::Owned(ChatUsage { extra, ..base }))
|
||||
}
|
||||
|
||||
fn maps_cost(usage: Option<&ChatUsage>, resolved: ResolvedModelInfo<'_>) -> f64 {
|
||||
let Some(usage) = usage.filter(|usage| google_maps_grounding_requests(Some(usage)).is_some())
|
||||
else {
|
||||
return 0.0;
|
||||
};
|
||||
match resolved {
|
||||
ResolvedModelInfo {
|
||||
model_info: Some(model_info),
|
||||
provider: Some(provider),
|
||||
} => get_cost_for_google_maps_grounding_request(provider, usage, model_info).unwrap_or(0.0),
|
||||
_ => 0.0,
|
||||
}
|
||||
}
|
||||
|
||||
fn web_search_cost(request: BuiltInToolCostRequest<'_>, resolved: ResolvedModelInfo<'_>) -> f64 {
|
||||
let usage =
|
||||
usage_with_anthropic_web_search(request.usage, request.response, request.response_kind);
|
||||
let routed = match (resolved.model_info, usage.as_deref(), resolved.provider) {
|
||||
(Some(model_info), Some(usage), Some(provider)) => {
|
||||
get_cost_for_web_search_request(provider, usage, model_info, request.defaults)
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
routed.unwrap_or_else(|| {
|
||||
get_cost_for_web_search(
|
||||
request.params.get("web_search_options"),
|
||||
resolved.model_info,
|
||||
) * count_web_search_calls(request.response, request.response_kind) as f64
|
||||
})
|
||||
}
|
||||
|
||||
fn file_search_cost(request: BuiltInToolCostRequest<'_>, model_info: Option<&Value>) -> f64 {
|
||||
let file_search = request
|
||||
.params
|
||||
|
|
@ -179,24 +237,23 @@ fn azure_assistant_cost(request: BuiltInToolCostRequest<'_>, model_info: Option<
|
|||
}
|
||||
|
||||
pub fn get_cost_for_built_in_tools(
|
||||
catalog: &ModelInfoCatalog,
|
||||
model: &str,
|
||||
region: Option<&str>,
|
||||
request: BuiltInToolCostRequest<'_>,
|
||||
model_info: Option<&Value>,
|
||||
) -> f64 {
|
||||
let maps = maps_cost(request, model_info);
|
||||
let resolved = resolve_model_info(catalog, model, request.provider, region);
|
||||
let maps = maps_cost(request.usage, resolved);
|
||||
let web_search =
|
||||
response_object_includes_web_search_call(request.response, request.response_kind, None)
|
||||
|| (request.response_kind != ResponseKind::Responses
|
||||
&& request.usage.is_some_and(usage_reports_web_search));
|
||||
if web_search {
|
||||
let routed =
|
||||
model_info.and_then(|model_info| provider_web_search_cost(request, model_info));
|
||||
let fallback =
|
||||
get_cost_for_web_search(request.params.get("web_search_options"), model_info)
|
||||
* count_web_search_calls(request.response, request.response_kind) as f64;
|
||||
return maps + routed.unwrap_or(fallback);
|
||||
return maps + web_search_cost(request, resolved);
|
||||
}
|
||||
let direct = catalog.entry(model, request.provider, region);
|
||||
if response_object_includes_file_search_call(request.response, request.response_kind) {
|
||||
return maps + file_search_cost(request, model_info);
|
||||
return maps + file_search_cost(request, direct);
|
||||
}
|
||||
maps + azure_assistant_cost(request, model_info)
|
||||
maps + azure_assistant_cost(request, direct)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -83,6 +83,55 @@ pub fn is_truthy(value: &Value) -> bool {
|
|||
}
|
||||
}
|
||||
|
||||
fn python_int_literal(text: &str) -> Option<i64> {
|
||||
let digits = text.strip_prefix(['+', '-']).unwrap_or(text);
|
||||
let well_formed = digits
|
||||
.split('_')
|
||||
.all(|group| !group.is_empty() && group.bytes().all(|byte| byte.is_ascii_digit()));
|
||||
if !well_formed {
|
||||
return None;
|
||||
}
|
||||
text.replace('_', "").parse().ok()
|
||||
}
|
||||
|
||||
pub fn py_int(value: &Value) -> Option<i64> {
|
||||
match value {
|
||||
Value::Number(number) => number.as_i64().or_else(|| {
|
||||
number
|
||||
.as_f64()
|
||||
.map(f64::trunc)
|
||||
.filter(|float| float.abs() < i64::MAX as f64)
|
||||
.map(|float| float as i64)
|
||||
}),
|
||||
Value::Bool(flag) => Some(i64::from(*flag)),
|
||||
Value::String(text) => python_int_literal(text.trim()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn py_float(value: &Value) -> Option<f64> {
|
||||
match value {
|
||||
Value::Number(number) => number.as_f64(),
|
||||
Value::Bool(flag) => Some(f64::from(u8::from(*flag))),
|
||||
Value::String(text) => text.trim().parse().ok(),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn lax_int(value: &Value) -> Option<i64> {
|
||||
match value {
|
||||
Value::Number(number) => number.as_i64().or_else(|| {
|
||||
number
|
||||
.as_f64()
|
||||
.filter(|float| float.fract() == 0.0 && float.abs() < i64::MAX as f64)
|
||||
.map(|float| float as i64)
|
||||
}),
|
||||
Value::Bool(flag) => Some(i64::from(*flag)),
|
||||
Value::String(text) => python_int_literal(text.trim()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn rate(value: Option<&Value>) -> Rate {
|
||||
match value {
|
||||
None => Rate::Missing,
|
||||
|
|
@ -838,6 +887,52 @@ mod tests {
|
|||
))
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case(serde_json::json!(3), Some(3))]
|
||||
#[case(serde_json::json!(3.9), Some(3))]
|
||||
#[case(serde_json::json!(-3.9), Some(-3))]
|
||||
#[case(serde_json::json!(true), Some(1))]
|
||||
#[case(serde_json::json!(" 12 "), Some(12))]
|
||||
#[case(serde_json::json!("1_0"), Some(10))]
|
||||
#[case(serde_json::json!("3.5"), None)]
|
||||
#[case(serde_json::json!(null), None)]
|
||||
#[case(serde_json::json!({}), None)]
|
||||
fn py_int_follows_python_int(#[case] value: Value, #[case] expected: Option<i64>) {
|
||||
assert_eq!(py_int(&value), expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case(serde_json::json!(2), Some(2.0))]
|
||||
#[case(serde_json::json!(0.25), Some(0.25))]
|
||||
#[case(serde_json::json!(false), Some(0.0))]
|
||||
#[case(serde_json::json!(" 0.5 "), Some(0.5))]
|
||||
#[case(serde_json::json!("abc"), None)]
|
||||
#[case(serde_json::json!(null), None)]
|
||||
#[case(serde_json::json!([1]), None)]
|
||||
fn py_float_follows_python_float(#[case] value: Value, #[case] expected: Option<f64>) {
|
||||
assert_eq!(py_float(&value), expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case(serde_json::json!(3), Some(3))]
|
||||
#[case(serde_json::json!(-1), Some(-1))]
|
||||
#[case(serde_json::json!(2.0), Some(2))]
|
||||
#[case(serde_json::json!(2.5), None)]
|
||||
#[case(serde_json::json!(true), Some(1))]
|
||||
#[case(serde_json::json!("3"), Some(3))]
|
||||
#[case(serde_json::json!(" 3 "), Some(3))]
|
||||
#[case(serde_json::json!("-4"), Some(-4))]
|
||||
#[case(serde_json::json!("1_000"), Some(1000))]
|
||||
#[case(serde_json::json!("1__000"), None)]
|
||||
#[case(serde_json::json!("_1"), None)]
|
||||
#[case(serde_json::json!("3.0"), None)]
|
||||
#[case(serde_json::json!("x"), None)]
|
||||
#[case(serde_json::json!(null), None)]
|
||||
#[case(serde_json::json!([3]), None)]
|
||||
fn lax_int_follows_pydantic_lax_int(#[case] value: Value, #[case] expected: Option<i64>) {
|
||||
assert_eq!(lax_int(&value), expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case(serde_json::json!(null), false)]
|
||||
#[case(serde_json::json!(false), false)]
|
||||
|
|
|
|||
|
|
@ -17,6 +17,17 @@ use litellm_cost::usage_dispatch::get_usage_object;
|
|||
use rstest::rstest;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
fn priced(request: BuiltInToolCostRequest<'_>, model_info: Option<&Value>) -> f64 {
|
||||
let provider = request.provider.expect("tests price a provider");
|
||||
let catalog = ModelInfoCatalog::new(
|
||||
model_info
|
||||
.map(|info| (format!("{provider}/model"), info.clone()))
|
||||
.into_iter()
|
||||
.collect(),
|
||||
);
|
||||
get_cost_for_built_in_tools(&catalog, "model", None, request)
|
||||
}
|
||||
|
||||
fn usage(value: &Value) -> litellm_cost::responses_usage::ChatUsage {
|
||||
get_usage_object(&json!({"usage": value})).unwrap().unwrap()
|
||||
}
|
||||
|
|
@ -91,7 +102,10 @@ fn get_anthropic_web_search_requests_from_response_reads_nested_usage(
|
|||
#[case(json!({"web_search_requests": 0}), Some(0))]
|
||||
#[case(json!({"web_search_requests": 3}), Some(3))]
|
||||
#[case(json!({}), None)]
|
||||
#[case(json!({"web_search_requests": "3"}), None)]
|
||||
#[case(json!({"web_search_requests": "3"}), Some(3))]
|
||||
#[case(json!({"web_search_requests": 2.0}), Some(2))]
|
||||
#[case(json!({"web_search_requests": 2.5}), None)]
|
||||
#[case(json!({"web_search_requests": null}), None)]
|
||||
fn get_web_search_requests_reads_serialized_server_tool_use(
|
||||
#[case] server_tool_use: Value,
|
||||
#[case] expected: Option<i64>,
|
||||
|
|
@ -114,7 +128,7 @@ fn get_cost_for_anthropic_web_search_prices_server_tool_requests() {
|
|||
get_cost_for_anthropic_web_search(Some(&model_info), None),
|
||||
0.0
|
||||
);
|
||||
let dispatched = get_cost_for_built_in_tools(
|
||||
let dispatched = priced(
|
||||
BuiltInToolCostRequest {
|
||||
response: &json!({}),
|
||||
response_kind: ResponseKind::Anthropic,
|
||||
|
|
|
|||
|
|
@ -1,12 +1,13 @@
|
|||
#![allow(clippy::disallowed_types)]
|
||||
// mirrors: test_litellm/test_cost_calculator.py::test_cost_per_token_duplicate_openai_prefix_matches_model_cost
|
||||
// mirrors: test_litellm/test_utils.py::test_check_provider_match_azure_ai_allows_openai_and_azure
|
||||
use litellm_cost::cost_calculator::{cost_per_token, cost_per_token_for_call};
|
||||
use litellm_cost::error::CostError;
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use jiff::Timestamp;
|
||||
use litellm_cost::catalog::{ModelCostRequest, ModelInfoCatalog};
|
||||
use litellm_cost::catalog::{ModelCostRequest, ModelInfoCatalog, check_provider_match};
|
||||
use litellm_cost::usage_dispatch::get_usage_object;
|
||||
use rstest::rstest;
|
||||
use serde_json::json;
|
||||
|
|
@ -209,3 +210,57 @@ fn model_info_catalog_applies_xai_inclusive_threshold_policy() {
|
|||
assert!((actual.0 - 128_000.0 * 5e-6).abs() < 1e-12);
|
||||
assert!((actual.1 - 10.0 * 7e-6).abs() < 1e-12);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::no_requested_provider(Some("gemini"), None, true)]
|
||||
#[case::empty_requested_provider(Some("gemini"), Some(""), true)]
|
||||
#[case::entry_without_provider(None, Some("openai"), true)]
|
||||
#[case::same_provider(Some("openai"), Some("openai"), true)]
|
||||
#[case::different_provider(Some("gemini"), Some("openai"), false)]
|
||||
#[case::vertex_family(Some("vertex_ai-language-models"), Some("vertex_ai"), true)]
|
||||
#[case::vertex_family_is_one_way(Some("vertex_ai"), Some("vertex_ai-language-models"), false)]
|
||||
#[case::fireworks_family(Some("fireworks_ai-embedding-models"), Some("fireworks_ai"), true)]
|
||||
#[case::bedrock_family(Some("bedrock_converse"), Some("bedrock"), true)]
|
||||
#[case::litellm_proxy_matches_anything(Some("anthropic"), Some("litellm_proxy"), true)]
|
||||
#[case::azure_ai_falls_back_to_azure(Some("azure"), Some("azure_ai"), true)]
|
||||
#[case::azure_ai_falls_back_to_openai(Some("openai"), Some("azure_ai"), true)]
|
||||
#[case::azure_does_not_fall_back_to_azure_ai(Some("azure_ai"), Some("azure"), false)]
|
||||
#[case::github_reuses_any_entry(Some("openai"), Some("github"), true)]
|
||||
fn check_provider_match_follows_python(
|
||||
#[case] entry_provider: Option<&str>,
|
||||
#[case] requested: Option<&str>,
|
||||
#[case] expected: bool,
|
||||
) {
|
||||
let model_info = json!({"litellm_provider": entry_provider});
|
||||
assert_eq!(check_provider_match(&model_info, requested), expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::matching_provider(Some("gemini"), Some("gemini/gemini-x"))]
|
||||
#[case::no_provider(None, Some("gemini/gemini-x"))]
|
||||
#[case::mismatched_provider(Some("openai"), None)]
|
||||
fn select_model_key_skips_entries_owned_by_another_provider(
|
||||
#[case] provider: Option<&str>,
|
||||
#[case] expected: Option<&str>,
|
||||
) {
|
||||
let catalog = ModelInfoCatalog::new(HashMap::from([(
|
||||
"gemini/gemini-x".to_owned(),
|
||||
json!({"litellm_provider": "gemini", "input_cost_per_token": 1e-6}),
|
||||
)]));
|
||||
assert_eq!(
|
||||
catalog.select_model_key("gemini/gemini-x", provider, None),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn select_model_key_lets_vertex_ai_beta_use_vertex_entries() {
|
||||
let catalog = ModelInfoCatalog::new(HashMap::from([(
|
||||
"vertex_ai/gemini-x".to_owned(),
|
||||
json!({"litellm_provider": "vertex_ai-language-models"}),
|
||||
)]));
|
||||
assert_eq!(
|
||||
catalog.select_model_key("vertex_ai/gemini-x", Some("vertex_ai_beta"), None),
|
||||
Some("vertex_ai/gemini-x")
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,12 +2,28 @@
|
|||
|
||||
// mirrors: test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py::test_get_cost_for_built_in_tools_web_search
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use jiff::Timestamp;
|
||||
use litellm_cost::catalog::{ModelCostRequest, ModelInfoCatalog};
|
||||
use litellm_cost::cost_calculator::{BuiltInToolCharge, CompletionCostRequest};
|
||||
use litellm_cost::tool_call_cost_tracking::{DefaultToolRates, ResponseKind};
|
||||
use litellm_cost::tool_cost_dispatch::{BuiltInToolCostRequest, get_cost_for_built_in_tools};
|
||||
use litellm_cost::usage_dispatch::get_usage_object;
|
||||
use rstest::rstest;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
fn priced(request: BuiltInToolCostRequest<'_>, model_info: Option<&Value>) -> f64 {
|
||||
let provider = request.provider.expect("tests price a provider");
|
||||
let catalog = ModelInfoCatalog::new(
|
||||
model_info
|
||||
.map(|info| (format!("{provider}/model"), info.clone()))
|
||||
.into_iter()
|
||||
.collect(),
|
||||
);
|
||||
get_cost_for_built_in_tools(&catalog, "model", None, request)
|
||||
}
|
||||
|
||||
fn defaults() -> DefaultToolRates {
|
||||
DefaultToolRates {
|
||||
file_search_per_call: 0.25,
|
||||
|
|
@ -42,7 +58,7 @@ fn get_cost_for_built_in_tools_adds_gemini_search_and_maps(
|
|||
"search_context_cost_per_query": {"search_context_size_medium": 0.01},
|
||||
"google_maps_grounding_cost_per_query": 0.02
|
||||
});
|
||||
let actual = get_cost_for_built_in_tools(
|
||||
let actual = priced(
|
||||
BuiltInToolCostRequest {
|
||||
response: &json!({"choices": []}),
|
||||
response_kind: ResponseKind::Chat,
|
||||
|
|
@ -62,7 +78,7 @@ fn get_cost_for_built_in_tools_prefers_reported_responses_search_count() {
|
|||
"output": [{"type": "web_search_call"}, {"type": "web_search_call"}],
|
||||
"tool_usage": {"web_search": {"num_requests": 1}}
|
||||
});
|
||||
let actual = get_cost_for_built_in_tools(
|
||||
let actual = priced(
|
||||
BuiltInToolCostRequest {
|
||||
response: &response,
|
||||
response_kind: ResponseKind::Responses,
|
||||
|
|
@ -79,7 +95,7 @@ fn get_cost_for_built_in_tools_prefers_reported_responses_search_count() {
|
|||
#[rstest]
|
||||
fn get_cost_for_built_in_tools_reads_anthropic_raw_web_search_usage() {
|
||||
let response = json!({"usage": {"server_tool_use": {"web_search_requests": 2}}});
|
||||
let actual = get_cost_for_built_in_tools(
|
||||
let actual = priced(
|
||||
BuiltInToolCostRequest {
|
||||
response: &response,
|
||||
response_kind: ResponseKind::Anthropic,
|
||||
|
|
@ -106,7 +122,7 @@ fn get_cost_for_built_in_tools_uses_xai_reported_cost_or_search_calls(
|
|||
"cost": reported,
|
||||
"server_side_tool_usage_details": {"web_search_calls": 2}
|
||||
}));
|
||||
let actual = get_cost_for_built_in_tools(
|
||||
let actual = priced(
|
||||
BuiltInToolCostRequest {
|
||||
response: &json!({"choices": []}),
|
||||
response_kind: ResponseKind::Chat,
|
||||
|
|
@ -127,7 +143,7 @@ fn get_cost_for_built_in_tools_prices_groq_search_and_browser_open() {
|
|||
"completion_tokens": 2,
|
||||
"server_tool_use": {"web_search_requests": 2, "browser_open_requests": 3}
|
||||
}));
|
||||
let actual = get_cost_for_built_in_tools(
|
||||
let actual = priced(
|
||||
BuiltInToolCostRequest {
|
||||
response: &json!({"choices": []}),
|
||||
response_kind: ResponseKind::Chat,
|
||||
|
|
@ -143,7 +159,7 @@ fn get_cost_for_built_in_tools_prices_groq_search_and_browser_open() {
|
|||
|
||||
#[rstest]
|
||||
fn get_cost_for_built_in_tools_prices_file_search_before_assistant_features() {
|
||||
let actual = get_cost_for_built_in_tools(
|
||||
let actual = priced(
|
||||
BuiltInToolCostRequest {
|
||||
response: &json!({"output": [{"type": "file_search_call"}]}),
|
||||
response_kind: ResponseKind::Responses,
|
||||
|
|
@ -162,7 +178,7 @@ fn get_cost_for_built_in_tools_prices_file_search_before_assistant_features() {
|
|||
|
||||
#[rstest]
|
||||
fn get_cost_for_built_in_tools_adds_azure_assistant_features() {
|
||||
let actual = get_cost_for_built_in_tools(
|
||||
let actual = priced(
|
||||
BuiltInToolCostRequest {
|
||||
response: &json!({"output": []}),
|
||||
response_kind: ResponseKind::Responses,
|
||||
|
|
@ -226,8 +242,167 @@ fn model_info_catalog_completion_cost_dispatches_built_in_tools() {
|
|||
.unwrap();
|
||||
assert!((actual.total - (100.0 * 2e-6 + 50.0 * 4e-6 + 2.0 * 0.01)).abs() < 1e-12);
|
||||
}
|
||||
use std::collections::HashMap;
|
||||
|
||||
use jiff::Timestamp;
|
||||
use litellm_cost::catalog::{ModelCostRequest, ModelInfoCatalog};
|
||||
use litellm_cost::cost_calculator::{BuiltInToolCharge, CompletionCostRequest};
|
||||
fn gemini_catalog() -> ModelInfoCatalog {
|
||||
ModelInfoCatalog::new(HashMap::from([(
|
||||
"gemini/gemini-x".to_owned(),
|
||||
json!({
|
||||
"litellm_provider": "gemini",
|
||||
"web_search_billing_unit": "per_query",
|
||||
"search_context_cost_per_query": {"search_context_size_medium": 0.014},
|
||||
"google_maps_grounding_cost_per_query": 0.0125
|
||||
}),
|
||||
)]))
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::no_provider(None)]
|
||||
#[case::provider_the_entry_does_not_belong_to(Some("openai"))]
|
||||
fn get_cost_for_built_in_tools_adopts_the_provider_of_the_resolved_entry(
|
||||
#[case] provider: Option<&str>,
|
||||
) {
|
||||
let usage = usage(&json!({
|
||||
"prompt_tokens": 10,
|
||||
"completion_tokens": 2,
|
||||
"prompt_tokens_details": {"web_search_requests": 3, "google_maps_grounding_requests": 2}
|
||||
}));
|
||||
let actual = get_cost_for_built_in_tools(
|
||||
&gemini_catalog(),
|
||||
"gemini/gemini-x",
|
||||
None,
|
||||
BuiltInToolCostRequest {
|
||||
response: &json!({"choices": []}),
|
||||
response_kind: ResponseKind::Chat,
|
||||
usage: Some(&usage),
|
||||
provider,
|
||||
params: &json!({}),
|
||||
defaults: defaults(),
|
||||
},
|
||||
);
|
||||
assert!((actual - (3.0 * 0.014 + 2.0 * 0.0125)).abs() < 1e-12);
|
||||
}
|
||||
|
||||
fn cited_chat_response() -> Value {
|
||||
json!({"choices": [{"message": {"annotations": [{"type": "url_citation"}]}}]})
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn anthropic_usage_without_a_search_count_bills_nothing_instead_of_the_per_call_price() {
|
||||
let usage = usage(&json!({"prompt_tokens": 10, "completion_tokens": 2}));
|
||||
let actual = priced(
|
||||
BuiltInToolCostRequest {
|
||||
response: &cited_chat_response(),
|
||||
response_kind: ResponseKind::Chat,
|
||||
usage: Some(&usage),
|
||||
provider: Some("anthropic"),
|
||||
params: &json!({}),
|
||||
defaults: defaults(),
|
||||
},
|
||||
Some(&json!({"search_context_cost_per_query": {"search_context_size_medium": 0.01}})),
|
||||
);
|
||||
assert_eq!(actual, 0.0);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::with_usage(true, 0.0)]
|
||||
#[case::without_usage(false, 0.01)]
|
||||
fn perplexity_search_is_free_only_when_usage_reaches_the_router(
|
||||
#[case] with_usage: bool,
|
||||
#[case] expected: f64,
|
||||
) {
|
||||
let usage = usage(&json!({"prompt_tokens": 10, "completion_tokens": 2}));
|
||||
let actual = priced(
|
||||
BuiltInToolCostRequest {
|
||||
response: &cited_chat_response(),
|
||||
response_kind: ResponseKind::Chat,
|
||||
usage: with_usage.then_some(&usage),
|
||||
provider: Some("perplexity"),
|
||||
params: &json!({}),
|
||||
defaults: defaults(),
|
||||
},
|
||||
Some(&json!({"search_context_cost_per_query": {"search_context_size_medium": 0.01}})),
|
||||
);
|
||||
assert_eq!(actual, expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::per_prompt(json!("per_prompt"), 0.01)]
|
||||
#[case::unset(json!(null), 0.01)]
|
||||
#[case::empty(json!(""), 0.01)]
|
||||
#[case::per_query(json!("per_query"), 0.03)]
|
||||
#[case::unrecognized(json!("per_request"), 0.03)]
|
||||
fn gemini_bills_one_search_only_for_per_prompt_units(
|
||||
#[case] billing_unit: Value,
|
||||
#[case] expected: f64,
|
||||
) {
|
||||
let usage = usage(&json!({
|
||||
"prompt_tokens": 10,
|
||||
"completion_tokens": 2,
|
||||
"prompt_tokens_details": {"web_search_requests": 3}
|
||||
}));
|
||||
let actual = priced(
|
||||
BuiltInToolCostRequest {
|
||||
response: &json!({"choices": []}),
|
||||
response_kind: ResponseKind::Chat,
|
||||
usage: Some(&usage),
|
||||
provider: Some("gemini"),
|
||||
params: &json!({}),
|
||||
defaults: defaults(),
|
||||
},
|
||||
Some(&json!({
|
||||
"web_search_billing_unit": billing_unit,
|
||||
"search_context_cost_per_query": {"search_context_size_medium": 0.01}
|
||||
})),
|
||||
);
|
||||
assert!((actual - expected).abs() < 1e-12);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::string_count(json!("3"), 0.03)]
|
||||
#[case::integral_float_count(json!(2.0), 0.02)]
|
||||
#[case::null_count_is_not_a_search(json!(null), 0.0)]
|
||||
fn raw_anthropic_search_counts_are_coerced_like_pydantic(
|
||||
#[case] requests: Value,
|
||||
#[case] expected: f64,
|
||||
) {
|
||||
let response = json!({"usage": {"server_tool_use": {"web_search_requests": requests}}});
|
||||
let actual = priced(
|
||||
BuiltInToolCostRequest {
|
||||
response: &response,
|
||||
response_kind: ResponseKind::Anthropic,
|
||||
usage: None,
|
||||
provider: Some("anthropic"),
|
||||
params: &json!({}),
|
||||
defaults: defaults(),
|
||||
},
|
||||
Some(&json!({"search_context_cost_per_query": {"search_context_size_medium": 0.01}})),
|
||||
);
|
||||
assert!((actual - expected).abs() < 1e-12);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::integer(json!(1), 0.015)]
|
||||
#[case::boolean_counts_as_one_call(json!(true), 0.015)]
|
||||
#[case::float_is_not_an_int(json!(1.0), 0.0)]
|
||||
fn xai_server_side_search_calls_follow_python_isinstance_int(
|
||||
#[case] calls: Value,
|
||||
#[case] expected: f64,
|
||||
) {
|
||||
let usage = usage(&json!({
|
||||
"prompt_tokens": 10,
|
||||
"completion_tokens": 2,
|
||||
"server_side_tool_usage_details": {"web_search_calls": calls}
|
||||
}));
|
||||
let actual = priced(
|
||||
BuiltInToolCostRequest {
|
||||
response: &json!({"choices": []}),
|
||||
response_kind: ResponseKind::Chat,
|
||||
usage: Some(&usage),
|
||||
provider: Some("xai"),
|
||||
params: &json!({}),
|
||||
defaults: defaults(),
|
||||
},
|
||||
Some(&json!({"search_context_cost_per_query": {"search_context_size_medium": 0.015}})),
|
||||
);
|
||||
assert!((actual - expected).abs() < 1e-12);
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue