mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
refactor(cost): form cost_calculator.rs as the public token entry surface
cost_per_token, cost_per_token_for_call, and their custom-pricing variants move out of ModelInfoCatalog into cost_calculator.rs as free functions mirroring cost_calculator.py's entry points, taking the catalog only for entry lookup and model-key selection. Together with the provider-module moves this starts the split; speech, realtime, image, and billed-rate entry points still to migrate
This commit is contained in:
parent
14201a6c2e
commit
a8189817fa
21 changed files with 623 additions and 1753 deletions
|
|
@ -3,14 +3,9 @@ use std::collections::HashMap;
|
||||||
use jiff::Timestamp;
|
use jiff::Timestamp;
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
|
|
||||||
use crate::anthropic_cost::fast_speed_multiplier;
|
|
||||||
use crate::azure_ai_cost::azure_ai_cost_per_token;
|
|
||||||
use crate::azure_ai_image_cost::{
|
use crate::azure_ai_image_cost::{
|
||||||
AzureAiImageRequest, cost_calculator as azure_ai_image_cost_calculator,
|
AzureAiImageRequest, cost_calculator as azure_ai_image_cost_calculator,
|
||||||
};
|
};
|
||||||
use crate::azure_cost::output_per_second_cost;
|
|
||||||
use crate::base_rate_selection::uses_inclusive_token_thresholds;
|
|
||||||
use crate::batch::batch_cost_from_model_info;
|
|
||||||
use crate::bedrock_image_cost::cost_calculator as bedrock_image_cost_calculator;
|
use crate::bedrock_image_cost::cost_calculator as bedrock_image_cost_calculator;
|
||||||
use crate::billed_token_rates::{
|
use crate::billed_token_rates::{
|
||||||
BilledRatesRequest, BilledTokenRates, TokenTypeCostBreakdown,
|
BilledRatesRequest, BilledTokenRates, TokenTypeCostBreakdown,
|
||||||
|
|
@ -23,43 +18,32 @@ use crate::completion_cost::{
|
||||||
use crate::completion_input::{
|
use crate::completion_input::{
|
||||||
CompletionInputRequest, PreparedCompletionInput, prepare_completion_input,
|
CompletionInputRequest, PreparedCompletionInput, prepare_completion_input,
|
||||||
};
|
};
|
||||||
use crate::custom_pricing::{CustomPricing, CustomTokenRates, cost_from_chat_usage};
|
use crate::cost_calculator::cost_per_token;
|
||||||
use crate::dashscope_cost::cost_per_token as dashscope_cost_per_token;
|
use crate::custom_pricing::CustomTokenRates;
|
||||||
use crate::databricks_cost::databricks_cost_per_token;
|
|
||||||
use crate::error::CostError;
|
use crate::error::CostError;
|
||||||
use crate::fal_ai_image_cost::{
|
use crate::fal_ai_image_cost::{
|
||||||
cost_calculator as fal_ai_image_cost_calculator,
|
cost_calculator as fal_ai_image_cost_calculator,
|
||||||
fal_ai_passthrough_cost as calculate_fal_ai_passthrough_cost,
|
fal_ai_passthrough_cost as calculate_fal_ai_passthrough_cost,
|
||||||
};
|
};
|
||||||
use crate::fireworks_cost::{
|
|
||||||
FireworksThresholds, cost_per_token as fireworks_cost_per_token, get_base_model_for_pricing,
|
|
||||||
};
|
|
||||||
use crate::generic_cost::calculate_generic_cost_from_model_info_with_region;
|
use crate::generic_cost::calculate_generic_cost_from_model_info_with_region;
|
||||||
use crate::image_response_cost::{
|
use crate::image_response_cost::{
|
||||||
calculate_image_response_cost_from_usage, flat_image_cost, gemini_image_edit_cost,
|
calculate_image_response_cost_from_usage, flat_image_cost, gemini_image_edit_cost,
|
||||||
gemini_image_generation_cost, resolve_image_model_info, vertex_image_edit_cost,
|
gemini_image_generation_cost, resolve_image_model_info, vertex_image_edit_cost,
|
||||||
vertex_image_generation_cost,
|
vertex_image_generation_cost,
|
||||||
};
|
};
|
||||||
use crate::lemonade_cost::lemonade_cost_per_token;
|
|
||||||
use crate::model_selection::{
|
use crate::model_selection::{
|
||||||
ModelSelectionRequest, get_provider_for_cost_calc, select_model_name_for_cost_calc,
|
ModelSelectionRequest, get_provider_for_cost_calc, select_model_name_for_cost_calc,
|
||||||
};
|
};
|
||||||
use crate::non_token::{ImageRates, ImageUsage, calculate_image};
|
use crate::non_token::{ImageRates, ImageUsage, calculate_image};
|
||||||
use crate::ocr_cost::ocr_cost;
|
|
||||||
use crate::openai_cost::video_generation_cost as calculate_video_generation_cost;
|
use crate::openai_cost::video_generation_cost as calculate_video_generation_cost;
|
||||||
use crate::openai_image_cost::cost_calculator as openai_image_cost_calculator;
|
use crate::openai_image_cost::cost_calculator as openai_image_cost_calculator;
|
||||||
use crate::per_second::{has_token_or_tiered_pricing, per_second_pricing_cost};
|
|
||||||
use crate::perplexity_cost::cost_per_token as perplexity_cost_per_token;
|
|
||||||
use crate::prompt_caching_savings::{
|
use crate::prompt_caching_savings::{
|
||||||
PromptCachingSavingsRequest, calculate_prompt_caching_savings,
|
PromptCachingSavingsRequest, calculate_prompt_caching_savings,
|
||||||
};
|
};
|
||||||
use crate::provider::LlmProviders;
|
|
||||||
use crate::provider_cache::apply_provider_cache_read_default;
|
|
||||||
use crate::realtime_cost::{
|
use crate::realtime_cost::{
|
||||||
combine_usage_objects, event_usage, get_transcription_model_name_from_results,
|
combine_usage_objects, event_usage, get_transcription_model_name_from_results,
|
||||||
partition_results_by_service_tier, transcription_usage_cost,
|
partition_results_by_service_tier, transcription_usage_cost,
|
||||||
};
|
};
|
||||||
use crate::regional_uplift::get_provider_specific_geo_multiplier;
|
|
||||||
use crate::responses_usage::ChatUsage;
|
use crate::responses_usage::ChatUsage;
|
||||||
use crate::retrieval_cost::{rerank_cost, vector_store_search_cost};
|
use crate::retrieval_cost::{rerank_cost, vector_store_search_cost};
|
||||||
use crate::search_cost::{
|
use crate::search_cost::{
|
||||||
|
|
@ -70,13 +54,7 @@ use crate::speech_cost::{
|
||||||
SpeechCostMetric, cost_per_second, generic_cost_per_character, lyria_generation_cost,
|
SpeechCostMetric, cost_per_second, generic_cost_per_character, lyria_generation_cost,
|
||||||
select_cost_metric_for_model, transcription_usage_has_token_details,
|
select_cost_metric_for_model, transcription_usage_has_token_details,
|
||||||
};
|
};
|
||||||
use crate::together_cost::{
|
|
||||||
TogetherThresholds, get_model_params_and_category, has_together_registry_pricing,
|
|
||||||
together_ai_cost_per_token,
|
|
||||||
};
|
|
||||||
use crate::tool_cost_dispatch::{BuiltInToolCostRequest, get_cost_for_built_in_tools};
|
use crate::tool_cost_dispatch::{BuiltInToolCostRequest, get_cost_for_built_in_tools};
|
||||||
use crate::vertex_cost::{cost_per_token as vertex_cost_per_token, vertex_cost};
|
|
||||||
use crate::xai_cost::{cost_per_token as xai_cost_per_token, reported_cost as xai_reported_cost};
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, Default)]
|
#[derive(Clone, Debug, Default)]
|
||||||
pub struct ModelInfoCatalog {
|
pub struct ModelInfoCatalog {
|
||||||
|
|
@ -329,130 +307,6 @@ impl ModelInfoCatalog {
|
||||||
prepare_completion_input(request, &self.entries)
|
prepare_completion_input(request, &self.entries)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn cost_per_token_with_custom(
|
|
||||||
&self,
|
|
||||||
request: ModelCostRequest<'_>,
|
|
||||||
pricing: CustomPricing,
|
|
||||||
) -> Result<(f64, f64), CostError> {
|
|
||||||
match cost_from_chat_usage(request.usage, pricing, request.response_time_ms)? {
|
|
||||||
Some(cost) => Ok((cost.input, cost.output)),
|
|
||||||
None => self.cost_per_token(request),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn cost_per_token_for_call_with_custom(
|
|
||||||
&self,
|
|
||||||
request: ModelCostRequest<'_>,
|
|
||||||
call: CostCall<'_>,
|
|
||||||
pricing: CustomPricing,
|
|
||||||
) -> Result<(f64, f64), CostError> {
|
|
||||||
match cost_from_chat_usage(request.usage, pricing, request.response_time_ms)? {
|
|
||||||
Some(cost) => Ok((cost.input, cost.output)),
|
|
||||||
None => self.cost_per_token_for_call(request, call),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn cost_per_token_for_call(
|
|
||||||
&self,
|
|
||||||
request: ModelCostRequest<'_>,
|
|
||||||
call: CostCall<'_>,
|
|
||||||
) -> Result<(f64, f64), CostError> {
|
|
||||||
let provider = request.provider.and_then(LlmProviders::parse);
|
|
||||||
match call {
|
|
||||||
CostCall::Token {
|
|
||||||
call_type,
|
|
||||||
prompt_characters,
|
|
||||||
completion_characters,
|
|
||||||
request_model,
|
|
||||||
} => {
|
|
||||||
if provider == Some(LlmProviders::VERTEX_AI) {
|
|
||||||
return vertex_cost(
|
|
||||||
self,
|
|
||||||
request,
|
|
||||||
call_type,
|
|
||||||
prompt_characters,
|
|
||||||
completion_characters,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
if provider == Some(LlmProviders::TOGETHER_AI)
|
|
||||||
&& matches!(
|
|
||||||
crate::call_type::CallTypes::parse(call_type),
|
|
||||||
Some(crate::call_type::CallTypes::embedding)
|
|
||||||
| Some(crate::call_type::CallTypes::aembedding)
|
|
||||||
)
|
|
||||||
{
|
|
||||||
return together_ai_cost_per_token(self, request, call_type);
|
|
||||||
}
|
|
||||||
if provider == Some(LlmProviders::AZURE_AI) {
|
|
||||||
return azure_ai_cost_per_token(self, request, request_model);
|
|
||||||
}
|
|
||||||
Ok(self.cost_per_token(request)?)
|
|
||||||
}
|
|
||||||
CostCall::Speech { prompt_characters } => {
|
|
||||||
Ok(self.speech_cost(request, prompt_characters)?)
|
|
||||||
}
|
|
||||||
CostCall::Transcription { duration_seconds } => {
|
|
||||||
Ok(self.transcription_cost(request, duration_seconds)?)
|
|
||||||
}
|
|
||||||
CostCall::Rerank { billed_units } => {
|
|
||||||
let provider = request.provider.ok_or(CostError::MissingProvider)?;
|
|
||||||
Ok(self.rerank_cost(request.model, provider, request.region, billed_units))
|
|
||||||
}
|
|
||||||
CostCall::VectorStoreSearch { api_type } => {
|
|
||||||
let provider = request.provider.ok_or(CostError::MissingProvider)?;
|
|
||||||
Ok(self.vector_store_search_cost(provider, api_type))
|
|
||||||
}
|
|
||||||
CostCall::Search {
|
|
||||||
number_of_queries,
|
|
||||||
optional_params,
|
|
||||||
} => Ok(self.search_provider_cost_per_query(
|
|
||||||
request.model,
|
|
||||||
request.provider,
|
|
||||||
number_of_queries.filter(|count| *count > 0).unwrap_or(1),
|
|
||||||
optional_params,
|
|
||||||
)?),
|
|
||||||
CostCall::Ocr {
|
|
||||||
response,
|
|
||||||
deployment_info,
|
|
||||||
} => {
|
|
||||||
let published = self
|
|
||||||
.select_model_key(request.model, request.provider, request.region)
|
|
||||||
.and_then(|key| self.entries.get(key));
|
|
||||||
Ok(ocr_cost(response, deployment_info, published)?)
|
|
||||||
}
|
|
||||||
CostCall::Batch { deployment_info } => {
|
|
||||||
let published = self
|
|
||||||
.select_model_key(request.model, request.provider, request.region)
|
|
||||||
.and_then(|key| self.entries.get(key));
|
|
||||||
let has_deployment_rates = deployment_info.is_some_and(|info| {
|
|
||||||
[
|
|
||||||
"input_cost_per_token_batches",
|
|
||||||
"input_cost_per_token",
|
|
||||||
"output_cost_per_token_batches",
|
|
||||||
"output_cost_per_token",
|
|
||||||
]
|
|
||||||
.into_iter()
|
|
||||||
.any(|key| info.get(key).is_some_and(|value| !value.is_null()))
|
|
||||||
});
|
|
||||||
let info = if has_deployment_rates {
|
|
||||||
deployment_info
|
|
||||||
} else {
|
|
||||||
published.or(deployment_info)
|
|
||||||
};
|
|
||||||
let Some(info) = info else {
|
|
||||||
return Ok((0.0, 0.0));
|
|
||||||
};
|
|
||||||
let cost = batch_cost_from_model_info(
|
|
||||||
info,
|
|
||||||
request.usage,
|
|
||||||
request.provider,
|
|
||||||
request.data_residency,
|
|
||||||
)?;
|
|
||||||
Ok((cost.prompt, cost.completion))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn speech_cost(
|
pub fn speech_cost(
|
||||||
&self,
|
&self,
|
||||||
request: ModelCostRequest<'_>,
|
request: ModelCostRequest<'_>,
|
||||||
|
|
@ -477,7 +331,7 @@ impl ModelInfoCatalog {
|
||||||
completion.unwrap_or(0.0),
|
completion.unwrap_or(0.0),
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
SpeechCostMetric::PerToken => Ok(self.cost_per_token(request)?),
|
SpeechCostMetric::PerToken => Ok(cost_per_token(self, request)?),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -594,170 +448,25 @@ impl ModelInfoCatalog {
|
||||||
.map(event_usage)
|
.map(event_usage)
|
||||||
.collect::<Result<Vec<_>, _>>()?,
|
.collect::<Result<Vec<_>, _>>()?,
|
||||||
)?;
|
)?;
|
||||||
let (prompt, completion) = self.cost_per_token(ModelCostRequest {
|
let (prompt, completion) = cost_per_token(
|
||||||
model,
|
self,
|
||||||
provider,
|
ModelCostRequest {
|
||||||
region,
|
model,
|
||||||
usage: &usage,
|
provider,
|
||||||
service_tier,
|
region,
|
||||||
data_residency,
|
usage: &usage,
|
||||||
vertex_location: None,
|
service_tier,
|
||||||
at,
|
data_residency,
|
||||||
response_time_ms: None,
|
vertex_location: None,
|
||||||
})?;
|
at,
|
||||||
|
response_time_ms: None,
|
||||||
|
},
|
||||||
|
)?;
|
||||||
Ok(prompt + completion)
|
Ok(prompt + completion)
|
||||||
})
|
})
|
||||||
.sum()
|
.sum()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn cost_per_token(&self, request: ModelCostRequest<'_>) -> Result<(f64, f64), CostError> {
|
|
||||||
if let Some(cost) = self
|
|
||||||
.select_model_key(request.model, request.provider, request.region)
|
|
||||||
.and_then(|key| self.entries.get(key))
|
|
||||||
.and_then(|info| per_second_pricing_cost(info, request.response_time_ms))
|
|
||||||
{
|
|
||||||
return Ok(cost);
|
|
||||||
}
|
|
||||||
let provider = request.provider.and_then(LlmProviders::parse);
|
|
||||||
if provider == Some(LlmProviders::AZURE_AI) {
|
|
||||||
return azure_ai_cost_per_token(self, request, None);
|
|
||||||
}
|
|
||||||
if provider == Some(LlmProviders::DATABRICKS) {
|
|
||||||
return databricks_cost_per_token(self, request);
|
|
||||||
}
|
|
||||||
if provider == Some(LlmProviders::LEMONADE) {
|
|
||||||
return Ok(lemonade_cost_per_token(self, request));
|
|
||||||
}
|
|
||||||
if provider == Some(LlmProviders::PERPLEXITY)
|
|
||||||
&& let Some(cost) = request.usage.cost
|
|
||||||
{
|
|
||||||
return Ok((0.0, cost));
|
|
||||||
}
|
|
||||||
if provider == Some(LlmProviders::XAI)
|
|
||||||
&& let Some(cost) = xai_reported_cost(request.usage)
|
|
||||||
{
|
|
||||||
return Ok((0.0, cost));
|
|
||||||
}
|
|
||||||
let needs_together_fallback = (provider == Some(LlmProviders::TOGETHER_AI)
|
|
||||||
|| request.model.contains("togethercomputer")
|
|
||||||
|| request.model.contains("together_ai"))
|
|
||||||
&& !has_together_registry_pricing(request.model, &self.entries);
|
|
||||||
let together_fallback = needs_together_fallback.then(|| {
|
|
||||||
get_model_params_and_category(
|
|
||||||
request.model,
|
|
||||||
"completion",
|
|
||||||
TogetherThresholds::default(),
|
|
||||||
)
|
|
||||||
});
|
|
||||||
let key = match together_fallback.as_deref() {
|
|
||||||
Some(category) => self.select_model_key(category, None, request.region),
|
|
||||||
None => self.select_model_key(request.model, request.provider, request.region),
|
|
||||||
}
|
|
||||||
.or_else(|| {
|
|
||||||
(request.provider == Some("fireworks_ai"))
|
|
||||||
.then(|| get_base_model_for_pricing(request.model, FireworksThresholds::default()))
|
|
||||||
.and_then(|category| {
|
|
||||||
self.select_model_key(category, request.provider, request.region)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
.ok_or(CostError::ModelNotFound)?;
|
|
||||||
let model_info = apply_provider_cache_read_default(&self.entries[key], request.provider);
|
|
||||||
if let Some(cost) = per_second_pricing_cost(&model_info, request.response_time_ms) {
|
|
||||||
return Ok(cost);
|
|
||||||
}
|
|
||||||
if provider == Some(LlmProviders::AZURE)
|
|
||||||
&& let Some(cost) = output_per_second_cost(&model_info, request.response_time_ms)
|
|
||||||
{
|
|
||||||
return Ok(cost);
|
|
||||||
}
|
|
||||||
match provider {
|
|
||||||
Some(LlmProviders::PERPLEXITY) => {
|
|
||||||
return Ok(perplexity_cost_per_token(
|
|
||||||
request.usage,
|
|
||||||
&model_info,
|
|
||||||
request.at,
|
|
||||||
));
|
|
||||||
}
|
|
||||||
Some(LlmProviders::XAI) => {
|
|
||||||
return Ok(xai_cost_per_token(request.usage, &model_info, request.at));
|
|
||||||
}
|
|
||||||
Some(
|
|
||||||
LlmProviders::DASHSCOPE | LlmProviders::QWENCLOUD | LlmProviders::QWEN_AI_PLATFORM,
|
|
||||||
) => {
|
|
||||||
return Ok(dashscope_cost_per_token(
|
|
||||||
request.usage,
|
|
||||||
&model_info,
|
|
||||||
request.at,
|
|
||||||
));
|
|
||||||
}
|
|
||||||
Some(LlmProviders::FIREWORKS_AI) => {
|
|
||||||
return Ok(fireworks_cost_per_token(
|
|
||||||
request.usage,
|
|
||||||
&self.entries[key],
|
|
||||||
request.at,
|
|
||||||
));
|
|
||||||
}
|
|
||||||
Some(LlmProviders::VERTEX_AI) => {
|
|
||||||
return Ok(vertex_cost_per_token(
|
|
||||||
request.usage,
|
|
||||||
&model_info,
|
|
||||||
request.service_tier,
|
|
||||||
request.vertex_location,
|
|
||||||
request.at,
|
|
||||||
));
|
|
||||||
}
|
|
||||||
_ => {}
|
|
||||||
}
|
|
||||||
let dispatches_before_the_gate = matches!(
|
|
||||||
provider,
|
|
||||||
Some(
|
|
||||||
LlmProviders::VERTEX_AI
|
|
||||||
| LlmProviders::ANTHROPIC
|
|
||||||
| LlmProviders::BEDROCK
|
|
||||||
| LlmProviders::OPENAI
|
|
||||||
| LlmProviders::DATABRICKS
|
|
||||||
| LlmProviders::FIREWORKS_AI
|
|
||||||
| LlmProviders::AZURE
|
|
||||||
| LlmProviders::GEMINI
|
|
||||||
| LlmProviders::DEEPSEEK
|
|
||||||
| LlmProviders::TENCENT
|
|
||||||
| LlmProviders::PERPLEXITY
|
|
||||||
| LlmProviders::XAI
|
|
||||||
| LlmProviders::LEMONADE
|
|
||||||
| LlmProviders::DASHSCOPE
|
|
||||||
| LlmProviders::QWENCLOUD
|
|
||||||
| LlmProviders::QWEN_AI_PLATFORM
|
|
||||||
| LlmProviders::AZURE_AI
|
|
||||||
)
|
|
||||||
);
|
|
||||||
if !dispatches_before_the_gate && !has_token_or_tiered_pricing(&model_info) {
|
|
||||||
return Ok((0.0, 0.0));
|
|
||||||
}
|
|
||||||
let cost = calculate_generic_cost_from_model_info_with_region(
|
|
||||||
request.usage,
|
|
||||||
&model_info,
|
|
||||||
request.service_tier,
|
|
||||||
uses_inclusive_token_thresholds(request.provider),
|
|
||||||
request.data_residency,
|
|
||||||
request.vertex_location,
|
|
||||||
request.at,
|
|
||||||
);
|
|
||||||
let speed = if provider == Some(LlmProviders::ANTHROPIC) {
|
|
||||||
fast_speed_multiplier(&model_info, request.usage)
|
|
||||||
* get_provider_specific_geo_multiplier(
|
|
||||||
&model_info,
|
|
||||||
request
|
|
||||||
.usage
|
|
||||||
.extra
|
|
||||||
.get("inference_geo")
|
|
||||||
.and_then(Value::as_str),
|
|
||||||
)
|
|
||||||
} else {
|
|
||||||
1.0
|
|
||||||
};
|
|
||||||
Ok((cost.0 * speed, cost.1 * speed))
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn get_billed_token_rates(
|
pub fn get_billed_token_rates(
|
||||||
&self,
|
&self,
|
||||||
request: ModelCostRequest<'_>,
|
request: ModelCostRequest<'_>,
|
||||||
|
|
@ -1150,7 +859,7 @@ impl ModelInfoCatalog {
|
||||||
&self,
|
&self,
|
||||||
request: CompletionCostRequest<'_>,
|
request: CompletionCostRequest<'_>,
|
||||||
) -> Result<CompletionCost, CostError> {
|
) -> Result<CompletionCost, CostError> {
|
||||||
let (prompt, output) = self.cost_per_token(request.token)?;
|
let (prompt, output) = cost_per_token(self, request.token)?;
|
||||||
let built_in_tools = match request.built_in_tools {
|
let built_in_tools = match request.built_in_tools {
|
||||||
BuiltInToolCharge::Provided(cost) => cost,
|
BuiltInToolCharge::Provided(cost) => cost,
|
||||||
BuiltInToolCharge::FromResponse(tool_request) => self.built_in_tool_cost(
|
BuiltInToolCharge::FromResponse(tool_request) => self.built_in_tool_cost(
|
||||||
|
|
|
||||||
|
|
@ -12,6 +12,7 @@ use crate::completion_cost::{
|
||||||
CompletionCost, completion_cost, get_response_cost_from_hidden_params,
|
CompletionCost, completion_cost, get_response_cost_from_hidden_params,
|
||||||
};
|
};
|
||||||
use crate::completion_input::{CompletionInputRequest, PreparedCompletionInput, ResponseKind};
|
use crate::completion_input::{CompletionInputRequest, PreparedCompletionInput, ResponseKind};
|
||||||
|
use crate::cost_calculator::{cost_per_token, cost_per_token_for_call};
|
||||||
use crate::custom_pricing::{CustomPricing, cost_from_chat_usage};
|
use crate::custom_pricing::{CustomPricing, cost_from_chat_usage};
|
||||||
use crate::error::CostError;
|
use crate::error::CostError;
|
||||||
use crate::image_cost_router::{
|
use crate::image_cost_router::{
|
||||||
|
|
@ -553,17 +554,20 @@ fn price_responses_websocket(
|
||||||
request.response_time_ms,
|
request.response_time_ms,
|
||||||
),
|
),
|
||||||
|model| {
|
|model| {
|
||||||
catalog.cost_per_token(ModelCostRequest {
|
cost_per_token(
|
||||||
model,
|
catalog,
|
||||||
provider,
|
ModelCostRequest {
|
||||||
region,
|
model,
|
||||||
usage,
|
provider,
|
||||||
service_tier: tier,
|
region,
|
||||||
data_residency: request.data_residency,
|
usage,
|
||||||
vertex_location: request.vertex_location,
|
service_tier: tier,
|
||||||
at: request.at,
|
data_residency: request.data_residency,
|
||||||
response_time_ms: None,
|
vertex_location: request.vertex_location,
|
||||||
})
|
at: request.at,
|
||||||
|
response_time_ms: None,
|
||||||
|
},
|
||||||
|
)
|
||||||
},
|
},
|
||||||
)?;
|
)?;
|
||||||
let built_in = if index == 0 {
|
let built_in = if index == 0 {
|
||||||
|
|
@ -806,7 +810,7 @@ pub fn completion_cost_from_response(
|
||||||
request.response_time_ms,
|
request.response_time_ms,
|
||||||
),
|
),
|
||||||
};
|
};
|
||||||
catalog.cost_per_token_for_call(cost_request, call)
|
cost_per_token_for_call(catalog, cost_request, call)
|
||||||
},
|
},
|
||||||
)?;
|
)?;
|
||||||
let router_fee = (!is_search && provider == Some("azure_ai") && !is_azure_model_router(&model))
|
let router_fee = (!is_search && provider == Some("azure_ai") && !is_azure_model_router(&model))
|
||||||
|
|
|
||||||
298
litellm-rust/crates/cost/src/cost_calculator.rs
Normal file
298
litellm-rust/crates/cost/src/cost_calculator.rs
Normal file
|
|
@ -0,0 +1,298 @@
|
||||||
|
//! The public cost entry points mirroring litellm/cost_calculator.py.
|
||||||
|
|
||||||
|
use serde_json::Value;
|
||||||
|
|
||||||
|
use crate::anthropic_cost::fast_speed_multiplier;
|
||||||
|
use crate::azure_ai_cost::azure_ai_cost_per_token;
|
||||||
|
use crate::azure_cost::output_per_second_cost;
|
||||||
|
use crate::base_rate_selection::uses_inclusive_token_thresholds;
|
||||||
|
use crate::batch::batch_cost_from_model_info;
|
||||||
|
use crate::catalog::{CostCall, ModelCostRequest, ModelInfoCatalog};
|
||||||
|
use crate::custom_pricing::{CustomPricing, cost_from_chat_usage};
|
||||||
|
use crate::dashscope_cost::cost_per_token as dashscope_cost_per_token;
|
||||||
|
use crate::databricks_cost::databricks_cost_per_token;
|
||||||
|
use crate::error::CostError;
|
||||||
|
use crate::fireworks_cost::{
|
||||||
|
FireworksThresholds, cost_per_token as fireworks_cost_per_token, get_base_model_for_pricing,
|
||||||
|
};
|
||||||
|
use crate::generic_cost::calculate_generic_cost_from_model_info_with_region;
|
||||||
|
use crate::lemonade_cost::lemonade_cost_per_token;
|
||||||
|
use crate::ocr_cost::ocr_cost;
|
||||||
|
use crate::per_second::{has_token_or_tiered_pricing, per_second_pricing_cost};
|
||||||
|
use crate::perplexity_cost::cost_per_token as perplexity_cost_per_token;
|
||||||
|
use crate::provider::LlmProviders;
|
||||||
|
use crate::provider_cache::apply_provider_cache_read_default;
|
||||||
|
use crate::regional_uplift::get_provider_specific_geo_multiplier;
|
||||||
|
use crate::together_cost::{
|
||||||
|
TogetherThresholds, get_model_params_and_category, has_together_registry_pricing,
|
||||||
|
together_ai_cost_per_token,
|
||||||
|
};
|
||||||
|
use crate::vertex_cost::{cost_per_token as vertex_cost_per_token, vertex_cost};
|
||||||
|
use crate::xai_cost::{cost_per_token as xai_cost_per_token, reported_cost as xai_reported_cost};
|
||||||
|
|
||||||
|
pub fn cost_per_token_with_custom(
|
||||||
|
catalog: &ModelInfoCatalog,
|
||||||
|
request: ModelCostRequest<'_>,
|
||||||
|
pricing: CustomPricing,
|
||||||
|
) -> Result<(f64, f64), CostError> {
|
||||||
|
match cost_from_chat_usage(request.usage, pricing, request.response_time_ms)? {
|
||||||
|
Some(cost) => Ok((cost.input, cost.output)),
|
||||||
|
None => cost_per_token(catalog, request),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn cost_per_token_for_call_with_custom(
|
||||||
|
catalog: &ModelInfoCatalog,
|
||||||
|
request: ModelCostRequest<'_>,
|
||||||
|
call: CostCall<'_>,
|
||||||
|
pricing: CustomPricing,
|
||||||
|
) -> Result<(f64, f64), CostError> {
|
||||||
|
match cost_from_chat_usage(request.usage, pricing, request.response_time_ms)? {
|
||||||
|
Some(cost) => Ok((cost.input, cost.output)),
|
||||||
|
None => cost_per_token_for_call(catalog, request, call),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn cost_per_token_for_call(
|
||||||
|
catalog: &ModelInfoCatalog,
|
||||||
|
request: ModelCostRequest<'_>,
|
||||||
|
call: CostCall<'_>,
|
||||||
|
) -> Result<(f64, f64), CostError> {
|
||||||
|
let provider = request.provider.and_then(LlmProviders::parse);
|
||||||
|
match call {
|
||||||
|
CostCall::Token {
|
||||||
|
call_type,
|
||||||
|
prompt_characters,
|
||||||
|
completion_characters,
|
||||||
|
request_model,
|
||||||
|
} => {
|
||||||
|
if provider == Some(LlmProviders::VERTEX_AI) {
|
||||||
|
return vertex_cost(
|
||||||
|
catalog,
|
||||||
|
request,
|
||||||
|
call_type,
|
||||||
|
prompt_characters,
|
||||||
|
completion_characters,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if provider == Some(LlmProviders::TOGETHER_AI)
|
||||||
|
&& matches!(
|
||||||
|
crate::call_type::CallTypes::parse(call_type),
|
||||||
|
Some(crate::call_type::CallTypes::embedding)
|
||||||
|
| Some(crate::call_type::CallTypes::aembedding)
|
||||||
|
)
|
||||||
|
{
|
||||||
|
return together_ai_cost_per_token(catalog, request, call_type);
|
||||||
|
}
|
||||||
|
if provider == Some(LlmProviders::AZURE_AI) {
|
||||||
|
return azure_ai_cost_per_token(catalog, request, request_model);
|
||||||
|
}
|
||||||
|
Ok(cost_per_token(catalog, request)?)
|
||||||
|
}
|
||||||
|
CostCall::Speech { prompt_characters } => {
|
||||||
|
Ok(catalog.speech_cost(request, prompt_characters)?)
|
||||||
|
}
|
||||||
|
CostCall::Transcription { duration_seconds } => {
|
||||||
|
Ok(catalog.transcription_cost(request, duration_seconds)?)
|
||||||
|
}
|
||||||
|
CostCall::Rerank { billed_units } => {
|
||||||
|
let provider = request.provider.ok_or(CostError::MissingProvider)?;
|
||||||
|
Ok(catalog.rerank_cost(request.model, provider, request.region, billed_units))
|
||||||
|
}
|
||||||
|
CostCall::VectorStoreSearch { api_type } => {
|
||||||
|
let provider = request.provider.ok_or(CostError::MissingProvider)?;
|
||||||
|
Ok(catalog.vector_store_search_cost(provider, api_type))
|
||||||
|
}
|
||||||
|
CostCall::Search {
|
||||||
|
number_of_queries,
|
||||||
|
optional_params,
|
||||||
|
} => Ok(catalog.search_provider_cost_per_query(
|
||||||
|
request.model,
|
||||||
|
request.provider,
|
||||||
|
number_of_queries.filter(|count| *count > 0).unwrap_or(1),
|
||||||
|
optional_params,
|
||||||
|
)?),
|
||||||
|
CostCall::Ocr {
|
||||||
|
response,
|
||||||
|
deployment_info,
|
||||||
|
} => {
|
||||||
|
let published = catalog.entry(request.model, request.provider, request.region);
|
||||||
|
Ok(ocr_cost(response, deployment_info, published)?)
|
||||||
|
}
|
||||||
|
CostCall::Batch { deployment_info } => {
|
||||||
|
let published = catalog.entry(request.model, request.provider, request.region);
|
||||||
|
let has_deployment_rates = deployment_info.is_some_and(|info| {
|
||||||
|
[
|
||||||
|
"input_cost_per_token_batches",
|
||||||
|
"input_cost_per_token",
|
||||||
|
"output_cost_per_token_batches",
|
||||||
|
"output_cost_per_token",
|
||||||
|
]
|
||||||
|
.into_iter()
|
||||||
|
.any(|key| info.get(key).is_some_and(|value| !value.is_null()))
|
||||||
|
});
|
||||||
|
let info = if has_deployment_rates {
|
||||||
|
deployment_info
|
||||||
|
} else {
|
||||||
|
published.or(deployment_info)
|
||||||
|
};
|
||||||
|
let Some(info) = info else {
|
||||||
|
return Ok((0.0, 0.0));
|
||||||
|
};
|
||||||
|
let cost = batch_cost_from_model_info(
|
||||||
|
info,
|
||||||
|
request.usage,
|
||||||
|
request.provider,
|
||||||
|
request.data_residency,
|
||||||
|
)?;
|
||||||
|
Ok((cost.prompt, cost.completion))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn cost_per_token(
|
||||||
|
catalog: &ModelInfoCatalog,
|
||||||
|
request: ModelCostRequest<'_>,
|
||||||
|
) -> Result<(f64, f64), CostError> {
|
||||||
|
if let Some(cost) = catalog
|
||||||
|
.entry(request.model, request.provider, request.region)
|
||||||
|
.and_then(|info| per_second_pricing_cost(info, request.response_time_ms))
|
||||||
|
{
|
||||||
|
return Ok(cost);
|
||||||
|
}
|
||||||
|
let provider = request.provider.and_then(LlmProviders::parse);
|
||||||
|
if provider == Some(LlmProviders::AZURE_AI) {
|
||||||
|
return azure_ai_cost_per_token(catalog, request, None);
|
||||||
|
}
|
||||||
|
if provider == Some(LlmProviders::DATABRICKS) {
|
||||||
|
return databricks_cost_per_token(catalog, request);
|
||||||
|
}
|
||||||
|
if provider == Some(LlmProviders::LEMONADE) {
|
||||||
|
return Ok(lemonade_cost_per_token(catalog, request));
|
||||||
|
}
|
||||||
|
if provider == Some(LlmProviders::PERPLEXITY)
|
||||||
|
&& let Some(cost) = request.usage.cost
|
||||||
|
{
|
||||||
|
return Ok((0.0, cost));
|
||||||
|
}
|
||||||
|
if provider == Some(LlmProviders::XAI)
|
||||||
|
&& let Some(cost) = xai_reported_cost(request.usage)
|
||||||
|
{
|
||||||
|
return Ok((0.0, cost));
|
||||||
|
}
|
||||||
|
let needs_together_fallback = (provider == Some(LlmProviders::TOGETHER_AI)
|
||||||
|
|| request.model.contains("togethercomputer")
|
||||||
|
|| request.model.contains("together_ai"))
|
||||||
|
&& !has_together_registry_pricing(request.model, catalog.entries());
|
||||||
|
let together_fallback = needs_together_fallback.then(|| {
|
||||||
|
get_model_params_and_category(request.model, "completion", TogetherThresholds::default())
|
||||||
|
});
|
||||||
|
let key = match together_fallback.as_deref() {
|
||||||
|
Some(category) => catalog.select_model_key(category, None, request.region),
|
||||||
|
None => catalog.select_model_key(request.model, request.provider, request.region),
|
||||||
|
}
|
||||||
|
.or_else(|| {
|
||||||
|
(request.provider == Some("fireworks_ai"))
|
||||||
|
.then(|| get_base_model_for_pricing(request.model, FireworksThresholds::default()))
|
||||||
|
.and_then(|category| {
|
||||||
|
catalog.select_model_key(category, request.provider, request.region)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.ok_or(CostError::ModelNotFound)?;
|
||||||
|
let model_info =
|
||||||
|
apply_provider_cache_read_default(catalog.entry_for_key(key), request.provider);
|
||||||
|
if let Some(cost) = per_second_pricing_cost(&model_info, request.response_time_ms) {
|
||||||
|
return Ok(cost);
|
||||||
|
}
|
||||||
|
if provider == Some(LlmProviders::AZURE)
|
||||||
|
&& let Some(cost) = output_per_second_cost(&model_info, request.response_time_ms)
|
||||||
|
{
|
||||||
|
return Ok(cost);
|
||||||
|
}
|
||||||
|
match provider {
|
||||||
|
Some(LlmProviders::PERPLEXITY) => {
|
||||||
|
return Ok(perplexity_cost_per_token(
|
||||||
|
request.usage,
|
||||||
|
&model_info,
|
||||||
|
request.at,
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Some(LlmProviders::XAI) => {
|
||||||
|
return Ok(xai_cost_per_token(request.usage, &model_info, request.at));
|
||||||
|
}
|
||||||
|
Some(
|
||||||
|
LlmProviders::DASHSCOPE | LlmProviders::QWENCLOUD | LlmProviders::QWEN_AI_PLATFORM,
|
||||||
|
) => {
|
||||||
|
return Ok(dashscope_cost_per_token(
|
||||||
|
request.usage,
|
||||||
|
&model_info,
|
||||||
|
request.at,
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Some(LlmProviders::FIREWORKS_AI) => {
|
||||||
|
return Ok(fireworks_cost_per_token(
|
||||||
|
request.usage,
|
||||||
|
catalog.entry_for_key(key),
|
||||||
|
request.at,
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Some(LlmProviders::VERTEX_AI) => {
|
||||||
|
return Ok(vertex_cost_per_token(
|
||||||
|
request.usage,
|
||||||
|
&model_info,
|
||||||
|
request.service_tier,
|
||||||
|
request.vertex_location,
|
||||||
|
request.at,
|
||||||
|
));
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
let dispatches_before_the_gate = matches!(
|
||||||
|
provider,
|
||||||
|
Some(
|
||||||
|
LlmProviders::VERTEX_AI
|
||||||
|
| LlmProviders::ANTHROPIC
|
||||||
|
| LlmProviders::BEDROCK
|
||||||
|
| LlmProviders::OPENAI
|
||||||
|
| LlmProviders::DATABRICKS
|
||||||
|
| LlmProviders::FIREWORKS_AI
|
||||||
|
| LlmProviders::AZURE
|
||||||
|
| LlmProviders::GEMINI
|
||||||
|
| LlmProviders::DEEPSEEK
|
||||||
|
| LlmProviders::TENCENT
|
||||||
|
| LlmProviders::PERPLEXITY
|
||||||
|
| LlmProviders::XAI
|
||||||
|
| LlmProviders::LEMONADE
|
||||||
|
| LlmProviders::DASHSCOPE
|
||||||
|
| LlmProviders::QWENCLOUD
|
||||||
|
| LlmProviders::QWEN_AI_PLATFORM
|
||||||
|
| LlmProviders::AZURE_AI
|
||||||
|
)
|
||||||
|
);
|
||||||
|
if !dispatches_before_the_gate && !has_token_or_tiered_pricing(&model_info) {
|
||||||
|
return Ok((0.0, 0.0));
|
||||||
|
}
|
||||||
|
let cost = calculate_generic_cost_from_model_info_with_region(
|
||||||
|
request.usage,
|
||||||
|
&model_info,
|
||||||
|
request.service_tier,
|
||||||
|
uses_inclusive_token_thresholds(request.provider),
|
||||||
|
request.data_residency,
|
||||||
|
request.vertex_location,
|
||||||
|
request.at,
|
||||||
|
);
|
||||||
|
let speed = if provider == Some(LlmProviders::ANTHROPIC) {
|
||||||
|
fast_speed_multiplier(&model_info, request.usage)
|
||||||
|
* get_provider_specific_geo_multiplier(
|
||||||
|
&model_info,
|
||||||
|
request
|
||||||
|
.usage
|
||||||
|
.extra
|
||||||
|
.get("inference_geo")
|
||||||
|
.and_then(Value::as_str),
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
1.0
|
||||||
|
};
|
||||||
|
Ok((cost.0 * speed, cost.1 * speed))
|
||||||
|
}
|
||||||
|
|
@ -30,6 +30,8 @@ pub mod completion_input;
|
||||||
pub mod completion_response;
|
pub mod completion_response;
|
||||||
pub mod cost_breakdown;
|
pub mod cost_breakdown;
|
||||||
#[allow(clippy::disallowed_types)]
|
#[allow(clippy::disallowed_types)]
|
||||||
|
pub mod cost_calculator;
|
||||||
|
#[allow(clippy::disallowed_types)]
|
||||||
pub mod custom_pricing;
|
pub mod custom_pricing;
|
||||||
#[allow(clippy::disallowed_types)]
|
#[allow(clippy::disallowed_types)]
|
||||||
pub mod dashscope_cost;
|
pub mod dashscope_cost;
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
use crate::catalog::{ModelCostRequest, ModelInfoCatalog};
|
use crate::catalog::{ModelCostRequest, ModelInfoCatalog};
|
||||||
|
use crate::cost_calculator::cost_per_token;
|
||||||
use crate::error::CostError;
|
use crate::error::CostError;
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::sync::LazyLock;
|
use std::sync::LazyLock;
|
||||||
|
|
@ -92,13 +93,16 @@ pub fn together_ai_cost_per_token(
|
||||||
call_type: &str,
|
call_type: &str,
|
||||||
) -> Result<(f64, f64), CostError> {
|
) -> Result<(f64, f64), CostError> {
|
||||||
if has_together_registry_pricing(request.model, catalog.entries()) {
|
if has_together_registry_pricing(request.model, catalog.entries()) {
|
||||||
return catalog.cost_per_token(request);
|
return cost_per_token(catalog, request);
|
||||||
}
|
}
|
||||||
let category =
|
let category =
|
||||||
get_model_params_and_category(request.model, call_type, TogetherThresholds::default());
|
get_model_params_and_category(request.model, call_type, TogetherThresholds::default());
|
||||||
catalog.cost_per_token(ModelCostRequest {
|
cost_per_token(
|
||||||
model: &category,
|
catalog,
|
||||||
provider: Some("together_ai"),
|
ModelCostRequest {
|
||||||
..request
|
model: &category,
|
||||||
})
|
provider: Some("together_ai"),
|
||||||
|
..request
|
||||||
|
},
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@ use jiff::Timestamp;
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
|
|
||||||
use crate::catalog::{ModelCostRequest, ModelInfoCatalog};
|
use crate::catalog::{ModelCostRequest, ModelInfoCatalog};
|
||||||
|
use crate::cost_calculator::cost_per_token as catalog_cost_per_token;
|
||||||
use crate::error::CostError;
|
use crate::error::CostError;
|
||||||
use crate::generic_cost::calculate_generic_cost_from_model_info_with_region;
|
use crate::generic_cost::calculate_generic_cost_from_model_info_with_region;
|
||||||
use crate::per_second::per_second_pricing_cost;
|
use crate::per_second::per_second_pricing_cost;
|
||||||
|
|
@ -141,7 +142,7 @@ pub fn vertex_cost(
|
||||||
}
|
}
|
||||||
if cost_router(request.model, request.provider.unwrap_or(""), call_type) == CostRoute::PerToken
|
if cost_router(request.model, request.provider.unwrap_or(""), call_type) == CostRoute::PerToken
|
||||||
{
|
{
|
||||||
return catalog.cost_per_token(request);
|
return catalog_cost_per_token(catalog, request);
|
||||||
}
|
}
|
||||||
let entry = catalog
|
let entry = catalog
|
||||||
.entry(request.model, request.provider, request.region)
|
.entry(request.model, request.provider, request.region)
|
||||||
|
|
|
||||||
|
|
@ -1,136 +1,3 @@
|
||||||
#![allow(clippy::disallowed_types)]
|
#![allow(clippy::disallowed_types)]
|
||||||
|
|
||||||
// mirrors: test_litellm/test_cost_calculator.py::test_anthropic_geo_and_fast_multipliers_compose
|
// mirrors: test_litellm/test_cost_calculator.py::test_anthropic_geo_and_fast_multipliers_compose
|
||||||
// mirrors: test_litellm/llms/anthropic/test_cost_calculation_dict_safety.py::test_get_cost_for_anthropic_web_search_with_dict_server_tool_use
|
// mirrors: test_litellm/llms/anthropic/test_cost_calculation_dict_safety.py::test_get_cost_for_anthropic_web_search_with_dict_server_tool_use
|
||||||
|
|
||||||
use std::collections::HashMap;
|
|
||||||
|
|
||||||
use jiff::Timestamp;
|
|
||||||
use litellm_cost::anthropic_cost::{
|
|
||||||
fast_speed_multiplier, get_anthropic_web_search_requests_from_response,
|
|
||||||
get_cost_for_anthropic_web_search, get_web_search_requests, get_web_search_requests_from_usage,
|
|
||||||
};
|
|
||||||
use litellm_cost::catalog::{ModelCostRequest, ModelInfoCatalog};
|
|
||||||
use litellm_cost::tool_call_cost_tracking::{DefaultToolRates, ResponseKind};
|
|
||||||
use litellm_cost::tool_cost_dispatch::{BuiltInToolCostRequest, get_cost_for_built_in_tools};
|
|
||||||
use litellm_cost::usage_dispatch::get_usage_object;
|
|
||||||
use rstest::rstest;
|
|
||||||
use serde_json::{Value, json};
|
|
||||||
|
|
||||||
fn usage(value: &Value) -> litellm_cost::responses_usage::ChatUsage {
|
|
||||||
get_usage_object(&json!({"usage": value})).unwrap().unwrap()
|
|
||||||
}
|
|
||||||
|
|
||||||
#[rstest]
|
|
||||||
fn anthropic_cost_per_token_composes_served_tier_cache_geo_and_fast_once() {
|
|
||||||
let model_info = json!({
|
|
||||||
"input_cost_per_token": 0.003,
|
|
||||||
"output_cost_per_token": 0.015,
|
|
||||||
"cache_read_input_token_cost": 0.0003,
|
|
||||||
"input_cost_per_token_priority": 0.006,
|
|
||||||
"output_cost_per_token_priority": 0.030,
|
|
||||||
"cache_read_input_token_cost_priority": 0.0006,
|
|
||||||
"provider_specific_entry": {"fast": 2.0, "us": 1.1}
|
|
||||||
});
|
|
||||||
let catalog = ModelInfoCatalog::new(HashMap::from([(
|
|
||||||
"anthropic/model".to_owned(),
|
|
||||||
model_info.clone(),
|
|
||||||
)]));
|
|
||||||
let usage = usage(&json!({
|
|
||||||
"prompt_tokens": 1000,
|
|
||||||
"completion_tokens": 500,
|
|
||||||
"prompt_tokens_details": {"cached_tokens": 200},
|
|
||||||
"speed": "fast",
|
|
||||||
"inference_geo": "us"
|
|
||||||
}));
|
|
||||||
let at: Timestamp = "2026-09-22T12:00:00Z".parse().unwrap();
|
|
||||||
let (prompt, completion) = catalog
|
|
||||||
.cost_per_token(ModelCostRequest {
|
|
||||||
model: "model",
|
|
||||||
provider: Some("anthropic"),
|
|
||||||
region: None,
|
|
||||||
usage: &usage,
|
|
||||||
service_tier: Some("priority"),
|
|
||||||
data_residency: None,
|
|
||||||
vertex_location: None,
|
|
||||||
at,
|
|
||||||
response_time_ms: None,
|
|
||||||
})
|
|
||||||
.unwrap();
|
|
||||||
let expected_prompt = (800.0 * 0.006 + 200.0 * 0.0006) * 1.1 * 2.0;
|
|
||||||
let expected_completion = 500.0 * 0.030 * 1.1 * 2.0;
|
|
||||||
assert!((prompt - expected_prompt).abs() < 1e-12);
|
|
||||||
assert!((completion - expected_completion).abs() < 1e-12);
|
|
||||||
assert_eq!(fast_speed_multiplier(&model_info, &usage), 2.0);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[rstest]
|
|
||||||
fn anthropic_standard_speed_preserves_geo_without_fast_multiplier() {
|
|
||||||
let info = json!({"provider_specific_entry": {"fast": 2.0, "us": 1.1}});
|
|
||||||
let usage = usage(&json!({"prompt_tokens": 100, "speed": "standard", "inference_geo": "us"}));
|
|
||||||
assert_eq!(fast_speed_multiplier(&info, &usage), 1.0);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[rstest]
|
|
||||||
#[case(json!({"usage": {"server_tool_use": {"web_search_requests": 4}}}), Some(4))]
|
|
||||||
#[case(json!({"usage": {"server_tool_use": {}}}), None)]
|
|
||||||
#[case(json!({"usage": null}), None)]
|
|
||||||
fn get_anthropic_web_search_requests_from_response_reads_nested_usage(
|
|
||||||
#[case] response: Value,
|
|
||||||
#[case] expected: Option<i64>,
|
|
||||||
) {
|
|
||||||
assert_eq!(
|
|
||||||
get_anthropic_web_search_requests_from_response(&response),
|
|
||||||
expected
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[rstest]
|
|
||||||
#[case(json!({"web_search_requests": 0}), Some(0))]
|
|
||||||
#[case(json!({"web_search_requests": 3}), Some(3))]
|
|
||||||
#[case(json!({}), None)]
|
|
||||||
#[case(json!({"web_search_requests": "3"}), None)]
|
|
||||||
fn get_web_search_requests_reads_serialized_server_tool_use(
|
|
||||||
#[case] server_tool_use: Value,
|
|
||||||
#[case] expected: Option<i64>,
|
|
||||||
) {
|
|
||||||
assert_eq!(get_web_search_requests(Some(&server_tool_use)), expected);
|
|
||||||
assert_eq!(get_web_search_requests(None), None);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[rstest]
|
|
||||||
fn get_cost_for_anthropic_web_search_prices_server_tool_requests() {
|
|
||||||
let model_info =
|
|
||||||
json!({"search_context_cost_per_query": {"search_context_size_medium": 0.003}});
|
|
||||||
let usage = usage(&json!({"prompt_tokens": 10, "server_tool_use": {"web_search_requests": 3}}));
|
|
||||||
assert_eq!(get_web_search_requests_from_usage(&usage), Some(3));
|
|
||||||
assert!(
|
|
||||||
(get_cost_for_anthropic_web_search(Some(&model_info), Some(&usage)) - 0.009).abs() < 1e-12
|
|
||||||
);
|
|
||||||
assert_eq!(get_cost_for_anthropic_web_search(None, Some(&usage)), 0.0);
|
|
||||||
assert_eq!(
|
|
||||||
get_cost_for_anthropic_web_search(Some(&model_info), None),
|
|
||||||
0.0
|
|
||||||
);
|
|
||||||
let dispatched = get_cost_for_built_in_tools(
|
|
||||||
BuiltInToolCostRequest {
|
|
||||||
response: &json!({}),
|
|
||||||
response_kind: ResponseKind::Anthropic,
|
|
||||||
usage: Some(&usage),
|
|
||||||
provider: Some("anthropic"),
|
|
||||||
params: &json!({}),
|
|
||||||
defaults: DefaultToolRates {
|
|
||||||
file_search_per_call: 0.0,
|
|
||||||
azure_file_search_per_gb_day: 0.0,
|
|
||||||
azure_vector_store_per_gb_day: 0.0,
|
|
||||||
azure_computer_input_per_1k_tokens: 0.0,
|
|
||||||
azure_computer_output_per_1k_tokens: 0.0,
|
|
||||||
code_interpreter_per_session: None,
|
|
||||||
xai_web_search_per_call: 0.0,
|
|
||||||
groq_browser_open_per_call: 0.0,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
Some(&model_info),
|
|
||||||
);
|
|
||||||
assert!((dispatched - 0.009).abs() < 1e-12);
|
|
||||||
}
|
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
#![allow(clippy::disallowed_types)]
|
#![allow(clippy::disallowed_types)]
|
||||||
// mirrors: test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py::TestAzureModelRouterFlatCost
|
// mirrors: test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py::TestAzureModelRouterFlatCost
|
||||||
use litellm_cost::azure_ai_cost::azure_ai_cost_per_token;
|
use litellm_cost::azure_ai_cost::azure_ai_cost_per_token;
|
||||||
|
use litellm_cost::cost_calculator::cost_per_token;
|
||||||
use litellm_cost::error::CostError;
|
use litellm_cost::error::CostError;
|
||||||
|
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
|
|
@ -124,9 +125,7 @@ fn azure_ai_cost_per_token_adds_fee_for_routed_response_model_only_once() {
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let catalog = catalog();
|
let catalog = catalog();
|
||||||
let response_only = catalog
|
let response_only = cost_per_token(&catalog, request("routed", &usage, None)).unwrap();
|
||||||
.cost_per_token(request("routed", &usage, None))
|
|
||||||
.unwrap();
|
|
||||||
assert_eq!(response_only, (0.2, 0.06));
|
assert_eq!(response_only, (0.2, 0.06));
|
||||||
let routed = azure_ai_cost_per_token(
|
let routed = azure_ai_cost_per_token(
|
||||||
&catalog,
|
&catalog,
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
#![allow(clippy::disallowed_types)]
|
#![allow(clippy::disallowed_types)]
|
||||||
// mirrors: test_litellm/test_cost_calculator.py::test_per_query_priced_rerank_deployment_completion_cost_is_nonzero
|
// mirrors: test_litellm/test_cost_calculator.py::test_per_query_priced_rerank_deployment_completion_cost_is_nonzero
|
||||||
|
use litellm_cost::cost_calculator::cost_per_token_for_call;
|
||||||
use litellm_cost::error::CostError;
|
use litellm_cost::error::CostError;
|
||||||
|
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
|
|
@ -61,34 +62,34 @@ fn token_call_routes_vertex_character_and_preserves_wall_clock_priority() {
|
||||||
]));
|
]));
|
||||||
let usage = usage();
|
let usage = usage();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
catalog
|
cost_per_token_for_call(
|
||||||
.cost_per_token_for_call(
|
&catalog,
|
||||||
request("character", Some("vertex_ai"), &usage),
|
request("character", Some("vertex_ai"), &usage),
|
||||||
CostCall::Token {
|
CostCall::Token {
|
||||||
call_type: "completion",
|
call_type: "completion",
|
||||||
prompt_characters: Some(10.0),
|
prompt_characters: Some(10.0),
|
||||||
completion_characters: Some(20.0),
|
completion_characters: Some(20.0),
|
||||||
request_model: None,
|
request_model: None,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.unwrap(),
|
.unwrap(),
|
||||||
(0.02, 0.06)
|
(0.02, 0.06)
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
catalog
|
cost_per_token_for_call(
|
||||||
.cost_per_token_for_call(
|
&catalog,
|
||||||
ModelCostRequest {
|
ModelCostRequest {
|
||||||
response_time_ms: Some(2000.0),
|
response_time_ms: Some(2000.0),
|
||||||
..request("duration", Some("vertex_ai"), &usage)
|
..request("duration", Some("vertex_ai"), &usage)
|
||||||
},
|
},
|
||||||
CostCall::Token {
|
CostCall::Token {
|
||||||
call_type: "completion",
|
call_type: "completion",
|
||||||
prompt_characters: Some(10.0),
|
prompt_characters: Some(10.0),
|
||||||
completion_characters: Some(20.0),
|
completion_characters: Some(20.0),
|
||||||
request_model: None,
|
request_model: None,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.unwrap(),
|
.unwrap(),
|
||||||
(1.0, 0.0)
|
(1.0, 0.0)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
@ -111,20 +112,20 @@ fn token_call_routes_together_embedding_and_azure_router() {
|
||||||
]));
|
]));
|
||||||
let usage = usage();
|
let usage = usage();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
catalog
|
cost_per_token_for_call(
|
||||||
.cost_per_token_for_call(
|
&catalog,
|
||||||
request("model-200m", Some("together_ai"), &usage),
|
request("model-200m", Some("together_ai"), &usage),
|
||||||
token("aembedding", None),
|
token("aembedding", None),
|
||||||
)
|
)
|
||||||
.unwrap(),
|
.unwrap(),
|
||||||
(4e-6, 0.0)
|
(4e-6, 0.0)
|
||||||
);
|
);
|
||||||
let (prompt, completion) = catalog
|
let (prompt, completion) = cost_per_token_for_call(
|
||||||
.cost_per_token_for_call(
|
&catalog,
|
||||||
request("routed", Some("azure_ai"), &usage),
|
request("routed", Some("azure_ai"), &usage),
|
||||||
token("completion", Some("azure_ai/model_router")),
|
token("completion", Some("azure_ai/model_router")),
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert!((prompt - 0.3).abs() < 1e-12);
|
assert!((prompt - 0.3).abs() < 1e-12);
|
||||||
assert!((completion - 0.06).abs() < 1e-12);
|
assert!((completion - 0.06).abs() < 1e-12);
|
||||||
}
|
}
|
||||||
|
|
@ -143,36 +144,36 @@ fn speech_and_transcription_calls_select_character_token_or_duration_rates() {
|
||||||
]));
|
]));
|
||||||
let usage = usage();
|
let usage = usage();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
catalog
|
cost_per_token_for_call(
|
||||||
.cost_per_token_for_call(
|
&catalog,
|
||||||
request("speech", Some("openai"), &usage),
|
request("speech", Some("openai"), &usage),
|
||||||
CostCall::Speech {
|
CostCall::Speech {
|
||||||
prompt_characters: Some(5.0),
|
prompt_characters: Some(5.0),
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.unwrap(),
|
.unwrap(),
|
||||||
(0.01, 0.0)
|
(0.01, 0.0)
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
catalog
|
cost_per_token_for_call(
|
||||||
.cost_per_token_for_call(
|
&catalog,
|
||||||
request("transcribe", Some("openai"), &usage),
|
request("transcribe", Some("openai"), &usage),
|
||||||
CostCall::Transcription {
|
CostCall::Transcription {
|
||||||
duration_seconds: 2.0,
|
duration_seconds: 2.0,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.unwrap(),
|
.unwrap(),
|
||||||
(0.3, 0.08)
|
(0.3, 0.08)
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
catalog
|
cost_per_token_for_call(
|
||||||
.cost_per_token_for_call(
|
&catalog,
|
||||||
request("transcribe", Some("openai"), &ChatUsage::default()),
|
request("transcribe", Some("openai"), &ChatUsage::default()),
|
||||||
CostCall::Transcription {
|
CostCall::Transcription {
|
||||||
duration_seconds: 2.0,
|
duration_seconds: 2.0,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.unwrap(),
|
.unwrap(),
|
||||||
(0.02, 0.0)
|
(0.02, 0.0)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
@ -195,41 +196,42 @@ fn retrieval_and_search_calls_use_their_reported_units() {
|
||||||
]));
|
]));
|
||||||
let usage = usage();
|
let usage = usage();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
catalog
|
cost_per_token_for_call(
|
||||||
.cost_per_token_for_call(
|
&catalog,
|
||||||
request("rerank", Some("cohere"), &usage),
|
request("rerank", Some("cohere"), &usage),
|
||||||
CostCall::Rerank {
|
CostCall::Rerank {
|
||||||
billed_units: Some(&json!({"search_units": 3})),
|
billed_units: Some(&json!({"search_units": 3})),
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.unwrap(),
|
.unwrap(),
|
||||||
(0.75, 0.0)
|
(0.75, 0.0)
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
catalog
|
cost_per_token_for_call(
|
||||||
.cost_per_token_for_call(
|
&catalog,
|
||||||
request("search_api", Some("vertex_ai"), &usage),
|
request("search_api", Some("vertex_ai"), &usage),
|
||||||
CostCall::VectorStoreSearch {
|
CostCall::VectorStoreSearch {
|
||||||
api_type: Some("search_api"),
|
api_type: Some("search_api"),
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.unwrap(),
|
.unwrap(),
|
||||||
(0.4, 0.0)
|
(0.4, 0.0)
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
catalog
|
cost_per_token_for_call(
|
||||||
.cost_per_token_for_call(
|
&catalog,
|
||||||
request("search", Some("exa_ai"), &usage),
|
request("search", Some("exa_ai"), &usage),
|
||||||
CostCall::Search {
|
CostCall::Search {
|
||||||
number_of_queries: Some(0),
|
number_of_queries: Some(0),
|
||||||
optional_params: &json!({}),
|
optional_params: &json!({}),
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.unwrap(),
|
.unwrap(),
|
||||||
(0.003, 0.0)
|
(0.003, 0.0)
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
catalog.cost_per_token_for_call(
|
cost_per_token_for_call(
|
||||||
|
&catalog,
|
||||||
request("rerank", None, &usage),
|
request("rerank", None, &usage),
|
||||||
CostCall::Rerank { billed_units: None },
|
CostCall::Rerank { billed_units: None },
|
||||||
),
|
),
|
||||||
|
|
@ -255,30 +257,30 @@ fn ocr_call_layers_deployment_and_published_rates_with_credit_priority() {
|
||||||
"pages_processed_annotation": 2
|
"pages_processed_annotation": 2
|
||||||
}});
|
}});
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
catalog
|
cost_per_token_for_call(
|
||||||
.cost_per_token_for_call(
|
&catalog,
|
||||||
request("ocr", Some("mistral"), &usage),
|
request("ocr", Some("mistral"), &usage),
|
||||||
CostCall::Ocr {
|
CostCall::Ocr {
|
||||||
response: &credit_response,
|
response: &credit_response,
|
||||||
deployment_info: Some(&deployment),
|
deployment_info: Some(&deployment),
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.unwrap(),
|
.unwrap(),
|
||||||
(1.0, 0.0)
|
(1.0, 0.0)
|
||||||
);
|
);
|
||||||
let page_response = json!({"usage_info": {
|
let page_response = json!({"usage_info": {
|
||||||
"pages_processed": 3,
|
"pages_processed": 3,
|
||||||
"pages_processed_annotation": 2
|
"pages_processed_annotation": 2
|
||||||
}});
|
}});
|
||||||
let (prompt, completion) = catalog
|
let (prompt, completion) = cost_per_token_for_call(
|
||||||
.cost_per_token_for_call(
|
&catalog,
|
||||||
request("ocr", Some("mistral"), &usage),
|
request("ocr", Some("mistral"), &usage),
|
||||||
CostCall::Ocr {
|
CostCall::Ocr {
|
||||||
response: &page_response,
|
response: &page_response,
|
||||||
deployment_info: Some(&deployment),
|
deployment_info: Some(&deployment),
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert!((prompt - 0.17).abs() < 1e-12);
|
assert!((prompt - 0.17).abs() < 1e-12);
|
||||||
assert_eq!(completion, 0.0);
|
assert_eq!(completion, 0.0);
|
||||||
}
|
}
|
||||||
|
|
@ -292,7 +294,8 @@ fn ocr_call_distinguishes_missing_pages_from_unpriced_usage() {
|
||||||
let usage = usage();
|
let usage = usage();
|
||||||
let missing_pages = json!({"usage_info": {}});
|
let missing_pages = json!({"usage_info": {}});
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
catalog.cost_per_token_for_call(
|
cost_per_token_for_call(
|
||||||
|
&catalog,
|
||||||
request("priced", Some("mistral"), &usage),
|
request("priced", Some("mistral"), &usage),
|
||||||
CostCall::Ocr {
|
CostCall::Ocr {
|
||||||
response: &missing_pages,
|
response: &missing_pages,
|
||||||
|
|
@ -302,19 +305,20 @@ fn ocr_call_distinguishes_missing_pages_from_unpriced_usage() {
|
||||||
Err(CostError::MissingPages)
|
Err(CostError::MissingPages)
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
catalog
|
cost_per_token_for_call(
|
||||||
.cost_per_token_for_call(
|
&catalog,
|
||||||
request("unpriced", Some("mistral"), &usage),
|
request("unpriced", Some("mistral"), &usage),
|
||||||
CostCall::Ocr {
|
CostCall::Ocr {
|
||||||
response: &missing_pages,
|
response: &missing_pages,
|
||||||
deployment_info: None,
|
deployment_info: None,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.unwrap(),
|
.unwrap(),
|
||||||
(0.0, 0.0)
|
(0.0, 0.0)
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
catalog.cost_per_token_for_call(
|
cost_per_token_for_call(
|
||||||
|
&catalog,
|
||||||
request("priced", Some("mistral"), &usage),
|
request("priced", Some("mistral"), &usage),
|
||||||
CostCall::Ocr {
|
CostCall::Ocr {
|
||||||
response: &json!({}),
|
response: &json!({}),
|
||||||
|
|
@ -346,17 +350,17 @@ fn batch_call_selects_independent_thresholds_modalities_and_region() {
|
||||||
}}))
|
}}))
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let (prompt, completion) = catalog
|
let (prompt, completion) = cost_per_token_for_call(
|
||||||
.cost_per_token_for_call(
|
&catalog,
|
||||||
ModelCostRequest {
|
ModelCostRequest {
|
||||||
data_residency: Some("eu"),
|
data_residency: Some("eu"),
|
||||||
..request("batch", Some("openai"), &usage)
|
..request("batch", Some("openai"), &usage)
|
||||||
},
|
},
|
||||||
CostCall::Batch {
|
CostCall::Batch {
|
||||||
deployment_info: None,
|
deployment_info: None,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert!((prompt - (90.0 * 2e-6 + 10.0 * 5e-6 + 20.0 * 0.4e-6) * 1.1).abs() < 1e-12);
|
assert!((prompt - (90.0 * 2e-6 + 10.0 * 5e-6 + 20.0 * 0.4e-6) * 1.1).abs() < 1e-12);
|
||||||
assert!((completion - 30.0 * 3e-6 * 1.1).abs() < 1e-12);
|
assert!((completion - 30.0 * 3e-6 * 1.1).abs() < 1e-12);
|
||||||
}
|
}
|
||||||
|
|
@ -378,25 +382,25 @@ fn batch_call_falls_back_from_unpriced_deployment_and_uses_inclusive_xai_tier()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
catalog
|
cost_per_token_for_call(
|
||||||
.cost_per_token_for_call(
|
&catalog,
|
||||||
request("batch", Some("xai"), &usage),
|
request("batch", Some("xai"), &usage),
|
||||||
CostCall::Batch {
|
CostCall::Batch {
|
||||||
deployment_info: Some(&json!({"id": "deployment"})),
|
deployment_info: Some(&json!({"id": "deployment"})),
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.unwrap(),
|
.unwrap(),
|
||||||
(100.0 * 2e-6, 20.0 * 3e-6)
|
(100.0 * 2e-6, 20.0 * 3e-6)
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
catalog
|
cost_per_token_for_call(
|
||||||
.cost_per_token_for_call(
|
&catalog,
|
||||||
request("unknown", Some("xai"), &usage),
|
request("unknown", Some("xai"), &usage),
|
||||||
CostCall::Batch {
|
CostCall::Batch {
|
||||||
deployment_info: None,
|
deployment_info: None,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.unwrap(),
|
.unwrap(),
|
||||||
(0.0, 0.0)
|
(0.0, 0.0)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
@ -417,14 +421,14 @@ fn batch_call_parses_k_threshold_and_numeric_string_rate() {
|
||||||
}}))
|
}}))
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let (prompt, completion) = catalog
|
let (prompt, completion) = cost_per_token_for_call(
|
||||||
.cost_per_token_for_call(
|
&catalog,
|
||||||
request("batch", Some("openai"), &usage),
|
request("batch", Some("openai"), &usage),
|
||||||
CostCall::Batch {
|
CostCall::Batch {
|
||||||
deployment_info: None,
|
deployment_info: None,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert!((prompt - 100_001.0 * 2e-6).abs() < 1e-12);
|
assert!((prompt - 100_001.0 * 2e-6).abs() < 1e-12);
|
||||||
assert!((completion - 2.0 * 3e-6).abs() < 1e-12);
|
assert!((completion - 2.0 * 3e-6).abs() < 1e-12);
|
||||||
}
|
}
|
||||||
|
|
@ -446,14 +450,14 @@ fn batch_call_invalid_highest_tier_falls_back_to_flat_rate() {
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
catalog
|
cost_per_token_for_call(
|
||||||
.cost_per_token_for_call(
|
&catalog,
|
||||||
request("batch", Some("openai"), &usage),
|
request("batch", Some("openai"), &usage),
|
||||||
CostCall::Batch {
|
CostCall::Batch {
|
||||||
deployment_info: None,
|
deployment_info: None,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.unwrap(),
|
.unwrap(),
|
||||||
(201.0 * 1e-6, 0.0)
|
(201.0 * 1e-6, 0.0)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
@ -473,21 +477,21 @@ fn generic_token_cost_is_gated_on_token_or_tiered_pricing() {
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
catalog
|
cost_per_token_for_call(
|
||||||
.cost_per_token_for_call(
|
&catalog,
|
||||||
request("audio-only/model", Some("mistral"), &usage),
|
request("audio-only/model", Some("mistral"), &usage),
|
||||||
token("completion", None),
|
token("completion", None),
|
||||||
)
|
)
|
||||||
.unwrap(),
|
.unwrap(),
|
||||||
(0.0, 0.0),
|
(0.0, 0.0),
|
||||||
"providers without a dedicated calculator bill zero when no token or tiered pricing exists"
|
"providers without a dedicated calculator bill zero when no token or tiered pricing exists"
|
||||||
);
|
);
|
||||||
let (input, output) = catalog
|
let (input, output) = cost_per_token_for_call(
|
||||||
.cost_per_token_for_call(
|
&catalog,
|
||||||
request("audio-only/model", Some("anthropic"), &usage),
|
request("audio-only/model", Some("anthropic"), &usage),
|
||||||
token("completion", None),
|
token("completion", None),
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert!(
|
assert!(
|
||||||
(input - 100.0 * 1e-5).abs() < 1e-12 && (output - 20.0 * 2e-5).abs() < 1e-12,
|
(input - 100.0 * 1e-5).abs() < 1e-12 && (output - 20.0 * 2e-5).abs() < 1e-12,
|
||||||
"anthropic dispatches before the gate in Python and still bills modality rates"
|
"anthropic dispatches before the gate in Python and still bills modality rates"
|
||||||
|
|
@ -511,12 +515,12 @@ fn anthropic_totals_apply_the_geo_multiplier_other_providers_do_not() {
|
||||||
}}))
|
}}))
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let (input, output) = catalog
|
let (input, output) = cost_per_token_for_call(
|
||||||
.cost_per_token_for_call(
|
&catalog,
|
||||||
request("model", Some("anthropic"), &usage),
|
request("model", Some("anthropic"), &usage),
|
||||||
token("completion", None),
|
token("completion", None),
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert!((input - 100.0 * 2e-6 * 1.4).abs() < 1e-12);
|
assert!((input - 100.0 * 2e-6 * 1.4).abs() < 1e-12);
|
||||||
assert!((output - 50.0 * 4e-6 * 1.4).abs() < 1e-12);
|
assert!((output - 50.0 * 4e-6 * 1.4).abs() < 1e-12);
|
||||||
let catalog = ModelInfoCatalog::new(HashMap::from([(
|
let catalog = ModelInfoCatalog::new(HashMap::from([(
|
||||||
|
|
@ -527,12 +531,12 @@ fn anthropic_totals_apply_the_geo_multiplier_other_providers_do_not() {
|
||||||
"provider_specific_entry": {"us": 1.4}
|
"provider_specific_entry": {"us": 1.4}
|
||||||
}),
|
}),
|
||||||
)]));
|
)]));
|
||||||
let (input, output) = catalog
|
let (input, output) = cost_per_token_for_call(
|
||||||
.cost_per_token_for_call(
|
&catalog,
|
||||||
request("model", Some("openai"), &usage),
|
request("model", Some("openai"), &usage),
|
||||||
token("completion", None),
|
token("completion", None),
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert!((input - 100.0 * 2e-6).abs() < 1e-12);
|
assert!((input - 100.0 * 2e-6).abs() < 1e-12);
|
||||||
assert!((output - 50.0 * 4e-6).abs() < 1e-12);
|
assert!((output - 50.0 * 4e-6).abs() < 1e-12);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,159 +1,3 @@
|
||||||
#![allow(clippy::disallowed_types)]
|
#![allow(clippy::disallowed_types)]
|
||||||
|
|
||||||
// mirrors: test_litellm/test_cost_calculator.py::test_completion_cost_extracts_service_tier_from_response
|
// mirrors: test_litellm/test_cost_calculator.py::test_completion_cost_extracts_service_tier_from_response
|
||||||
// mirrors: test_litellm/llms/gemini/test_cost_calculator.py::test_map_traffic_type_to_service_tier
|
// mirrors: test_litellm/llms/gemini/test_cost_calculator.py::test_map_traffic_type_to_service_tier
|
||||||
|
|
||||||
use std::collections::HashMap;
|
|
||||||
|
|
||||||
use jiff::Timestamp;
|
|
||||||
use litellm_cost::catalog::{ModelCostRequest, ModelInfoCatalog};
|
|
||||||
use litellm_cost::completion_input::{
|
|
||||||
CompletionInputRequest, ResponseKind, infer_call_type, map_traffic_type_to_service_tier,
|
|
||||||
select_service_tier,
|
|
||||||
};
|
|
||||||
use litellm_cost::model_selection::ModelSelectionRequest;
|
|
||||||
use rstest::rstest;
|
|
||||||
use serde_json::{Value, json};
|
|
||||||
|
|
||||||
const PROVIDERS: &[&str] = &["anthropic", "vertex_ai"];
|
|
||||||
|
|
||||||
#[rstest]
|
|
||||||
#[case("ON_DEMAND_PRIORITY", Some("priority"))]
|
|
||||||
#[case("on_demand_flex", Some("flex"))]
|
|
||||||
#[case("BATCH", Some("flex"))]
|
|
||||||
#[case("FLEX", Some("flex"))]
|
|
||||||
#[case("ON_DEMAND", None)]
|
|
||||||
#[case("unknown", None)]
|
|
||||||
fn traffic_type_maps_only_billable_tiers(#[case] raw: &str, #[case] expected: Option<&str>) {
|
|
||||||
assert_eq!(map_traffic_type_to_service_tier(Some(raw)), expected);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[rstest]
|
|
||||||
fn request_auto_defers_to_served_usage_tier() {
|
|
||||||
let explicit = json!("auto");
|
|
||||||
let optional = json!({"service_tier": "flex"});
|
|
||||||
let response = json!({"service_tier": {"name": "auto"}, "usage": {"service_tier": "priority"}});
|
|
||||||
assert_eq!(
|
|
||||||
select_service_tier(Some(&explicit), Some(&optional), Some(&response), None),
|
|
||||||
Some("priority".to_owned())
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[rstest]
|
|
||||||
#[case(json!("priority"), "priority")]
|
|
||||||
#[case(json!({"name": "auto"}), "flex")]
|
|
||||||
#[case(json!("AUTO"), "flex")]
|
|
||||||
fn request_tier_precedes_response_when_billable(#[case] requested: Value, #[case] expected: &str) {
|
|
||||||
let response = json!({"service_tier": "flex", "usage": {"service_tier": "priority"}});
|
|
||||||
assert_eq!(
|
|
||||||
select_service_tier(Some(&requested), None, Some(&response), None),
|
|
||||||
Some(expected.to_owned())
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[rstest]
|
|
||||||
fn traffic_type_is_last_tier_fallback() {
|
|
||||||
let response = json!({"usage": {"service_tier": "auto"}});
|
|
||||||
let hidden = json!({"provider_specific_fields": {"traffic_type": "ON_DEMAND_FLEX"}});
|
|
||||||
assert_eq!(
|
|
||||||
select_service_tier(None, None, Some(&response), Some(&hidden)),
|
|
||||||
Some("flex".to_owned())
|
|
||||||
);
|
|
||||||
let served = json!({"usage": {"service_tier": "priority"}});
|
|
||||||
assert_eq!(
|
|
||||||
select_service_tier(None, None, Some(&served), Some(&hidden)),
|
|
||||||
Some("priority".to_owned())
|
|
||||||
);
|
|
||||||
assert_eq!(select_service_tier(None, None, None, Some(&hidden)), None);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[rstest]
|
|
||||||
#[case(Some("batch"), Some(ResponseKind::Completion), Some("batch"))]
|
|
||||||
#[case(None, Some(ResponseKind::Embedding), Some("embedding"))]
|
|
||||||
#[case(None, Some(ResponseKind::ImageGeneration), Some("image_generation"))]
|
|
||||||
#[case(None, None, None)]
|
|
||||||
fn call_type_prefers_explicit_value(
|
|
||||||
#[case] explicit: Option<&str>,
|
|
||||||
#[case] kind: Option<ResponseKind>,
|
|
||||||
#[case] expected: Option<&str>,
|
|
||||||
) {
|
|
||||||
assert_eq!(infer_call_type(explicit, kind), expected);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[rstest]
|
|
||||||
fn prepared_input_prices_served_priority_rate() {
|
|
||||||
let catalog = ModelInfoCatalog::new(HashMap::from([(
|
|
||||||
"anthropic/served".to_owned(),
|
|
||||||
json!({
|
|
||||||
"litellm_provider": "anthropic",
|
|
||||||
"input_cost_per_token": 0.01,
|
|
||||||
"output_cost_per_token": 0.02,
|
|
||||||
"input_cost_per_token_priority": 0.03,
|
|
||||||
"output_cost_per_token_priority": 0.04
|
|
||||||
}),
|
|
||||||
)]));
|
|
||||||
let response = json!({
|
|
||||||
"model": "served",
|
|
||||||
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "service_tier": "priority"}
|
|
||||||
});
|
|
||||||
let request_tier = json!("auto");
|
|
||||||
let prepared = catalog
|
|
||||||
.prepare_completion_input(CompletionInputRequest {
|
|
||||||
model_selection: ModelSelectionRequest {
|
|
||||||
model: Some("requested"),
|
|
||||||
response: Some(&response),
|
|
||||||
hidden_params: None,
|
|
||||||
base_model: None,
|
|
||||||
custom_pricing: false,
|
|
||||||
provider: Some("anthropic"),
|
|
||||||
router_model_id: None,
|
|
||||||
region_name: None,
|
|
||||||
known_providers: PROVIDERS,
|
|
||||||
},
|
|
||||||
call_type: None,
|
|
||||||
response_kind: Some(ResponseKind::Completion),
|
|
||||||
service_tier: Some(&request_tier),
|
|
||||||
optional_params: None,
|
|
||||||
})
|
|
||||||
.unwrap();
|
|
||||||
assert_eq!(prepared.call_type, "completion");
|
|
||||||
assert_eq!(
|
|
||||||
prepared.model_candidates[0].as_deref(),
|
|
||||||
Some("anthropic/served")
|
|
||||||
);
|
|
||||||
assert_eq!(prepared.service_tier.as_deref(), Some("priority"));
|
|
||||||
let (prompt, output) = catalog
|
|
||||||
.cost_per_token(ModelCostRequest {
|
|
||||||
model: prepared.model_candidates[0].as_deref().unwrap(),
|
|
||||||
provider: Some("anthropic"),
|
|
||||||
region: None,
|
|
||||||
usage: prepared.usage.as_ref().unwrap(),
|
|
||||||
service_tier: prepared.service_tier.as_deref(),
|
|
||||||
data_residency: None,
|
|
||||||
vertex_location: None,
|
|
||||||
at: Timestamp::from_second(0).unwrap(),
|
|
||||||
response_time_ms: None,
|
|
||||||
})
|
|
||||||
.unwrap();
|
|
||||||
assert!((prompt - 0.3).abs() < 1e-10);
|
|
||||||
assert!((output - 0.2).abs() < 1e-10);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[rstest]
|
|
||||||
fn non_string_response_tier_defers_to_the_served_usage_tier() {
|
|
||||||
let response =
|
|
||||||
json!({"service_tier": {"name": "priority"}, "usage": {"service_tier": "priority"}});
|
|
||||||
assert_eq!(
|
|
||||||
select_service_tier(None, None, Some(&response), None),
|
|
||||||
Some("priority".to_owned())
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[rstest]
|
|
||||||
#[case(json!({"name": "priority"}))]
|
|
||||||
#[case(json!(5))]
|
|
||||||
#[case(json!(true))]
|
|
||||||
fn non_string_usage_tier_falls_back_to_standard(#[case] usage_tier: Value) {
|
|
||||||
let response = json!({"service_tier": {"name": "auto"}, "usage": {"service_tier": usage_tier}});
|
|
||||||
assert_eq!(select_service_tier(None, None, Some(&response), None), None);
|
|
||||||
}
|
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,8 @@
|
||||||
#![allow(clippy::disallowed_types)]
|
#![allow(clippy::disallowed_types)]
|
||||||
// mirrors: test_litellm/test_cost_calculator.py::test_custom_pricing_anthropic_style_cache_tokens_not_double_counted
|
// mirrors: test_litellm/test_cost_calculator.py::test_custom_pricing_anthropic_style_cache_tokens_not_double_counted
|
||||||
|
use litellm_cost::cost_calculator::{
|
||||||
|
cost_per_token_for_call_with_custom, cost_per_token_with_custom,
|
||||||
|
};
|
||||||
use litellm_cost::error::CostError;
|
use litellm_cost::error::CostError;
|
||||||
|
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
|
|
@ -203,30 +206,28 @@ fn catalog_custom_rates_override_lookup_and_call_dispatch() {
|
||||||
token: Some(rates(0.01, 0.02, Some(0.001), Some(0.03))),
|
token: Some(rates(0.01, 0.02, Some(0.001), Some(0.03))),
|
||||||
per_second: Some(1.0),
|
per_second: Some(1.0),
|
||||||
};
|
};
|
||||||
let direct = catalog
|
let direct = cost_per_token_with_custom(&catalog, request, pricing).unwrap();
|
||||||
.cost_per_token_with_custom(request, pricing)
|
let dispatched = cost_per_token_for_call_with_custom(
|
||||||
.unwrap();
|
&catalog,
|
||||||
let dispatched = catalog
|
request,
|
||||||
.cost_per_token_for_call_with_custom(
|
CostCall::Speech {
|
||||||
request,
|
prompt_characters: None,
|
||||||
CostCall::Speech {
|
},
|
||||||
prompt_characters: None,
|
pricing,
|
||||||
},
|
)
|
||||||
pricing,
|
.unwrap();
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
assert!((direct.0 - 0.104).abs() < 1e-12);
|
assert!((direct.0 - 0.104).abs() < 1e-12);
|
||||||
assert!((direct.1 - 0.06).abs() < 1e-12);
|
assert!((direct.1 - 0.06).abs() < 1e-12);
|
||||||
assert_eq!(direct, dispatched);
|
assert_eq!(direct, dispatched);
|
||||||
let seconds = catalog
|
let seconds = cost_per_token_with_custom(
|
||||||
.cost_per_token_with_custom(
|
&catalog,
|
||||||
request,
|
request,
|
||||||
CustomPricing {
|
CustomPricing {
|
||||||
token: None,
|
token: None,
|
||||||
per_second: Some(0.08),
|
per_second: Some(0.08),
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(seconds.0, 0.0);
|
assert_eq!(seconds.0, 0.0);
|
||||||
assert!((seconds.1 - 0.2).abs() < 1e-12);
|
assert!((seconds.1 - 0.2).abs() < 1e-12);
|
||||||
}
|
}
|
||||||
|
|
@ -249,13 +250,12 @@ fn catalog_custom_pricing_falls_back_and_rejects_invalid_rates() {
|
||||||
at: "2026-01-01T12:00Z".parse().unwrap(),
|
at: "2026-01-01T12:00Z".parse().unwrap(),
|
||||||
response_time_ms: None,
|
response_time_ms: None,
|
||||||
};
|
};
|
||||||
let fallback = catalog
|
let fallback = cost_per_token_with_custom(&catalog, request, CustomPricing::NONE).unwrap();
|
||||||
.cost_per_token_with_custom(request, CustomPricing::NONE)
|
|
||||||
.unwrap();
|
|
||||||
assert!((fallback.0 - 0.1).abs() < 1e-12);
|
assert!((fallback.0 - 0.1).abs() < 1e-12);
|
||||||
assert!((fallback.1 - 0.06).abs() < 1e-12);
|
assert!((fallback.1 - 0.06).abs() < 1e-12);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
catalog.cost_per_token_with_custom(
|
cost_per_token_with_custom(
|
||||||
|
&catalog,
|
||||||
request,
|
request,
|
||||||
CustomPricing {
|
CustomPricing {
|
||||||
token: Some(rates(-0.01, 0.02, None, None)),
|
token: Some(rates(-0.01, 0.02, None, None)),
|
||||||
|
|
|
||||||
|
|
@ -1,171 +1,2 @@
|
||||||
#![allow(clippy::disallowed_types)]
|
#![allow(clippy::disallowed_types)]
|
||||||
|
|
||||||
// mirrors: test_litellm/llms/dashscope/test_dashscope_cost_calculator.py::TestDashscopeCostCalculator
|
// mirrors: test_litellm/llms/dashscope/test_dashscope_cost_calculator.py::TestDashscopeCostCalculator
|
||||||
|
|
||||||
use std::collections::HashMap;
|
|
||||||
|
|
||||||
use jiff::Timestamp;
|
|
||||||
use litellm_cost::catalog::{ModelCostRequest, ModelInfoCatalog};
|
|
||||||
use litellm_cost::dashscope_cost::{cost_per_token, extract_token_breakdown};
|
|
||||||
use litellm_cost::usage_dispatch::get_usage_object;
|
|
||||||
use rstest::rstest;
|
|
||||||
use serde_json::{Value, json};
|
|
||||||
|
|
||||||
fn at(value: &str) -> Timestamp {
|
|
||||||
value.parse().unwrap()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn usage(value: Value) -> litellm_cost::responses_usage::ChatUsage {
|
|
||||||
get_usage_object(&json!({"usage": value})).unwrap().unwrap()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn tiered_model() -> Value {
|
|
||||||
json!({
|
|
||||||
"output_cost_per_reasoning_token": 7e-6,
|
|
||||||
"tiered_pricing": [
|
|
||||||
{"range": [0, 1000], "input_cost_per_token": "1e-6", "output_cost_per_token": "2e-6", "cache_read_input_token_cost": 0.2e-6, "cache_creation_input_token_cost": 1.5e-6},
|
|
||||||
{"range": [1000, 2000], "input_cost_per_token": "3e-6", "output_cost_per_token": "4e-6", "cache_read_input_token_cost": 0.6e-6, "cache_creation_input_token_cost": 4.5e-6}
|
|
||||||
]
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
#[rstest]
|
|
||||||
#[case(1000, 1e-6, 2e-6)]
|
|
||||||
#[case(1001, 3e-6, 4e-6)]
|
|
||||||
#[case(2500, 3e-6, 4e-6)]
|
|
||||||
fn dashscope_cost_per_token_uses_one_input_selected_tier_for_both_directions(
|
|
||||||
#[case] prompt_tokens: u64,
|
|
||||||
#[case] input_rate: f64,
|
|
||||||
#[case] output_rate: f64,
|
|
||||||
) {
|
|
||||||
let usage = usage(json!({
|
|
||||||
"prompt_tokens": prompt_tokens,
|
|
||||||
"completion_tokens": 3000,
|
|
||||||
"total_tokens": prompt_tokens + 3000
|
|
||||||
}));
|
|
||||||
let actual = cost_per_token(&usage, &tiered_model(), at("2026-01-01T12:00Z"));
|
|
||||||
assert!((actual.0 - prompt_tokens as f64 * input_rate).abs() < 1e-12);
|
|
||||||
assert!((actual.1 - 3000.0 * output_rate).abs() < 1e-12);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[rstest]
|
|
||||||
fn dashscope_cost_per_token_prices_nested_cache_creation_and_reasoning_at_selected_tier() {
|
|
||||||
let usage = usage(json!({
|
|
||||||
"prompt_tokens": 1500,
|
|
||||||
"completion_tokens": 200,
|
|
||||||
"total_tokens": 1700,
|
|
||||||
"prompt_tokens_details": {
|
|
||||||
"text_tokens": 1500,
|
|
||||||
"cached_tokens": 400,
|
|
||||||
"cache_creation_input_tokens": 600
|
|
||||||
},
|
|
||||||
"completion_tokens_details": {"reasoning_tokens": 50}
|
|
||||||
}));
|
|
||||||
let breakdown = extract_token_breakdown(&usage);
|
|
||||||
assert_eq!(breakdown.text_tokens, 500);
|
|
||||||
assert_eq!(breakdown.total_input_tokens(), 1500);
|
|
||||||
let actual = cost_per_token(&usage, &tiered_model(), at("2026-01-01T12:00Z"));
|
|
||||||
assert!((actual.0 - (500.0 * 3e-6 + 400.0 * 0.6e-6 + 600.0 * 4.5e-6)).abs() < 1e-12);
|
|
||||||
assert!((actual.1 - 200.0 * 4e-6).abs() < 1e-12);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[rstest]
|
|
||||||
#[case(json!({"output_cost_per_token": 2e-6, "output_cost_per_reasoning_token": 0}), 50.0 * 2e-6)]
|
|
||||||
#[case(json!({"output_cost_per_token": 2e-6}), 200.0 * 2e-6)]
|
|
||||||
#[case(json!({"input_cost_per_token": 1e-6}), 50.0 * 2e-6 + 150.0 * 7e-6)]
|
|
||||||
fn dashscope_tier_reasoning_fallback_and_explicit_zero(
|
|
||||||
#[case] tier_fields: Value,
|
|
||||||
#[case] expected_completion: f64,
|
|
||||||
) {
|
|
||||||
let tier = Value::Object(
|
|
||||||
json!({"range": [0, 1000], "input_cost_per_token": 1e-6})
|
|
||||||
.as_object()
|
|
||||||
.unwrap()
|
|
||||||
.iter()
|
|
||||||
.chain(tier_fields.as_object().unwrap().iter())
|
|
||||||
.map(|(key, value)| (key.clone(), value.clone()))
|
|
||||||
.collect(),
|
|
||||||
);
|
|
||||||
let model = json!({
|
|
||||||
"output_cost_per_token": 2e-6,
|
|
||||||
"output_cost_per_reasoning_token": 7e-6,
|
|
||||||
"tiered_pricing": [tier]
|
|
||||||
});
|
|
||||||
let usage = usage(json!({
|
|
||||||
"prompt_tokens": 500,
|
|
||||||
"completion_tokens": 200,
|
|
||||||
"total_tokens": 700,
|
|
||||||
"completion_tokens_details": {"reasoning_tokens": 150}
|
|
||||||
}));
|
|
||||||
let actual = cost_per_token(&usage, &model, at("2026-01-01T12:00Z"));
|
|
||||||
assert!((actual.1 - expected_completion).abs() < 1e-12);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[rstest]
|
|
||||||
#[case("2026-09-03T17:25Z", 0.1e-6, 0.4e-6, 0.8e-6)]
|
|
||||||
#[case("2026-09-03T09:00Z", 3e-6, 4e-6, 4e-6)]
|
|
||||||
fn dashscope_off_peak_rates_override_selected_tier_and_reasoning(
|
|
||||||
#[case] billed_at: &str,
|
|
||||||
#[case] input_rate: f64,
|
|
||||||
#[case] output_rate: f64,
|
|
||||||
#[case] reasoning_rate: f64,
|
|
||||||
) {
|
|
||||||
let model = json!({
|
|
||||||
"tiered_pricing": [{"range": [0, 1000], "input_cost_per_token": 3e-6, "output_cost_per_token": 4e-6}],
|
|
||||||
"off_peak_pricing": {
|
|
||||||
"hours_utc": "14:00-00:00",
|
|
||||||
"input_cost_per_token": 0.1e-6,
|
|
||||||
"output_cost_per_token": 0.4e-6,
|
|
||||||
"output_cost_per_reasoning_token": 0.8e-6
|
|
||||||
}
|
|
||||||
});
|
|
||||||
let usage = usage(json!({
|
|
||||||
"prompt_tokens": 500,
|
|
||||||
"completion_tokens": 100,
|
|
||||||
"total_tokens": 600,
|
|
||||||
"completion_tokens_details": {"reasoning_tokens": 40}
|
|
||||||
}));
|
|
||||||
let actual = cost_per_token(&usage, &model, at(billed_at));
|
|
||||||
assert!((actual.0 - 500.0 * input_rate).abs() < 1e-12);
|
|
||||||
assert!((actual.1 - (60.0 * output_rate + 40.0 * reasoning_rate)).abs() < 1e-12);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[rstest]
|
|
||||||
#[case("dashscope")]
|
|
||||||
#[case("qwencloud")]
|
|
||||||
#[case("qwen_ai_platform")]
|
|
||||||
fn dashscope_cost_per_token_dispatches_brand_aliases(#[case] provider: &str) {
|
|
||||||
let model_info = json!({
|
|
||||||
"input_cost_per_token": 2e-6,
|
|
||||||
"output_cost_per_token": 4e-6,
|
|
||||||
"cache_read_input_token_cost": 0.5e-6
|
|
||||||
});
|
|
||||||
let usage = usage(json!({
|
|
||||||
"prompt_tokens": 100,
|
|
||||||
"completion_tokens": 20,
|
|
||||||
"total_tokens": 120,
|
|
||||||
"prompt_tokens_details": {"cached_tokens": 40}
|
|
||||||
}));
|
|
||||||
let catalog = ModelInfoCatalog::new(HashMap::from([(
|
|
||||||
format!("{provider}/model"),
|
|
||||||
model_info.clone(),
|
|
||||||
)]));
|
|
||||||
let actual = catalog
|
|
||||||
.cost_per_token(ModelCostRequest {
|
|
||||||
model: "model",
|
|
||||||
provider: Some(provider),
|
|
||||||
region: None,
|
|
||||||
usage: &usage,
|
|
||||||
service_tier: None,
|
|
||||||
data_residency: None,
|
|
||||||
vertex_location: None,
|
|
||||||
at: at("2026-01-01T12:00Z"),
|
|
||||||
response_time_ms: None,
|
|
||||||
})
|
|
||||||
.unwrap();
|
|
||||||
assert_eq!(
|
|
||||||
actual,
|
|
||||||
cost_per_token(&usage, &model_info, at("2026-01-01T12:00Z"))
|
|
||||||
);
|
|
||||||
assert!((actual.0 - (60.0 * 2e-6 + 40.0 * 0.5e-6)).abs() < 1e-12);
|
|
||||||
}
|
|
||||||
|
|
|
||||||
|
|
@ -1,131 +1,2 @@
|
||||||
#![allow(clippy::disallowed_types)]
|
#![allow(clippy::disallowed_types)]
|
||||||
|
|
||||||
// mirrors: unit/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py
|
// mirrors: unit/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py
|
||||||
|
|
||||||
use std::collections::HashMap;
|
|
||||||
|
|
||||||
use jiff::Timestamp;
|
|
||||||
use litellm_cost::catalog::{ModelCostRequest, ModelInfoCatalog};
|
|
||||||
use litellm_cost::fireworks_cost::{
|
|
||||||
FireworksThresholds, cost_per_token, get_base_model_for_pricing,
|
|
||||||
};
|
|
||||||
use litellm_cost::usage_dispatch::get_usage_object;
|
|
||||||
use rstest::rstest;
|
|
||||||
use serde_json::{Value, json};
|
|
||||||
|
|
||||||
fn at(value: &str) -> Timestamp {
|
|
||||||
value.parse().unwrap()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn usage(value: Value) -> litellm_cost::responses_usage::ChatUsage {
|
|
||||||
get_usage_object(&json!({"usage": value})).unwrap().unwrap()
|
|
||||||
}
|
|
||||||
|
|
||||||
#[rstest]
|
|
||||||
#[case("MODEL-4B", "fireworks-ai-up-to-4b")]
|
|
||||||
#[case("model-16b", "fireworks-ai-4.1b-to-16b")]
|
|
||||||
#[case("model-17b", "fireworks-ai-above-16b")]
|
|
||||||
#[case("model-7x8b", "fireworks-ai-moe-up-to-56b")]
|
|
||||||
#[case("model-8x22b", "fireworks-ai-56b-to-176b")]
|
|
||||||
#[case("model-8x30b", "fireworks-ai-above-16b")]
|
|
||||||
#[case("model-without-size", "fireworks-ai-default")]
|
|
||||||
fn fireworks_model_size_category_matches_python_pattern_order(
|
|
||||||
#[case] model: &str,
|
|
||||||
#[case] expected: &str,
|
|
||||||
) {
|
|
||||||
assert_eq!(
|
|
||||||
get_base_model_for_pricing(model, FireworksThresholds::default()),
|
|
||||||
expected
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[rstest]
|
|
||||||
fn fireworks_model_size_categories_accept_configured_thresholds() {
|
|
||||||
let thresholds = FireworksThresholds {
|
|
||||||
small: 8,
|
|
||||||
medium: 32,
|
|
||||||
moe_small: 80,
|
|
||||||
moe_medium: 200,
|
|
||||||
};
|
|
||||||
assert_eq!(
|
|
||||||
get_base_model_for_pricing("model-8b", thresholds),
|
|
||||||
"fireworks-ai-up-to-4b"
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
get_base_model_for_pricing("model-8x10b", thresholds),
|
|
||||||
"fireworks-ai-moe-up-to-56b"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[rstest]
|
|
||||||
#[case(false, 2e-6)]
|
|
||||||
#[case(true, 5e-6)]
|
|
||||||
fn fireworks_catalog_prefers_exact_model_then_falls_back_to_size_category(
|
|
||||||
#[case] exact_model_exists: bool,
|
|
||||||
#[case] selected_input_rate: f64,
|
|
||||||
) {
|
|
||||||
let category = (
|
|
||||||
"fireworks_ai/fireworks-ai-moe-up-to-56b".to_owned(),
|
|
||||||
json!({"input_cost_per_token": 2e-6, "output_cost_per_token": 4e-6}),
|
|
||||||
);
|
|
||||||
let exact = exact_model_exists.then(|| {
|
|
||||||
(
|
|
||||||
"fireworks_ai/accounts/models/model-7x8b".to_owned(),
|
|
||||||
json!({"input_cost_per_token": 5e-6, "output_cost_per_token": 4e-6}),
|
|
||||||
)
|
|
||||||
});
|
|
||||||
let catalog = ModelInfoCatalog::new(
|
|
||||||
[Some(category), exact]
|
|
||||||
.into_iter()
|
|
||||||
.flatten()
|
|
||||||
.collect::<HashMap<_, _>>(),
|
|
||||||
);
|
|
||||||
let usage = usage(json!({
|
|
||||||
"prompt_tokens": 100,
|
|
||||||
"completion_tokens": 20,
|
|
||||||
"total_tokens": 120
|
|
||||||
}));
|
|
||||||
let (prompt, completion) = catalog
|
|
||||||
.cost_per_token(ModelCostRequest {
|
|
||||||
model: "accounts/models/model-7x8b",
|
|
||||||
provider: Some("fireworks_ai"),
|
|
||||||
region: None,
|
|
||||||
usage: &usage,
|
|
||||||
service_tier: None,
|
|
||||||
data_residency: None,
|
|
||||||
vertex_location: None,
|
|
||||||
at: at("2026-01-01T12:00Z"),
|
|
||||||
response_time_ms: None,
|
|
||||||
})
|
|
||||||
.unwrap();
|
|
||||||
assert!((prompt - 100.0 * selected_input_rate).abs() < 1e-12);
|
|
||||||
assert!((completion - 20.0 * 4e-6).abs() < 1e-12);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[rstest]
|
|
||||||
#[case("2026-01-01T18:00Z", 1e-6, 2e-6)]
|
|
||||||
#[case("2026-01-01T12:00Z", 2e-6, 4e-6)]
|
|
||||||
fn fireworks_cost_per_token_derives_cache_read_from_selected_window(
|
|
||||||
#[case] billed_at: &str,
|
|
||||||
#[case] input_rate: f64,
|
|
||||||
#[case] output_rate: f64,
|
|
||||||
) {
|
|
||||||
let model_info = json!({
|
|
||||||
"input_cost_per_token": 2e-6,
|
|
||||||
"output_cost_per_token": 4e-6,
|
|
||||||
"off_peak_pricing": {
|
|
||||||
"hours_utc": "16:30-00:30",
|
|
||||||
"input_cost_per_token": 1e-6,
|
|
||||||
"output_cost_per_token": 2e-6
|
|
||||||
}
|
|
||||||
});
|
|
||||||
let usage = usage(json!({
|
|
||||||
"prompt_tokens": 100,
|
|
||||||
"completion_tokens": 20,
|
|
||||||
"total_tokens": 120,
|
|
||||||
"prompt_tokens_details": {"cached_tokens": 40}
|
|
||||||
}));
|
|
||||||
let (prompt, completion) = cost_per_token(&usage, &model_info, at(billed_at));
|
|
||||||
assert!((prompt - (60.0 * input_rate + 40.0 * input_rate * 0.5)).abs() < 1e-12);
|
|
||||||
assert!((completion - 20.0 * output_rate).abs() < 1e-12);
|
|
||||||
}
|
|
||||||
|
|
|
||||||
|
|
@ -1,51 +1,2 @@
|
||||||
#![allow(clippy::disallowed_types)]
|
#![allow(clippy::disallowed_types)]
|
||||||
|
|
||||||
// mirrors: local_testing/test_completion_cost.py::test_gemini_completion_cost
|
// mirrors: local_testing/test_completion_cost.py::test_gemini_completion_cost
|
||||||
|
|
||||||
use std::collections::HashMap;
|
|
||||||
|
|
||||||
use litellm_cost::catalog::{ModelCostRequest, ModelInfoCatalog};
|
|
||||||
use litellm_cost::usage_dispatch::get_usage_object;
|
|
||||||
use rstest::rstest;
|
|
||||||
use serde_json::json;
|
|
||||||
|
|
||||||
#[rstest]
|
|
||||||
#[case("openai")]
|
|
||||||
#[case("bedrock")]
|
|
||||||
#[case("gemini")]
|
|
||||||
#[case("deepseek")]
|
|
||||||
#[case("tencent")]
|
|
||||||
#[case("amazon_nova")]
|
|
||||||
fn generic_provider_wrappers_price_cached_and_output_tokens(#[case] provider: &str) {
|
|
||||||
let catalog = ModelInfoCatalog::new(HashMap::from([(
|
|
||||||
format!("{provider}/model"),
|
|
||||||
json!({
|
|
||||||
"input_cost_per_token": 2e-6,
|
|
||||||
"output_cost_per_token": 3e-6,
|
|
||||||
"cache_read_input_token_cost": 0.5e-6
|
|
||||||
}),
|
|
||||||
)]));
|
|
||||||
let usage = get_usage_object(&json!({"usage": {
|
|
||||||
"prompt_tokens": 100,
|
|
||||||
"completion_tokens": 10,
|
|
||||||
"total_tokens": 110,
|
|
||||||
"prompt_tokens_details": {"cached_tokens": 20}
|
|
||||||
}}))
|
|
||||||
.unwrap()
|
|
||||||
.unwrap();
|
|
||||||
let (prompt, completion) = catalog
|
|
||||||
.cost_per_token(ModelCostRequest {
|
|
||||||
model: "model",
|
|
||||||
provider: Some(provider),
|
|
||||||
region: None,
|
|
||||||
usage: &usage,
|
|
||||||
service_tier: None,
|
|
||||||
data_residency: None,
|
|
||||||
vertex_location: None,
|
|
||||||
at: "2026-01-01T12:00Z".parse().unwrap(),
|
|
||||||
response_time_ms: None,
|
|
||||||
})
|
|
||||||
.unwrap();
|
|
||||||
assert!((prompt - (80.0 * 2e-6 + 20.0 * 0.5e-6)).abs() < 1e-12);
|
|
||||||
assert!((completion - 10.0 * 3e-6).abs() < 1e-12);
|
|
||||||
}
|
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
#![allow(clippy::disallowed_types)]
|
#![allow(clippy::disallowed_types)]
|
||||||
// mirrors: test_litellm/test_cost_calculator.py::test_cost_per_token_duplicate_openai_prefix_matches_model_cost
|
// mirrors: test_litellm/test_cost_calculator.py::test_cost_per_token_duplicate_openai_prefix_matches_model_cost
|
||||||
|
use litellm_cost::cost_calculator::{cost_per_token, cost_per_token_for_call};
|
||||||
use litellm_cost::error::CostError;
|
use litellm_cost::error::CostError;
|
||||||
|
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
|
|
@ -90,22 +91,22 @@ fn cost_per_token_resolves_model_key_in_python_order(
|
||||||
#[rstest]
|
#[rstest]
|
||||||
fn cost_per_token_uses_the_selected_regional_prices() {
|
fn cost_per_token_uses_the_selected_regional_prices() {
|
||||||
let usage = usage();
|
let usage = usage();
|
||||||
let (input, output) = catalog()
|
let (input, output) = cost_per_token_for_call(
|
||||||
.cost_per_token_for_call(
|
&catalog(),
|
||||||
request(
|
request(
|
||||||
"bedrock_mantle/model",
|
"bedrock_mantle/model",
|
||||||
Some("bedrock_mantle"),
|
Some("bedrock_mantle"),
|
||||||
Some("us-gov-west-1"),
|
Some("us-gov-west-1"),
|
||||||
&usage,
|
&usage,
|
||||||
),
|
),
|
||||||
litellm_cost::catalog::CostCall::Token {
|
litellm_cost::catalog::CostCall::Token {
|
||||||
call_type: "completion",
|
call_type: "completion",
|
||||||
prompt_characters: None,
|
prompt_characters: None,
|
||||||
completion_characters: None,
|
completion_characters: None,
|
||||||
request_model: None,
|
request_model: None,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert!((input - 100.0 * 5e-6).abs() < 1e-12);
|
assert!((input - 100.0 * 5e-6).abs() < 1e-12);
|
||||||
assert!((output - 50.0 * 6e-6).abs() < 1e-12);
|
assert!((output - 50.0 * 6e-6).abs() < 1e-12);
|
||||||
}
|
}
|
||||||
|
|
@ -113,7 +114,10 @@ fn cost_per_token_uses_the_selected_regional_prices() {
|
||||||
#[rstest]
|
#[rstest]
|
||||||
fn cost_per_token_reports_an_unmapped_model() {
|
fn cost_per_token_reports_an_unmapped_model() {
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
catalog().cost_per_token(request("missing", Some("openai"), None, &usage())),
|
cost_per_token(
|
||||||
|
&catalog(),
|
||||||
|
request("missing", Some("openai"), None, &usage())
|
||||||
|
),
|
||||||
Err(CostError::ModelNotFound)
|
Err(CostError::ModelNotFound)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
@ -149,8 +153,9 @@ fn model_info_catalog_prices_selected_model_with_off_peak_and_region() {
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let at: Timestamp = "2026-01-01T18:00Z".parse().unwrap();
|
let at: Timestamp = "2026-01-01T18:00Z".parse().unwrap();
|
||||||
let actual = catalog
|
let actual = cost_per_token(
|
||||||
.cost_per_token(ModelCostRequest {
|
&catalog,
|
||||||
|
ModelCostRequest {
|
||||||
model: "openai/openai/model",
|
model: "openai/openai/model",
|
||||||
provider: Some("openai"),
|
provider: Some("openai"),
|
||||||
region: None,
|
region: None,
|
||||||
|
|
@ -160,8 +165,9 @@ fn model_info_catalog_prices_selected_model_with_off_peak_and_region() {
|
||||||
vertex_location: None,
|
vertex_location: None,
|
||||||
at,
|
at,
|
||||||
response_time_ms: None,
|
response_time_ms: None,
|
||||||
})
|
},
|
||||||
.unwrap();
|
)
|
||||||
|
.unwrap();
|
||||||
assert!((actual.0 - (80.0 * 1e-6 + 20.0 * 0.25e-6) * 1.2).abs() < 1e-12);
|
assert!((actual.0 - (80.0 * 1e-6 + 20.0 * 0.25e-6) * 1.2).abs() < 1e-12);
|
||||||
assert!((actual.1 - 50.0 * 2e-6 * 1.2).abs() < 1e-12);
|
assert!((actual.1 - 50.0 * 2e-6 * 1.2).abs() < 1e-12);
|
||||||
}
|
}
|
||||||
|
|
@ -185,8 +191,9 @@ fn model_info_catalog_applies_xai_inclusive_threshold_policy() {
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let at: Timestamp = "2026-01-01T18:00Z".parse().unwrap();
|
let at: Timestamp = "2026-01-01T18:00Z".parse().unwrap();
|
||||||
let actual = catalog
|
let actual = cost_per_token(
|
||||||
.cost_per_token(ModelCostRequest {
|
&catalog,
|
||||||
|
ModelCostRequest {
|
||||||
model: "model",
|
model: "model",
|
||||||
provider: Some("xai"),
|
provider: Some("xai"),
|
||||||
region: None,
|
region: None,
|
||||||
|
|
@ -196,8 +203,9 @@ fn model_info_catalog_applies_xai_inclusive_threshold_policy() {
|
||||||
vertex_location: None,
|
vertex_location: None,
|
||||||
at,
|
at,
|
||||||
response_time_ms: None,
|
response_time_ms: None,
|
||||||
})
|
},
|
||||||
.unwrap();
|
)
|
||||||
|
.unwrap();
|
||||||
assert!((actual.0 - 128_000.0 * 5e-6).abs() < 1e-12);
|
assert!((actual.0 - 128_000.0 * 5e-6).abs() < 1e-12);
|
||||||
assert!((actual.1 - 10.0 * 7e-6).abs() < 1e-12);
|
assert!((actual.1 - 10.0 * 7e-6).abs() < 1e-12);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,127 +1,2 @@
|
||||||
#![allow(clippy::disallowed_types)]
|
#![allow(clippy::disallowed_types)]
|
||||||
|
|
||||||
// mirrors: test_litellm/test_cost_calculator.py::test_cost_per_token_per_second_pricing
|
// mirrors: test_litellm/test_cost_calculator.py::test_cost_per_token_per_second_pricing
|
||||||
|
|
||||||
use std::collections::HashMap;
|
|
||||||
|
|
||||||
use jiff::Timestamp;
|
|
||||||
use litellm_cost::catalog::{ModelCostRequest, ModelInfoCatalog};
|
|
||||||
use litellm_cost::per_second::get_replicate_completion_pricing;
|
|
||||||
use litellm_cost::per_second::{
|
|
||||||
bills_wall_clock_seconds, has_token_or_tiered_pricing, per_second_pricing_cost,
|
|
||||||
};
|
|
||||||
use litellm_cost::usage_dispatch::get_usage_object;
|
|
||||||
use rstest::rstest;
|
|
||||||
use serde_json::json;
|
|
||||||
|
|
||||||
#[rstest]
|
|
||||||
#[case(json!({"input_cost_per_second": 0.5, "output_cost_per_second": 1.0}), Some((1.0, 2.0)))]
|
|
||||||
#[case(json!({"input_cost_per_second": 0.5}), Some((1.0, 0.0)))]
|
|
||||||
#[case(json!({"output_cost_per_second": 1.0}), Some((0.0, 2.0)))]
|
|
||||||
#[case(json!({"input_cost_per_second": 0.5, "input_cost_per_token": 1e-6}), None)]
|
|
||||||
#[case(json!({"input_cost_per_second": 0.5, "tiered_pricing": []}), None)]
|
|
||||||
#[case(json!({"input_cost_per_second": 0.5, "mode": "image_generation"}), None)]
|
|
||||||
#[case(json!({"input_cost_per_token": 1e-6}), None)]
|
|
||||||
fn per_second_pricing_cost_only_prices_eligible_models(
|
|
||||||
#[case] model_info: serde_json::Value,
|
|
||||||
#[case] expected: Option<(f64, f64)>,
|
|
||||||
) {
|
|
||||||
assert_eq!(per_second_pricing_cost(&model_info, Some(2000.0)), expected);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[rstest]
|
|
||||||
fn per_second_pricing_cost_uses_zero_for_missing_response_time() {
|
|
||||||
assert_eq!(
|
|
||||||
per_second_pricing_cost(&json!({"input_cost_per_second": 0.5}), None),
|
|
||||||
Some((0.0, 0.0))
|
|
||||||
);
|
|
||||||
assert!(!has_token_or_tiered_pricing(
|
|
||||||
&json!({"input_cost_per_token": 0.0})
|
|
||||||
));
|
|
||||||
assert!(bills_wall_clock_seconds(&json!({"mode": "responses"})));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[rstest]
|
|
||||||
fn model_info_catalog_routes_wall_clock_pricing_before_tokens() {
|
|
||||||
let catalog = ModelInfoCatalog::new(HashMap::from([(
|
|
||||||
"provider/model".to_owned(),
|
|
||||||
json!({"mode": "responses", "input_cost_per_second": 0.5, "output_cost_per_second": 1.0}),
|
|
||||||
)]));
|
|
||||||
let usage = get_usage_object(&json!({"usage": {
|
|
||||||
"prompt_tokens": 100,
|
|
||||||
"completion_tokens": 50,
|
|
||||||
"total_tokens": 150
|
|
||||||
}}))
|
|
||||||
.unwrap()
|
|
||||||
.unwrap();
|
|
||||||
let at: Timestamp = "2026-01-01T18:00Z".parse().unwrap();
|
|
||||||
assert_eq!(
|
|
||||||
catalog
|
|
||||||
.cost_per_token(ModelCostRequest {
|
|
||||||
model: "model",
|
|
||||||
provider: Some("provider"),
|
|
||||||
region: None,
|
|
||||||
usage: &usage,
|
|
||||||
service_tier: None,
|
|
||||||
data_residency: None,
|
|
||||||
vertex_location: None,
|
|
||||||
at,
|
|
||||||
response_time_ms: Some(2000.0),
|
|
||||||
})
|
|
||||||
.unwrap(),
|
|
||||||
(1.0, 2.0)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[rstest]
|
|
||||||
#[case("perplexity")]
|
|
||||||
#[case("xai")]
|
|
||||||
fn wall_clock_pricing_precedes_provider_reported_cost(#[case] provider: &str) {
|
|
||||||
let catalog = ModelInfoCatalog::new(HashMap::from([(
|
|
||||||
format!("{provider}/model"),
|
|
||||||
json!({"mode": "responses", "input_cost_per_second": 0.5, "output_cost_per_second": 1.0}),
|
|
||||||
)]));
|
|
||||||
let usage = get_usage_object(&json!({"usage": {
|
|
||||||
"prompt_tokens": 100,
|
|
||||||
"completion_tokens": 50,
|
|
||||||
"cost": 0.7
|
|
||||||
}}))
|
|
||||||
.unwrap()
|
|
||||||
.unwrap();
|
|
||||||
assert_eq!(
|
|
||||||
catalog
|
|
||||||
.cost_per_token(ModelCostRequest {
|
|
||||||
model: "model",
|
|
||||||
provider: Some(provider),
|
|
||||||
region: None,
|
|
||||||
usage: &usage,
|
|
||||||
service_tier: None,
|
|
||||||
data_residency: None,
|
|
||||||
vertex_location: None,
|
|
||||||
at: "2026-01-01T18:00Z".parse().unwrap(),
|
|
||||||
response_time_ms: Some(2000.0),
|
|
||||||
})
|
|
||||||
.unwrap(),
|
|
||||||
(1.0, 2.0)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
#[rstest]
|
|
||||||
#[case(2500.0, None, None, 100.0, 0.05)]
|
|
||||||
#[case(0.0, Some(100.0), Some(102.0), 200.0, 0.00004)]
|
|
||||||
#[case(0.0, None, None, 200.0, 0.0)]
|
|
||||||
fn replicate_completion_pricing_matches_python_duration_units(
|
|
||||||
#[case] total_time_ms: f64,
|
|
||||||
#[case] created_seconds: Option<f64>,
|
|
||||||
#[case] ended_seconds: Option<f64>,
|
|
||||||
#[case] now_seconds: f64,
|
|
||||||
#[case] expected: f64,
|
|
||||||
) {
|
|
||||||
let actual = get_replicate_completion_pricing(
|
|
||||||
total_time_ms,
|
|
||||||
created_seconds,
|
|
||||||
ended_seconds,
|
|
||||||
now_seconds,
|
|
||||||
0.02,
|
|
||||||
);
|
|
||||||
assert!((actual - expected).abs() < 1e-12);
|
|
||||||
}
|
|
||||||
|
|
|
||||||
|
|
@ -1,139 +1,2 @@
|
||||||
#![allow(clippy::disallowed_types)]
|
#![allow(clippy::disallowed_types)]
|
||||||
|
|
||||||
// mirrors: test_litellm/llms/perplexity/test_perplexity_cost_calculator.py::TestPerplexityCostCalculator
|
// mirrors: test_litellm/llms/perplexity/test_perplexity_cost_calculator.py::TestPerplexityCostCalculator
|
||||||
|
|
||||||
use std::collections::HashMap;
|
|
||||||
|
|
||||||
use jiff::Timestamp;
|
|
||||||
use litellm_cost::catalog::{ModelCostRequest, ModelInfoCatalog};
|
|
||||||
use litellm_cost::perplexity_cost::cost_per_token;
|
|
||||||
use litellm_cost::usage_dispatch::get_usage_object;
|
|
||||||
use rstest::rstest;
|
|
||||||
use serde_json::{Value, json};
|
|
||||||
|
|
||||||
fn at(value: &str) -> Timestamp {
|
|
||||||
value.parse().unwrap()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn usage(value: Value) -> litellm_cost::responses_usage::ChatUsage {
|
|
||||||
get_usage_object(&json!({"usage": value})).unwrap().unwrap()
|
|
||||||
}
|
|
||||||
|
|
||||||
#[rstest]
|
|
||||||
#[case("2026-09-03T17:25Z", 1e-7, 2e-7)]
|
|
||||||
#[case("2026-09-03T09:00Z", 1e-6, 1e-6)]
|
|
||||||
fn perplexity_cost_per_token_keeps_citation_search_and_reasoning_fees_during_off_peak(
|
|
||||||
#[case] billed_at: &str,
|
|
||||||
#[case] input_rate: f64,
|
|
||||||
#[case] output_rate: f64,
|
|
||||||
) {
|
|
||||||
let model_info = json!({
|
|
||||||
"input_cost_per_token": 1e-6,
|
|
||||||
"output_cost_per_token": 1e-6,
|
|
||||||
"output_cost_per_reasoning_token": 3e-6,
|
|
||||||
"citation_cost_per_token": 2e-6,
|
|
||||||
"search_context_cost_per_query": {"search_context_size_low": 0.005},
|
|
||||||
"off_peak_pricing": {
|
|
||||||
"hours_utc": "14:00-00:00",
|
|
||||||
"input_cost_per_token": 1e-7,
|
|
||||||
"output_cost_per_token": 2e-7
|
|
||||||
}
|
|
||||||
});
|
|
||||||
let usage = usage(json!({
|
|
||||||
"prompt_tokens": 1000,
|
|
||||||
"completion_tokens": 200,
|
|
||||||
"total_tokens": 1200,
|
|
||||||
"citation_tokens": 100,
|
|
||||||
"prompt_tokens_details": {"web_search_requests": 2},
|
|
||||||
"completion_tokens_details": {"reasoning_tokens": 50}
|
|
||||||
}));
|
|
||||||
let expected = (
|
|
||||||
1000.0 * input_rate + 100.0 * 2e-6,
|
|
||||||
150.0 * output_rate + 50.0 * 3e-6 + 2.0 * 0.005,
|
|
||||||
);
|
|
||||||
let actual = cost_per_token(&usage, &model_info, at(billed_at));
|
|
||||||
assert!((actual.0 - expected.0).abs() < 1e-12);
|
|
||||||
assert!((actual.1 - expected.1).abs() < 1e-12);
|
|
||||||
let catalog =
|
|
||||||
ModelInfoCatalog::new(HashMap::from([("perplexity/model".to_owned(), model_info)]));
|
|
||||||
assert_eq!(
|
|
||||||
catalog
|
|
||||||
.cost_per_token(ModelCostRequest {
|
|
||||||
model: "model",
|
|
||||||
provider: Some("perplexity"),
|
|
||||||
region: None,
|
|
||||||
usage: &usage,
|
|
||||||
service_tier: None,
|
|
||||||
data_residency: None,
|
|
||||||
vertex_location: None,
|
|
||||||
at: at(billed_at),
|
|
||||||
response_time_ms: None,
|
|
||||||
})
|
|
||||||
.unwrap(),
|
|
||||||
actual
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[rstest]
|
|
||||||
#[case(json!(0.008))]
|
|
||||||
#[case(json!({"input_tokens_cost": 0.001, "request_cost": 0.007, "total_cost": 0.008}))]
|
|
||||||
fn perplexity_cost_per_token_prefers_provider_total_without_model_pricing(#[case] cost: Value) {
|
|
||||||
let usage = usage(json!({
|
|
||||||
"prompt_tokens": 100,
|
|
||||||
"completion_tokens": 50,
|
|
||||||
"total_tokens": 150,
|
|
||||||
"cost": cost
|
|
||||||
}));
|
|
||||||
assert_eq!(usage.cost, Some(0.008));
|
|
||||||
let catalog = ModelInfoCatalog::new(HashMap::new());
|
|
||||||
assert_eq!(
|
|
||||||
catalog
|
|
||||||
.cost_per_token(ModelCostRequest {
|
|
||||||
model: "unmapped",
|
|
||||||
provider: Some("perplexity"),
|
|
||||||
region: None,
|
|
||||||
usage: &usage,
|
|
||||||
service_tier: None,
|
|
||||||
data_residency: None,
|
|
||||||
vertex_location: None,
|
|
||||||
at: at("2026-09-03T17:25Z"),
|
|
||||||
response_time_ms: None,
|
|
||||||
})
|
|
||||||
.unwrap(),
|
|
||||||
(0.0, 0.008)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[rstest]
|
|
||||||
fn perplexity_response_usage_preserves_provider_cost_object() {
|
|
||||||
let usage = usage(json!({
|
|
||||||
"input_tokens": 100,
|
|
||||||
"output_tokens": 50,
|
|
||||||
"cost": {"total_cost": 0.008}
|
|
||||||
}));
|
|
||||||
assert_eq!(usage.cost, Some(0.008));
|
|
||||||
assert_eq!(
|
|
||||||
cost_per_token(&usage, &json!({}), at("2026-09-03T17:25Z")),
|
|
||||||
(0.0, 0.008)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[rstest]
|
|
||||||
fn perplexity_cost_per_token_uses_flat_search_rate_and_basic_fallbacks() {
|
|
||||||
let usage = usage(json!({
|
|
||||||
"prompt_tokens": 100,
|
|
||||||
"completion_tokens": 50,
|
|
||||||
"total_tokens": 150,
|
|
||||||
"citation_tokens": 25,
|
|
||||||
"prompt_tokens_details": {"web_search_requests": 2}
|
|
||||||
}));
|
|
||||||
let model_info = json!({
|
|
||||||
"input_cost_per_token": "0.000002",
|
|
||||||
"output_cost_per_token": 8e-6,
|
|
||||||
"search_queries_cost_per_query": 0.003,
|
|
||||||
"search_context_cost_per_query": {"search_context_size_low": 0.005}
|
|
||||||
});
|
|
||||||
let actual = cost_per_token(&usage, &model_info, at("2026-09-03T17:25Z"));
|
|
||||||
assert!((actual.0 - 100.0 * 2e-6).abs() < 1e-12);
|
|
||||||
assert!((actual.1 - (50.0 * 8e-6 + 2.0 * 0.003)).abs() < 1e-12);
|
|
||||||
}
|
|
||||||
|
|
|
||||||
|
|
@ -1,68 +1,2 @@
|
||||||
#![allow(clippy::disallowed_types)]
|
#![allow(clippy::disallowed_types)]
|
||||||
|
|
||||||
// mirrors: unit/llms/fireworks_ai/test_fireworks_ai_cache_pricing.py::test_missing_cache_read_rate_is_derived_for_standard_and_off_peak_without_mutating_the_entry
|
// mirrors: unit/llms/fireworks_ai/test_fireworks_ai_cache_pricing.py::test_missing_cache_read_rate_is_derived_for_standard_and_off_peak_without_mutating_the_entry
|
||||||
|
|
||||||
use std::collections::HashMap;
|
|
||||||
|
|
||||||
use jiff::Timestamp;
|
|
||||||
use litellm_cost::catalog::{ModelCostRequest, ModelInfoCatalog};
|
|
||||||
use litellm_cost::provider_cache::with_default_cache_read_rate;
|
|
||||||
use litellm_cost::usage_dispatch::get_usage_object;
|
|
||||||
use rstest::rstest;
|
|
||||||
use serde_json::json;
|
|
||||||
|
|
||||||
#[rstest]
|
|
||||||
fn with_default_cache_read_rate_derives_base_and_off_peak_rates() {
|
|
||||||
let info = json!({
|
|
||||||
"input_cost_per_token": 2e-6,
|
|
||||||
"off_peak_pricing": {"hours_utc": "16:00-20:00", "input_cost_per_token": 1e-6}
|
|
||||||
});
|
|
||||||
let actual = with_default_cache_read_rate(&info);
|
|
||||||
assert_eq!(actual["cache_read_input_token_cost"], json!(1e-6));
|
|
||||||
assert_eq!(
|
|
||||||
actual["off_peak_pricing"]["cache_read_input_token_cost"],
|
|
||||||
json!(0.5e-6)
|
|
||||||
);
|
|
||||||
assert!(info.get("cache_read_input_token_cost").is_none());
|
|
||||||
}
|
|
||||||
|
|
||||||
#[rstest]
|
|
||||||
fn with_default_cache_read_rate_preserves_explicit_cache_rates() {
|
|
||||||
let info = json!({
|
|
||||||
"input_cost_per_token": 2e-6,
|
|
||||||
"cache_read_input_token_cost": 0.25e-6,
|
|
||||||
"off_peak_pricing": {"input_cost_per_token": 1e-6}
|
|
||||||
});
|
|
||||||
assert_eq!(with_default_cache_read_rate(&info), info);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[rstest]
|
|
||||||
fn model_info_catalog_uses_fireworks_default_for_cached_tokens() {
|
|
||||||
let catalog = ModelInfoCatalog::new(HashMap::from([(
|
|
||||||
"fireworks_ai/model".to_owned(),
|
|
||||||
json!({"input_cost_per_token": 2e-6, "output_cost_per_token": 4e-6}),
|
|
||||||
)]));
|
|
||||||
let usage = get_usage_object(&json!({"usage": {
|
|
||||||
"prompt_tokens": 100,
|
|
||||||
"completion_tokens": 0,
|
|
||||||
"total_tokens": 100,
|
|
||||||
"prompt_tokens_details": {"cached_tokens": 40}
|
|
||||||
}}))
|
|
||||||
.unwrap()
|
|
||||||
.unwrap();
|
|
||||||
let at: Timestamp = "2026-01-01T18:00Z".parse().unwrap();
|
|
||||||
let actual = catalog
|
|
||||||
.cost_per_token(ModelCostRequest {
|
|
||||||
model: "model",
|
|
||||||
provider: Some("fireworks_ai"),
|
|
||||||
region: None,
|
|
||||||
usage: &usage,
|
|
||||||
service_tier: None,
|
|
||||||
data_residency: None,
|
|
||||||
vertex_location: None,
|
|
||||||
at,
|
|
||||||
response_time_ms: None,
|
|
||||||
})
|
|
||||||
.unwrap();
|
|
||||||
assert!((actual.0 - (60.0 * 2e-6 + 40.0 * 1e-6)).abs() < 1e-12);
|
|
||||||
}
|
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
#![allow(clippy::disallowed_types)]
|
#![allow(clippy::disallowed_types)]
|
||||||
// mirrors: test_litellm/llms/openai/test_cost_calculation.py::test_shipped_per_second_models_bill_a_non_zero_cost
|
// mirrors: test_litellm/llms/openai/test_cost_calculation.py::test_shipped_per_second_models_bill_a_non_zero_cost
|
||||||
// mirrors: test_litellm/llms/databricks/test_databricks_cost_calculator.py
|
// mirrors: test_litellm/llms/databricks/test_databricks_cost_calculator.py
|
||||||
|
use litellm_cost::cost_calculator::cost_per_token;
|
||||||
use litellm_cost::lemonade_cost::lemonade_cost_per_token;
|
use litellm_cost::lemonade_cost::lemonade_cost_per_token;
|
||||||
|
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
|
|
@ -52,15 +53,15 @@ fn azure_provider_output_seconds_override_token_rates_when_present() {
|
||||||
get_usage_object(&json!({"usage": {"prompt_tokens": 100, "completion_tokens": 20}}))
|
get_usage_object(&json!({"usage": {"prompt_tokens": 100, "completion_tokens": 20}}))
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let seconds = catalog
|
let seconds = cost_per_token(
|
||||||
.cost_per_token(ModelCostRequest {
|
&catalog,
|
||||||
|
ModelCostRequest {
|
||||||
response_time_ms: Some(1500.0),
|
response_time_ms: Some(1500.0),
|
||||||
..request("speech", "azure", &usage)
|
..request("speech", "azure", &usage)
|
||||||
})
|
},
|
||||||
.unwrap();
|
)
|
||||||
let tokens = catalog
|
.unwrap();
|
||||||
.cost_per_token(request("speech", "azure", &usage))
|
let tokens = cost_per_token(&catalog, request("speech", "azure", &usage)).unwrap();
|
||||||
.unwrap();
|
|
||||||
assert_eq!(seconds, (0.0, 0.03));
|
assert_eq!(seconds, (0.0, 0.03));
|
||||||
assert_eq!(tokens, (0.1, 0.04));
|
assert_eq!(tokens, (0.1, 0.04));
|
||||||
}
|
}
|
||||||
|
|
@ -92,15 +93,19 @@ fn databricks_provider_uses_legacy_registry_price_and_preserves_original_duratio
|
||||||
get_usage_object(&json!({"usage": {"prompt_tokens": 100, "completion_tokens": 20}}))
|
get_usage_object(&json!({"usage": {"prompt_tokens": 100, "completion_tokens": 20}}))
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let alias = catalog
|
let alias = cost_per_token(
|
||||||
.cost_per_token(request("dbrx-instruct-slow", "databricks", &usage))
|
&catalog,
|
||||||
.unwrap();
|
request("dbrx-instruct-slow", "databricks", &usage),
|
||||||
let duration = catalog
|
)
|
||||||
.cost_per_token(ModelCostRequest {
|
.unwrap();
|
||||||
|
let duration = cost_per_token(
|
||||||
|
&catalog,
|
||||||
|
ModelCostRequest {
|
||||||
response_time_ms: Some(2000.0),
|
response_time_ms: Some(2000.0),
|
||||||
..request("dbrx-instruct-fast", "databricks", &usage)
|
..request("dbrx-instruct-fast", "databricks", &usage)
|
||||||
})
|
},
|
||||||
.unwrap();
|
)
|
||||||
|
.unwrap();
|
||||||
assert_eq!(alias, (0.2, 0.06));
|
assert_eq!(alias, (0.2, 0.06));
|
||||||
assert_eq!(duration, (0.1, 0.0));
|
assert_eq!(duration, (0.1, 0.0));
|
||||||
}
|
}
|
||||||
|
|
@ -122,26 +127,24 @@ fn lemonade_provider_is_free_for_unmapped_and_token_priced_models() {
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
catalog
|
cost_per_token(&catalog, request("unknown", "lemonade", &usage)).unwrap(),
|
||||||
.cost_per_token(request("unknown", "lemonade", &usage))
|
|
||||||
.unwrap(),
|
|
||||||
(0.0, 0.0)
|
(0.0, 0.0)
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
catalog
|
cost_per_token(&catalog, request("priced", "lemonade", &usage)).unwrap(),
|
||||||
.cost_per_token(request("priced", "lemonade", &usage))
|
|
||||||
.unwrap(),
|
|
||||||
(0.0, 0.0)
|
(0.0, 0.0)
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
lemonade_cost_per_token(&catalog, request("duration", "lemonade", &usage)),
|
lemonade_cost_per_token(&catalog, request("duration", "lemonade", &usage)),
|
||||||
(0.0, 0.0)
|
(0.0, 0.0)
|
||||||
);
|
);
|
||||||
let elapsed = catalog
|
let elapsed = cost_per_token(
|
||||||
.cost_per_token(ModelCostRequest {
|
&catalog,
|
||||||
|
ModelCostRequest {
|
||||||
response_time_ms: Some(2000.0),
|
response_time_ms: Some(2000.0),
|
||||||
..request("duration", "lemonade", &usage)
|
..request("duration", "lemonade", &usage)
|
||||||
})
|
},
|
||||||
.unwrap();
|
)
|
||||||
|
.unwrap();
|
||||||
assert_eq!(elapsed, (0.1, 0.0));
|
assert_eq!(elapsed, (0.1, 0.0));
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
#![allow(clippy::disallowed_types)]
|
#![allow(clippy::disallowed_types)]
|
||||||
// mirrors: local_testing/test_completion_cost.py::test_together_ai_qwen_completion_cost
|
// mirrors: local_testing/test_completion_cost.py::test_together_ai_qwen_completion_cost
|
||||||
|
use litellm_cost::cost_calculator::cost_per_token;
|
||||||
use litellm_cost::error::CostError;
|
use litellm_cost::error::CostError;
|
||||||
use litellm_cost::together_cost::together_ai_cost_per_token;
|
use litellm_cost::together_cost::together_ai_cost_per_token;
|
||||||
|
|
||||||
|
|
@ -131,7 +132,7 @@ fn catalog_uses_exact_price_or_chat_category(
|
||||||
),
|
),
|
||||||
]));
|
]));
|
||||||
let usage = usage();
|
let usage = usage();
|
||||||
let (prompt, completion) = catalog.cost_per_token(request("model-7b", &usage)).unwrap();
|
let (prompt, completion) = cost_per_token(&catalog, request("model-7b", &usage)).unwrap();
|
||||||
assert!((prompt - 100.0 * expected_input_rate).abs() < 1e-12);
|
assert!((prompt - 100.0 * expected_input_rate).abs() < 1e-12);
|
||||||
let output_rate = if exact_has_input_rate { 6e-6 } else { 3e-6 };
|
let output_rate = if exact_has_input_rate { 6e-6 } else { 3e-6 };
|
||||||
assert!((completion - 20.0 * output_rate).abs() < 1e-12);
|
assert!((completion - 20.0 * output_rate).abs() < 1e-12);
|
||||||
|
|
@ -158,7 +159,7 @@ fn catalog_does_not_price_metadata_only_exact_row_when_category_is_missing() {
|
||||||
)]));
|
)]));
|
||||||
let usage = usage();
|
let usage = usage();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
catalog.cost_per_token(request("model-7b", &usage)),
|
cost_per_token(&catalog, request("model-7b", &usage)),
|
||||||
Err(CostError::ModelNotFound)
|
Err(CostError::ModelNotFound)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,201 +1,2 @@
|
||||||
#![allow(clippy::disallowed_types)]
|
#![allow(clippy::disallowed_types)]
|
||||||
|
|
||||||
// mirrors: test_litellm/llms/xai/test_xai_cost_calculator.py::TestXAICostCalculator
|
// mirrors: test_litellm/llms/xai/test_xai_cost_calculator.py::TestXAICostCalculator
|
||||||
|
|
||||||
use std::collections::HashMap;
|
|
||||||
|
|
||||||
use jiff::Timestamp;
|
|
||||||
use litellm_cost::catalog::{ModelCostRequest, ModelInfoCatalog};
|
|
||||||
use litellm_cost::usage_dispatch::get_usage_object;
|
|
||||||
use litellm_cost::xai_cost::{
|
|
||||||
apply_server_side_tool_usage_details_to_usage, cost_per_token, cost_per_web_search_request,
|
|
||||||
reported_cost, web_search_cost_per_call_from_model_info,
|
|
||||||
};
|
|
||||||
use rstest::rstest;
|
|
||||||
use serde_json::{Value, json};
|
|
||||||
|
|
||||||
fn at() -> Timestamp {
|
|
||||||
"2026-01-01T12:00Z".parse().unwrap()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn usage(value: Value) -> litellm_cost::responses_usage::ChatUsage {
|
|
||||||
get_usage_object(&json!({"usage": value})).unwrap().unwrap()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn model_info() -> Value {
|
|
||||||
json!({
|
|
||||||
"input_cost_per_token": 1e-6,
|
|
||||||
"output_cost_per_token": 2e-6,
|
|
||||||
"input_cost_per_token_above_200k_tokens": 3e-6,
|
|
||||||
"output_cost_per_token_above_200k_tokens": 4e-6
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
#[rstest]
|
|
||||||
#[case(50, 200_070)]
|
|
||||||
#[case(70, 200_070)]
|
|
||||||
fn xai_cost_per_token_bills_raw_and_normalized_reasoning_once_at_inclusive_threshold(
|
|
||||||
#[case] completion_tokens: u64,
|
|
||||||
#[case] total_tokens: u64,
|
|
||||||
) {
|
|
||||||
let usage = usage(json!({
|
|
||||||
"prompt_tokens": 200_000,
|
|
||||||
"completion_tokens": completion_tokens,
|
|
||||||
"total_tokens": total_tokens,
|
|
||||||
"completion_tokens_details": {"reasoning_tokens": 20}
|
|
||||||
}));
|
|
||||||
let direct = cost_per_token(&usage, &model_info(), at());
|
|
||||||
assert!((direct.0 - 200_000.0 * 3e-6).abs() < 1e-12);
|
|
||||||
assert!((direct.1 - 70.0 * 4e-6).abs() < 1e-12);
|
|
||||||
let catalog = ModelInfoCatalog::new(HashMap::from([("xai/model".to_owned(), model_info())]));
|
|
||||||
assert_eq!(
|
|
||||||
catalog
|
|
||||||
.cost_per_token(ModelCostRequest {
|
|
||||||
model: "model",
|
|
||||||
provider: Some("xai"),
|
|
||||||
region: None,
|
|
||||||
usage: &usage,
|
|
||||||
service_tier: None,
|
|
||||||
data_residency: None,
|
|
||||||
vertex_location: None,
|
|
||||||
at: at(),
|
|
||||||
response_time_ms: None,
|
|
||||||
})
|
|
||||||
.unwrap(),
|
|
||||||
direct
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[rstest]
|
|
||||||
#[case(0.0037756)]
|
|
||||||
#[case(0.0)]
|
|
||||||
fn xai_cost_per_token_prefers_nonnegative_provider_total_without_model_metadata(
|
|
||||||
#[case] provider_cost: f64,
|
|
||||||
) {
|
|
||||||
let usage = usage(json!({
|
|
||||||
"prompt_tokens": 100,
|
|
||||||
"completion_tokens": 50,
|
|
||||||
"total_tokens": 150,
|
|
||||||
"cost": provider_cost
|
|
||||||
}));
|
|
||||||
assert_eq!(reported_cost(&usage), Some(provider_cost));
|
|
||||||
let catalog = ModelInfoCatalog::new(HashMap::new());
|
|
||||||
assert_eq!(
|
|
||||||
catalog
|
|
||||||
.cost_per_token(ModelCostRequest {
|
|
||||||
model: "unmapped",
|
|
||||||
provider: Some("xai"),
|
|
||||||
region: None,
|
|
||||||
usage: &usage,
|
|
||||||
service_tier: None,
|
|
||||||
data_residency: None,
|
|
||||||
vertex_location: None,
|
|
||||||
at: at(),
|
|
||||||
response_time_ms: None,
|
|
||||||
})
|
|
||||||
.unwrap(),
|
|
||||||
(0.0, provider_cost)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[rstest]
|
|
||||||
fn xai_cost_per_token_ignores_negative_provider_total() {
|
|
||||||
let usage = usage(json!({
|
|
||||||
"prompt_tokens": 100,
|
|
||||||
"completion_tokens": 50,
|
|
||||||
"total_tokens": 150,
|
|
||||||
"cost": -0.5
|
|
||||||
}));
|
|
||||||
assert_eq!(reported_cost(&usage), None);
|
|
||||||
let (prompt, completion) = cost_per_token(&usage, &model_info(), at());
|
|
||||||
assert!((prompt - 100e-6).abs() < 1e-12);
|
|
||||||
assert!((completion - 100e-6).abs() < 1e-12);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[rstest]
|
|
||||||
#[case(json!({"web_search_calls": 2}), Some(2))]
|
|
||||||
#[case(json!({"web_search_calls": "3"}), Some(3))]
|
|
||||||
#[case(json!({"web_search_calls": "invalid"}), None)]
|
|
||||||
fn apply_server_side_tool_usage_details_to_usage_mirrors_positive_web_search_calls(
|
|
||||||
#[case] details: Value,
|
|
||||||
#[case] expected_web_search_requests: Option<u64>,
|
|
||||||
) {
|
|
||||||
let usage = usage(json!({
|
|
||||||
"prompt_tokens": 10,
|
|
||||||
"completion_tokens": 5,
|
|
||||||
"total_tokens": 15
|
|
||||||
}));
|
|
||||||
let updated = apply_server_side_tool_usage_details_to_usage(&usage, Some(&details));
|
|
||||||
assert_eq!(
|
|
||||||
updated
|
|
||||||
.prompt_tokens_details
|
|
||||||
.as_ref()
|
|
||||||
.and_then(|details| details.web_search_requests),
|
|
||||||
expected_web_search_requests
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
updated.extra.get("server_side_tool_usage_details"),
|
|
||||||
Some(&details)
|
|
||||||
);
|
|
||||||
assert!(
|
|
||||||
(cost_per_web_search_request(&updated, &json!({}), 0.002)
|
|
||||||
- expected_web_search_requests.unwrap_or(0) as f64 * 0.002)
|
|
||||||
.abs()
|
|
||||||
< 1e-12
|
|
||||||
);
|
|
||||||
assert_eq!(usage.extra.get("server_side_tool_usage_details"), None);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[rstest]
|
|
||||||
#[case(json!({"search_context_size_medium": 0.009, "search_context_size_low": 0.003}), 0.009)]
|
|
||||||
#[case(json!({"search_context_size_medium": 0, "search_context_size_low": "0.003"}), 0.003)]
|
|
||||||
#[case(json!({"search_context_size_medium": "invalid", "search_context_size_high": 0.007}), 0.007)]
|
|
||||||
#[case(json!({"search_context_size_medium": 0, "search_context_size_low": "invalid"}), 0.002)]
|
|
||||||
fn xai_web_search_price_selects_first_positive_context_rate(
|
|
||||||
#[case] prices: Value,
|
|
||||||
#[case] expected: f64,
|
|
||||||
) {
|
|
||||||
let model_info = json!({"search_context_cost_per_query": prices});
|
|
||||||
assert_eq!(
|
|
||||||
web_search_cost_per_call_from_model_info(&model_info, 0.002),
|
|
||||||
expected
|
|
||||||
);
|
|
||||||
let usage = usage(json!({
|
|
||||||
"prompt_tokens": 10,
|
|
||||||
"completion_tokens": 5,
|
|
||||||
"total_tokens": 15,
|
|
||||||
"server_side_tool_usage_details": {"web_search_calls": 3}
|
|
||||||
}));
|
|
||||||
assert!(
|
|
||||||
(cost_per_web_search_request(&usage, &model_info, 0.002) - 3.0 * expected).abs() < 1e-12
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[rstest]
|
|
||||||
fn xai_web_search_cost_is_included_in_a_reported_total() {
|
|
||||||
let usage = usage(json!({
|
|
||||||
"prompt_tokens": 10,
|
|
||||||
"completion_tokens": 5,
|
|
||||||
"total_tokens": 15,
|
|
||||||
"cost": 0.03,
|
|
||||||
"server_side_tool_usage_details": {"web_search_calls": 3}
|
|
||||||
}));
|
|
||||||
assert_eq!(cost_per_web_search_request(&usage, &json!({}), 0.002), 0.0);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[rstest]
|
|
||||||
fn apply_server_side_tool_usage_details_preserves_existing_cache_usage() {
|
|
||||||
let usage = usage(json!({
|
|
||||||
"prompt_tokens": 10,
|
|
||||||
"completion_tokens": 5,
|
|
||||||
"total_tokens": 15,
|
|
||||||
"prompt_tokens_details": {"cached_tokens": 7}
|
|
||||||
}));
|
|
||||||
let updated = apply_server_side_tool_usage_details_to_usage(
|
|
||||||
&usage,
|
|
||||||
Some(&json!({"web_search_calls": 4})),
|
|
||||||
);
|
|
||||||
let prompt = updated.prompt_tokens_details.unwrap();
|
|
||||||
assert_eq!(prompt.cached_tokens, 7);
|
|
||||||
assert_eq!(prompt.web_search_requests, Some(4));
|
|
||||||
}
|
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue