mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(cost): pass each provider only the pricing flags Python forwards
A new generic_cost_per_token mirrors Python's: it applies the provider cache read default and inclusive thresholds itself. cost_per_token forwards the service tier to anthropic, bedrock, azure, and gemini but not data residency, drops both for deepseek and tencent, and never forwards the Vertex location outside the Vertex route. Speech priced per token, transcription token usage, and realtime candidates go through generic_cost_per_token like Python Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
f74ee2b856
commit
29a8f4ce83
4 changed files with 164 additions and 27 deletions
|
|
@ -6,7 +6,6 @@ use serde_json::Value;
|
|||
use crate::anthropic_cost::fast_speed_multiplier;
|
||||
use crate::azure_ai_cost::azure_ai_cost_per_token;
|
||||
use crate::azure_cost::output_per_second_cost;
|
||||
use crate::base_rate_selection::uses_inclusive_token_thresholds;
|
||||
use crate::batch::batch_cost_from_model_info;
|
||||
use crate::catalog::{CostCall, ModelCostRequest, ModelInfoCatalog};
|
||||
use crate::completion_cost::{
|
||||
|
|
@ -20,7 +19,7 @@ use crate::error::CostError;
|
|||
use crate::fireworks_cost::{
|
||||
FireworksThresholds, cost_per_token as fireworks_cost_per_token, get_base_model_for_pricing,
|
||||
};
|
||||
use crate::generic_cost::calculate_generic_cost_from_model_info_with_region;
|
||||
use crate::generic_cost::{GenericCostRequest, generic_cost_per_token};
|
||||
use crate::lemonade_cost::lemonade_cost_per_token;
|
||||
use crate::ocr_cost::ocr_cost;
|
||||
use crate::openai_cost::video_generation_cost;
|
||||
|
|
@ -321,15 +320,25 @@ pub fn cost_per_token(
|
|||
if !dispatches_before_the_gate && !has_token_or_tiered_pricing(&model_info) {
|
||||
return Ok((0.0, 0.0));
|
||||
}
|
||||
let cost = calculate_generic_cost_from_model_info_with_region(
|
||||
request.usage,
|
||||
&model_info,
|
||||
request.service_tier,
|
||||
uses_inclusive_token_thresholds(request.provider),
|
||||
request.data_residency,
|
||||
request.vertex_location,
|
||||
request.at,
|
||||
);
|
||||
let (service_tier, data_residency) = match provider {
|
||||
Some(
|
||||
LlmProviders::ANTHROPIC
|
||||
| LlmProviders::BEDROCK
|
||||
| LlmProviders::AZURE
|
||||
| LlmProviders::GEMINI,
|
||||
) => (request.service_tier, None),
|
||||
Some(LlmProviders::DEEPSEEK | LlmProviders::TENCENT) => (None, None),
|
||||
_ => (request.service_tier, request.data_residency),
|
||||
};
|
||||
let cost = generic_cost_per_token(GenericCostRequest {
|
||||
model_info: catalog.entry_for_key(key),
|
||||
usage: request.usage,
|
||||
provider: request.provider,
|
||||
service_tier,
|
||||
data_residency,
|
||||
vertex_location: None,
|
||||
at: request.at,
|
||||
});
|
||||
let speed = if provider == Some(LlmProviders::ANTHROPIC) {
|
||||
fast_speed_multiplier(&model_info, request.usage)
|
||||
* get_provider_specific_geo_multiplier(
|
||||
|
|
@ -374,7 +383,15 @@ pub fn speech_cost(
|
|||
completion.unwrap_or(0.0),
|
||||
))
|
||||
}
|
||||
SpeechCostMetric::PerToken => cost_per_token(catalog, request),
|
||||
SpeechCostMetric::PerToken => Ok(generic_cost_per_token(GenericCostRequest {
|
||||
model_info,
|
||||
usage: request.usage,
|
||||
provider: request.provider,
|
||||
service_tier: request.service_tier,
|
||||
data_residency: request.data_residency,
|
||||
vertex_location: None,
|
||||
at: request.at,
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -387,15 +404,15 @@ pub fn transcription_cost(
|
|||
.entry(request.model, request.provider, request.region)
|
||||
.ok_or(CostError::ModelNotFound)?;
|
||||
if transcription_usage_has_token_details(request.usage) {
|
||||
return Ok(calculate_generic_cost_from_model_info_with_region(
|
||||
request.usage,
|
||||
return Ok(generic_cost_per_token(GenericCostRequest {
|
||||
model_info,
|
||||
request.service_tier,
|
||||
false,
|
||||
request.data_residency,
|
||||
request.vertex_location,
|
||||
request.at,
|
||||
));
|
||||
usage: request.usage,
|
||||
provider: request.provider,
|
||||
service_tier: request.service_tier,
|
||||
data_residency: request.data_residency,
|
||||
vertex_location: None,
|
||||
at: request.at,
|
||||
}));
|
||||
}
|
||||
Ok(cost_per_second(model_info, duration_seconds))
|
||||
}
|
||||
|
|
@ -453,15 +470,15 @@ pub fn handle_realtime_stream_cost_calculation(
|
|||
.chain(std::iter::once(requested_model))
|
||||
.find_map(|model| {
|
||||
let model_info = catalog.entry(model, Some(provider), None)?;
|
||||
let cost = calculate_generic_cost_from_model_info_with_region(
|
||||
combined_usage,
|
||||
let cost = generic_cost_per_token(GenericCostRequest {
|
||||
model_info,
|
||||
None,
|
||||
false,
|
||||
usage: combined_usage,
|
||||
provider: Some(provider),
|
||||
service_tier: None,
|
||||
data_residency,
|
||||
None,
|
||||
vertex_location: None,
|
||||
at,
|
||||
);
|
||||
});
|
||||
(cost.0 + cost.1 > 0.0 || cost_map_entry_declares_pricing(catalog, model, provider))
|
||||
.then_some(cost)
|
||||
})
|
||||
|
|
|
|||
|
|
@ -3,12 +3,13 @@ use serde_json::Value;
|
|||
|
||||
use crate::base_rate_selection::{
|
||||
TokenBaseRates, get_tiered_reasoning_rate, get_token_base_cost,
|
||||
get_token_base_cost_without_off_peak,
|
||||
get_token_base_cost_without_off_peak, uses_inclusive_token_thresholds,
|
||||
};
|
||||
use crate::generic_input::{InputBaseRates, calculate_input_cost};
|
||||
use crate::generic_output::{calculate_output_cost, resolve_reasoning_token_cost};
|
||||
use crate::generic_usage::{ParsedPromptDetails, parse_prompt_tokens_details};
|
||||
use crate::off_peak::{open_off_peak_block, parse_off_peak_rate};
|
||||
use crate::provider_cache::apply_provider_cache_read_default;
|
||||
use crate::regional_uplift::regional_totals_multiplier;
|
||||
use crate::responses_usage::ChatUsage;
|
||||
|
||||
|
|
@ -187,3 +188,27 @@ pub fn calculate_generic_cost_from_model_info_with_region(
|
|||
at,
|
||||
)
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct GenericCostRequest<'a> {
|
||||
pub model_info: &'a Value,
|
||||
pub usage: &'a ChatUsage,
|
||||
pub provider: Option<&'a str>,
|
||||
pub service_tier: Option<&'a str>,
|
||||
pub data_residency: Option<&'a str>,
|
||||
pub vertex_location: Option<&'a str>,
|
||||
pub at: Timestamp,
|
||||
}
|
||||
|
||||
pub fn generic_cost_per_token(request: GenericCostRequest<'_>) -> (f64, f64) {
|
||||
let model_info = apply_provider_cache_read_default(request.model_info, request.provider);
|
||||
calculate_generic_cost_from_model_info_with_region(
|
||||
request.usage,
|
||||
&model_info,
|
||||
request.service_tier,
|
||||
uses_inclusive_token_thresholds(request.provider),
|
||||
request.data_residency,
|
||||
request.vertex_location,
|
||||
request.at,
|
||||
)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -148,3 +148,52 @@ fn lemonade_provider_is_free_for_unmapped_and_token_priced_models() {
|
|||
.unwrap();
|
||||
assert_eq!(elapsed, (0.1, 0.0));
|
||||
}
|
||||
|
||||
fn flagged_rates() -> Value {
|
||||
json!({
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 2e-6,
|
||||
"input_cost_per_token_flex": 0.5e-6,
|
||||
"output_cost_per_token_flex": 1e-6,
|
||||
"regional_processing_uplift_multiplier_eu": 1.5,
|
||||
"regional_endpoint_uplift_multiplier": 3.0
|
||||
})
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::openai_takes_tier_and_residency("openai", (0.5e-3 * 1.5, 1e-3 * 1.5))]
|
||||
#[case::custom_provider_takes_tier_and_residency("custom", (0.5e-3 * 1.5, 1e-3 * 1.5))]
|
||||
#[case::anthropic_takes_only_the_tier("anthropic", (0.5e-3, 1e-3))]
|
||||
#[case::bedrock_takes_only_the_tier("bedrock", (0.5e-3, 1e-3))]
|
||||
#[case::azure_takes_only_the_tier("azure", (0.5e-3, 1e-3))]
|
||||
#[case::gemini_takes_only_the_tier("gemini", (0.5e-3, 1e-3))]
|
||||
#[case::deepseek_takes_neither("deepseek", (1e-3, 2e-3))]
|
||||
#[case::tencent_takes_neither("tencent", (1e-3, 2e-3))]
|
||||
fn cost_per_token_forwards_only_the_pricing_flags_each_python_provider_passes(
|
||||
#[case] provider: &str,
|
||||
#[case] expected: (f64, f64),
|
||||
) {
|
||||
let catalog = ModelInfoCatalog::new(HashMap::from([(
|
||||
format!("{provider}/model"),
|
||||
flagged_rates(),
|
||||
)]));
|
||||
let usage =
|
||||
get_usage_object(&json!({"usage": {"prompt_tokens": 1000, "completion_tokens": 1000}}))
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let (prompt, completion) = cost_per_token(
|
||||
&catalog,
|
||||
ModelCostRequest {
|
||||
service_tier: Some("flex"),
|
||||
data_residency: Some("eu"),
|
||||
vertex_location: Some("us-east5"),
|
||||
..request("model", provider, &usage)
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
assert!((prompt - expected.0).abs() < 1e-15, "{provider}: {prompt}");
|
||||
assert!(
|
||||
(completion - expected.1).abs() < 1e-15,
|
||||
"{provider}: {completion}"
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -174,3 +174,49 @@ fn transcription_cost_selects_token_usage_or_duration_pricing() {
|
|||
(0.05, 0.0)
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn speech_priced_per_token_uses_generic_rates_not_the_provider_per_second_route() {
|
||||
let catalog = ModelInfoCatalog::new(HashMap::from([(
|
||||
"azure/tts".to_owned(),
|
||||
json!({"input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6, "output_cost_per_second": 0.01}),
|
||||
)]));
|
||||
let usage =
|
||||
get_usage_object(&json!({"usage": {"prompt_tokens": 1000, "completion_tokens": 1000}}))
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let (prompt, completion) = litellm_cost::cost_calculator::speech_cost(
|
||||
&catalog,
|
||||
ModelCostRequest {
|
||||
response_time_ms: Some(1000.0),
|
||||
..request("tts", "azure", &usage)
|
||||
},
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
assert!((prompt - 1e-3).abs() < 1e-15);
|
||||
assert!((completion - 2e-3).abs() < 1e-15);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn transcription_token_pricing_ignores_the_vertex_location() {
|
||||
let catalog = ModelInfoCatalog::new(HashMap::from([(
|
||||
"openai/transcribe".to_owned(),
|
||||
json!({"input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6, "regional_endpoint_uplift_multiplier": 3.0}),
|
||||
)]));
|
||||
let usage =
|
||||
get_usage_object(&json!({"usage": {"prompt_tokens": 1000, "completion_tokens": 1000}}))
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let (prompt, completion) = litellm_cost::cost_calculator::transcription_cost(
|
||||
&catalog,
|
||||
ModelCostRequest {
|
||||
vertex_location: Some("us-east5"),
|
||||
..request("transcribe", "openai", &usage)
|
||||
},
|
||||
0.0,
|
||||
)
|
||||
.unwrap();
|
||||
assert!((prompt - 1e-3).abs() < 1e-15);
|
||||
assert!((completion - 2e-3).abs() < 1e-15);
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue