mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
refactor(cost): delete the Pricing scaffold and run everything through the live path
The lib.rs scaffold (Pricing, compile, calculate, PricingPlan, CostCatalog, Request, Cost, and their error type) is gone; non-token, OCR, batch, and image rates move onto pricing::Rate, batch owns ThresholdPolicy, and ChatUsage loses its scaffold bridge. The python_reference.tsv replay now exercises calculate_generic_cost_from_model_info_with_region instead of the scaffold, retargeting it at the live generic path, and the scaffold-based test blocks in the anthropic, responses, and model-lookup files replay through the generic path with identical expectations. Rate::checked plus a seeded CostError carry the deliberate negative, NaN, and infinity strictness with pins, replacing the compile-time rejection the scaffold provided
This commit is contained in:
parent
0bdf8abf1e
commit
0f4ce30103
20 changed files with 238 additions and 1114 deletions
|
|
@ -17,9 +17,5 @@ criterion.workspace = true
|
|||
proptest.workspace = true
|
||||
rstest.workspace = true
|
||||
|
||||
[[bench]]
|
||||
name = "calculate"
|
||||
harness = false
|
||||
|
||||
[lints.clippy]
|
||||
disallowed-types = "deny"
|
||||
|
|
|
|||
|
|
@ -1,66 +0,0 @@
|
|||
use criterion::{Criterion, criterion_group, criterion_main};
|
||||
use litellm_cost::{
|
||||
Pricing, PromptConvention, Rate, Rates, Request, ServiceTier, ThresholdPolicy, ThresholdRates,
|
||||
Usage, calculate, compile,
|
||||
};
|
||||
use std::hint::black_box;
|
||||
|
||||
fn bench(c: &mut Criterion) {
|
||||
let pricing = Pricing {
|
||||
standard: Rates {
|
||||
input: Rate::Value(0.000002),
|
||||
output: Rate::Value(0.000008),
|
||||
cache_read: Rate::Value(0.0000005),
|
||||
cache_write: Rate::Missing,
|
||||
cache_write_1h: Rate::Missing,
|
||||
},
|
||||
tiers: &[],
|
||||
thresholds: &[],
|
||||
off_peak: None,
|
||||
};
|
||||
let request = Request {
|
||||
usage: Usage {
|
||||
prompt_tokens: 1000,
|
||||
completion_tokens: 200,
|
||||
cache_read_tokens: 250,
|
||||
cache_write_tokens: 0,
|
||||
cache_write_5m_tokens: None,
|
||||
cache_write_1h_tokens: None,
|
||||
prompt_convention: PromptConvention::IncludesCache,
|
||||
},
|
||||
service_tier: ServiceTier::Standard,
|
||||
threshold_policy: ThresholdPolicy::Exclusive,
|
||||
region_multiplier: None,
|
||||
billed_at_utc_minute: None,
|
||||
};
|
||||
let plan = compile(&pricing).unwrap();
|
||||
c.bench_function("native_compiled_calculation", |b| {
|
||||
b.iter(|| black_box(plan.calculate(black_box(&request)).unwrap()))
|
||||
});
|
||||
c.bench_function("native_full_wrapper", |b| {
|
||||
b.iter(|| black_box(calculate(black_box(&pricing), black_box(&request)).unwrap()))
|
||||
});
|
||||
c.bench_function("native_rate_compilation", |b| {
|
||||
b.iter(|| black_box(compile(black_box(&pricing)).unwrap()))
|
||||
});
|
||||
let threshold = ThresholdRates {
|
||||
above_prompt_tokens: 1000,
|
||||
standard: Rates {
|
||||
input: Rate::Value(0.000004),
|
||||
output: Rate::Value(0.000016),
|
||||
..Rates::EMPTY
|
||||
},
|
||||
tiers: &[],
|
||||
};
|
||||
let threshold_pricing = Pricing {
|
||||
thresholds: &[threshold],
|
||||
..pricing
|
||||
};
|
||||
let threshold_plan = compile(&threshold_pricing).unwrap();
|
||||
c.bench_function("native_threshold_boundary", |b| {
|
||||
b.iter(|| black_box(threshold_plan.calculate(black_box(&request)).unwrap()))
|
||||
});
|
||||
}
|
||||
|
||||
criterion_group!(benches, bench);
|
||||
criterion_main!(benches);
|
||||
|
|
@ -1,10 +1,10 @@
|
|||
use jiff::Timestamp;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::Rate;
|
||||
use crate::generic_input::get_cost_per_unit;
|
||||
use crate::image_response_cost::calculate_image_response_cost_from_usage;
|
||||
use crate::non_token::{Error, ImageRates, ImageUsage, calculate_image};
|
||||
use crate::pricing::Rate;
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct AzureAiImageRequest<'a> {
|
||||
|
|
|
|||
|
|
@ -2,9 +2,15 @@ use serde_json::Value;
|
|||
|
||||
use crate::base_rate_selection::uses_inclusive_token_thresholds;
|
||||
use crate::generic_usage::{get_billable_input_tokens, parse_prompt_tokens_details};
|
||||
use crate::pricing::Rate;
|
||||
use crate::regional_uplift::get_regional_uplift_multiplier;
|
||||
use crate::responses_usage::ChatUsage;
|
||||
use crate::{Rate, ThresholdPolicy};
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum ThresholdPolicy {
|
||||
Exclusive,
|
||||
Inclusive,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct BatchCostRates {
|
||||
|
|
@ -71,10 +77,7 @@ pub enum BatchError {
|
|||
}
|
||||
|
||||
fn value(rate: Rate) -> Option<f64> {
|
||||
match rate {
|
||||
Rate::Value(value) => Some(value),
|
||||
Rate::Missing | Rate::Null => None,
|
||||
}
|
||||
rate.value()
|
||||
}
|
||||
|
||||
fn selected_rate(
|
||||
|
|
@ -258,7 +261,7 @@ fn selected_model_rate(
|
|||
.max_by_key(|(threshold, _)| *threshold)
|
||||
.map_or(flat, |(_, key)| match model_rate(model_info, key) {
|
||||
Rate::Value(rate) => Rate::Value(rate),
|
||||
Rate::Missing | Rate::Null => flat,
|
||||
Rate::Missing | Rate::Null | Rate::Invalid => flat,
|
||||
})
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -81,12 +81,6 @@ use crate::vertex_cost::{
|
|||
cost_per_token as vertex_cost_per_token, cost_router as vertex_cost_router,
|
||||
};
|
||||
use crate::xai_cost::{cost_per_token as xai_cost_per_token, reported_cost as xai_reported_cost};
|
||||
use crate::{Cost, Pricing, PricingError, Rates, Request, calculate};
|
||||
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct CostCatalog {
|
||||
entries: HashMap<String, Rates>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct ModelInfoCatalog {
|
||||
|
|
@ -185,7 +179,6 @@ pub struct DefaultImageCostRequest<'a> {
|
|||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum CatalogError {
|
||||
ModelNotFound,
|
||||
Pricing(PricingError),
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
|
|
@ -295,12 +288,6 @@ impl From<UsageError> for RealtimeCostError {
|
|||
}
|
||||
}
|
||||
|
||||
impl From<PricingError> for CatalogError {
|
||||
fn from(value: PricingError) -> Self {
|
||||
Self::Pricing(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<CatalogError> for CatalogResponseError {
|
||||
fn from(value: CatalogError) -> Self {
|
||||
Self::Catalog(value)
|
||||
|
|
@ -360,40 +347,6 @@ fn select_model_key<'a, T>(
|
|||
})
|
||||
}
|
||||
|
||||
impl CostCatalog {
|
||||
pub fn new(entries: HashMap<String, Rates>) -> Self {
|
||||
Self { entries }
|
||||
}
|
||||
|
||||
pub fn select_model_key<'a>(
|
||||
&'a self,
|
||||
model: &str,
|
||||
provider: Option<&str>,
|
||||
region: Option<&str>,
|
||||
) -> Option<&'a str> {
|
||||
select_model_key(&self.entries, model, provider, region)
|
||||
}
|
||||
|
||||
pub fn cost_per_token(
|
||||
&self,
|
||||
model: &str,
|
||||
provider: Option<&str>,
|
||||
region: Option<&str>,
|
||||
request: &Request,
|
||||
) -> Result<Cost, CatalogError> {
|
||||
let key = self
|
||||
.select_model_key(model, provider, region)
|
||||
.ok_or(CatalogError::ModelNotFound)?;
|
||||
let pricing = Pricing {
|
||||
standard: self.entries[key],
|
||||
tiers: &[],
|
||||
thresholds: &[],
|
||||
off_peak: None,
|
||||
};
|
||||
Ok(calculate(&pricing, request)?)
|
||||
}
|
||||
}
|
||||
|
||||
impl ModelInfoCatalog {
|
||||
pub fn new(entries: HashMap<String, Value>) -> Self {
|
||||
Self { entries }
|
||||
|
|
@ -1347,7 +1300,7 @@ impl ModelInfoCatalog {
|
|||
}
|
||||
let rate = |info: &Value, key: &str| {
|
||||
crate::generic_input::get_cost_per_unit(info, key, None)
|
||||
.map_or(crate::Rate::Missing, crate::Rate::Value)
|
||||
.map_or(crate::pricing::Rate::Missing, crate::pricing::Rate::Value)
|
||||
};
|
||||
let tables = [request.supplied_model_info, shared]
|
||||
.into_iter()
|
||||
|
|
|
|||
7
litellm-rust/crates/cost/src/error.rs
Normal file
7
litellm-rust/crates/cost/src/error.rs
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum CostError {
|
||||
InvalidRate,
|
||||
TokenCountOverflow,
|
||||
InvalidUsage,
|
||||
ModelNotFound,
|
||||
}
|
||||
|
|
@ -33,6 +33,7 @@ pub mod custom_pricing;
|
|||
#[allow(clippy::disallowed_types)]
|
||||
pub mod dashscope_cost;
|
||||
pub mod databricks_cost;
|
||||
pub mod error;
|
||||
#[allow(clippy::disallowed_types)]
|
||||
pub mod fal_ai_image_cost;
|
||||
#[allow(clippy::disallowed_types)]
|
||||
|
|
@ -113,408 +114,4 @@ pub mod xai_cost;
|
|||
#[allow(clippy::disallowed_types)]
|
||||
pub mod zero_cost_diagnostic;
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub enum Rate {
|
||||
Missing,
|
||||
Null,
|
||||
Value(f64),
|
||||
}
|
||||
|
||||
impl Rate {
|
||||
fn value(self) -> Option<f64> {
|
||||
match self {
|
||||
Self::Value(value) => Some(value),
|
||||
Self::Missing | Self::Null => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn or(self, fallback: Self) -> Self {
|
||||
if self.value().is_some() {
|
||||
self
|
||||
} else {
|
||||
fallback
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct Rates {
|
||||
pub input: Rate,
|
||||
pub output: Rate,
|
||||
pub cache_read: Rate,
|
||||
pub cache_write: Rate,
|
||||
pub cache_write_1h: Rate,
|
||||
}
|
||||
|
||||
impl Rates {
|
||||
pub const EMPTY: Self = Self {
|
||||
input: Rate::Missing,
|
||||
output: Rate::Missing,
|
||||
cache_read: Rate::Missing,
|
||||
cache_write: Rate::Missing,
|
||||
cache_write_1h: Rate::Missing,
|
||||
};
|
||||
|
||||
fn overlay(self, base: Self) -> Self {
|
||||
Self {
|
||||
input: self.input.or(base.input),
|
||||
output: self.output.or(base.output),
|
||||
cache_read: self.cache_read.or(base.cache_read),
|
||||
cache_write: self.cache_write.or(base.cache_write),
|
||||
cache_write_1h: self.cache_write_1h.or(base.cache_write_1h),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum ServiceTier {
|
||||
Standard,
|
||||
Flex,
|
||||
Priority,
|
||||
Fast,
|
||||
Ultrafast,
|
||||
Unknown,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum ThresholdPolicy {
|
||||
Exclusive,
|
||||
Inclusive,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum PromptConvention {
|
||||
IncludesCache,
|
||||
ExcludesCache,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct Usage {
|
||||
pub prompt_tokens: u64,
|
||||
pub completion_tokens: u64,
|
||||
pub cache_read_tokens: u64,
|
||||
pub cache_write_tokens: u64,
|
||||
pub cache_write_5m_tokens: Option<u64>,
|
||||
pub cache_write_1h_tokens: Option<u64>,
|
||||
pub prompt_convention: PromptConvention,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct TierRates {
|
||||
pub tier: ServiceTier,
|
||||
pub rates: Rates,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct ThresholdRates<'a> {
|
||||
pub above_prompt_tokens: u64,
|
||||
pub standard: Rates,
|
||||
pub tiers: &'a [TierRates],
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct OffPeakRates {
|
||||
pub start_utc_minute: u16,
|
||||
pub end_utc_minute: u16,
|
||||
pub rates: Rates,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct Pricing<'a> {
|
||||
pub standard: Rates,
|
||||
pub tiers: &'a [TierRates],
|
||||
pub thresholds: &'a [ThresholdRates<'a>],
|
||||
pub off_peak: Option<OffPeakRates>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct Request {
|
||||
pub usage: Usage,
|
||||
pub service_tier: ServiceTier,
|
||||
pub threshold_policy: ThresholdPolicy,
|
||||
pub region_multiplier: Option<f64>,
|
||||
pub billed_at_utc_minute: Option<u16>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct Cost {
|
||||
pub uncached_input: f64,
|
||||
pub cache_read: f64,
|
||||
pub cache_write_5m: f64,
|
||||
pub cache_write_1h: f64,
|
||||
pub output: f64,
|
||||
pub multiplier: f64,
|
||||
pub rates: EffectiveRates,
|
||||
}
|
||||
|
||||
impl Cost {
|
||||
pub fn input(self) -> f64 {
|
||||
(self.uncached_input + self.cache_read + self.cache_write_5m + self.cache_write_1h)
|
||||
* self.multiplier
|
||||
}
|
||||
|
||||
pub fn output(self) -> f64 {
|
||||
self.output * self.multiplier
|
||||
}
|
||||
|
||||
pub fn total(self) -> f64 {
|
||||
self.input() + self.output()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct EffectiveRates {
|
||||
pub input: f64,
|
||||
pub output: f64,
|
||||
pub cache_read: f64,
|
||||
pub cache_write_5m: f64,
|
||||
pub cache_write_1h: f64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum PricingError {
|
||||
MissingInputRate,
|
||||
MissingOutputRate,
|
||||
InvalidRate,
|
||||
InvalidRegionMultiplier,
|
||||
InvalidBillingTime,
|
||||
InvalidOffPeakWindow,
|
||||
CacheExceedsPrompt,
|
||||
InvalidCacheWriteDetails,
|
||||
TokenCountOverflow,
|
||||
DuplicateTier,
|
||||
DuplicateThreshold,
|
||||
DuplicateThresholdTier,
|
||||
}
|
||||
|
||||
fn selected_tier(tier: ServiceTier) -> ServiceTier {
|
||||
if tier == ServiceTier::Fast {
|
||||
ServiceTier::Priority
|
||||
} else {
|
||||
tier
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct CompiledThreshold {
|
||||
above_prompt_tokens: u64,
|
||||
standard: Rates,
|
||||
tiers: Vec<TierRates>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct PricingPlan {
|
||||
standard: Rates,
|
||||
tiers: Vec<TierRates>,
|
||||
thresholds: Vec<CompiledThreshold>,
|
||||
off_peak: Option<OffPeakRates>,
|
||||
}
|
||||
|
||||
fn valid_rates(rates: Rates) -> bool {
|
||||
[
|
||||
rates.input,
|
||||
rates.output,
|
||||
rates.cache_read,
|
||||
rates.cache_write,
|
||||
rates.cache_write_1h,
|
||||
]
|
||||
.into_iter()
|
||||
.all(|rate| {
|
||||
rate.value()
|
||||
.is_none_or(|value| value.is_finite() && value >= 0.0)
|
||||
})
|
||||
}
|
||||
|
||||
fn validate_tiers(tiers: &[TierRates], duplicate: PricingError) -> Result<(), PricingError> {
|
||||
if tiers.iter().any(|entry| !valid_rates(entry.rates)) {
|
||||
return Err(PricingError::InvalidRate);
|
||||
}
|
||||
if tiers.iter().enumerate().any(|(index, entry)| {
|
||||
matches!(
|
||||
entry.tier,
|
||||
ServiceTier::Standard | ServiceTier::Unknown | ServiceTier::Fast
|
||||
) || tiers[..index]
|
||||
.iter()
|
||||
.any(|previous| previous.tier == entry.tier)
|
||||
}) {
|
||||
return Err(duplicate);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn compile(pricing: &Pricing<'_>) -> Result<PricingPlan, PricingError> {
|
||||
if !valid_rates(pricing.standard) {
|
||||
return Err(PricingError::InvalidRate);
|
||||
}
|
||||
validate_tiers(pricing.tiers, PricingError::DuplicateTier)?;
|
||||
if let Some(window) = pricing.off_peak {
|
||||
if window.start_utc_minute >= 1440
|
||||
|| window.end_utc_minute > 1440
|
||||
|| window.start_utc_minute >= window.end_utc_minute
|
||||
{
|
||||
return Err(PricingError::InvalidOffPeakWindow);
|
||||
}
|
||||
if !valid_rates(window.rates) {
|
||||
return Err(PricingError::InvalidRate);
|
||||
}
|
||||
}
|
||||
let mut thresholds: Vec<_> = pricing
|
||||
.thresholds
|
||||
.iter()
|
||||
.map(|entry| {
|
||||
if !valid_rates(entry.standard) {
|
||||
return Err(PricingError::InvalidRate);
|
||||
}
|
||||
validate_tiers(entry.tiers, PricingError::DuplicateThresholdTier)?;
|
||||
Ok(CompiledThreshold {
|
||||
above_prompt_tokens: entry.above_prompt_tokens,
|
||||
standard: entry.standard,
|
||||
tiers: entry.tiers.to_vec(),
|
||||
})
|
||||
})
|
||||
.collect::<Result<_, _>>()?;
|
||||
thresholds.sort_unstable_by_key(|entry| entry.above_prompt_tokens);
|
||||
if thresholds
|
||||
.windows(2)
|
||||
.any(|pair| pair[0].above_prompt_tokens == pair[1].above_prompt_tokens)
|
||||
{
|
||||
return Err(PricingError::DuplicateThreshold);
|
||||
}
|
||||
Ok(PricingPlan {
|
||||
standard: pricing.standard,
|
||||
tiers: pricing.tiers.to_vec(),
|
||||
thresholds,
|
||||
off_peak: pricing.off_peak,
|
||||
})
|
||||
}
|
||||
|
||||
impl PricingPlan {
|
||||
fn resolve_rates(
|
||||
&self,
|
||||
request: &Request,
|
||||
threshold_tokens: u64,
|
||||
) -> Result<Rates, PricingError> {
|
||||
let tier = selected_tier(request.service_tier);
|
||||
let base = self
|
||||
.tiers
|
||||
.iter()
|
||||
.find(|entry| tier != ServiceTier::Standard && entry.tier == tier)
|
||||
.map_or(self.standard, |entry| entry.rates.overlay(self.standard));
|
||||
let threshold = self.thresholds.iter().rev().find(|entry| {
|
||||
threshold_tokens > entry.above_prompt_tokens
|
||||
|| (request.threshold_policy == ThresholdPolicy::Inclusive
|
||||
&& threshold_tokens == entry.above_prompt_tokens)
|
||||
});
|
||||
let selected = threshold.map_or(base, |entry| {
|
||||
let standard = entry.standard.overlay(base);
|
||||
entry
|
||||
.tiers
|
||||
.iter()
|
||||
.find(|specific| tier != ServiceTier::Standard && specific.tier == tier)
|
||||
.map_or(standard, |specific| specific.rates.overlay(standard))
|
||||
});
|
||||
match self.off_peak {
|
||||
None => Ok(selected),
|
||||
Some(window) => {
|
||||
if window.start_utc_minute >= 1440
|
||||
|| window.end_utc_minute > 1440
|
||||
|| window.start_utc_minute >= window.end_utc_minute
|
||||
{
|
||||
return Err(PricingError::InvalidOffPeakWindow);
|
||||
}
|
||||
let minute = request
|
||||
.billed_at_utc_minute
|
||||
.ok_or(PricingError::InvalidBillingTime)?;
|
||||
if minute >= 1440 {
|
||||
return Err(PricingError::InvalidBillingTime);
|
||||
}
|
||||
if (window.start_utc_minute..window.end_utc_minute).contains(&minute) {
|
||||
Ok(window.rates.overlay(selected))
|
||||
} else {
|
||||
Ok(selected)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn checked_rate(rate: Rate, missing: PricingError) -> Result<f64, PricingError> {
|
||||
let value = rate.value().ok_or(missing)?;
|
||||
if !value.is_finite() || value < 0.0 {
|
||||
return Err(PricingError::InvalidRate);
|
||||
}
|
||||
Ok(value)
|
||||
}
|
||||
|
||||
pub fn calculate(&self, request: &Request) -> Result<Cost, PricingError> {
|
||||
let usage = request.usage;
|
||||
let cached = usage
|
||||
.cache_read_tokens
|
||||
.checked_add(usage.cache_write_tokens)
|
||||
.ok_or(PricingError::TokenCountOverflow)?;
|
||||
let (regular, threshold_tokens) = match usage.prompt_convention {
|
||||
PromptConvention::IncludesCache => (
|
||||
usage
|
||||
.prompt_tokens
|
||||
.checked_sub(cached)
|
||||
.ok_or(PricingError::CacheExceedsPrompt)?,
|
||||
usage.prompt_tokens,
|
||||
),
|
||||
PromptConvention::ExcludesCache => (
|
||||
usage.prompt_tokens,
|
||||
usage
|
||||
.prompt_tokens
|
||||
.checked_add(cached)
|
||||
.ok_or(PricingError::TokenCountOverflow)?,
|
||||
),
|
||||
};
|
||||
let writes = match (usage.cache_write_5m_tokens, usage.cache_write_1h_tokens) {
|
||||
(None, None) => (usage.cache_write_tokens, 0),
|
||||
(Some(five), Some(one)) if five.checked_add(one) == Some(usage.cache_write_tokens) => {
|
||||
(five, one)
|
||||
}
|
||||
_ => return Err(PricingError::InvalidCacheWriteDetails),
|
||||
};
|
||||
let rates = self.resolve_rates(request, threshold_tokens)?;
|
||||
let input = Self::checked_rate(rates.input, PricingError::MissingInputRate)?;
|
||||
let output = Self::checked_rate(rates.output, PricingError::MissingOutputRate)?;
|
||||
let read = Self::checked_rate(
|
||||
rates.cache_read.or(rates.input),
|
||||
PricingError::MissingInputRate,
|
||||
)?;
|
||||
let write = Self::checked_rate(
|
||||
rates.cache_write.or(rates.input),
|
||||
PricingError::MissingInputRate,
|
||||
)?;
|
||||
let write_1h = Self::checked_rate(
|
||||
rates.cache_write_1h.or(rates.cache_write).or(rates.input),
|
||||
PricingError::MissingInputRate,
|
||||
)?;
|
||||
let multiplier = request.region_multiplier.unwrap_or(1.0);
|
||||
if !multiplier.is_finite() || multiplier <= 0.0 {
|
||||
return Err(PricingError::InvalidRegionMultiplier);
|
||||
}
|
||||
let cost = Cost {
|
||||
uncached_input: regular as f64 * input,
|
||||
cache_read: usage.cache_read_tokens as f64 * read,
|
||||
cache_write_5m: writes.0 as f64 * write,
|
||||
cache_write_1h: writes.1 as f64 * write_1h,
|
||||
output: usage.completion_tokens as f64 * output,
|
||||
multiplier,
|
||||
rates: EffectiveRates {
|
||||
input,
|
||||
output,
|
||||
cache_read: read,
|
||||
cache_write_5m: write,
|
||||
cache_write_1h: write_1h,
|
||||
},
|
||||
};
|
||||
if !cost.total().is_finite() {
|
||||
return Err(PricingError::TokenCountOverflow);
|
||||
}
|
||||
Ok(cost)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn calculate(pricing: &Pricing<'_>, request: &Request) -> Result<Cost, PricingError> {
|
||||
compile(pricing)?.calculate(request)
|
||||
}
|
||||
pub use pricing::Rate;
|
||||
|
|
|
|||
|
|
@ -84,7 +84,7 @@ pub struct VideoRates<'a> {
|
|||
fn priced(rate: Rate) -> Option<f64> {
|
||||
match rate {
|
||||
Rate::Value(value) => Some(value),
|
||||
Rate::Missing | Rate::Null => None,
|
||||
Rate::Missing | Rate::Null | Rate::Invalid => None,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -283,4 +283,4 @@ pub fn video_resolution_to_cost_field_suffix(resolution: &str) -> Option<String>
|
|||
.collect();
|
||||
(!suffix.is_empty() && suffix.chars().count() <= 24).then_some(suffix)
|
||||
}
|
||||
use crate::Rate;
|
||||
use crate::pricing::Rate;
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
use serde_json::Value;
|
||||
|
||||
use crate::Rate;
|
||||
use crate::non_token::{Error as NonTokenError, OcrRates, OcrUsage, calculate_ocr_with_tables};
|
||||
use crate::pricing::Rate;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum OcrCostError {
|
||||
|
|
@ -28,7 +28,7 @@ fn rates(model_info: &Value) -> OcrRates {
|
|||
fn value(rate: Rate) -> Option<f64> {
|
||||
match rate {
|
||||
Rate::Value(value) => Some(value),
|
||||
Rate::Missing | Rate::Null => None,
|
||||
Rate::Missing | Rate::Null | Rate::Invalid => None,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
use std::collections::BTreeMap;
|
||||
|
||||
use crate::error::CostError;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq)]
|
||||
pub enum Rate {
|
||||
#[default]
|
||||
|
|
@ -16,6 +18,13 @@ impl Rate {
|
|||
Self::Missing | Self::Null | Self::Invalid => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn checked(self) -> Result<f64, CostError> {
|
||||
match self {
|
||||
Self::Value(value) if value.is_finite() && value >= 0.0 => Ok(value),
|
||||
_ => Err(CostError::InvalidRate),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
|
||||
|
|
|
|||
|
|
@ -3,8 +3,6 @@ use std::collections::BTreeMap;
|
|||
use serde::Deserialize;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::{PromptConvention, Usage};
|
||||
|
||||
#[derive(Clone, Debug, Default, Deserialize, PartialEq)]
|
||||
pub struct CachedTokenDetails {
|
||||
pub text_tokens: Option<u64>,
|
||||
|
|
@ -60,33 +58,6 @@ pub struct ChatUsage {
|
|||
pub extra: BTreeMap<String, Value>,
|
||||
}
|
||||
|
||||
impl ChatUsage {
|
||||
pub fn token_usage(&self) -> Usage {
|
||||
let cache_creation_details = self
|
||||
.prompt_tokens_details
|
||||
.as_ref()
|
||||
.and_then(|details| details.cache_creation_token_details.as_ref());
|
||||
Usage {
|
||||
prompt_tokens: self.prompt_tokens,
|
||||
completion_tokens: self.completion_tokens,
|
||||
cache_read_tokens: self
|
||||
.prompt_tokens_details
|
||||
.as_ref()
|
||||
.map_or(0, |details| details.cached_tokens),
|
||||
cache_write_tokens: self
|
||||
.prompt_tokens_details
|
||||
.as_ref()
|
||||
.and_then(|details| details.cache_write_tokens)
|
||||
.unwrap_or(0),
|
||||
cache_write_5m_tokens: cache_creation_details
|
||||
.map(|details| details.ephemeral_5m_input_tokens.unwrap_or(0)),
|
||||
cache_write_1h_tokens: cache_creation_details
|
||||
.map(|details| details.ephemeral_1h_input_tokens.unwrap_or(0)),
|
||||
prompt_convention: PromptConvention::IncludesCache,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum UsageError {
|
||||
InvalidShape,
|
||||
|
|
|
|||
|
|
@ -1,464 +0,0 @@
|
|||
use litellm_cost::{
|
||||
OffPeakRates, Pricing, PricingError, PromptConvention, Rate, Rates, Request, ServiceTier,
|
||||
ThresholdPolicy, ThresholdRates, TierRates, Usage, calculate, compile,
|
||||
};
|
||||
|
||||
fn rates(input: Rate, output: Rate) -> Rates {
|
||||
Rates {
|
||||
input,
|
||||
output,
|
||||
..Rates::EMPTY
|
||||
}
|
||||
}
|
||||
|
||||
fn request() -> Request {
|
||||
Request {
|
||||
usage: Usage {
|
||||
prompt_tokens: 100,
|
||||
completion_tokens: 20,
|
||||
cache_read_tokens: 25,
|
||||
cache_write_tokens: 10,
|
||||
cache_write_5m_tokens: None,
|
||||
cache_write_1h_tokens: None,
|
||||
prompt_convention: PromptConvention::IncludesCache,
|
||||
},
|
||||
service_tier: ServiceTier::Standard,
|
||||
threshold_policy: ThresholdPolicy::Exclusive,
|
||||
region_multiplier: None,
|
||||
billed_at_utc_minute: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn pricing(standard: Rates) -> Pricing<'static> {
|
||||
Pricing {
|
||||
standard,
|
||||
tiers: &[],
|
||||
thresholds: &[],
|
||||
off_peak: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn breakdown_and_total_agree() {
|
||||
let standard = Rates {
|
||||
cache_read: Rate::Value(0.5),
|
||||
cache_write: Rate::Value(3.0),
|
||||
..rates(Rate::Value(2.0), Rate::Value(4.0))
|
||||
};
|
||||
let result = calculate(&pricing(standard), &request()).unwrap();
|
||||
assert_eq!(result.uncached_input, 65.0 * 2.0);
|
||||
assert_eq!(result.cache_read, 25.0 * 0.5);
|
||||
assert_eq!(result.cache_write_5m, 10.0 * 3.0);
|
||||
assert_eq!(result.output(), 20.0 * 4.0);
|
||||
assert_eq!(result.total(), result.input() + result.output());
|
||||
assert_eq!(result.rates.cache_read, 0.5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn absent_null_and_zero_cache_rates_are_distinct() {
|
||||
let base = rates(Rate::Value(2.0), Rate::Value(4.0));
|
||||
for read in [Rate::Missing, Rate::Null] {
|
||||
let standard = Rates {
|
||||
cache_read: read,
|
||||
..base
|
||||
};
|
||||
assert_eq!(
|
||||
calculate(&pricing(standard), &request()).unwrap().input(),
|
||||
200.0
|
||||
);
|
||||
}
|
||||
let standard = Rates {
|
||||
cache_read: Rate::Value(0.0),
|
||||
cache_write: Rate::Value(0.0),
|
||||
..base
|
||||
};
|
||||
assert_eq!(
|
||||
calculate(&pricing(standard), &request()).unwrap().input(),
|
||||
130.0
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn equivalent_prompt_conventions_select_the_same_threshold() {
|
||||
let threshold = ThresholdRates {
|
||||
above_prompt_tokens: 90,
|
||||
standard: rates(Rate::Value(5.0), Rate::Value(8.0)),
|
||||
tiers: &[],
|
||||
};
|
||||
let specification = Pricing {
|
||||
standard: rates(Rate::Value(2.0), Rate::Value(4.0)),
|
||||
tiers: &[],
|
||||
thresholds: &[threshold],
|
||||
off_peak: None,
|
||||
};
|
||||
let included = request();
|
||||
let excluded = Request {
|
||||
usage: Usage {
|
||||
prompt_tokens: 65,
|
||||
prompt_convention: PromptConvention::ExcludesCache,
|
||||
..included.usage
|
||||
},
|
||||
..included
|
||||
};
|
||||
let plan = compile(&specification).unwrap();
|
||||
assert_eq!(plan.calculate(&included), plan.calculate(&excluded));
|
||||
assert_eq!(plan.calculate(&included).unwrap().rates.input, 5.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn split_writes_and_invalid_accounting() {
|
||||
let standard = Rates {
|
||||
cache_read: Rate::Value(0.5),
|
||||
cache_write: Rate::Value(3.0),
|
||||
cache_write_1h: Rate::Value(5.0),
|
||||
..rates(Rate::Value(2.0), Rate::Value(4.0))
|
||||
};
|
||||
let base = request();
|
||||
let split = Request {
|
||||
usage: Usage {
|
||||
cache_write_5m_tokens: Some(4),
|
||||
cache_write_1h_tokens: Some(6),
|
||||
..base.usage
|
||||
},
|
||||
..base
|
||||
};
|
||||
let result = calculate(&pricing(standard), &split).unwrap();
|
||||
assert_eq!(result.cache_write_5m, 12.0);
|
||||
assert_eq!(result.cache_write_1h, 30.0);
|
||||
let overlapping = Request {
|
||||
usage: Usage {
|
||||
prompt_tokens: 30,
|
||||
..split.usage
|
||||
},
|
||||
..split
|
||||
};
|
||||
assert_eq!(
|
||||
calculate(&pricing(standard), &overlapping),
|
||||
Err(PricingError::CacheExceedsPrompt)
|
||||
);
|
||||
let incomplete = Request {
|
||||
usage: Usage {
|
||||
cache_write_1h_tokens: None,
|
||||
..split.usage
|
||||
},
|
||||
..split
|
||||
};
|
||||
assert_eq!(
|
||||
calculate(&pricing(standard), &incomplete),
|
||||
Err(PricingError::InvalidCacheWriteDetails)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn threshold_tiers_and_boundaries() {
|
||||
let priority = TierRates {
|
||||
tier: ServiceTier::Priority,
|
||||
rates: rates(Rate::Value(3.0), Rate::Missing),
|
||||
};
|
||||
let threshold = ThresholdRates {
|
||||
above_prompt_tokens: 100,
|
||||
standard: rates(Rate::Value(5.0), Rate::Value(8.0)),
|
||||
tiers: &[
|
||||
TierRates {
|
||||
tier: ServiceTier::Priority,
|
||||
rates: rates(Rate::Value(7.0), Rate::Missing),
|
||||
},
|
||||
TierRates {
|
||||
tier: ServiceTier::Flex,
|
||||
rates: rates(Rate::Value(6.0), Rate::Missing),
|
||||
},
|
||||
],
|
||||
};
|
||||
let specification = Pricing {
|
||||
standard: rates(Rate::Value(2.0), Rate::Value(4.0)),
|
||||
tiers: &[priority],
|
||||
thresholds: &[threshold],
|
||||
off_peak: None,
|
||||
};
|
||||
let base = request();
|
||||
let no_cache = Request {
|
||||
usage: Usage {
|
||||
cache_read_tokens: 0,
|
||||
cache_write_tokens: 0,
|
||||
..base.usage
|
||||
},
|
||||
..base
|
||||
};
|
||||
let fast = Request {
|
||||
service_tier: ServiceTier::Fast,
|
||||
..no_cache
|
||||
};
|
||||
let inclusive = Request {
|
||||
threshold_policy: ThresholdPolicy::Inclusive,
|
||||
..fast
|
||||
};
|
||||
let flex = Request {
|
||||
service_tier: ServiceTier::Flex,
|
||||
..inclusive
|
||||
};
|
||||
assert_eq!(calculate(&specification, &no_cache).unwrap().input(), 200.0);
|
||||
assert_eq!(calculate(&specification, &fast).unwrap().input(), 300.0);
|
||||
assert_eq!(
|
||||
calculate(&specification, &inclusive).unwrap().input(),
|
||||
700.0
|
||||
);
|
||||
assert_eq!(calculate(&specification, &flex).unwrap().input(), 600.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compile_rejects_ambiguous_rates() {
|
||||
let duplicate = ThresholdRates {
|
||||
above_prompt_tokens: 100,
|
||||
standard: Rates::EMPTY,
|
||||
tiers: &[],
|
||||
};
|
||||
let specification = Pricing {
|
||||
standard: rates(Rate::Value(1.0), Rate::Value(1.0)),
|
||||
tiers: &[],
|
||||
thresholds: &[duplicate, duplicate],
|
||||
off_peak: None,
|
||||
};
|
||||
assert_eq!(
|
||||
compile(&specification).err(),
|
||||
Some(PricingError::DuplicateThreshold)
|
||||
);
|
||||
let invalid = pricing(rates(Rate::Value(f64::NAN), Rate::Value(1.0)));
|
||||
assert_eq!(compile(&invalid).err(), Some(PricingError::InvalidRate));
|
||||
let negative = pricing(rates(Rate::Value(-1.0), Rate::Value(1.0)));
|
||||
assert_eq!(
|
||||
compile(&negative).err(),
|
||||
Some(PricingError::InvalidRate),
|
||||
"divergence: Python bills negative rates as negative costs; Rust refuses them"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn off_peak_is_one_non_wrapping_utc_window() {
|
||||
let specification = Pricing {
|
||||
standard: rates(Rate::Value(2.0), Rate::Value(4.0)),
|
||||
tiers: &[],
|
||||
thresholds: &[],
|
||||
off_peak: Some(OffPeakRates {
|
||||
start_utc_minute: 60,
|
||||
end_utc_minute: 120,
|
||||
rates: rates(Rate::Value(1.0), Rate::Value(2.0)),
|
||||
}),
|
||||
};
|
||||
let base = request();
|
||||
let start = Request {
|
||||
billed_at_utc_minute: Some(60),
|
||||
..base
|
||||
};
|
||||
let end = Request {
|
||||
billed_at_utc_minute: Some(120),
|
||||
..base
|
||||
};
|
||||
assert_eq!(
|
||||
calculate(&specification, &base),
|
||||
Err(PricingError::InvalidBillingTime)
|
||||
);
|
||||
assert_eq!(calculate(&specification, &start).unwrap().input(), 100.0);
|
||||
assert_eq!(calculate(&specification, &end).unwrap().input(), 200.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_rates_and_free_rates_remain_distinct() {
|
||||
let base = request();
|
||||
let empty = Request {
|
||||
usage: Usage {
|
||||
prompt_tokens: 0,
|
||||
completion_tokens: 0,
|
||||
cache_read_tokens: 0,
|
||||
cache_write_tokens: 0,
|
||||
..base.usage
|
||||
},
|
||||
..base
|
||||
};
|
||||
assert_eq!(
|
||||
calculate(&pricing(Rates::EMPTY), &empty),
|
||||
Err(PricingError::MissingInputRate)
|
||||
);
|
||||
assert_eq!(
|
||||
calculate(&pricing(rates(Rate::Value(0.0), Rate::Missing)), &empty),
|
||||
Err(PricingError::MissingOutputRate)
|
||||
);
|
||||
assert_eq!(
|
||||
calculate(&pricing(rates(Rate::Value(0.0), Rate::Value(0.0))), &empty)
|
||||
.unwrap()
|
||||
.total(),
|
||||
0.0
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn matches_executed_python_reference_cases() {
|
||||
for row in include_str!("python_reference.tsv")
|
||||
.lines()
|
||||
.filter(|line| !line.starts_with('#'))
|
||||
{
|
||||
let fields: Vec<_> = row.split('\t').collect();
|
||||
let count = |index: usize| fields[index].parse::<u64>().unwrap();
|
||||
let number = |index: usize| fields[index].parse::<f64>().unwrap();
|
||||
let optional_rate = |index: usize| {
|
||||
if fields[index].is_empty() {
|
||||
Rate::Missing
|
||||
} else {
|
||||
Rate::Value(number(index))
|
||||
}
|
||||
};
|
||||
let threshold = ThresholdRates {
|
||||
above_prompt_tokens: if fields[9].is_empty() { 0 } else { count(9) },
|
||||
standard: rates(optional_rate(10), optional_rate(11)),
|
||||
tiers: &[],
|
||||
};
|
||||
let thresholds = if fields[9].is_empty() {
|
||||
&[][..]
|
||||
} else {
|
||||
std::slice::from_ref(&threshold)
|
||||
};
|
||||
let specification = Pricing {
|
||||
standard: Rates {
|
||||
cache_read: optional_rate(7),
|
||||
cache_write: optional_rate(8),
|
||||
..rates(Rate::Value(number(5)), Rate::Value(number(6)))
|
||||
},
|
||||
tiers: &[],
|
||||
thresholds,
|
||||
off_peak: None,
|
||||
};
|
||||
let base = request();
|
||||
let input = Request {
|
||||
usage: Usage {
|
||||
prompt_tokens: count(1),
|
||||
completion_tokens: count(2),
|
||||
cache_read_tokens: count(3),
|
||||
cache_write_tokens: count(4),
|
||||
..base.usage
|
||||
},
|
||||
..base
|
||||
};
|
||||
let actual = calculate(&specification, &input).unwrap();
|
||||
assert_eq!(actual.input(), number(12), "{}", fields[0]);
|
||||
assert_eq!(actual.output(), number(13), "{}", fields[0]);
|
||||
}
|
||||
}
|
||||
|
||||
proptest::proptest! {
|
||||
#[test]
|
||||
fn equivalent_usage_conventions_and_breakdown_agree(
|
||||
regular in 0_u64..1000,
|
||||
read in 0_u64..1000,
|
||||
write in 0_u64..1000,
|
||||
output in 0_u64..1000,
|
||||
) {
|
||||
let threshold = ThresholdRates {
|
||||
above_prompt_tokens: 1000,
|
||||
standard: rates(Rate::Value(5.0), Rate::Value(8.0)),
|
||||
tiers: &[],
|
||||
};
|
||||
let specification = Pricing {
|
||||
standard: rates(Rate::Value(2.0), Rate::Value(4.0)),
|
||||
tiers: &[],
|
||||
thresholds: &[threshold],
|
||||
off_peak: None,
|
||||
};
|
||||
let base = request();
|
||||
let included = Request {
|
||||
usage: Usage {
|
||||
prompt_tokens: regular + read + write,
|
||||
completion_tokens: output,
|
||||
cache_read_tokens: read,
|
||||
cache_write_tokens: write,
|
||||
..base.usage
|
||||
},
|
||||
..base
|
||||
};
|
||||
let excluded = Request {
|
||||
usage: Usage {
|
||||
prompt_tokens: regular,
|
||||
prompt_convention: PromptConvention::ExcludesCache,
|
||||
..included.usage
|
||||
},
|
||||
..included
|
||||
};
|
||||
let plan = compile(&specification).unwrap();
|
||||
let left = plan.calculate(&included).unwrap();
|
||||
let right = plan.calculate(&excluded).unwrap();
|
||||
proptest::prop_assert_eq!(left, right);
|
||||
proptest::prop_assert_eq!(left.total(), left.input() + left.output());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn regional_multiplier_applies_after_input_components_are_summed() {
|
||||
let standard = Rates {
|
||||
cache_read: Rate::Value(0.5),
|
||||
cache_write: Rate::Value(3.0),
|
||||
..rates(Rate::Value(2.0), Rate::Value(4.0))
|
||||
};
|
||||
let base = request();
|
||||
let regional = Request {
|
||||
region_multiplier: Some(1.1),
|
||||
..base
|
||||
};
|
||||
let result = calculate(&pricing(standard), ®ional).unwrap();
|
||||
assert_eq!(result.input(), (65.0 * 2.0 + 25.0 * 0.5 + 10.0 * 3.0) * 1.1);
|
||||
assert_eq!(result.output(), 20.0 * 4.0 * 1.1);
|
||||
let invalid = Request {
|
||||
region_multiplier: Some(f64::NAN),
|
||||
..base
|
||||
};
|
||||
assert_eq!(
|
||||
calculate(&pricing(standard), &invalid),
|
||||
Err(PricingError::InvalidRegionMultiplier)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compilation_sorts_thresholds_and_rejects_duplicate_tiers() {
|
||||
let high = ThresholdRates {
|
||||
above_prompt_tokens: 200,
|
||||
standard: rates(Rate::Value(7.0), Rate::Missing),
|
||||
tiers: &[],
|
||||
};
|
||||
let low = ThresholdRates {
|
||||
above_prompt_tokens: 100,
|
||||
standard: rates(Rate::Value(5.0), Rate::Missing),
|
||||
tiers: &[],
|
||||
};
|
||||
let specification = Pricing {
|
||||
standard: rates(Rate::Value(2.0), Rate::Value(4.0)),
|
||||
tiers: &[],
|
||||
thresholds: &[high, low],
|
||||
off_peak: None,
|
||||
};
|
||||
let base = request();
|
||||
let above_both = Request {
|
||||
usage: Usage {
|
||||
prompt_tokens: 201,
|
||||
cache_read_tokens: 0,
|
||||
cache_write_tokens: 0,
|
||||
..base.usage
|
||||
},
|
||||
..base
|
||||
};
|
||||
assert_eq!(
|
||||
compile(&specification)
|
||||
.unwrap()
|
||||
.calculate(&above_both)
|
||||
.unwrap()
|
||||
.rates
|
||||
.input,
|
||||
7.0
|
||||
);
|
||||
let duplicate = TierRates {
|
||||
tier: ServiceTier::Flex,
|
||||
rates: Rates::EMPTY,
|
||||
};
|
||||
let invalid = Pricing {
|
||||
tiers: &[duplicate, duplicate],
|
||||
thresholds: &[],
|
||||
..specification
|
||||
};
|
||||
assert_eq!(compile(&invalid).err(), Some(PricingError::DuplicateTier));
|
||||
}
|
||||
|
|
@ -179,3 +179,28 @@ fn parsed_pricing_buckets_a_synthetic_overlay_entry() {
|
|||
assert!(pricing.extra.is_empty(), "recognized non-rate keys drop");
|
||||
let _: LazyLock<()> = LazyLock::new(|| ());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checked_rates_reject_negative_and_non_finite_where_python_bills_them() {
|
||||
use litellm_cost::error::CostError;
|
||||
use litellm_cost::pricing::Rate;
|
||||
assert_eq!(Rate::Value(3e-6).checked(), Ok(3e-6));
|
||||
assert_eq!(Rate::Value(0.0).checked(), Ok(0.0));
|
||||
assert_eq!(
|
||||
Rate::Value(-1.0).checked(),
|
||||
Err(CostError::InvalidRate),
|
||||
"divergence: Python bills negative rates as negative costs; Rust refuses them"
|
||||
);
|
||||
assert_eq!(
|
||||
Rate::Value(f64::NAN).checked(),
|
||||
Err(CostError::InvalidRate),
|
||||
"divergence: Python propagates NaN costs; Rust refuses them"
|
||||
);
|
||||
assert_eq!(
|
||||
Rate::Value(f64::INFINITY).checked(),
|
||||
Err(CostError::InvalidRate)
|
||||
);
|
||||
assert_eq!(Rate::Missing.checked(), Err(CostError::InvalidRate));
|
||||
assert_eq!(Rate::Null.checked(), Err(CostError::InvalidRate));
|
||||
assert_eq!(Rate::Invalid.checked(), Err(CostError::InvalidRate));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,8 +5,8 @@
|
|||
use litellm_cost::anthropic_usage::{
|
||||
is_anthropic_usage_object, transform_anthropic_usage_to_chat_usage,
|
||||
};
|
||||
use litellm_cost::generic_cost::calculate_generic_cost_from_model_info_with_region;
|
||||
use litellm_cost::responses_usage::UsageError;
|
||||
use litellm_cost::{Pricing, Rate, Rates, Request, ServiceTier, ThresholdPolicy, calculate};
|
||||
use rstest::rstest;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
|
|
@ -82,29 +82,22 @@ fn calculate_usage_aggregates_iteration_cache_write_details(
|
|||
assert_eq!(usage.prompt_tokens, first_write + second_write);
|
||||
assert_eq!(details.ephemeral_5m_input_tokens, Some(expected_5m));
|
||||
assert_eq!(details.ephemeral_1h_input_tokens, Some(expected_1h));
|
||||
let pricing = Pricing {
|
||||
standard: Rates {
|
||||
input: Rate::Value(1e-6),
|
||||
output: Rate::Value(2e-6),
|
||||
cache_read: Rate::Missing,
|
||||
cache_write: Rate::Value(1.25e-6),
|
||||
cache_write_1h: Rate::Value(2e-6),
|
||||
},
|
||||
tiers: &[],
|
||||
thresholds: &[],
|
||||
off_peak: None,
|
||||
};
|
||||
let request = Request {
|
||||
usage: usage.token_usage(),
|
||||
service_tier: ServiceTier::Standard,
|
||||
threshold_policy: ThresholdPolicy::Exclusive,
|
||||
region_multiplier: None,
|
||||
billed_at_utc_minute: None,
|
||||
};
|
||||
let cost = calculate(&pricing, &request).unwrap();
|
||||
assert!(
|
||||
(cost.input() - (expected_5m as f64 * 1.25e-6 + expected_1h as f64 * 2e-6)).abs() < 1e-12
|
||||
let model_info = json!({
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 2e-6,
|
||||
"cache_creation_input_token_cost": 1.25e-6,
|
||||
"cache_creation_input_token_cost_above_1hr": 2e-6
|
||||
});
|
||||
let (input, _) = calculate_generic_cost_from_model_info_with_region(
|
||||
&usage,
|
||||
&model_info,
|
||||
None,
|
||||
false,
|
||||
None,
|
||||
None,
|
||||
"2026-01-01T12:00Z".parse().unwrap(),
|
||||
);
|
||||
assert!((input - (expected_5m as f64 * 1.25e-6 + expected_1h as f64 * 2e-6)).abs() < 1e-12);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
|
|
|
|||
|
|
@ -5,6 +5,8 @@
|
|||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use litellm_cost::Rate;
|
||||
use litellm_cost::batch::ThresholdPolicy;
|
||||
use litellm_cost::batch::{
|
||||
BatchCostRates, BatchPricing, BatchTier, BatchUsage, ModalityRates, batch_cost_calculator,
|
||||
batch_cost_from_model_info, get_batch_cost_rates,
|
||||
|
|
@ -15,7 +17,6 @@ use litellm_cost::non_token::{
|
|||
calculate_ocr_batch, calculate_ocr_with_tables, calculate_video,
|
||||
};
|
||||
use litellm_cost::responses_usage::{ChatUsage, PromptTokenDetails};
|
||||
use litellm_cost::{Rate, ThresholdPolicy};
|
||||
use rstest::rstest;
|
||||
use serde_json::json;
|
||||
|
||||
|
|
|
|||
|
|
@ -149,10 +149,7 @@ fn batch_cost_rates_match_executed_python_fixtures() {
|
|||
prompt_tokens,
|
||||
uses_inclusive_token_thresholds(Some(provider)),
|
||||
);
|
||||
let as_option = |rate: Rate| match rate {
|
||||
Rate::Value(value) => Some(value),
|
||||
Rate::Missing | Rate::Null => None,
|
||||
};
|
||||
let as_option = |rate: Rate| rate.value();
|
||||
let option_equals = |actual: Option<f64>, key: &str| match (actual, row.get(key)) {
|
||||
(Some(got), Some(want)) => {
|
||||
assert_close(got, want.as_f64().unwrap(), &format!("{id}/{key}"))
|
||||
|
|
|
|||
93
litellm-rust/crates/cost/tests/python_generic_reference.rs
Normal file
93
litellm-rust/crates/cost/tests/python_generic_reference.rs
Normal file
|
|
@ -0,0 +1,93 @@
|
|||
// mirrors: crates/cost/tests/generate_python_reference.py
|
||||
|
||||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
use jiff::Timestamp;
|
||||
use litellm_cost::generic_cost::calculate_generic_cost_from_model_info_with_region;
|
||||
use litellm_cost::responses_usage::ChatUsage;
|
||||
use serde_json::Value;
|
||||
|
||||
fn chat_usage(prompt: u64, completion: u64, cache_read: u64, cache_write: u64) -> ChatUsage {
|
||||
ChatUsage {
|
||||
prompt_tokens: prompt,
|
||||
completion_tokens: completion,
|
||||
total_tokens: prompt + completion,
|
||||
prompt_tokens_details: (cache_read > 0 || cache_write > 0).then(|| {
|
||||
litellm_cost::responses_usage::PromptTokenDetails {
|
||||
cached_tokens: cache_read,
|
||||
cache_write_tokens: Some(cache_write),
|
||||
..Default::default()
|
||||
}
|
||||
}),
|
||||
completion_tokens_details: None,
|
||||
cost: None,
|
||||
extra: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
fn model_info(fields: &[(&str, String)]) -> Value {
|
||||
let object = fields
|
||||
.iter()
|
||||
.map(|(key, value)| {
|
||||
(
|
||||
key.to_string(),
|
||||
serde_json::from_str::<Value>(value).expect("rate parses"),
|
||||
)
|
||||
})
|
||||
.collect::<serde_json::Map<_, _>>();
|
||||
Value::Object(object)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn generic_path_matches_executed_python_reference_cases() {
|
||||
for row in include_str!("python_reference.tsv")
|
||||
.lines()
|
||||
.filter(|line| !line.starts_with('#'))
|
||||
{
|
||||
let fields: Vec<_> = row.split('\t').collect();
|
||||
let count = |index: usize| fields[index].parse::<u64>().unwrap();
|
||||
let number = |index: usize| fields[index].parse::<f64>().unwrap();
|
||||
let rate_field = |index: usize| {
|
||||
fields[index]
|
||||
.parse::<f64>()
|
||||
.ok()
|
||||
.map(|rate| rate.to_string())
|
||||
};
|
||||
let mut entries = vec![
|
||||
("input_cost_per_token", number(5).to_string()),
|
||||
("output_cost_per_token", number(6).to_string()),
|
||||
];
|
||||
let threshold_keys: Vec<String>;
|
||||
if let Some(cache_read) = rate_field(7) {
|
||||
entries.push(("cache_read_input_token_cost", cache_read));
|
||||
}
|
||||
if let Some(cache_write) = rate_field(8) {
|
||||
entries.push(("cache_creation_input_token_cost", cache_write));
|
||||
}
|
||||
if !fields[9].is_empty() {
|
||||
let threshold = count(9);
|
||||
threshold_keys = vec![
|
||||
format!("input_cost_per_token_above_{threshold}_tokens"),
|
||||
format!("output_cost_per_token_above_{threshold}_tokens"),
|
||||
];
|
||||
entries.push((threshold_keys[0].as_str(), number(10).to_string()));
|
||||
entries.push((threshold_keys[1].as_str(), number(11).to_string()));
|
||||
}
|
||||
let usage = chat_usage(count(1), count(2), count(3), count(4));
|
||||
let (input, output) = calculate_generic_cost_from_model_info_with_region(
|
||||
&usage,
|
||||
&model_info(&entries),
|
||||
None,
|
||||
false,
|
||||
None,
|
||||
None,
|
||||
"2026-01-01T12:00Z".parse::<Timestamp>().unwrap(),
|
||||
);
|
||||
let name = &fields[0];
|
||||
assert!((input - number(12)).abs() < 1e-9, "{name}: input {input}");
|
||||
assert!(
|
||||
(output - number(13)).abs() < 1e-9,
|
||||
"{name}: output {output}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -5,22 +5,17 @@
|
|||
use std::collections::HashMap;
|
||||
|
||||
use jiff::Timestamp;
|
||||
use litellm_cost::catalog::{CatalogError, CostCatalog, ModelCostRequest, ModelInfoCatalog};
|
||||
use litellm_cost::catalog::{CatalogError, ModelCostRequest, ModelInfoCatalog};
|
||||
use litellm_cost::usage_dispatch::get_usage_object;
|
||||
use litellm_cost::{PromptConvention, Rate, Rates, Request, ServiceTier, ThresholdPolicy, Usage};
|
||||
use rstest::rstest;
|
||||
use serde_json::json;
|
||||
|
||||
fn rates(input: f64, output: f64) -> Rates {
|
||||
Rates {
|
||||
input: Rate::Value(input),
|
||||
output: Rate::Value(output),
|
||||
..Rates::EMPTY
|
||||
}
|
||||
fn rates(input: f64, output: f64) -> serde_json::Value {
|
||||
json!({"input_cost_per_token": input, "output_cost_per_token": output})
|
||||
}
|
||||
|
||||
fn catalog() -> CostCatalog {
|
||||
CostCatalog::new(HashMap::from([
|
||||
fn catalog() -> ModelInfoCatalog {
|
||||
ModelInfoCatalog::new(HashMap::from([
|
||||
("openai/model".to_owned(), rates(1e-6, 2e-6)),
|
||||
("openai/openai/model".to_owned(), rates(9e-6, 9e-6)),
|
||||
("bedrock_mantle/model".to_owned(), rates(3e-6, 4e-6)),
|
||||
|
|
@ -32,21 +27,28 @@ fn catalog() -> CostCatalog {
|
|||
]))
|
||||
}
|
||||
|
||||
fn request() -> Request {
|
||||
Request {
|
||||
usage: Usage {
|
||||
prompt_tokens: 100,
|
||||
completion_tokens: 50,
|
||||
cache_read_tokens: 0,
|
||||
cache_write_tokens: 0,
|
||||
cache_write_5m_tokens: None,
|
||||
cache_write_1h_tokens: None,
|
||||
prompt_convention: PromptConvention::IncludesCache,
|
||||
},
|
||||
service_tier: ServiceTier::Standard,
|
||||
threshold_policy: ThresholdPolicy::Exclusive,
|
||||
region_multiplier: None,
|
||||
billed_at_utc_minute: None,
|
||||
fn usage() -> litellm_cost::responses_usage::ChatUsage {
|
||||
get_usage_object(&json!({"usage": {"prompt_tokens": 100, "completion_tokens": 50}}))
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn request<'a>(
|
||||
model: &'a str,
|
||||
provider: Option<&'a str>,
|
||||
region: Option<&'a str>,
|
||||
usage: &'a litellm_cost::responses_usage::ChatUsage,
|
||||
) -> ModelCostRequest<'a> {
|
||||
ModelCostRequest {
|
||||
model,
|
||||
provider,
|
||||
region,
|
||||
usage,
|
||||
service_tier: None,
|
||||
data_residency: None,
|
||||
vertex_location: None,
|
||||
at: "2026-01-01T12:00Z".parse().unwrap(),
|
||||
response_time_ms: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -87,22 +89,31 @@ fn cost_per_token_resolves_model_key_in_python_order(
|
|||
|
||||
#[rstest]
|
||||
fn cost_per_token_uses_the_selected_regional_prices() {
|
||||
let cost = catalog()
|
||||
.cost_per_token(
|
||||
"bedrock_mantle/model",
|
||||
Some("bedrock_mantle"),
|
||||
Some("us-gov-west-1"),
|
||||
&request(),
|
||||
let usage = usage();
|
||||
let (input, output) = catalog()
|
||||
.cost_per_token_for_call(
|
||||
request(
|
||||
"bedrock_mantle/model",
|
||||
Some("bedrock_mantle"),
|
||||
Some("us-gov-west-1"),
|
||||
&usage,
|
||||
),
|
||||
litellm_cost::catalog::CostCall::Token {
|
||||
call_type: "completion",
|
||||
prompt_characters: None,
|
||||
completion_characters: None,
|
||||
request_model: None,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
assert!((cost.input() - 100.0 * 5e-6).abs() < 1e-12);
|
||||
assert!((cost.output() - 50.0 * 6e-6).abs() < 1e-12);
|
||||
assert!((input - 100.0 * 5e-6).abs() < 1e-12);
|
||||
assert!((output - 50.0 * 6e-6).abs() < 1e-12);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn cost_per_token_reports_an_unmapped_model() {
|
||||
assert_eq!(
|
||||
catalog().cost_per_token("missing", Some("openai"), None, &request()),
|
||||
catalog().cost_per_token(request("missing", Some("openai"), None, &usage())),
|
||||
Err(CatalogError::ModelNotFound)
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,11 +2,11 @@
|
|||
|
||||
// mirrors: test_litellm/responses/test_responses_utils.py::TestResponseAPILoggingUtils
|
||||
|
||||
use litellm_cost::generic_cost::calculate_generic_cost_from_model_info_with_region;
|
||||
use litellm_cost::responses_usage::{
|
||||
UsageError, is_response_api_usage, text_tokens_without_nested_reasoning,
|
||||
transform_response_api_usage_to_chat_usage,
|
||||
};
|
||||
use litellm_cost::{Pricing, Rate, Rates, Request, ServiceTier, ThresholdPolicy, calculate};
|
||||
use rstest::rstest;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
|
|
@ -181,26 +181,21 @@ fn transformed_responses_usage_reaches_cache_aware_token_calculation() {
|
|||
"input_tokens_details": {"cached_tokens": 200, "cache_write_tokens": 100}
|
||||
});
|
||||
let usage = transform_response_api_usage_to_chat_usage(&raw).unwrap();
|
||||
let pricing = Pricing {
|
||||
standard: Rates {
|
||||
input: Rate::Value(1e-6),
|
||||
output: Rate::Value(2e-6),
|
||||
cache_read: Rate::Value(0.2e-6),
|
||||
cache_write: Rate::Value(1.25e-6),
|
||||
cache_write_1h: Rate::Missing,
|
||||
},
|
||||
tiers: &[],
|
||||
thresholds: &[],
|
||||
off_peak: None,
|
||||
};
|
||||
let request = Request {
|
||||
usage: usage.token_usage(),
|
||||
service_tier: ServiceTier::Standard,
|
||||
threshold_policy: ThresholdPolicy::Exclusive,
|
||||
region_multiplier: None,
|
||||
billed_at_utc_minute: None,
|
||||
};
|
||||
let cost = calculate(&pricing, &request).unwrap();
|
||||
assert!((cost.input() - (700.0 * 1e-6 + 200.0 * 0.2e-6 + 100.0 * 1.25e-6)).abs() < 1e-12);
|
||||
assert!((cost.output() - 100.0 * 2e-6).abs() < 1e-12);
|
||||
let model_info = json!({
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 2e-6,
|
||||
"cache_read_input_token_cost": 0.2e-6,
|
||||
"cache_creation_input_token_cost": 1.25e-6
|
||||
});
|
||||
let (input, output) = calculate_generic_cost_from_model_info_with_region(
|
||||
&usage,
|
||||
&model_info,
|
||||
None,
|
||||
false,
|
||||
None,
|
||||
None,
|
||||
"2026-01-01T12:00Z".parse().unwrap(),
|
||||
);
|
||||
assert!((input - (700.0 * 1e-6 + 200.0 * 0.2e-6 + 100.0 * 1.25e-6)).abs() < 1e-12);
|
||||
assert!((output - 100.0 * 2e-6).abs() < 1e-12);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,7 +16,10 @@ from pathlib import Path
|
|||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
COST_TESTS = REPO_ROOT / "litellm-rust" / "crates" / "cost" / "tests"
|
||||
PYTHON_TESTS = REPO_ROOT / "tests"
|
||||
GENERATOR = "crates/cost/tests/generate_python_fixtures.py"
|
||||
GENERATORS = {
|
||||
"crates/cost/tests/generate_python_fixtures.py",
|
||||
"crates/cost/tests/generate_python_reference.py",
|
||||
}
|
||||
MIRROR_RE = re.compile(r"^//\s*mirrors:\s*(\S.*)$", re.MULTILINE)
|
||||
TARGET_RE = re.compile(r"^([\w./-]+\.py)(?:::[\w.-]+)*$")
|
||||
|
||||
|
|
@ -48,7 +51,7 @@ def main() -> int:
|
|||
offenders.append(f"{rust_file.name}: malformed mirrors target {target!r}")
|
||||
continue
|
||||
path = match.group(1)
|
||||
if path == GENERATOR:
|
||||
if path in GENERATORS:
|
||||
continue
|
||||
if not (PYTHON_TESTS / path.removeprefix("litellm/tests/")).is_file():
|
||||
offenders.append(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue