From 0f4ce30103244d670c2ba275dfaa599cca99fbf2 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Wed, 23 Sep 2026 17:14:55 -0700 Subject: [PATCH] refactor(cost): delete the Pricing scaffold and run everything through the live path The lib.rs scaffold (Pricing, compile, calculate, PricingPlan, CostCatalog, Request, Cost, and their error type) is gone; non-token, OCR, batch, and image rates move onto pricing::Rate, batch owns ThresholdPolicy, and ChatUsage loses its scaffold bridge. The python_reference.tsv replay now exercises calculate_generic_cost_from_model_info_with_region instead of the scaffold, retargeting it at the live generic path, and the scaffold-based test blocks in the anthropic, responses, and model-lookup files replay through the generic path with identical expectations. Rate::checked plus a seeded CostError carry the deliberate negative, NaN, and infinity strictness with pins, replacing the compile-time rejection the scaffold provided --- litellm-rust/crates/cost/Cargo.toml | 4 - litellm-rust/crates/cost/benches/calculate.rs | 66 --- .../crates/cost/src/azure_ai_image_cost.rs | 2 +- litellm-rust/crates/cost/src/batch.rs | 15 +- litellm-rust/crates/cost/src/catalog.rs | 49 +- litellm-rust/crates/cost/src/error.rs | 7 + litellm-rust/crates/cost/src/lib.rs | 407 +-------------- litellm-rust/crates/cost/src/non_token.rs | 4 +- litellm-rust/crates/cost/src/ocr_cost.rs | 4 +- litellm-rust/crates/cost/src/pricing.rs | 9 + .../crates/cost/src/responses_usage.rs | 29 -- litellm-rust/crates/cost/tests/calculation.rs | 464 ------------------ .../crates/cost/tests/pricing_grammar.rs | 25 + .../cost/tests/python_anthropic_usage.rs | 39 +- .../cost/tests/python_cost_calculator.rs | 3 +- .../cost/tests/python_fixture_replay.rs | 5 +- .../cost/tests/python_generic_reference.rs | 93 ++++ .../crates/cost/tests/python_model_lookup.rs | 79 +-- .../cost/tests/python_responses_usage.rs | 41 +- scripts/check_cost_test_mirrors.py | 7 +- 20 files changed, 238 insertions(+), 1114 deletions(-) delete mode 100644 litellm-rust/crates/cost/benches/calculate.rs create mode 100644 litellm-rust/crates/cost/src/error.rs delete mode 100644 litellm-rust/crates/cost/tests/calculation.rs create mode 100644 litellm-rust/crates/cost/tests/python_generic_reference.rs diff --git a/litellm-rust/crates/cost/Cargo.toml b/litellm-rust/crates/cost/Cargo.toml index 02342798847..5910e3ab9f2 100644 --- a/litellm-rust/crates/cost/Cargo.toml +++ b/litellm-rust/crates/cost/Cargo.toml @@ -17,9 +17,5 @@ criterion.workspace = true proptest.workspace = true rstest.workspace = true -[[bench]] -name = "calculate" -harness = false - [lints.clippy] disallowed-types = "deny" diff --git a/litellm-rust/crates/cost/benches/calculate.rs b/litellm-rust/crates/cost/benches/calculate.rs deleted file mode 100644 index 2561b606e6c..00000000000 --- a/litellm-rust/crates/cost/benches/calculate.rs +++ /dev/null @@ -1,66 +0,0 @@ -use criterion::{Criterion, criterion_group, criterion_main}; -use litellm_cost::{ - Pricing, PromptConvention, Rate, Rates, Request, ServiceTier, ThresholdPolicy, ThresholdRates, - Usage, calculate, compile, -}; -use std::hint::black_box; - -fn bench(c: &mut Criterion) { - let pricing = Pricing { - standard: Rates { - input: Rate::Value(0.000002), - output: Rate::Value(0.000008), - cache_read: Rate::Value(0.0000005), - cache_write: Rate::Missing, - cache_write_1h: Rate::Missing, - }, - tiers: &[], - thresholds: &[], - off_peak: None, - }; - let request = Request { - usage: Usage { - prompt_tokens: 1000, - completion_tokens: 200, - cache_read_tokens: 250, - cache_write_tokens: 0, - cache_write_5m_tokens: None, - cache_write_1h_tokens: None, - prompt_convention: PromptConvention::IncludesCache, - }, - service_tier: ServiceTier::Standard, - threshold_policy: ThresholdPolicy::Exclusive, - region_multiplier: None, - billed_at_utc_minute: None, - }; - let plan = compile(&pricing).unwrap(); - c.bench_function("native_compiled_calculation", |b| { - b.iter(|| black_box(plan.calculate(black_box(&request)).unwrap())) - }); - c.bench_function("native_full_wrapper", |b| { - b.iter(|| black_box(calculate(black_box(&pricing), black_box(&request)).unwrap())) - }); - c.bench_function("native_rate_compilation", |b| { - b.iter(|| black_box(compile(black_box(&pricing)).unwrap())) - }); - let threshold = ThresholdRates { - above_prompt_tokens: 1000, - standard: Rates { - input: Rate::Value(0.000004), - output: Rate::Value(0.000016), - ..Rates::EMPTY - }, - tiers: &[], - }; - let threshold_pricing = Pricing { - thresholds: &[threshold], - ..pricing - }; - let threshold_plan = compile(&threshold_pricing).unwrap(); - c.bench_function("native_threshold_boundary", |b| { - b.iter(|| black_box(threshold_plan.calculate(black_box(&request)).unwrap())) - }); -} - -criterion_group!(benches, bench); -criterion_main!(benches); diff --git a/litellm-rust/crates/cost/src/azure_ai_image_cost.rs b/litellm-rust/crates/cost/src/azure_ai_image_cost.rs index 779bcff3668..bb83b77ae0e 100644 --- a/litellm-rust/crates/cost/src/azure_ai_image_cost.rs +++ b/litellm-rust/crates/cost/src/azure_ai_image_cost.rs @@ -1,10 +1,10 @@ use jiff::Timestamp; use serde_json::Value; -use crate::Rate; use crate::generic_input::get_cost_per_unit; use crate::image_response_cost::calculate_image_response_cost_from_usage; use crate::non_token::{Error, ImageRates, ImageUsage, calculate_image}; +use crate::pricing::Rate; #[derive(Clone, Copy, Debug)] pub struct AzureAiImageRequest<'a> { diff --git a/litellm-rust/crates/cost/src/batch.rs b/litellm-rust/crates/cost/src/batch.rs index 487947f7e86..a10115cd66d 100644 --- a/litellm-rust/crates/cost/src/batch.rs +++ b/litellm-rust/crates/cost/src/batch.rs @@ -2,9 +2,15 @@ use serde_json::Value; use crate::base_rate_selection::uses_inclusive_token_thresholds; use crate::generic_usage::{get_billable_input_tokens, parse_prompt_tokens_details}; +use crate::pricing::Rate; use crate::regional_uplift::get_regional_uplift_multiplier; use crate::responses_usage::ChatUsage; -use crate::{Rate, ThresholdPolicy}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ThresholdPolicy { + Exclusive, + Inclusive, +} #[derive(Clone, Copy, Debug, PartialEq)] pub struct BatchCostRates { @@ -71,10 +77,7 @@ pub enum BatchError { } fn value(rate: Rate) -> Option { - match rate { - Rate::Value(value) => Some(value), - Rate::Missing | Rate::Null => None, - } + rate.value() } fn selected_rate( @@ -258,7 +261,7 @@ fn selected_model_rate( .max_by_key(|(threshold, _)| *threshold) .map_or(flat, |(_, key)| match model_rate(model_info, key) { Rate::Value(rate) => Rate::Value(rate), - Rate::Missing | Rate::Null => flat, + Rate::Missing | Rate::Null | Rate::Invalid => flat, }) } diff --git a/litellm-rust/crates/cost/src/catalog.rs b/litellm-rust/crates/cost/src/catalog.rs index e0e0288267b..549843c4c26 100644 --- a/litellm-rust/crates/cost/src/catalog.rs +++ b/litellm-rust/crates/cost/src/catalog.rs @@ -81,12 +81,6 @@ use crate::vertex_cost::{ cost_per_token as vertex_cost_per_token, cost_router as vertex_cost_router, }; use crate::xai_cost::{cost_per_token as xai_cost_per_token, reported_cost as xai_reported_cost}; -use crate::{Cost, Pricing, PricingError, Rates, Request, calculate}; - -#[derive(Clone, Debug, Default)] -pub struct CostCatalog { - entries: HashMap, -} #[derive(Clone, Debug, Default)] pub struct ModelInfoCatalog { @@ -185,7 +179,6 @@ pub struct DefaultImageCostRequest<'a> { #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum CatalogError { ModelNotFound, - Pricing(PricingError), } #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -295,12 +288,6 @@ impl From for RealtimeCostError { } } -impl From for CatalogError { - fn from(value: PricingError) -> Self { - Self::Pricing(value) - } -} - impl From for CatalogResponseError { fn from(value: CatalogError) -> Self { Self::Catalog(value) @@ -360,40 +347,6 @@ fn select_model_key<'a, T>( }) } -impl CostCatalog { - pub fn new(entries: HashMap) -> Self { - Self { entries } - } - - pub fn select_model_key<'a>( - &'a self, - model: &str, - provider: Option<&str>, - region: Option<&str>, - ) -> Option<&'a str> { - select_model_key(&self.entries, model, provider, region) - } - - pub fn cost_per_token( - &self, - model: &str, - provider: Option<&str>, - region: Option<&str>, - request: &Request, - ) -> Result { - let key = self - .select_model_key(model, provider, region) - .ok_or(CatalogError::ModelNotFound)?; - let pricing = Pricing { - standard: self.entries[key], - tiers: &[], - thresholds: &[], - off_peak: None, - }; - Ok(calculate(&pricing, request)?) - } -} - impl ModelInfoCatalog { pub fn new(entries: HashMap) -> Self { Self { entries } @@ -1347,7 +1300,7 @@ impl ModelInfoCatalog { } let rate = |info: &Value, key: &str| { crate::generic_input::get_cost_per_unit(info, key, None) - .map_or(crate::Rate::Missing, crate::Rate::Value) + .map_or(crate::pricing::Rate::Missing, crate::pricing::Rate::Value) }; let tables = [request.supplied_model_info, shared] .into_iter() diff --git a/litellm-rust/crates/cost/src/error.rs b/litellm-rust/crates/cost/src/error.rs new file mode 100644 index 00000000000..b3c3477e7f1 --- /dev/null +++ b/litellm-rust/crates/cost/src/error.rs @@ -0,0 +1,7 @@ +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum CostError { + InvalidRate, + TokenCountOverflow, + InvalidUsage, + ModelNotFound, +} diff --git a/litellm-rust/crates/cost/src/lib.rs b/litellm-rust/crates/cost/src/lib.rs index 5fc2fc82030..0efce8b4046 100644 --- a/litellm-rust/crates/cost/src/lib.rs +++ b/litellm-rust/crates/cost/src/lib.rs @@ -33,6 +33,7 @@ pub mod custom_pricing; #[allow(clippy::disallowed_types)] pub mod dashscope_cost; pub mod databricks_cost; +pub mod error; #[allow(clippy::disallowed_types)] pub mod fal_ai_image_cost; #[allow(clippy::disallowed_types)] @@ -113,408 +114,4 @@ pub mod xai_cost; #[allow(clippy::disallowed_types)] pub mod zero_cost_diagnostic; -#[derive(Clone, Copy, Debug, PartialEq)] -pub enum Rate { - Missing, - Null, - Value(f64), -} - -impl Rate { - fn value(self) -> Option { - match self { - Self::Value(value) => Some(value), - Self::Missing | Self::Null => None, - } - } - - fn or(self, fallback: Self) -> Self { - if self.value().is_some() { - self - } else { - fallback - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq)] -pub struct Rates { - pub input: Rate, - pub output: Rate, - pub cache_read: Rate, - pub cache_write: Rate, - pub cache_write_1h: Rate, -} - -impl Rates { - pub const EMPTY: Self = Self { - input: Rate::Missing, - output: Rate::Missing, - cache_read: Rate::Missing, - cache_write: Rate::Missing, - cache_write_1h: Rate::Missing, - }; - - fn overlay(self, base: Self) -> Self { - Self { - input: self.input.or(base.input), - output: self.output.or(base.output), - cache_read: self.cache_read.or(base.cache_read), - cache_write: self.cache_write.or(base.cache_write), - cache_write_1h: self.cache_write_1h.or(base.cache_write_1h), - } - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum ServiceTier { - Standard, - Flex, - Priority, - Fast, - Ultrafast, - Unknown, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum ThresholdPolicy { - Exclusive, - Inclusive, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum PromptConvention { - IncludesCache, - ExcludesCache, -} - -#[derive(Clone, Copy, Debug, PartialEq)] -pub struct Usage { - pub prompt_tokens: u64, - pub completion_tokens: u64, - pub cache_read_tokens: u64, - pub cache_write_tokens: u64, - pub cache_write_5m_tokens: Option, - pub cache_write_1h_tokens: Option, - pub prompt_convention: PromptConvention, -} - -#[derive(Clone, Copy, Debug, PartialEq)] -pub struct TierRates { - pub tier: ServiceTier, - pub rates: Rates, -} - -#[derive(Clone, Copy, Debug, PartialEq)] -pub struct ThresholdRates<'a> { - pub above_prompt_tokens: u64, - pub standard: Rates, - pub tiers: &'a [TierRates], -} - -#[derive(Clone, Copy, Debug, PartialEq)] -pub struct OffPeakRates { - pub start_utc_minute: u16, - pub end_utc_minute: u16, - pub rates: Rates, -} - -#[derive(Clone, Copy, Debug, PartialEq)] -pub struct Pricing<'a> { - pub standard: Rates, - pub tiers: &'a [TierRates], - pub thresholds: &'a [ThresholdRates<'a>], - pub off_peak: Option, -} - -#[derive(Clone, Copy, Debug, PartialEq)] -pub struct Request { - pub usage: Usage, - pub service_tier: ServiceTier, - pub threshold_policy: ThresholdPolicy, - pub region_multiplier: Option, - pub billed_at_utc_minute: Option, -} - -#[derive(Clone, Copy, Debug, PartialEq)] -pub struct Cost { - pub uncached_input: f64, - pub cache_read: f64, - pub cache_write_5m: f64, - pub cache_write_1h: f64, - pub output: f64, - pub multiplier: f64, - pub rates: EffectiveRates, -} - -impl Cost { - pub fn input(self) -> f64 { - (self.uncached_input + self.cache_read + self.cache_write_5m + self.cache_write_1h) - * self.multiplier - } - - pub fn output(self) -> f64 { - self.output * self.multiplier - } - - pub fn total(self) -> f64 { - self.input() + self.output() - } -} - -#[derive(Clone, Copy, Debug, PartialEq)] -pub struct EffectiveRates { - pub input: f64, - pub output: f64, - pub cache_read: f64, - pub cache_write_5m: f64, - pub cache_write_1h: f64, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum PricingError { - MissingInputRate, - MissingOutputRate, - InvalidRate, - InvalidRegionMultiplier, - InvalidBillingTime, - InvalidOffPeakWindow, - CacheExceedsPrompt, - InvalidCacheWriteDetails, - TokenCountOverflow, - DuplicateTier, - DuplicateThreshold, - DuplicateThresholdTier, -} - -fn selected_tier(tier: ServiceTier) -> ServiceTier { - if tier == ServiceTier::Fast { - ServiceTier::Priority - } else { - tier - } -} - -#[derive(Clone, Debug)] -struct CompiledThreshold { - above_prompt_tokens: u64, - standard: Rates, - tiers: Vec, -} - -#[derive(Clone, Debug)] -pub struct PricingPlan { - standard: Rates, - tiers: Vec, - thresholds: Vec, - off_peak: Option, -} - -fn valid_rates(rates: Rates) -> bool { - [ - rates.input, - rates.output, - rates.cache_read, - rates.cache_write, - rates.cache_write_1h, - ] - .into_iter() - .all(|rate| { - rate.value() - .is_none_or(|value| value.is_finite() && value >= 0.0) - }) -} - -fn validate_tiers(tiers: &[TierRates], duplicate: PricingError) -> Result<(), PricingError> { - if tiers.iter().any(|entry| !valid_rates(entry.rates)) { - return Err(PricingError::InvalidRate); - } - if tiers.iter().enumerate().any(|(index, entry)| { - matches!( - entry.tier, - ServiceTier::Standard | ServiceTier::Unknown | ServiceTier::Fast - ) || tiers[..index] - .iter() - .any(|previous| previous.tier == entry.tier) - }) { - return Err(duplicate); - } - Ok(()) -} - -pub fn compile(pricing: &Pricing<'_>) -> Result { - if !valid_rates(pricing.standard) { - return Err(PricingError::InvalidRate); - } - validate_tiers(pricing.tiers, PricingError::DuplicateTier)?; - if let Some(window) = pricing.off_peak { - if window.start_utc_minute >= 1440 - || window.end_utc_minute > 1440 - || window.start_utc_minute >= window.end_utc_minute - { - return Err(PricingError::InvalidOffPeakWindow); - } - if !valid_rates(window.rates) { - return Err(PricingError::InvalidRate); - } - } - let mut thresholds: Vec<_> = pricing - .thresholds - .iter() - .map(|entry| { - if !valid_rates(entry.standard) { - return Err(PricingError::InvalidRate); - } - validate_tiers(entry.tiers, PricingError::DuplicateThresholdTier)?; - Ok(CompiledThreshold { - above_prompt_tokens: entry.above_prompt_tokens, - standard: entry.standard, - tiers: entry.tiers.to_vec(), - }) - }) - .collect::>()?; - thresholds.sort_unstable_by_key(|entry| entry.above_prompt_tokens); - if thresholds - .windows(2) - .any(|pair| pair[0].above_prompt_tokens == pair[1].above_prompt_tokens) - { - return Err(PricingError::DuplicateThreshold); - } - Ok(PricingPlan { - standard: pricing.standard, - tiers: pricing.tiers.to_vec(), - thresholds, - off_peak: pricing.off_peak, - }) -} - -impl PricingPlan { - fn resolve_rates( - &self, - request: &Request, - threshold_tokens: u64, - ) -> Result { - let tier = selected_tier(request.service_tier); - let base = self - .tiers - .iter() - .find(|entry| tier != ServiceTier::Standard && entry.tier == tier) - .map_or(self.standard, |entry| entry.rates.overlay(self.standard)); - let threshold = self.thresholds.iter().rev().find(|entry| { - threshold_tokens > entry.above_prompt_tokens - || (request.threshold_policy == ThresholdPolicy::Inclusive - && threshold_tokens == entry.above_prompt_tokens) - }); - let selected = threshold.map_or(base, |entry| { - let standard = entry.standard.overlay(base); - entry - .tiers - .iter() - .find(|specific| tier != ServiceTier::Standard && specific.tier == tier) - .map_or(standard, |specific| specific.rates.overlay(standard)) - }); - match self.off_peak { - None => Ok(selected), - Some(window) => { - if window.start_utc_minute >= 1440 - || window.end_utc_minute > 1440 - || window.start_utc_minute >= window.end_utc_minute - { - return Err(PricingError::InvalidOffPeakWindow); - } - let minute = request - .billed_at_utc_minute - .ok_or(PricingError::InvalidBillingTime)?; - if minute >= 1440 { - return Err(PricingError::InvalidBillingTime); - } - if (window.start_utc_minute..window.end_utc_minute).contains(&minute) { - Ok(window.rates.overlay(selected)) - } else { - Ok(selected) - } - } - } - } - - fn checked_rate(rate: Rate, missing: PricingError) -> Result { - let value = rate.value().ok_or(missing)?; - if !value.is_finite() || value < 0.0 { - return Err(PricingError::InvalidRate); - } - Ok(value) - } - - pub fn calculate(&self, request: &Request) -> Result { - let usage = request.usage; - let cached = usage - .cache_read_tokens - .checked_add(usage.cache_write_tokens) - .ok_or(PricingError::TokenCountOverflow)?; - let (regular, threshold_tokens) = match usage.prompt_convention { - PromptConvention::IncludesCache => ( - usage - .prompt_tokens - .checked_sub(cached) - .ok_or(PricingError::CacheExceedsPrompt)?, - usage.prompt_tokens, - ), - PromptConvention::ExcludesCache => ( - usage.prompt_tokens, - usage - .prompt_tokens - .checked_add(cached) - .ok_or(PricingError::TokenCountOverflow)?, - ), - }; - let writes = match (usage.cache_write_5m_tokens, usage.cache_write_1h_tokens) { - (None, None) => (usage.cache_write_tokens, 0), - (Some(five), Some(one)) if five.checked_add(one) == Some(usage.cache_write_tokens) => { - (five, one) - } - _ => return Err(PricingError::InvalidCacheWriteDetails), - }; - let rates = self.resolve_rates(request, threshold_tokens)?; - let input = Self::checked_rate(rates.input, PricingError::MissingInputRate)?; - let output = Self::checked_rate(rates.output, PricingError::MissingOutputRate)?; - let read = Self::checked_rate( - rates.cache_read.or(rates.input), - PricingError::MissingInputRate, - )?; - let write = Self::checked_rate( - rates.cache_write.or(rates.input), - PricingError::MissingInputRate, - )?; - let write_1h = Self::checked_rate( - rates.cache_write_1h.or(rates.cache_write).or(rates.input), - PricingError::MissingInputRate, - )?; - let multiplier = request.region_multiplier.unwrap_or(1.0); - if !multiplier.is_finite() || multiplier <= 0.0 { - return Err(PricingError::InvalidRegionMultiplier); - } - let cost = Cost { - uncached_input: regular as f64 * input, - cache_read: usage.cache_read_tokens as f64 * read, - cache_write_5m: writes.0 as f64 * write, - cache_write_1h: writes.1 as f64 * write_1h, - output: usage.completion_tokens as f64 * output, - multiplier, - rates: EffectiveRates { - input, - output, - cache_read: read, - cache_write_5m: write, - cache_write_1h: write_1h, - }, - }; - if !cost.total().is_finite() { - return Err(PricingError::TokenCountOverflow); - } - Ok(cost) - } -} - -pub fn calculate(pricing: &Pricing<'_>, request: &Request) -> Result { - compile(pricing)?.calculate(request) -} +pub use pricing::Rate; diff --git a/litellm-rust/crates/cost/src/non_token.rs b/litellm-rust/crates/cost/src/non_token.rs index 7e57413c744..dd3b16384bc 100644 --- a/litellm-rust/crates/cost/src/non_token.rs +++ b/litellm-rust/crates/cost/src/non_token.rs @@ -84,7 +84,7 @@ pub struct VideoRates<'a> { fn priced(rate: Rate) -> Option { match rate { Rate::Value(value) => Some(value), - Rate::Missing | Rate::Null => None, + Rate::Missing | Rate::Null | Rate::Invalid => None, } } @@ -283,4 +283,4 @@ pub fn video_resolution_to_cost_field_suffix(resolution: &str) -> Option .collect(); (!suffix.is_empty() && suffix.chars().count() <= 24).then_some(suffix) } -use crate::Rate; +use crate::pricing::Rate; diff --git a/litellm-rust/crates/cost/src/ocr_cost.rs b/litellm-rust/crates/cost/src/ocr_cost.rs index 11a8f2ab03e..681b15b96c4 100644 --- a/litellm-rust/crates/cost/src/ocr_cost.rs +++ b/litellm-rust/crates/cost/src/ocr_cost.rs @@ -1,7 +1,7 @@ use serde_json::Value; -use crate::Rate; use crate::non_token::{Error as NonTokenError, OcrRates, OcrUsage, calculate_ocr_with_tables}; +use crate::pricing::Rate; #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum OcrCostError { @@ -28,7 +28,7 @@ fn rates(model_info: &Value) -> OcrRates { fn value(rate: Rate) -> Option { match rate { Rate::Value(value) => Some(value), - Rate::Missing | Rate::Null => None, + Rate::Missing | Rate::Null | Rate::Invalid => None, } } diff --git a/litellm-rust/crates/cost/src/pricing.rs b/litellm-rust/crates/cost/src/pricing.rs index 733712177b3..30af002f113 100644 --- a/litellm-rust/crates/cost/src/pricing.rs +++ b/litellm-rust/crates/cost/src/pricing.rs @@ -1,5 +1,7 @@ use std::collections::BTreeMap; +use crate::error::CostError; + #[derive(Clone, Copy, Debug, Default, PartialEq)] pub enum Rate { #[default] @@ -16,6 +18,13 @@ impl Rate { Self::Missing | Self::Null | Self::Invalid => None, } } + + pub fn checked(self) -> Result { + match self { + Self::Value(value) if value.is_finite() && value >= 0.0 => Ok(value), + _ => Err(CostError::InvalidRate), + } + } } #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] diff --git a/litellm-rust/crates/cost/src/responses_usage.rs b/litellm-rust/crates/cost/src/responses_usage.rs index 50b476c7153..fe2edddbfa8 100644 --- a/litellm-rust/crates/cost/src/responses_usage.rs +++ b/litellm-rust/crates/cost/src/responses_usage.rs @@ -3,8 +3,6 @@ use std::collections::BTreeMap; use serde::Deserialize; use serde_json::Value; -use crate::{PromptConvention, Usage}; - #[derive(Clone, Debug, Default, Deserialize, PartialEq)] pub struct CachedTokenDetails { pub text_tokens: Option, @@ -60,33 +58,6 @@ pub struct ChatUsage { pub extra: BTreeMap, } -impl ChatUsage { - pub fn token_usage(&self) -> Usage { - let cache_creation_details = self - .prompt_tokens_details - .as_ref() - .and_then(|details| details.cache_creation_token_details.as_ref()); - Usage { - prompt_tokens: self.prompt_tokens, - completion_tokens: self.completion_tokens, - cache_read_tokens: self - .prompt_tokens_details - .as_ref() - .map_or(0, |details| details.cached_tokens), - cache_write_tokens: self - .prompt_tokens_details - .as_ref() - .and_then(|details| details.cache_write_tokens) - .unwrap_or(0), - cache_write_5m_tokens: cache_creation_details - .map(|details| details.ephemeral_5m_input_tokens.unwrap_or(0)), - cache_write_1h_tokens: cache_creation_details - .map(|details| details.ephemeral_1h_input_tokens.unwrap_or(0)), - prompt_convention: PromptConvention::IncludesCache, - } - } -} - #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum UsageError { InvalidShape, diff --git a/litellm-rust/crates/cost/tests/calculation.rs b/litellm-rust/crates/cost/tests/calculation.rs deleted file mode 100644 index f5234a80f13..00000000000 --- a/litellm-rust/crates/cost/tests/calculation.rs +++ /dev/null @@ -1,464 +0,0 @@ -use litellm_cost::{ - OffPeakRates, Pricing, PricingError, PromptConvention, Rate, Rates, Request, ServiceTier, - ThresholdPolicy, ThresholdRates, TierRates, Usage, calculate, compile, -}; - -fn rates(input: Rate, output: Rate) -> Rates { - Rates { - input, - output, - ..Rates::EMPTY - } -} - -fn request() -> Request { - Request { - usage: Usage { - prompt_tokens: 100, - completion_tokens: 20, - cache_read_tokens: 25, - cache_write_tokens: 10, - cache_write_5m_tokens: None, - cache_write_1h_tokens: None, - prompt_convention: PromptConvention::IncludesCache, - }, - service_tier: ServiceTier::Standard, - threshold_policy: ThresholdPolicy::Exclusive, - region_multiplier: None, - billed_at_utc_minute: None, - } -} - -fn pricing(standard: Rates) -> Pricing<'static> { - Pricing { - standard, - tiers: &[], - thresholds: &[], - off_peak: None, - } -} - -#[test] -fn breakdown_and_total_agree() { - let standard = Rates { - cache_read: Rate::Value(0.5), - cache_write: Rate::Value(3.0), - ..rates(Rate::Value(2.0), Rate::Value(4.0)) - }; - let result = calculate(&pricing(standard), &request()).unwrap(); - assert_eq!(result.uncached_input, 65.0 * 2.0); - assert_eq!(result.cache_read, 25.0 * 0.5); - assert_eq!(result.cache_write_5m, 10.0 * 3.0); - assert_eq!(result.output(), 20.0 * 4.0); - assert_eq!(result.total(), result.input() + result.output()); - assert_eq!(result.rates.cache_read, 0.5); -} - -#[test] -fn absent_null_and_zero_cache_rates_are_distinct() { - let base = rates(Rate::Value(2.0), Rate::Value(4.0)); - for read in [Rate::Missing, Rate::Null] { - let standard = Rates { - cache_read: read, - ..base - }; - assert_eq!( - calculate(&pricing(standard), &request()).unwrap().input(), - 200.0 - ); - } - let standard = Rates { - cache_read: Rate::Value(0.0), - cache_write: Rate::Value(0.0), - ..base - }; - assert_eq!( - calculate(&pricing(standard), &request()).unwrap().input(), - 130.0 - ); -} - -#[test] -fn equivalent_prompt_conventions_select_the_same_threshold() { - let threshold = ThresholdRates { - above_prompt_tokens: 90, - standard: rates(Rate::Value(5.0), Rate::Value(8.0)), - tiers: &[], - }; - let specification = Pricing { - standard: rates(Rate::Value(2.0), Rate::Value(4.0)), - tiers: &[], - thresholds: &[threshold], - off_peak: None, - }; - let included = request(); - let excluded = Request { - usage: Usage { - prompt_tokens: 65, - prompt_convention: PromptConvention::ExcludesCache, - ..included.usage - }, - ..included - }; - let plan = compile(&specification).unwrap(); - assert_eq!(plan.calculate(&included), plan.calculate(&excluded)); - assert_eq!(plan.calculate(&included).unwrap().rates.input, 5.0); -} - -#[test] -fn split_writes_and_invalid_accounting() { - let standard = Rates { - cache_read: Rate::Value(0.5), - cache_write: Rate::Value(3.0), - cache_write_1h: Rate::Value(5.0), - ..rates(Rate::Value(2.0), Rate::Value(4.0)) - }; - let base = request(); - let split = Request { - usage: Usage { - cache_write_5m_tokens: Some(4), - cache_write_1h_tokens: Some(6), - ..base.usage - }, - ..base - }; - let result = calculate(&pricing(standard), &split).unwrap(); - assert_eq!(result.cache_write_5m, 12.0); - assert_eq!(result.cache_write_1h, 30.0); - let overlapping = Request { - usage: Usage { - prompt_tokens: 30, - ..split.usage - }, - ..split - }; - assert_eq!( - calculate(&pricing(standard), &overlapping), - Err(PricingError::CacheExceedsPrompt) - ); - let incomplete = Request { - usage: Usage { - cache_write_1h_tokens: None, - ..split.usage - }, - ..split - }; - assert_eq!( - calculate(&pricing(standard), &incomplete), - Err(PricingError::InvalidCacheWriteDetails) - ); -} - -#[test] -fn threshold_tiers_and_boundaries() { - let priority = TierRates { - tier: ServiceTier::Priority, - rates: rates(Rate::Value(3.0), Rate::Missing), - }; - let threshold = ThresholdRates { - above_prompt_tokens: 100, - standard: rates(Rate::Value(5.0), Rate::Value(8.0)), - tiers: &[ - TierRates { - tier: ServiceTier::Priority, - rates: rates(Rate::Value(7.0), Rate::Missing), - }, - TierRates { - tier: ServiceTier::Flex, - rates: rates(Rate::Value(6.0), Rate::Missing), - }, - ], - }; - let specification = Pricing { - standard: rates(Rate::Value(2.0), Rate::Value(4.0)), - tiers: &[priority], - thresholds: &[threshold], - off_peak: None, - }; - let base = request(); - let no_cache = Request { - usage: Usage { - cache_read_tokens: 0, - cache_write_tokens: 0, - ..base.usage - }, - ..base - }; - let fast = Request { - service_tier: ServiceTier::Fast, - ..no_cache - }; - let inclusive = Request { - threshold_policy: ThresholdPolicy::Inclusive, - ..fast - }; - let flex = Request { - service_tier: ServiceTier::Flex, - ..inclusive - }; - assert_eq!(calculate(&specification, &no_cache).unwrap().input(), 200.0); - assert_eq!(calculate(&specification, &fast).unwrap().input(), 300.0); - assert_eq!( - calculate(&specification, &inclusive).unwrap().input(), - 700.0 - ); - assert_eq!(calculate(&specification, &flex).unwrap().input(), 600.0); -} - -#[test] -fn compile_rejects_ambiguous_rates() { - let duplicate = ThresholdRates { - above_prompt_tokens: 100, - standard: Rates::EMPTY, - tiers: &[], - }; - let specification = Pricing { - standard: rates(Rate::Value(1.0), Rate::Value(1.0)), - tiers: &[], - thresholds: &[duplicate, duplicate], - off_peak: None, - }; - assert_eq!( - compile(&specification).err(), - Some(PricingError::DuplicateThreshold) - ); - let invalid = pricing(rates(Rate::Value(f64::NAN), Rate::Value(1.0))); - assert_eq!(compile(&invalid).err(), Some(PricingError::InvalidRate)); - let negative = pricing(rates(Rate::Value(-1.0), Rate::Value(1.0))); - assert_eq!( - compile(&negative).err(), - Some(PricingError::InvalidRate), - "divergence: Python bills negative rates as negative costs; Rust refuses them" - ); -} - -#[test] -fn off_peak_is_one_non_wrapping_utc_window() { - let specification = Pricing { - standard: rates(Rate::Value(2.0), Rate::Value(4.0)), - tiers: &[], - thresholds: &[], - off_peak: Some(OffPeakRates { - start_utc_minute: 60, - end_utc_minute: 120, - rates: rates(Rate::Value(1.0), Rate::Value(2.0)), - }), - }; - let base = request(); - let start = Request { - billed_at_utc_minute: Some(60), - ..base - }; - let end = Request { - billed_at_utc_minute: Some(120), - ..base - }; - assert_eq!( - calculate(&specification, &base), - Err(PricingError::InvalidBillingTime) - ); - assert_eq!(calculate(&specification, &start).unwrap().input(), 100.0); - assert_eq!(calculate(&specification, &end).unwrap().input(), 200.0); -} - -#[test] -fn missing_rates_and_free_rates_remain_distinct() { - let base = request(); - let empty = Request { - usage: Usage { - prompt_tokens: 0, - completion_tokens: 0, - cache_read_tokens: 0, - cache_write_tokens: 0, - ..base.usage - }, - ..base - }; - assert_eq!( - calculate(&pricing(Rates::EMPTY), &empty), - Err(PricingError::MissingInputRate) - ); - assert_eq!( - calculate(&pricing(rates(Rate::Value(0.0), Rate::Missing)), &empty), - Err(PricingError::MissingOutputRate) - ); - assert_eq!( - calculate(&pricing(rates(Rate::Value(0.0), Rate::Value(0.0))), &empty) - .unwrap() - .total(), - 0.0 - ); -} - -#[test] -fn matches_executed_python_reference_cases() { - for row in include_str!("python_reference.tsv") - .lines() - .filter(|line| !line.starts_with('#')) - { - let fields: Vec<_> = row.split('\t').collect(); - let count = |index: usize| fields[index].parse::().unwrap(); - let number = |index: usize| fields[index].parse::().unwrap(); - let optional_rate = |index: usize| { - if fields[index].is_empty() { - Rate::Missing - } else { - Rate::Value(number(index)) - } - }; - let threshold = ThresholdRates { - above_prompt_tokens: if fields[9].is_empty() { 0 } else { count(9) }, - standard: rates(optional_rate(10), optional_rate(11)), - tiers: &[], - }; - let thresholds = if fields[9].is_empty() { - &[][..] - } else { - std::slice::from_ref(&threshold) - }; - let specification = Pricing { - standard: Rates { - cache_read: optional_rate(7), - cache_write: optional_rate(8), - ..rates(Rate::Value(number(5)), Rate::Value(number(6))) - }, - tiers: &[], - thresholds, - off_peak: None, - }; - let base = request(); - let input = Request { - usage: Usage { - prompt_tokens: count(1), - completion_tokens: count(2), - cache_read_tokens: count(3), - cache_write_tokens: count(4), - ..base.usage - }, - ..base - }; - let actual = calculate(&specification, &input).unwrap(); - assert_eq!(actual.input(), number(12), "{}", fields[0]); - assert_eq!(actual.output(), number(13), "{}", fields[0]); - } -} - -proptest::proptest! { - #[test] - fn equivalent_usage_conventions_and_breakdown_agree( - regular in 0_u64..1000, - read in 0_u64..1000, - write in 0_u64..1000, - output in 0_u64..1000, - ) { - let threshold = ThresholdRates { - above_prompt_tokens: 1000, - standard: rates(Rate::Value(5.0), Rate::Value(8.0)), - tiers: &[], - }; - let specification = Pricing { - standard: rates(Rate::Value(2.0), Rate::Value(4.0)), - tiers: &[], - thresholds: &[threshold], - off_peak: None, - }; - let base = request(); - let included = Request { - usage: Usage { - prompt_tokens: regular + read + write, - completion_tokens: output, - cache_read_tokens: read, - cache_write_tokens: write, - ..base.usage - }, - ..base - }; - let excluded = Request { - usage: Usage { - prompt_tokens: regular, - prompt_convention: PromptConvention::ExcludesCache, - ..included.usage - }, - ..included - }; - let plan = compile(&specification).unwrap(); - let left = plan.calculate(&included).unwrap(); - let right = plan.calculate(&excluded).unwrap(); - proptest::prop_assert_eq!(left, right); - proptest::prop_assert_eq!(left.total(), left.input() + left.output()); - } -} - -#[test] -fn regional_multiplier_applies_after_input_components_are_summed() { - let standard = Rates { - cache_read: Rate::Value(0.5), - cache_write: Rate::Value(3.0), - ..rates(Rate::Value(2.0), Rate::Value(4.0)) - }; - let base = request(); - let regional = Request { - region_multiplier: Some(1.1), - ..base - }; - let result = calculate(&pricing(standard), ®ional).unwrap(); - assert_eq!(result.input(), (65.0 * 2.0 + 25.0 * 0.5 + 10.0 * 3.0) * 1.1); - assert_eq!(result.output(), 20.0 * 4.0 * 1.1); - let invalid = Request { - region_multiplier: Some(f64::NAN), - ..base - }; - assert_eq!( - calculate(&pricing(standard), &invalid), - Err(PricingError::InvalidRegionMultiplier) - ); -} - -#[test] -fn compilation_sorts_thresholds_and_rejects_duplicate_tiers() { - let high = ThresholdRates { - above_prompt_tokens: 200, - standard: rates(Rate::Value(7.0), Rate::Missing), - tiers: &[], - }; - let low = ThresholdRates { - above_prompt_tokens: 100, - standard: rates(Rate::Value(5.0), Rate::Missing), - tiers: &[], - }; - let specification = Pricing { - standard: rates(Rate::Value(2.0), Rate::Value(4.0)), - tiers: &[], - thresholds: &[high, low], - off_peak: None, - }; - let base = request(); - let above_both = Request { - usage: Usage { - prompt_tokens: 201, - cache_read_tokens: 0, - cache_write_tokens: 0, - ..base.usage - }, - ..base - }; - assert_eq!( - compile(&specification) - .unwrap() - .calculate(&above_both) - .unwrap() - .rates - .input, - 7.0 - ); - let duplicate = TierRates { - tier: ServiceTier::Flex, - rates: Rates::EMPTY, - }; - let invalid = Pricing { - tiers: &[duplicate, duplicate], - thresholds: &[], - ..specification - }; - assert_eq!(compile(&invalid).err(), Some(PricingError::DuplicateTier)); -} diff --git a/litellm-rust/crates/cost/tests/pricing_grammar.rs b/litellm-rust/crates/cost/tests/pricing_grammar.rs index 12a50ebc644..cb8b66011f2 100644 --- a/litellm-rust/crates/cost/tests/pricing_grammar.rs +++ b/litellm-rust/crates/cost/tests/pricing_grammar.rs @@ -179,3 +179,28 @@ fn parsed_pricing_buckets_a_synthetic_overlay_entry() { assert!(pricing.extra.is_empty(), "recognized non-rate keys drop"); let _: LazyLock<()> = LazyLock::new(|| ()); } + +#[test] +fn checked_rates_reject_negative_and_non_finite_where_python_bills_them() { + use litellm_cost::error::CostError; + use litellm_cost::pricing::Rate; + assert_eq!(Rate::Value(3e-6).checked(), Ok(3e-6)); + assert_eq!(Rate::Value(0.0).checked(), Ok(0.0)); + assert_eq!( + Rate::Value(-1.0).checked(), + Err(CostError::InvalidRate), + "divergence: Python bills negative rates as negative costs; Rust refuses them" + ); + assert_eq!( + Rate::Value(f64::NAN).checked(), + Err(CostError::InvalidRate), + "divergence: Python propagates NaN costs; Rust refuses them" + ); + assert_eq!( + Rate::Value(f64::INFINITY).checked(), + Err(CostError::InvalidRate) + ); + assert_eq!(Rate::Missing.checked(), Err(CostError::InvalidRate)); + assert_eq!(Rate::Null.checked(), Err(CostError::InvalidRate)); + assert_eq!(Rate::Invalid.checked(), Err(CostError::InvalidRate)); +} diff --git a/litellm-rust/crates/cost/tests/python_anthropic_usage.rs b/litellm-rust/crates/cost/tests/python_anthropic_usage.rs index b22cebe6237..37550368ee1 100644 --- a/litellm-rust/crates/cost/tests/python_anthropic_usage.rs +++ b/litellm-rust/crates/cost/tests/python_anthropic_usage.rs @@ -5,8 +5,8 @@ use litellm_cost::anthropic_usage::{ is_anthropic_usage_object, transform_anthropic_usage_to_chat_usage, }; +use litellm_cost::generic_cost::calculate_generic_cost_from_model_info_with_region; use litellm_cost::responses_usage::UsageError; -use litellm_cost::{Pricing, Rate, Rates, Request, ServiceTier, ThresholdPolicy, calculate}; use rstest::rstest; use serde_json::{Value, json}; @@ -82,29 +82,22 @@ fn calculate_usage_aggregates_iteration_cache_write_details( assert_eq!(usage.prompt_tokens, first_write + second_write); assert_eq!(details.ephemeral_5m_input_tokens, Some(expected_5m)); assert_eq!(details.ephemeral_1h_input_tokens, Some(expected_1h)); - let pricing = Pricing { - standard: Rates { - input: Rate::Value(1e-6), - output: Rate::Value(2e-6), - cache_read: Rate::Missing, - cache_write: Rate::Value(1.25e-6), - cache_write_1h: Rate::Value(2e-6), - }, - tiers: &[], - thresholds: &[], - off_peak: None, - }; - let request = Request { - usage: usage.token_usage(), - service_tier: ServiceTier::Standard, - threshold_policy: ThresholdPolicy::Exclusive, - region_multiplier: None, - billed_at_utc_minute: None, - }; - let cost = calculate(&pricing, &request).unwrap(); - assert!( - (cost.input() - (expected_5m as f64 * 1.25e-6 + expected_1h as f64 * 2e-6)).abs() < 1e-12 + let model_info = json!({ + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "cache_creation_input_token_cost": 1.25e-6, + "cache_creation_input_token_cost_above_1hr": 2e-6 + }); + let (input, _) = calculate_generic_cost_from_model_info_with_region( + &usage, + &model_info, + None, + false, + None, + None, + "2026-01-01T12:00Z".parse().unwrap(), ); + assert!((input - (expected_5m as f64 * 1.25e-6 + expected_1h as f64 * 2e-6)).abs() < 1e-12); } #[rstest] diff --git a/litellm-rust/crates/cost/tests/python_cost_calculator.rs b/litellm-rust/crates/cost/tests/python_cost_calculator.rs index b7036f885e4..0e7a96b505b 100644 --- a/litellm-rust/crates/cost/tests/python_cost_calculator.rs +++ b/litellm-rust/crates/cost/tests/python_cost_calculator.rs @@ -5,6 +5,8 @@ use std::collections::HashMap; +use litellm_cost::Rate; +use litellm_cost::batch::ThresholdPolicy; use litellm_cost::batch::{ BatchCostRates, BatchPricing, BatchTier, BatchUsage, ModalityRates, batch_cost_calculator, batch_cost_from_model_info, get_batch_cost_rates, @@ -15,7 +17,6 @@ use litellm_cost::non_token::{ calculate_ocr_batch, calculate_ocr_with_tables, calculate_video, }; use litellm_cost::responses_usage::{ChatUsage, PromptTokenDetails}; -use litellm_cost::{Rate, ThresholdPolicy}; use rstest::rstest; use serde_json::json; diff --git a/litellm-rust/crates/cost/tests/python_fixture_replay.rs b/litellm-rust/crates/cost/tests/python_fixture_replay.rs index 2a797f22bf4..bc54042e611 100644 --- a/litellm-rust/crates/cost/tests/python_fixture_replay.rs +++ b/litellm-rust/crates/cost/tests/python_fixture_replay.rs @@ -149,10 +149,7 @@ fn batch_cost_rates_match_executed_python_fixtures() { prompt_tokens, uses_inclusive_token_thresholds(Some(provider)), ); - let as_option = |rate: Rate| match rate { - Rate::Value(value) => Some(value), - Rate::Missing | Rate::Null => None, - }; + let as_option = |rate: Rate| rate.value(); let option_equals = |actual: Option, key: &str| match (actual, row.get(key)) { (Some(got), Some(want)) => { assert_close(got, want.as_f64().unwrap(), &format!("{id}/{key}")) diff --git a/litellm-rust/crates/cost/tests/python_generic_reference.rs b/litellm-rust/crates/cost/tests/python_generic_reference.rs new file mode 100644 index 00000000000..ad83f672b05 --- /dev/null +++ b/litellm-rust/crates/cost/tests/python_generic_reference.rs @@ -0,0 +1,93 @@ +// mirrors: crates/cost/tests/generate_python_reference.py + +#![allow(clippy::disallowed_types)] + +use jiff::Timestamp; +use litellm_cost::generic_cost::calculate_generic_cost_from_model_info_with_region; +use litellm_cost::responses_usage::ChatUsage; +use serde_json::Value; + +fn chat_usage(prompt: u64, completion: u64, cache_read: u64, cache_write: u64) -> ChatUsage { + ChatUsage { + prompt_tokens: prompt, + completion_tokens: completion, + total_tokens: prompt + completion, + prompt_tokens_details: (cache_read > 0 || cache_write > 0).then(|| { + litellm_cost::responses_usage::PromptTokenDetails { + cached_tokens: cache_read, + cache_write_tokens: Some(cache_write), + ..Default::default() + } + }), + completion_tokens_details: None, + cost: None, + extra: Default::default(), + } +} + +fn model_info(fields: &[(&str, String)]) -> Value { + let object = fields + .iter() + .map(|(key, value)| { + ( + key.to_string(), + serde_json::from_str::(value).expect("rate parses"), + ) + }) + .collect::>(); + Value::Object(object) +} + +#[test] +fn generic_path_matches_executed_python_reference_cases() { + for row in include_str!("python_reference.tsv") + .lines() + .filter(|line| !line.starts_with('#')) + { + let fields: Vec<_> = row.split('\t').collect(); + let count = |index: usize| fields[index].parse::().unwrap(); + let number = |index: usize| fields[index].parse::().unwrap(); + let rate_field = |index: usize| { + fields[index] + .parse::() + .ok() + .map(|rate| rate.to_string()) + }; + let mut entries = vec![ + ("input_cost_per_token", number(5).to_string()), + ("output_cost_per_token", number(6).to_string()), + ]; + let threshold_keys: Vec; + if let Some(cache_read) = rate_field(7) { + entries.push(("cache_read_input_token_cost", cache_read)); + } + if let Some(cache_write) = rate_field(8) { + entries.push(("cache_creation_input_token_cost", cache_write)); + } + if !fields[9].is_empty() { + let threshold = count(9); + threshold_keys = vec![ + format!("input_cost_per_token_above_{threshold}_tokens"), + format!("output_cost_per_token_above_{threshold}_tokens"), + ]; + entries.push((threshold_keys[0].as_str(), number(10).to_string())); + entries.push((threshold_keys[1].as_str(), number(11).to_string())); + } + let usage = chat_usage(count(1), count(2), count(3), count(4)); + let (input, output) = calculate_generic_cost_from_model_info_with_region( + &usage, + &model_info(&entries), + None, + false, + None, + None, + "2026-01-01T12:00Z".parse::().unwrap(), + ); + let name = &fields[0]; + assert!((input - number(12)).abs() < 1e-9, "{name}: input {input}"); + assert!( + (output - number(13)).abs() < 1e-9, + "{name}: output {output}" + ); + } +} diff --git a/litellm-rust/crates/cost/tests/python_model_lookup.rs b/litellm-rust/crates/cost/tests/python_model_lookup.rs index 1fd389cdcb7..a69416c9f94 100644 --- a/litellm-rust/crates/cost/tests/python_model_lookup.rs +++ b/litellm-rust/crates/cost/tests/python_model_lookup.rs @@ -5,22 +5,17 @@ use std::collections::HashMap; use jiff::Timestamp; -use litellm_cost::catalog::{CatalogError, CostCatalog, ModelCostRequest, ModelInfoCatalog}; +use litellm_cost::catalog::{CatalogError, ModelCostRequest, ModelInfoCatalog}; use litellm_cost::usage_dispatch::get_usage_object; -use litellm_cost::{PromptConvention, Rate, Rates, Request, ServiceTier, ThresholdPolicy, Usage}; use rstest::rstest; use serde_json::json; -fn rates(input: f64, output: f64) -> Rates { - Rates { - input: Rate::Value(input), - output: Rate::Value(output), - ..Rates::EMPTY - } +fn rates(input: f64, output: f64) -> serde_json::Value { + json!({"input_cost_per_token": input, "output_cost_per_token": output}) } -fn catalog() -> CostCatalog { - CostCatalog::new(HashMap::from([ +fn catalog() -> ModelInfoCatalog { + ModelInfoCatalog::new(HashMap::from([ ("openai/model".to_owned(), rates(1e-6, 2e-6)), ("openai/openai/model".to_owned(), rates(9e-6, 9e-6)), ("bedrock_mantle/model".to_owned(), rates(3e-6, 4e-6)), @@ -32,21 +27,28 @@ fn catalog() -> CostCatalog { ])) } -fn request() -> Request { - Request { - usage: Usage { - prompt_tokens: 100, - completion_tokens: 50, - cache_read_tokens: 0, - cache_write_tokens: 0, - cache_write_5m_tokens: None, - cache_write_1h_tokens: None, - prompt_convention: PromptConvention::IncludesCache, - }, - service_tier: ServiceTier::Standard, - threshold_policy: ThresholdPolicy::Exclusive, - region_multiplier: None, - billed_at_utc_minute: None, +fn usage() -> litellm_cost::responses_usage::ChatUsage { + get_usage_object(&json!({"usage": {"prompt_tokens": 100, "completion_tokens": 50}})) + .unwrap() + .unwrap() +} + +fn request<'a>( + model: &'a str, + provider: Option<&'a str>, + region: Option<&'a str>, + usage: &'a litellm_cost::responses_usage::ChatUsage, +) -> ModelCostRequest<'a> { + ModelCostRequest { + model, + provider, + region, + usage, + service_tier: None, + data_residency: None, + vertex_location: None, + at: "2026-01-01T12:00Z".parse().unwrap(), + response_time_ms: None, } } @@ -87,22 +89,31 @@ fn cost_per_token_resolves_model_key_in_python_order( #[rstest] fn cost_per_token_uses_the_selected_regional_prices() { - let cost = catalog() - .cost_per_token( - "bedrock_mantle/model", - Some("bedrock_mantle"), - Some("us-gov-west-1"), - &request(), + let usage = usage(); + let (input, output) = catalog() + .cost_per_token_for_call( + request( + "bedrock_mantle/model", + Some("bedrock_mantle"), + Some("us-gov-west-1"), + &usage, + ), + litellm_cost::catalog::CostCall::Token { + call_type: "completion", + prompt_characters: None, + completion_characters: None, + request_model: None, + }, ) .unwrap(); - assert!((cost.input() - 100.0 * 5e-6).abs() < 1e-12); - assert!((cost.output() - 50.0 * 6e-6).abs() < 1e-12); + assert!((input - 100.0 * 5e-6).abs() < 1e-12); + assert!((output - 50.0 * 6e-6).abs() < 1e-12); } #[rstest] fn cost_per_token_reports_an_unmapped_model() { assert_eq!( - catalog().cost_per_token("missing", Some("openai"), None, &request()), + catalog().cost_per_token(request("missing", Some("openai"), None, &usage())), Err(CatalogError::ModelNotFound) ); } diff --git a/litellm-rust/crates/cost/tests/python_responses_usage.rs b/litellm-rust/crates/cost/tests/python_responses_usage.rs index 24848d218d9..e60b5f533f8 100644 --- a/litellm-rust/crates/cost/tests/python_responses_usage.rs +++ b/litellm-rust/crates/cost/tests/python_responses_usage.rs @@ -2,11 +2,11 @@ // mirrors: test_litellm/responses/test_responses_utils.py::TestResponseAPILoggingUtils +use litellm_cost::generic_cost::calculate_generic_cost_from_model_info_with_region; use litellm_cost::responses_usage::{ UsageError, is_response_api_usage, text_tokens_without_nested_reasoning, transform_response_api_usage_to_chat_usage, }; -use litellm_cost::{Pricing, Rate, Rates, Request, ServiceTier, ThresholdPolicy, calculate}; use rstest::rstest; use serde_json::{Value, json}; @@ -181,26 +181,21 @@ fn transformed_responses_usage_reaches_cache_aware_token_calculation() { "input_tokens_details": {"cached_tokens": 200, "cache_write_tokens": 100} }); let usage = transform_response_api_usage_to_chat_usage(&raw).unwrap(); - let pricing = Pricing { - standard: Rates { - input: Rate::Value(1e-6), - output: Rate::Value(2e-6), - cache_read: Rate::Value(0.2e-6), - cache_write: Rate::Value(1.25e-6), - cache_write_1h: Rate::Missing, - }, - tiers: &[], - thresholds: &[], - off_peak: None, - }; - let request = Request { - usage: usage.token_usage(), - service_tier: ServiceTier::Standard, - threshold_policy: ThresholdPolicy::Exclusive, - region_multiplier: None, - billed_at_utc_minute: None, - }; - let cost = calculate(&pricing, &request).unwrap(); - assert!((cost.input() - (700.0 * 1e-6 + 200.0 * 0.2e-6 + 100.0 * 1.25e-6)).abs() < 1e-12); - assert!((cost.output() - 100.0 * 2e-6).abs() < 1e-12); + let model_info = json!({ + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "cache_read_input_token_cost": 0.2e-6, + "cache_creation_input_token_cost": 1.25e-6 + }); + let (input, output) = calculate_generic_cost_from_model_info_with_region( + &usage, + &model_info, + None, + false, + None, + None, + "2026-01-01T12:00Z".parse().unwrap(), + ); + assert!((input - (700.0 * 1e-6 + 200.0 * 0.2e-6 + 100.0 * 1.25e-6)).abs() < 1e-12); + assert!((output - 100.0 * 2e-6).abs() < 1e-12); } diff --git a/scripts/check_cost_test_mirrors.py b/scripts/check_cost_test_mirrors.py index 5c701040e5d..0ec545e2842 100644 --- a/scripts/check_cost_test_mirrors.py +++ b/scripts/check_cost_test_mirrors.py @@ -16,7 +16,10 @@ from pathlib import Path REPO_ROOT = Path(__file__).resolve().parent.parent COST_TESTS = REPO_ROOT / "litellm-rust" / "crates" / "cost" / "tests" PYTHON_TESTS = REPO_ROOT / "tests" -GENERATOR = "crates/cost/tests/generate_python_fixtures.py" +GENERATORS = { + "crates/cost/tests/generate_python_fixtures.py", + "crates/cost/tests/generate_python_reference.py", +} MIRROR_RE = re.compile(r"^//\s*mirrors:\s*(\S.*)$", re.MULTILINE) TARGET_RE = re.compile(r"^([\w./-]+\.py)(?:::[\w.-]+)*$") @@ -48,7 +51,7 @@ def main() -> int: offenders.append(f"{rust_file.name}: malformed mirrors target {target!r}") continue path = match.group(1) - if path == GENERATOR: + if path in GENERATORS: continue if not (PYTHON_TESTS / path.removeprefix("litellm/tests/")).is_file(): offenders.append(