diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 5fbca55f479..1cbcbed5a3e 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -3694,11 +3694,13 @@ name = "litellm-cost" version = "0.1.0" dependencies = [ "criterion", + "jiff", "proptest", "rstest", "schemars 1.2.2", "serde", "serde_json", + "time", ] [[package]] diff --git a/litellm-rust/crates/cost/Cargo.toml b/litellm-rust/crates/cost/Cargo.toml index fb4c7141953..66cf0602337 100644 --- a/litellm-rust/crates/cost/Cargo.toml +++ b/litellm-rust/crates/cost/Cargo.toml @@ -10,6 +10,8 @@ serde = ["dep:serde"] schema = ["serde", "dep:schemars"] [dependencies] +jiff = { version = "0.2.37", default-features = false, features = ["std", "tzdb-bundle-always"] } +time.workspace = true schemars = { workspace = true, optional = true } serde = { workspace = true, optional = true } diff --git a/litellm-rust/crates/cost/README.md b/litellm-rust/crates/cost/README.md index bc216ca5291..b8987e41c93 100644 --- a/litellm-rust/crates/cost/README.md +++ b/litellm-rust/crates/cost/README.md @@ -1,15 +1,20 @@ # litellm-cost -This crate calculates text token charges from rates and usage supplied by its caller. It is standalone and has no Python bridge or proxy integration - -Call `compile(&pricing)` once for an immutable plan, then `plan.calculate(&request)` for each supported request. `calculate(&pricing, &request)` compiles on each call. A successful result exposes pre-multiplier component costs, selected rates, the multiplier, and derived `input()`, `output()`, and `total()` values - -The caller states whether `prompt_tokens` includes cache tokens. Threshold selection uses total input tokens for either convention and selects one rate for the whole request. Thresholds are sorted when compiled, and duplicate thresholds or tier overrides fail deterministically. `Fast` selects priority rates; unknown tiers use standard rates - -`Rate::Missing`, `Rate::Null`, and `Rate::Value(0.0)` remain distinct. Missing cache rates fall back to the selected input rate, and an absent one-hour write rate falls back to the selected write rate. Missing input or output rates return typed errors, including for zero usage. Python's sparse-entry behavior remains outside this native contract - -The supported off-peak shape is one non-wrapping UTC daily window. The caller supplies the applicable regional multiplier after provider-specific selection. Negative or non-finite rates, ambiguous rules, inconsistent cache counts, incomplete write splits, invalid windows and overflow return errors. Callers must decline unsupported inputs before native execution if their public contract accepts those shapes - -This crate does not select models, read catalogs, fetch provider prices, normalize multimodal usage, process provider-reported costs, or calculate non-token charges. It does not change proxy behavior. The reference fixture was generated by `tests/generate_python_reference.py` against the Python implementation at the commit recorded in `tests/python_reference.tsv`, using synthetic rates and fixed usage - -`money`, `price` and `schedule` hold the typed price model that `litellm-model-catalog` lowers each catalog entry into. `UnitPrice` is USD per unit and `Usd` is an amount, and only the operations that make sense between them exist, so a price is never added to an amount. `Prices::band` and `Prices::rate` resolve a token rate the way Python's `_get_token_base_cost` does. `calculate` does not read this model yet +- `calculate(&prices, &usage, &when)` consumes the same `Prices` that `litellm-model-catalog` produces. There is no separate pricing specification or compilation step, and no runtime caller is migrated here +- `Usage::quantities` contains disjoint billable quantities keyed by `Meter`, including text, audio, image, reasoning, cache reads, writes by TTL, characters, seconds, images and queries. Text counts exclude the quantities assigned to sub-meters. Provider response normalization stays with the caller + - `prompt_tokens` is the total prompt count, including cache and modalities, used only for selecting a prompt band. It is not billed a second time + - `inclusive_threshold` chooses `>=` instead of `>` at prompt-band boundaries + - `search_results` selects a query result band. Query quantity remains the number of queries. Video-second quantity selects a duration interval +- `When` supplies the service tier, channel and billing timestamp. `price::When` remains the catalog key for a tier and band +- `Priced::Known` contains a `Breakdown`. `Priced::Unpriced` lists every used meter without a price and retains the priced lines and their partial total. Zero usage needs no price. An explicit zero price remains a known zero-cost line +- Every line records its billed meter, quantity, unit price and amount. When a sub-meter falls back, `fallback_from` identifies the original usage meter and `meter` identifies the price source + - Audio, image and video tokens fall back to text on the same side, and reasoning falls back to text output + - Cache reads fall back to text input. Long-TTL writes fall back through the default write price to text input + - Missing non-token prices never fall back to token prices +- Online lookup preserves R1's `(tier, band)`, `(standard, band)`, `(tier, base)`, `(standard, base)` order before sub-meter fallback + - Off-peak overrides only declared rates during matching UTC windows. Windows can wrap midnight and restrict weekdays in an IANA timezone + - Batch prices select bands per meter. Explicit batch prices win, missing input sub-meters use the batch input rate, and missing batch token rates can use half the online base rate + - Decisions use their own channel and never silently borrow online prices +- `Breakdown::input()` sums non-output lines and `output()` sums output lines. `total` sums all lines. Prices and amounts remain money newtypes with `f64` internally +- Regional adjustments remain separate from prices. The caller selects and applies `Adjustments` when needed +- Run `cargo run -p litellm-cost --example charge` for a normalized usage example. The existing Python reference TSV is still tested unchanged. Broader fixture generation and full Python pricing parity are deferred diff --git a/litellm-rust/crates/cost/benches/calculate.rs b/litellm-rust/crates/cost/benches/calculate.rs index 2561b606e6c..7c0aad16642 100644 --- a/litellm-rust/crates/cost/benches/calculate.rs +++ b/litellm-rust/crates/cost/benches/calculate.rs @@ -1,64 +1,75 @@ -use criterion::{Criterion, criterion_group, criterion_main}; -use litellm_cost::{ - Pricing, PromptConvention, Rate, Rates, Request, ServiceTier, ThresholdPolicy, ThresholdRates, - Usage, calculate, compile, -}; use std::hint::black_box; +use criterion::{Criterion, criterion_group, criterion_main}; +use litellm_cost::money::UnitPrice; +use litellm_cost::price::{ + Band, CacheReadMode, Channel, InputUnit, Meter, OutputUnit, Price, Prices, ServiceTier, + TokenKind, When as PriceWhen, +}; +use litellm_cost::{Usage, When, calculate}; +use time::OffsetDateTime; + fn bench(c: &mut Criterion) { - let pricing = Pricing { - standard: Rates { - input: Rate::Value(0.000002), - output: Rate::Value(0.000008), - cache_read: Rate::Value(0.0000005), - cache_write: Rate::Missing, - cache_write_1h: Rate::Missing, - }, - tiers: &[], - thresholds: &[], - off_peak: None, + let input = Meter::Input(InputUnit::Token(TokenKind::Text)); + let output = Meter::Output(OutputUnit::Token(TokenKind::Text)); + let cache = Meter::CacheRead { + kind: TokenKind::Text, + mode: CacheReadMode::Implicit, }; - let request = Request { - usage: Usage { - prompt_tokens: 1000, - completion_tokens: 200, - cache_read_tokens: 250, - cache_write_tokens: 0, - cache_write_5m_tokens: None, - cache_write_1h_tokens: None, - prompt_convention: PromptConvention::IncludesCache, - }, - service_tier: ServiceTier::Standard, - threshold_policy: ThresholdPolicy::Exclusive, - region_multiplier: None, - billed_at_utc_minute: None, + let prices = Prices::new( + [ + (input.clone(), Band::Base, 0.000002), + (output.clone(), Band::Base, 0.000008), + (cache.clone(), Band::Base, 0.0000005), + (input.clone(), Band::over_prompt_tokens(1000), 0.000004), + ] + .into_iter() + .map(|(meter, band, rate)| { + ( + Price::online( + meter, + PriceWhen { + band, + ..PriceWhen::default() + }, + ), + UnitPrice::new(rate).unwrap(), + ) + }) + .collect(), + None, + ); + let usage = Usage { + quantities: [(input, 750), (output, 200), (cache, 250)].into(), + prompt_tokens: 1000, + ..Usage::default() }; - let plan = compile(&pricing).unwrap(); - c.bench_function("native_compiled_calculation", |b| { - b.iter(|| black_box(plan.calculate(black_box(&request)).unwrap())) + let when = When { + tier: ServiceTier::Standard, + channel: Channel::Online, + at: OffsetDateTime::UNIX_EPOCH, + }; + c.bench_function("catalog_calculation", |b| { + b.iter(|| { + black_box(calculate( + black_box(&prices), + black_box(&usage), + black_box(&when), + )) + }) }); - c.bench_function("native_full_wrapper", |b| { - b.iter(|| black_box(calculate(black_box(&pricing), black_box(&request)).unwrap())) - }); - c.bench_function("native_rate_compilation", |b| { - b.iter(|| black_box(compile(black_box(&pricing)).unwrap())) - }); - let threshold = ThresholdRates { - above_prompt_tokens: 1000, - standard: Rates { - input: Rate::Value(0.000004), - output: Rate::Value(0.000016), - ..Rates::EMPTY - }, - tiers: &[], + let above = Usage { + prompt_tokens: 1001, + ..usage }; - let threshold_pricing = Pricing { - thresholds: &[threshold], - ..pricing - }; - let threshold_plan = compile(&threshold_pricing).unwrap(); - c.bench_function("native_threshold_boundary", |b| { - b.iter(|| black_box(threshold_plan.calculate(black_box(&request)).unwrap())) + c.bench_function("catalog_threshold_calculation", |b| { + b.iter(|| { + black_box(calculate( + black_box(&prices), + black_box(&above), + black_box(&when), + )) + }) }); } diff --git a/litellm-rust/crates/cost/examples/charge.rs b/litellm-rust/crates/cost/examples/charge.rs index a3a83febd74..9eed1e3b186 100644 --- a/litellm-rust/crates/cost/examples/charge.rs +++ b/litellm-rust/crates/cost/examples/charge.rs @@ -1,40 +1,52 @@ -use litellm_cost::{ - Pricing, PromptConvention, Rate, Rates, Request, ServiceTier, ThresholdPolicy, Usage, compile, +use litellm_cost::money::UnitPrice; +use litellm_cost::price::{ + CacheReadMode, Channel, InputUnit, Meter, OutputUnit, Price, Prices, ServiceTier, TokenKind, }; +use litellm_cost::{Priced, Usage, When, calculate}; +use time::OffsetDateTime; fn main() { - let pricing = Pricing { - standard: Rates { - input: Rate::Value(2.0), - output: Rate::Value(4.0), - cache_read: Rate::Value(0.5), - cache_write: Rate::Value(3.0), - cache_write_1h: Rate::Missing, - }, - tiers: &[], - thresholds: &[], - off_peak: None, + let input = Meter::Input(InputUnit::Token(TokenKind::Text)); + let output = Meter::Output(OutputUnit::Token(TokenKind::Text)); + let cache = Meter::CacheRead { + kind: TokenKind::Text, + mode: CacheReadMode::Implicit, }; - let request = Request { - usage: Usage { - prompt_tokens: 100, - completion_tokens: 20, - cache_read_tokens: 25, - cache_write_tokens: 10, - cache_write_5m_tokens: None, - cache_write_1h_tokens: None, - prompt_convention: PromptConvention::IncludesCache, - }, - service_tier: ServiceTier::Standard, - threshold_policy: ThresholdPolicy::Exclusive, - region_multiplier: None, - billed_at_utc_minute: None, - }; - let cost = compile(&pricing).unwrap().calculate(&request).unwrap(); - println!( - "input={} output={} total={}", - cost.input(), - cost.output(), - cost.total() + let prices = Prices::new( + [ + (input.clone(), 2.0), + (output.clone(), 4.0), + (cache.clone(), 0.5), + ] + .into_iter() + .map(|(meter, rate)| { + ( + Price::online(meter, Default::default()), + UnitPrice::new(rate).unwrap(), + ) + }) + .collect(), + None, ); + let usage = Usage { + quantities: [(input, 75), (cache, 25), (output, 20)].into(), + prompt_tokens: 100, + ..Usage::default() + }; + let when = When { + tier: ServiceTier::Standard, + channel: Channel::Online, + at: OffsetDateTime::UNIX_EPOCH, + }; + match calculate(&prices, &usage, &when) { + Priced::Known(cost) => println!( + "input={} output={} total={}", + cost.input().get(), + cost.output().get(), + cost.total.get() + ), + Priced::Unpriced { missing, partial } => { + println!("unpriced={missing:?} partial={}", partial.total.get()) + } + } } diff --git a/litellm-rust/crates/cost/src/calculate.rs b/litellm-rust/crates/cost/src/calculate.rs index 32cc755ce15..56d14dc14b4 100644 --- a/litellm-rust/crates/cost/src/calculate.rs +++ b/litellm-rust/crates/cost/src/calculate.rs @@ -1,405 +1,850 @@ -#[derive(Clone, Copy, Debug, PartialEq)] -pub enum Rate { - Missing, - Null, - Value(f64), +use std::collections::BTreeMap; + +use time::OffsetDateTime; + +use crate::money::{UnitPrice, Usd}; +use crate::price::{Band, Channel, InputUnit, Meter, OutputUnit, Prices, ServiceTier}; + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Usage { + pub quantities: BTreeMap, + pub prompt_tokens: u64, + pub inclusive_threshold: bool, + pub search_results: u64, } -impl Rate { - fn value(self) -> Option { - match self { - Self::Value(value) => Some(value), - Self::Missing | Self::Null => None, - } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct When { + pub tier: ServiceTier, + pub channel: Channel, + pub at: OffsetDateTime, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum Priced { + Known(Breakdown), + Unpriced { + missing: Vec, + partial: Breakdown, + }, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Breakdown { + pub lines: Vec, + pub total: Usd, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Line { + pub meter: Meter, + pub quantity: u64, + pub unit_price: UnitPrice, + pub amount: Usd, + pub fallback_from: Option, +} + +impl Breakdown { + pub fn input(&self) -> Usd { + self.lines + .iter() + .filter(|line| !matches!(line.meter, Meter::Output(_))) + .map(|line| line.amount) + .sum() } - fn or(self, fallback: Self) -> Self { - if self.value().is_some() { - self - } else { - fallback + pub fn output(&self) -> Usd { + self.lines + .iter() + .filter(|line| matches!(line.meter, Meter::Output(_))) + .map(|line| line.amount) + .sum() + } +} + +pub fn calculate(prices: &Prices, usage: &Usage, when: &When) -> Priced { + let band = prices.channel_band( + when.channel, + when.tier, + usage.prompt_tokens, + usage.inclusive_threshold, + ); + let mut lines = Vec::new(); + let mut missing = Vec::new(); + for (meter, &quantity) in &usage.quantities { + if quantity == 0 { + continue; + } + let selected = select_meter(prices, meter, quantity, usage, when); + match resolve(prices, &selected, band, usage, when) { + Some((priced_meter, unit_price)) => lines.push(Line { + fallback_from: (priced_meter != selected).then(|| meter.clone()), + meter: priced_meter, + quantity, + unit_price, + amount: unit_price * quantity, + }), + None => missing.push(meter.clone()), + } + } + let breakdown = Breakdown { + total: lines.iter().map(|line| line.amount).sum(), + lines, + }; + if missing.is_empty() { + Priced::Known(breakdown) + } else { + Priced::Unpriced { + missing, + partial: breakdown, } } } -#[derive(Clone, Copy, Debug, PartialEq)] -pub struct Rates { - pub input: Rate, - pub output: Rate, - pub cache_read: Rate, - pub cache_write: Rate, - pub cache_write_1h: Rate, +fn select_meter( + prices: &Prices, + meter: &Meter, + quantity: u64, + usage: &Usage, + when: &When, +) -> Meter { + if !matches!( + meter, + Meter::Input( + InputUnit::Query { over_results: None } | InputUnit::VideoSecond { over_seconds: None } + ) + ) { + return meter.clone(); + } + prices + .iter() + .filter(|(price, _)| { + price.channel == when.channel + && (price.when.tier == when.tier || price.when.tier == ServiceTier::Standard) + }) + .filter_map(|(price, _)| match (meter, &price.meter) { + ( + Meter::Input(InputUnit::Query { over_results: None }), + Meter::Input(InputUnit::Query { + over_results: Some(floor), + }), + ) if usage.search_results > floor.get() => Some((floor, &price.meter)), + ( + Meter::Input(InputUnit::VideoSecond { over_seconds: None }), + Meter::Input(InputUnit::VideoSecond { + over_seconds: Some(floor), + }), + ) if quantity > floor.get() => Some((floor, &price.meter)), + _ => None, + }) + .max_by_key(|(floor, _)| *floor) + .map_or_else(|| meter.clone(), |(_, selected)| selected.clone()) } -impl Rates { - pub const EMPTY: Self = Self { - input: Rate::Missing, - output: Rate::Missing, - cache_read: Rate::Missing, - cache_write: Rate::Missing, - cache_write_1h: Rate::Missing, +fn resolve( + prices: &Prices, + meter: &Meter, + band: Band, + usage: &Usage, + when: &When, +) -> Option<(Meter, UnitPrice)> { + if when.channel == Channel::Batch { + return batch_rate(prices, meter, usage); + } + let overlay = (when.channel == Channel::Online) + .then(|| prices.off_peak()) + .flatten() + .filter(|off_peak| off_peak.schedule().contains(when.at)) + .and_then(|off_peak| off_peak.rates().get(meter)); + overlay + .or_else(|| prices.direct_rate(meter, when.channel, when.tier, band)) + .map(|rate| (meter.clone(), rate)) + .or_else(|| resolve(prices, &meter.fallback()?, band, usage, when)) +} + +fn batch_direct_rate(prices: &Prices, meter: &Meter, usage: &Usage) -> Option { + let band = prices + .iter() + .filter(|(price, _)| price.channel == Channel::Batch && price.meter == *meter) + .filter_map(|(price, _)| match price.when.band { + Band::OverPromptTokens(floor) + if usage.prompt_tokens > floor.get() + || (usage.inclusive_threshold && usage.prompt_tokens == floor.get()) => + { + Some(price.when.band) + } + _ => None, + }) + .max() + .unwrap_or_default(); + prices.direct_rate(meter, Channel::Batch, ServiceTier::Standard, band) +} + +fn batch_rate(prices: &Prices, meter: &Meter, usage: &Usage) -> Option<(Meter, UnitPrice)> { + batch_direct_rate(prices, meter, usage) + .map(|rate| (meter.clone(), rate)) + .or_else(|| { + let online = + prices.direct_rate(meter, Channel::Online, ServiceTier::Standard, Band::Base); + let batch_input = batch_direct_rate(prices, &Meter::TEXT_INPUT, usage).is_some(); + if (matches!(meter, Meter::Output(_)) || !batch_input) + && matches!( + meter, + Meter::Input(InputUnit::Token(_)) + | Meter::Output(OutputUnit::Token(_) | OutputUnit::ReasoningToken) + | Meter::CacheRead { .. } + | Meter::CacheWrite { .. } + ) + && let Some(rate) = online + { + return Some((meter.clone(), rate.half())); + } + batch_rate(prices, &meter.fallback()?, usage) + }) +} + +#[cfg(test)] +mod tests { + use std::num::NonZeroU64; + + use rstest::{fixture, rstest}; + use time::format_description::well_known::Rfc3339; + + use super::*; + use crate::price::{ + CacheReadMode, CacheTtl, Feature, ImageSize, OffPeak, OffPeakRates, Price, + SearchContextSize, TokenKind, When as PriceWhen, + }; + use crate::schedule::{Schedule, Window}; + + const INPUT: Meter = Meter::TEXT_INPUT; + const OUTPUT: Meter = Meter::Output(OutputUnit::Token(TokenKind::Text)); + const READ: Meter = Meter::CacheRead { + kind: TokenKind::Text, + mode: CacheReadMode::Implicit, + }; + const WRITE: Meter = Meter::CacheWrite { + kind: TokenKind::Text, + ttl: CacheTtl::Default, + }; + const HOUR: Meter = Meter::CacheWrite { + kind: TokenKind::Text, + ttl: CacheTtl::OneHour, }; - fn overlay(self, base: Self) -> Self { - Self { - input: self.input.or(base.input), - output: self.output.or(base.output), - cache_read: self.cache_read.or(base.cache_read), - cache_write: self.cache_write.or(base.cache_write), - cache_write_1h: self.cache_write_1h.or(base.cache_write_1h), + fn prices(rates: impl IntoIterator) -> Prices { + Prices::new( + rates + .into_iter() + .map(|(meter, rate)| { + ( + Price::online(meter, PriceWhen::default()), + UnitPrice::new(rate).unwrap(), + ) + }) + .collect(), + None, + ) + } + + fn known(result: Priced) -> Breakdown { + match result { + Priced::Known(breakdown) => breakdown, + Priced::Unpriced { missing, .. } => panic!("unexpected missing prices: {missing:?}"), } } -} -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum ServiceTier { - Standard, - Flex, - Priority, - Fast, - Ultrafast, - Unknown, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum ThresholdPolicy { - Exclusive, - Inclusive, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum PromptConvention { - IncludesCache, - ExcludesCache, -} - -#[derive(Clone, Copy, Debug, PartialEq)] -pub struct Usage { - pub prompt_tokens: u64, - pub completion_tokens: u64, - pub cache_read_tokens: u64, - pub cache_write_tokens: u64, - pub cache_write_5m_tokens: Option, - pub cache_write_1h_tokens: Option, - pub prompt_convention: PromptConvention, -} - -#[derive(Clone, Copy, Debug, PartialEq)] -pub struct TierRates { - pub tier: ServiceTier, - pub rates: Rates, -} - -#[derive(Clone, Copy, Debug, PartialEq)] -pub struct ThresholdRates<'a> { - pub above_prompt_tokens: u64, - pub standard: Rates, - pub tiers: &'a [TierRates], -} - -#[derive(Clone, Copy, Debug, PartialEq)] -pub struct OffPeakRates { - pub start_utc_minute: u16, - pub end_utc_minute: u16, - pub rates: Rates, -} - -#[derive(Clone, Copy, Debug, PartialEq)] -pub struct Pricing<'a> { - pub standard: Rates, - pub tiers: &'a [TierRates], - pub thresholds: &'a [ThresholdRates<'a>], - pub off_peak: Option, -} - -#[derive(Clone, Copy, Debug, PartialEq)] -pub struct Request { - pub usage: Usage, - pub service_tier: ServiceTier, - pub threshold_policy: ThresholdPolicy, - pub region_multiplier: Option, - pub billed_at_utc_minute: Option, -} - -#[derive(Clone, Copy, Debug, PartialEq)] -pub struct Cost { - pub uncached_input: f64, - pub cache_read: f64, - pub cache_write_5m: f64, - pub cache_write_1h: f64, - pub output: f64, - pub multiplier: f64, - pub rates: EffectiveRates, -} - -impl Cost { - pub fn input(self) -> f64 { - (self.uncached_input + self.cache_read + self.cache_write_5m + self.cache_write_1h) - * self.multiplier + #[fixture] + fn when() -> When { + When { + tier: ServiceTier::Standard, + channel: Channel::Online, + at: OffsetDateTime::UNIX_EPOCH, + } } - pub fn output(self) -> f64 { - self.output * self.multiplier - } - - pub fn total(self) -> f64 { - self.input() + self.output() - } -} - -#[derive(Clone, Copy, Debug, PartialEq)] -pub struct EffectiveRates { - pub input: f64, - pub output: f64, - pub cache_read: f64, - pub cache_write_5m: f64, - pub cache_write_1h: f64, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum PricingError { - MissingInputRate, - MissingOutputRate, - InvalidRate, - InvalidRegionMultiplier, - InvalidBillingTime, - InvalidOffPeakWindow, - CacheExceedsPrompt, - InvalidCacheWriteDetails, - TokenCountOverflow, - DuplicateTier, - DuplicateThreshold, - DuplicateThresholdTier, -} - -fn selected_tier(tier: ServiceTier) -> ServiceTier { - if tier == ServiceTier::Fast { - ServiceTier::Priority - } else { - tier - } -} - -#[derive(Clone, Debug)] -struct CompiledThreshold { - above_prompt_tokens: u64, - standard: Rates, - tiers: Vec, -} - -#[derive(Clone, Debug)] -pub struct PricingPlan { - standard: Rates, - tiers: Vec, - thresholds: Vec, - off_peak: Option, -} - -fn valid_rates(rates: Rates) -> bool { - [ - rates.input, - rates.output, - rates.cache_read, - rates.cache_write, - rates.cache_write_1h, - ] - .into_iter() - .all(|rate| { - rate.value() - .is_none_or(|value| value.is_finite() && value >= 0.0) - }) -} - -fn validate_tiers(tiers: &[TierRates], duplicate: PricingError) -> Result<(), PricingError> { - if tiers.iter().any(|entry| !valid_rates(entry.rates)) { - return Err(PricingError::InvalidRate); - } - if tiers.iter().enumerate().any(|(index, entry)| { - matches!( - entry.tier, - ServiceTier::Standard | ServiceTier::Unknown | ServiceTier::Fast - ) || tiers[..index] + #[rstest] + #[case::audio_input(Meter::Input(InputUnit::Token(TokenKind::Audio)), INPUT, 500)] + #[case::image_input(Meter::Input(InputUnit::Token(TokenKind::Image)), INPUT, 500)] + #[case::video_input(Meter::Input(InputUnit::Token(TokenKind::Video)), INPUT, 500)] + #[case::cache_read(READ, INPUT, 800)] + #[case::cache_write(WRITE, INPUT, 800)] + #[case::cache_write_hour(HOUR, INPUT, 800)] + #[case::audio_output(Meter::Output(OutputUnit::Token(TokenKind::Audio)), OUTPUT, 500)] + #[case::image_output(Meter::Output(OutputUnit::Token(TokenKind::Image)), OUTPUT, 500)] + #[case::reasoning(Meter::Output(OutputUnit::ReasoningToken), OUTPUT, 500)] + fn missing_submeters_bill_as_text_and_mark_the_fallback( + when: When, + #[case] meter: Meter, + #[case] base: Meter, + #[case] quantity: u64, + ) { + let usage = Usage { + quantities: [ + (INPUT, if base == INPUT { 1000 - quantity } else { 1000 }), + ( + OUTPUT, + if base == OUTPUT { + 1000 - quantity + } else { + 1000 + }, + ), + (meter.clone(), quantity), + ] + .into(), + prompt_tokens: 1000, + ..Usage::default() + }; + let result = known(calculate( + &prices([(INPUT, 0.000001), (OUTPUT, 0.000002)]), + &usage, + &when, + )); + assert!((result.input().get() - 0.001).abs() < 1e-12); + assert!((result.output().get() - 0.002).abs() < 1e-12); + assert!((result.total.get() - 0.003).abs() < 1e-12); + let fallback = result + .lines .iter() - .any(|previous| previous.tier == entry.tier) - }) { - return Err(duplicate); + .find(|line| line.fallback_from == Some(meter.clone())) + .unwrap(); + assert_eq!(fallback.meter, base); + assert_eq!(fallback.quantity, quantity); } - Ok(()) -} -pub fn compile(pricing: &Pricing<'_>) -> Result { - if !valid_rates(pricing.standard) { - return Err(PricingError::InvalidRate); + #[rstest] + #[case::input(INPUT)] + #[case::read(READ)] + #[case::write(WRITE)] + #[case::hour(HOUR)] + #[case::audio(Meter::Input(InputUnit::Token(TokenKind::Audio)))] + #[case::image(Meter::Output(OutputUnit::Token(TokenKind::Image)))] + fn an_explicit_zero_is_a_priced_line_not_a_fallback(when: When, #[case] meter: Meter) { + let row = prices([(INPUT, 1.0), (OUTPUT, 2.0), (meter.clone(), 0.0)]); + let usage = Usage { + quantities: [(meter.clone(), 100)].into(), + ..Usage::default() + }; + let result = known(calculate(&row, &usage, &when)); + assert_eq!(result.total, Usd::ZERO); + assert_eq!( + result.lines, + [Line { + meter, + quantity: 100, + unit_price: UnitPrice::new(0.0).unwrap(), + amount: Usd::ZERO, + fallback_from: None, + }] + ); } - validate_tiers(pricing.tiers, PricingError::DuplicateTier)?; - if let Some(window) = pricing.off_peak { - if window.start_utc_minute >= 1440 - || window.end_utc_minute > 1440 - || window.start_utc_minute >= window.end_utc_minute - { - return Err(PricingError::InvalidOffPeakWindow); - } - if !valid_rates(window.rates) { - return Err(PricingError::InvalidRate); - } + + #[rstest] + fn every_missing_meter_is_reported_and_priced_lines_survive(when: When) { + let audio = Meter::Input(InputUnit::Token(TokenKind::Audio)); + let usage = Usage { + quantities: [(INPUT, 3), (OUTPUT, 4), (audio.clone(), 2)].into(), + ..Usage::default() + }; + let Priced::Unpriced { missing, partial } = + calculate(&prices([(OUTPUT, 2.0)]), &usage, &when) + else { + panic!("unpriced input must not be free"); + }; + assert_eq!(missing, [INPUT, audio]); + assert_eq!(partial.total.get(), 8.0); + assert_eq!(partial.lines.len(), 1); + assert_eq!(partial.lines[0].meter, OUTPUT); + assert_eq!(partial.lines[0].quantity, 4); + assert_eq!(partial.output().get(), 8.0); } - let mut thresholds: Vec<_> = pricing - .thresholds - .iter() - .map(|entry| { - if !valid_rates(entry.standard) { - return Err(PricingError::InvalidRate); + + #[rstest] + fn an_unpriced_row_cannot_fall_back_to_zero(when: When) { + let usage = Usage { + quantities: [(INPUT, 1000), (OUTPUT, 1000)].into(), + ..Usage::default() + }; + assert_eq!( + calculate(&Prices::default(), &usage, &when), + Priced::Unpriced { + missing: vec![INPUT, OUTPUT], + partial: Breakdown { + lines: vec![], + total: Usd::ZERO + }, } - validate_tiers(entry.tiers, PricingError::DuplicateThresholdTier)?; - Ok(CompiledThreshold { - above_prompt_tokens: entry.above_prompt_tokens, - standard: entry.standard, - tiers: entry.tiers.to_vec(), + ); + } + + #[rstest] + fn unused_meters_need_no_price(when: When) { + let usage = Usage { + quantities: [(INPUT, 0), (OUTPUT, 0)].into(), + ..Usage::default() + }; + assert_eq!( + calculate(&Prices::default(), &usage, &when), + Priced::Known(Breakdown { + lines: vec![], + total: Usd::ZERO }) - }) - .collect::>()?; - thresholds.sort_unstable_by_key(|entry| entry.above_prompt_tokens); - if thresholds - .windows(2) - .any(|pair| pair[0].above_prompt_tokens == pair[1].above_prompt_tokens) - { - return Err(PricingError::DuplicateThreshold); - } - Ok(PricingPlan { - standard: pricing.standard, - tiers: pricing.tiers.to_vec(), - thresholds, - off_peak: pricing.off_peak, - }) -} - -impl PricingPlan { - fn resolve_rates( - &self, - request: &Request, - threshold_tokens: u64, - ) -> Result { - let tier = selected_tier(request.service_tier); - let base = self - .tiers - .iter() - .find(|entry| tier != ServiceTier::Standard && entry.tier == tier) - .map_or(self.standard, |entry| entry.rates.overlay(self.standard)); - let threshold = self.thresholds.iter().rev().find(|entry| { - threshold_tokens > entry.above_prompt_tokens - || (request.threshold_policy == ThresholdPolicy::Inclusive - && threshold_tokens == entry.above_prompt_tokens) - }); - let selected = threshold.map_or(base, |entry| { - let standard = entry.standard.overlay(base); - entry - .tiers - .iter() - .find(|specific| tier != ServiceTier::Standard && specific.tier == tier) - .map_or(standard, |specific| specific.rates.overlay(standard)) - }); - match self.off_peak { - None => Ok(selected), - Some(window) => { - if window.start_utc_minute >= 1440 - || window.end_utc_minute > 1440 - || window.start_utc_minute >= window.end_utc_minute - { - return Err(PricingError::InvalidOffPeakWindow); - } - let minute = request - .billed_at_utc_minute - .ok_or(PricingError::InvalidBillingTime)?; - if minute >= 1440 { - return Err(PricingError::InvalidBillingTime); - } - if (window.start_utc_minute..window.end_utc_minute).contains(&minute) { - Ok(window.rates.overlay(selected)) - } else { - Ok(selected) - } - } - } + ); } - fn checked_rate(rate: Rate, missing: PricingError) -> Result { - let value = rate.value().ok_or(missing)?; - if !value.is_finite() || value < 0.0 { - return Err(PricingError::InvalidRate); - } - Ok(value) - } - - pub fn calculate(&self, request: &Request) -> Result { - let usage = request.usage; - let cached = usage - .cache_read_tokens - .checked_add(usage.cache_write_tokens) - .ok_or(PricingError::TokenCountOverflow)?; - let (regular, threshold_tokens) = match usage.prompt_convention { - PromptConvention::IncludesCache => ( - usage - .prompt_tokens - .checked_sub(cached) - .ok_or(PricingError::CacheExceedsPrompt)?, - usage.prompt_tokens, - ), - PromptConvention::ExcludesCache => ( - usage.prompt_tokens, - usage - .prompt_tokens - .checked_add(cached) - .ok_or(PricingError::TokenCountOverflow)?, - ), + #[rstest] + fn one_hour_writes_fall_back_to_the_default_write_price(when: When) { + let usage = Usage { + quantities: [(WRITE, 4), (HOUR, 6)].into(), + ..Usage::default() }; - let writes = match (usage.cache_write_5m_tokens, usage.cache_write_1h_tokens) { - (None, None) => (usage.cache_write_tokens, 0), - (Some(five), Some(one)) if five.checked_add(one) == Some(usage.cache_write_tokens) => { - (five, one) - } - _ => return Err(PricingError::InvalidCacheWriteDetails), + let result = known(calculate( + &prices([(INPUT, 1.0), (WRITE, 3.0)]), + &usage, + &when, + )); + assert_eq!(result.total.get(), 30.0); + assert_eq!(result.lines[1].meter, WRITE); + assert_eq!(result.lines[1].fallback_from, Some(HOUR)); + } + + #[rstest] + #[case::tier_band(ServiceTier::Priority, Band::over_prompt_tokens(100), 2.0)] + #[case::standard_band(ServiceTier::Standard, Band::over_prompt_tokens(100), 2.0)] + #[case::tier_base(ServiceTier::Priority, Band::Base, 2.0)] + #[case::standard_base(ServiceTier::Standard, Band::Base, 1.0)] + fn calculator_uses_every_lookup_fallback_step( + when: When, + #[case] tier: ServiceTier, + #[case] band: Band, + #[case] expected: f64, + ) { + let row = Prices::new( + [ + ( + Price::online(INPUT, PriceWhen { tier, band }), + UnitPrice::new(2.0).unwrap(), + ), + ( + Price::online(INPUT, PriceWhen::default()), + UnitPrice::new(1.0).unwrap(), + ), + ( + Price::online( + OUTPUT, + PriceWhen { + tier: ServiceTier::Standard, + band: Band::over_prompt_tokens(100), + }, + ), + UnitPrice::new(3.0).unwrap(), + ), + ] + .into(), + None, + ); + let usage = Usage { + quantities: [(INPUT, 101)].into(), + prompt_tokens: 101, + ..Usage::default() }; - let rates = self.resolve_rates(request, threshold_tokens)?; - let input = Self::checked_rate(rates.input, PricingError::MissingInputRate)?; - let output = Self::checked_rate(rates.output, PricingError::MissingOutputRate)?; - let read = Self::checked_rate( - rates.cache_read.or(rates.input), - PricingError::MissingInputRate, - )?; - let write = Self::checked_rate( - rates.cache_write.or(rates.input), - PricingError::MissingInputRate, - )?; - let write_1h = Self::checked_rate( - rates.cache_write_1h.or(rates.cache_write).or(rates.input), - PricingError::MissingInputRate, - )?; - let multiplier = request.region_multiplier.unwrap_or(1.0); - if !multiplier.is_finite() || multiplier <= 0.0 { - return Err(PricingError::InvalidRegionMultiplier); - } - let cost = Cost { - uncached_input: regular as f64 * input, - cache_read: usage.cache_read_tokens as f64 * read, - cache_write_5m: writes.0 as f64 * write, - cache_write_1h: writes.1 as f64 * write_1h, - output: usage.completion_tokens as f64 * output, - multiplier, - rates: EffectiveRates { - input, - output, - cache_read: read, - cache_write_5m: write, - cache_write_1h: write_1h, + let result = known(calculate( + &row, + &usage, + &When { + tier: ServiceTier::Priority, + ..when }, + )); + assert_eq!(result.total.get(), 101.0 * expected); + assert_eq!(result.lines[0].fallback_from, None); + } + + #[rstest] + #[case::input_characters(Meter::Input(InputUnit::Character))] + #[case::output_characters(Meter::Output(OutputUnit::Character))] + #[case::audio_seconds(Meter::Input(InputUnit::AudioSecond))] + #[case::wall_clock(Meter::WallClockSecond)] + #[case::image_size(Meter::Output(OutputUnit::Image(ImageSize::new("1K"))))] + #[case::search(Meter::WebSearch(SearchContextSize::High))] + #[case::ocr(Meter::Feature(Feature::OcrPage))] + fn non_token_meters_use_their_own_units_and_never_fall_back_to_text( + when: When, + #[case] meter: Meter, + ) { + let usage = Usage { + quantities: [(meter.clone(), 7)].into(), + ..Usage::default() }; - if !cost.total().is_finite() { - return Err(PricingError::TokenCountOverflow); + let result = known(calculate(&prices([(meter.clone(), 0.25)]), &usage, &when)); + assert_eq!(result.total.get(), 1.75); + assert_eq!(result.lines[0].meter, meter); + assert_eq!(result.lines[0].fallback_from, None); + assert!( + matches!(calculate(&prices([(INPUT, 1.0), (OUTPUT, 2.0)]), &usage, &when), Priced::Unpriced { missing, .. } if missing == [meter]) + ); + } + + #[rstest] + #[case::before(9, 1.0)] + #[case::at(10, 1.0)] + #[case::after(11, 3.0)] + fn query_count_and_result_band_are_independent( + when: When, + #[case] results: u64, + #[case] rate: f64, + ) { + let query = Meter::Input(InputUnit::Query { over_results: None }); + let upper = Meter::Input(InputUnit::Query { + over_results: NonZeroU64::new(10), + }); + let usage = Usage { + quantities: [(query.clone(), 2)].into(), + search_results: results, + ..Usage::default() + }; + let result = known(calculate( + &prices([(query, 1.0), (upper, 3.0)]), + &usage, + &when, + )); + assert_eq!(result.total.get(), 2.0 * rate); + assert_eq!(result.lines[0].fallback_from, None); + } + + #[rstest] + #[case::at(10, 10.0)] + #[case::after(11, 33.0)] + fn video_duration_selects_the_interval( + when: When, + #[case] seconds: u64, + #[case] expected: f64, + ) { + let base = Meter::Input(InputUnit::VideoSecond { over_seconds: None }); + let upper = Meter::Input(InputUnit::VideoSecond { + over_seconds: NonZeroU64::new(10), + }); + let usage = Usage { + quantities: [(base.clone(), seconds)].into(), + ..Usage::default() + }; + assert_eq!( + known(calculate( + &prices([(base, 1.0), (upper, 3.0)]), + &usage, + &when + )) + .total + .get(), + expected + ); + } + + #[rstest] + #[case::inside("2026-01-05T23:00:00Z", 20.0)] + #[case::end_excluded("2026-01-05T01:00:00Z", 38.0)] + #[case::start_included("2026-01-05T22:00:00Z", 20.0)] + fn off_peak_overlays_only_declared_rates_and_propagates_fallbacks( + when: When, + #[case] at: &str, + #[case] expected: f64, + ) { + let base = prices([(INPUT, 2.0), (OUTPUT, 4.0), (READ, 3.0)]); + let row = Prices::new( + base.iter().map(|(key, rate)| (key.clone(), rate)).collect(), + OffPeak::new( + Schedule { + windows: vec![Window { + hours: vec!["22:00-01:00".parse().unwrap()], + weekdays: None, + }], + weekday_timezone: None, + }, + OffPeakRates { + input: UnitPrice::new(0.0), + ..OffPeakRates::default() + }, + ), + ); + let usage = Usage { + quantities: [(INPUT, 2), (READ, 4), (HOUR, 7), (OUTPUT, 2)].into(), + ..Usage::default() + }; + let result = known(calculate( + &row, + &usage, + &When { + at: OffsetDateTime::parse(at, &Rfc3339).unwrap(), + ..when + }, + )); + assert_eq!(result.total.get(), expected); + assert_eq!( + result + .lines + .iter() + .find(|line| line.fallback_from == Some(HOUR)) + .unwrap() + .meter, + INPUT + ); + } + + #[rstest] + #[case::explicit(0.5, 0.0, 2.0)] + #[case::free(0.0, 0.0, 0.0)] + fn batch_rates_preserve_explicit_zero( + when: When, + #[case] input: f64, + #[case] output: f64, + #[case] expected: f64, + ) { + let row = Prices::new( + [ + ( + Price { + meter: INPUT, + channel: Channel::Batch, + when: PriceWhen::default(), + }, + UnitPrice::new(input).unwrap(), + ), + ( + Price { + meter: OUTPUT, + channel: Channel::Batch, + when: PriceWhen::default(), + }, + UnitPrice::new(output).unwrap(), + ), + ( + Price::online(INPUT, PriceWhen::default()), + UnitPrice::new(20.0).unwrap(), + ), + ] + .into(), + None, + ); + let usage = Usage { + quantities: [(INPUT, 4), (OUTPUT, 2)].into(), + ..Usage::default() + }; + assert_eq!( + known(calculate( + &row, + &usage, + &When { + channel: Channel::Batch, + ..when + } + )) + .total + .get(), + expected + ); + } + + #[rstest] + fn absent_batch_rates_use_half_online_base_rates(when: When) { + let usage = Usage { + quantities: [(INPUT, 4), (OUTPUT, 2), (READ, 6)].into(), + ..Usage::default() + }; + let result = known(calculate( + &prices([(INPUT, 2.0), (OUTPUT, 4.0), (READ, 1.0)]), + &usage, + &When { + channel: Channel::Batch, + ..when + }, + )); + assert_eq!(result.total.get(), 11.0); + assert_eq!(result.input().get(), 7.0); + assert_eq!(result.output().get(), 4.0); + } + + #[rstest] + #[case::exclusive(100, false, 10.0)] + #[case::inclusive(100, true, 30.0)] + #[case::above(101, false, 30.0)] + fn batch_bands_are_per_meter_and_do_not_borrow_online_bands( + when: When, + #[case] prompt_tokens: u64, + #[case] inclusive_threshold: bool, + #[case] expected: f64, + ) { + let row = Prices::new( + [ + (INPUT, Channel::Batch, Band::Base, 2.0), + (INPUT, Channel::Batch, Band::over_prompt_tokens(100), 6.0), + (OUTPUT, Channel::Batch, Band::Base, 3.0), + (OUTPUT, Channel::Batch, Band::over_prompt_tokens(50), 5.0), + (INPUT, Channel::Online, Band::over_prompt_tokens(75), 100.0), + (READ, Channel::Online, Band::Base, 50.0), + ] + .into_iter() + .map(|(meter, channel, band, rate)| { + ( + Price { + meter, + channel, + when: PriceWhen { + band, + ..PriceWhen::default() + }, + }, + UnitPrice::new(rate).unwrap(), + ) + }) + .collect(), + None, + ); + let usage = Usage { + quantities: [(INPUT, 2), (READ, 3), (OUTPUT, 4)].into(), + prompt_tokens, + inclusive_threshold, + ..Usage::default() + }; + let result = known(calculate( + &row, + &usage, + &When { + channel: Channel::Batch, + ..when + }, + )); + assert_eq!(result.input().get(), expected); + assert_eq!(result.output().get(), 20.0); + assert_eq!(result.lines.last().unwrap().fallback_from, Some(READ)); + } + + #[rstest] + fn dedicated_modality_and_long_ttl_rates_win(when: When) { + let audio = Meter::Input(InputUnit::Token(TokenKind::Audio)); + let usage = Usage { + quantities: [(INPUT, 10), (audio.clone(), 5), (WRITE, 3), (HOUR, 2)].into(), + ..Usage::default() + }; + let result = known(calculate( + &prices([(INPUT, 1.0), (audio, 2.0), (WRITE, 4.0), (HOUR, 6.0)]), + &usage, + &when, + )); + assert_eq!(result.total.get(), 44.0); + assert_eq!( + result + .lines + .iter() + .filter(|line| line.fallback_from.is_some()) + .count(), + 0 + ); + } + + #[rstest] + fn decisions_prices_are_isolated_from_online_and_batch(when: When) { + let usage = Usage { + quantities: [(INPUT, 4)].into(), + ..Usage::default() + }; + assert!(matches!( + calculate( + &prices([(INPUT, 2.0)]), + &usage, + &When { + channel: Channel::Decisions, + ..when + } + ), + Priced::Unpriced { .. } + )); + let row = Prices::new( + [( + Price { + meter: INPUT, + channel: Channel::Decisions, + when: PriceWhen::default(), + }, + UnitPrice::new(3.0).unwrap(), + )] + .into(), + None, + ); + assert_eq!( + known(calculate( + &row, + &usage, + &When { + channel: Channel::Decisions, + ..when + } + )) + .total + .get(), + 12.0 + ); + } + + #[rstest] + fn existing_python_reference_still_agrees(when: When) { + for line in include_str!("../tests/python_reference.tsv") + .lines() + .filter(|line| !line.starts_with('#')) + { + let fields: Vec<_> = line.split('\t').collect(); + let count = |i: usize| fields[i].parse::().unwrap(); + let optional = |i: usize| fields[i].parse::().ok().and_then(UnitPrice::new); + let band = fields[9].parse::().ok().map(Band::over_prompt_tokens); + let row = Prices::new( + [ + (INPUT, Band::Base, optional(5)), + (OUTPUT, Band::Base, optional(6)), + (READ, Band::Base, optional(7)), + (WRITE, Band::Base, optional(8)), + (INPUT, band.unwrap_or_default(), optional(10)), + (OUTPUT, band.unwrap_or_default(), optional(11)), + ] + .into_iter() + .filter_map(|(meter, band, rate)| { + Some(( + Price::online( + meter, + PriceWhen { + band, + ..PriceWhen::default() + }, + ), + rate?, + )) + }) + .collect(), + None, + ); + let usage = Usage { + quantities: [ + (INPUT, count(1) - count(3) - count(4)), + (OUTPUT, count(2)), + (READ, count(3)), + (WRITE, count(4)), + ] + .into(), + prompt_tokens: count(1), + ..Usage::default() + }; + let result = known(calculate(&row, &usage, &when)); + assert_eq!( + result.input().get(), + fields[12].parse::().unwrap(), + "{}", + fields[0] + ); + assert_eq!( + result.output().get(), + fields[13].parse::().unwrap(), + "{}", + fields[0] + ); } - Ok(cost) } } - -pub fn calculate(pricing: &Pricing<'_>, request: &Request) -> Result { - compile(pricing)?.calculate(request) -} diff --git a/litellm-rust/crates/cost/src/money.rs b/litellm-rust/crates/cost/src/money.rs index 4c87d51962c..23f8d9925eb 100644 --- a/litellm-rust/crates/cost/src/money.rs +++ b/litellm-rust/crates/cost/src/money.rs @@ -17,6 +17,10 @@ impl UnitPrice { pub fn get(self) -> f64 { self.0 } + + pub(crate) fn half(self) -> Self { + Self(self.0 / 2.0) + } } impl Usd { diff --git a/litellm-rust/crates/cost/src/price.rs b/litellm-rust/crates/cost/src/price.rs index 7ead398722c..505cdc4796f 100644 --- a/litellm-rust/crates/cost/src/price.rs +++ b/litellm-rust/crates/cost/src/price.rs @@ -119,10 +119,20 @@ impl Prices { /// The online prompt band a request falls in. Every token meter's `_above_` prices count, /// and the highest band the prompt passes wins. pub fn band(&self, tier: ServiceTier, prompt_tokens: u64, inclusive: bool) -> Band { + self.channel_band(Channel::Online, tier, prompt_tokens, inclusive) + } + + pub(crate) fn channel_band( + &self, + channel: Channel, + tier: ServiceTier, + prompt_tokens: u64, + inclusive: bool, + ) -> Band { self.rates .keys() .filter(|price| { - price.channel == Channel::Online + price.channel == channel && price.meter.sets_prompt_band() && (price.when.tier == ServiceTier::Standard || price.when.tier == tier) }) @@ -139,6 +149,17 @@ impl Prices { /// and `(standard, base)` in that order. A missing cache price falls back to the input price /// resolved the same way. pub fn rate(&self, meter: &Meter, tier: ServiceTier, band: Band) -> Option { + self.direct_rate(meter, Channel::Online, tier, band) + .or_else(|| self.rate(&meter.fallback()?, tier, band)) + } + + pub(crate) fn direct_rate( + &self, + meter: &Meter, + channel: Channel, + tier: ServiceTier, + band: Band, + ) -> Option { [ When { tier, band }, When { @@ -152,8 +173,36 @@ impl Prices { When::default(), ] .into_iter() - .find_map(|when| self.get(&Price::online(meter.clone(), when))) - .or_else(|| self.rate(&meter.fallback()?, tier, band)) + .find_map(|when| { + self.get(&Price { + meter: meter.clone(), + channel, + when, + }) + }) + } +} + +impl OffPeakRates { + pub(crate) fn get(&self, meter: &Meter) -> Option { + match meter { + Meter::Input(InputUnit::Token(TokenKind::Text)) => self.input, + Meter::Output(OutputUnit::Token(TokenKind::Text)) => self.output, + Meter::Output(OutputUnit::ReasoningToken) => self.reasoning, + Meter::CacheRead { + kind: TokenKind::Text, + mode: CacheReadMode::Implicit, + } => self.cache_read, + Meter::CacheRead { + kind: TokenKind::Text, + mode: CacheReadMode::Explicit, + } => self.explicit_cache_read.or(self.cache_read), + Meter::CacheWrite { + kind: TokenKind::Text, + ttl: CacheTtl::Default, + } => self.cache_write, + _ => None, + } } } @@ -329,11 +378,11 @@ mod tests { Band::Base, Some(2.0) )] - #[case::no_fallback_for_other_meters( + #[case::audio_falls_back_to_input( Meter::Input(InputUnit::Token(TokenKind::Audio)), ServiceTier::Standard, Band::Base, - None + Some(1.0) )] fn rate_walks_tier_then_band( priority_with_a_standard_band: Prices, diff --git a/litellm-rust/crates/cost/src/price/meter.rs b/litellm-rust/crates/cost/src/price/meter.rs index 242e1068058..7cdcff10c8a 100644 --- a/litellm-rust/crates/cost/src/price/meter.rs +++ b/litellm-rust/crates/cost/src/price/meter.rs @@ -121,11 +121,28 @@ impl Meter { } => Some(Self::TEXT_INPUT), Self::CacheWrite { kind: TokenKind::Text, - ttl: CacheTtl::OneHour, + ttl: CacheTtl::OneHour | CacheTtl::TwelveHours, } => Some(Self::CacheWrite { kind: TokenKind::Text, ttl: CacheTtl::Default, }), + Self::CacheRead { .. } => Some(Self::CacheRead { + kind: TokenKind::Text, + mode: CacheReadMode::Implicit, + }), + Self::CacheWrite { kind, ttl } if *kind != TokenKind::Text => Some(Self::CacheWrite { + kind: TokenKind::Text, + ttl: *ttl, + }), + Self::Input(InputUnit::Token( + TokenKind::Audio | TokenKind::Image | TokenKind::Video, + )) => Some(Self::TEXT_INPUT), + Self::Output(OutputUnit::Token( + TokenKind::Audio | TokenKind::Image | TokenKind::Video, + )) + | Self::Output(OutputUnit::ReasoningToken) => { + Some(Self::Output(OutputUnit::Token(TokenKind::Text))) + } _ => None, } } diff --git a/litellm-rust/crates/cost/src/schedule.rs b/litellm-rust/crates/cost/src/schedule.rs index dd33301288f..3b7da9ec3c7 100644 --- a/litellm-rust/crates/cost/src/schedule.rs +++ b/litellm-rust/crates/cost/src/schedule.rs @@ -1,5 +1,8 @@ use std::str::FromStr; +use jiff::{Timestamp, tz::TimeZone}; +use time::{OffsetDateTime, UtcOffset}; + use crate::error::{InvalidUtcWindow, InvalidWeekday}; #[derive(Clone, Debug, PartialEq, Eq)] @@ -31,6 +34,35 @@ pub enum Weekday { Sunday, } +impl Schedule { + pub fn contains(&self, at: OffsetDateTime) -> bool { + let utc = at.to_offset(UtcOffset::UTC); + let minute = u16::from(utc.hour()) * 60 + u16::from(utc.minute()); + let weekday = Timestamp::from_second(at.unix_timestamp()) + .ok() + .and_then(|stamp| { + let zone = match self.weekday_timezone.as_deref() { + Some(name) => TimeZone::get(name).ok()?, + None => TimeZone::UTC, + }; + Weekday::from_iso(stamp.to_zoned(zone).weekday().to_monday_one_offset() as u8) + }); + self.windows.iter().any(|window| { + window + .weekdays + .as_ref() + .is_none_or(|days| weekday.is_some_and(|day| days.contains(&day))) + && window.hours.iter().any(|hours| { + if hours.start_minute < hours.end_minute { + (hours.start_minute..hours.end_minute).contains(&minute) + } else { + minute >= hours.start_minute || minute < hours.end_minute + } + }) + }) + } +} + impl FromStr for UtcWindow { type Err = InvalidUtcWindow; @@ -181,10 +213,67 @@ mod schema { #[cfg(test)] mod tests { + use time::format_description::well_known::Rfc3339; + use rstest::rstest; use super::*; + #[rstest] + #[case::start("2026-01-05T22:00:00Z", "22:00-01:00", None, None, true)] + #[case::wrapped("2026-01-06T00:59:59Z", "22:00-01:00", None, None, true)] + #[case::end("2026-01-06T01:00:00Z", "22:00-01:00", None, None, false)] + #[case::outside("2026-01-05T21:59:59Z", "22:00-01:00", None, None, false)] + #[case::whole_day("2026-01-05T12:00:00Z", "00:00-00:00", None, None, true)] + #[case::utc_monday( + "2026-01-05T01:00:00Z", + "00:00-02:00", + Some(Weekday::Monday), + None, + true + )] + #[case::local_sunday( + "2026-01-05T01:00:00Z", + "00:00-02:00", + Some(Weekday::Sunday), + Some("America/Los_Angeles"), + true + )] + #[case::not_local_monday( + "2026-01-05T01:00:00Z", + "00:00-02:00", + Some(Weekday::Monday), + Some("America/Los_Angeles"), + false + )] + #[case::summer_offset( + "2026-07-06T07:30:00Z", + "07:00-08:00", + Some(Weekday::Monday), + Some("America/Los_Angeles"), + true + )] + #[case::supplied_offset("2026-01-05T23:00:00+02:00", "21:00-22:00", None, None, true)] + fn windows_use_utc_hours_and_local_weekdays( + #[case] at: &str, + #[case] hours: &str, + #[case] weekday: Option, + #[case] zone: Option<&str>, + #[case] expected: bool, + ) { + let schedule = Schedule { + windows: vec![Window { + hours: vec![hours.parse().unwrap()], + weekdays: weekday.map(|day| vec![day]), + }], + weekday_timezone: zone.map(str::to_owned), + }; + assert_eq!( + schedule.contains(OffsetDateTime::parse(at, &Rfc3339).unwrap()), + expected + ); + } + #[rstest] #[case::same_day("10:00-18:30", Some((600, 1110)))] #[case::wraps_midnight("22:00-02:00", Some((1320, 120)))] diff --git a/litellm-rust/crates/cost/tests/calculation.rs b/litellm-rust/crates/cost/tests/calculation.rs deleted file mode 100644 index f8152483a7c..00000000000 --- a/litellm-rust/crates/cost/tests/calculation.rs +++ /dev/null @@ -1,535 +0,0 @@ -use litellm_cost::{ - OffPeakRates, Pricing, PricingError, PromptConvention, Rate, Rates, Request, ServiceTier, - ThresholdPolicy, ThresholdRates, TierRates, Usage, calculate, compile, -}; -use rstest::rstest; - -fn rates(input: Rate, output: Rate) -> Rates { - Rates { - input, - output, - ..Rates::EMPTY - } -} - -fn request() -> Request { - Request { - usage: Usage { - prompt_tokens: 100, - completion_tokens: 20, - cache_read_tokens: 25, - cache_write_tokens: 10, - cache_write_5m_tokens: None, - cache_write_1h_tokens: None, - prompt_convention: PromptConvention::IncludesCache, - }, - service_tier: ServiceTier::Standard, - threshold_policy: ThresholdPolicy::Exclusive, - region_multiplier: None, - billed_at_utc_minute: None, - } -} - -fn pricing(standard: Rates) -> Pricing<'static> { - Pricing { - standard, - tiers: &[], - thresholds: &[], - off_peak: None, - } -} - -#[test] -fn breakdown_and_total_agree() { - let standard = Rates { - cache_read: Rate::Value(0.5), - cache_write: Rate::Value(3.0), - ..rates(Rate::Value(2.0), Rate::Value(4.0)) - }; - let result = calculate(&pricing(standard), &request()).unwrap(); - assert_eq!(result.uncached_input, 65.0 * 2.0); - assert_eq!(result.cache_read, 25.0 * 0.5); - assert_eq!(result.cache_write_5m, 10.0 * 3.0); - assert_eq!(result.output(), 20.0 * 4.0); - assert_eq!(result.total(), result.input() + result.output()); - assert_eq!(result.rates.cache_read, 0.5); -} - -#[rstest] -#[case::missing(Rate::Missing, Rate::Missing, 200.0)] -#[case::null(Rate::Null, Rate::Missing, 200.0)] -#[case::zero(Rate::Value(0.0), Rate::Value(0.0), 130.0)] -fn absent_null_and_zero_cache_rates_are_distinct( - #[case] cache_read: Rate, - #[case] cache_write: Rate, - #[case] expected_input: f64, -) { - let base = rates(Rate::Value(2.0), Rate::Value(4.0)); - let standard = Rates { - cache_read, - cache_write, - ..base - }; - assert_eq!( - calculate(&pricing(standard), &request()).unwrap().input(), - expected_input - ); -} - -#[test] -fn equivalent_prompt_conventions_select_the_same_threshold() { - let threshold = ThresholdRates { - above_prompt_tokens: 90, - standard: rates(Rate::Value(5.0), Rate::Value(8.0)), - tiers: &[], - }; - let specification = Pricing { - standard: rates(Rate::Value(2.0), Rate::Value(4.0)), - tiers: &[], - thresholds: &[threshold], - off_peak: None, - }; - let included = request(); - let excluded = Request { - usage: Usage { - prompt_tokens: 65, - prompt_convention: PromptConvention::ExcludesCache, - ..included.usage - }, - ..included - }; - let plan = compile(&specification).unwrap(); - assert_eq!(plan.calculate(&included), plan.calculate(&excluded)); - assert_eq!(plan.calculate(&included).unwrap().rates.input, 5.0); -} - -#[test] -fn split_writes_and_invalid_accounting() { - let standard = Rates { - cache_read: Rate::Value(0.5), - cache_write: Rate::Value(3.0), - cache_write_1h: Rate::Value(5.0), - ..rates(Rate::Value(2.0), Rate::Value(4.0)) - }; - let base = request(); - let split = Request { - usage: Usage { - cache_write_5m_tokens: Some(4), - cache_write_1h_tokens: Some(6), - ..base.usage - }, - ..base - }; - let result = calculate(&pricing(standard), &split).unwrap(); - assert_eq!(result.cache_write_5m, 12.0); - assert_eq!(result.cache_write_1h, 30.0); - let overlapping = Request { - usage: Usage { - prompt_tokens: 30, - ..split.usage - }, - ..split - }; - assert_eq!( - calculate(&pricing(standard), &overlapping), - Err(PricingError::CacheExceedsPrompt) - ); - let incomplete = Request { - usage: Usage { - cache_write_1h_tokens: None, - ..split.usage - }, - ..split - }; - assert_eq!( - calculate(&pricing(standard), &incomplete), - Err(PricingError::InvalidCacheWriteDetails) - ); -} - -#[test] -fn threshold_tiers_and_boundaries() { - let priority = TierRates { - tier: ServiceTier::Priority, - rates: rates(Rate::Value(3.0), Rate::Missing), - }; - let threshold = ThresholdRates { - above_prompt_tokens: 100, - standard: rates(Rate::Value(5.0), Rate::Value(8.0)), - tiers: &[ - TierRates { - tier: ServiceTier::Priority, - rates: rates(Rate::Value(7.0), Rate::Missing), - }, - TierRates { - tier: ServiceTier::Flex, - rates: rates(Rate::Value(6.0), Rate::Missing), - }, - ], - }; - let specification = Pricing { - standard: rates(Rate::Value(2.0), Rate::Value(4.0)), - tiers: &[priority], - thresholds: &[threshold], - off_peak: None, - }; - let base = request(); - let no_cache = Request { - usage: Usage { - cache_read_tokens: 0, - cache_write_tokens: 0, - ..base.usage - }, - ..base - }; - let fast = Request { - service_tier: ServiceTier::Fast, - ..no_cache - }; - let inclusive = Request { - threshold_policy: ThresholdPolicy::Inclusive, - ..fast - }; - let flex = Request { - service_tier: ServiceTier::Flex, - ..inclusive - }; - assert_eq!(calculate(&specification, &no_cache).unwrap().input(), 200.0); - assert_eq!(calculate(&specification, &fast).unwrap().input(), 300.0); - assert_eq!( - calculate(&specification, &inclusive).unwrap().input(), - 700.0 - ); - assert_eq!(calculate(&specification, &flex).unwrap().input(), 600.0); -} - -#[rstest] -#[case::ultrafast_above_threshold(ServiceTier::Ultrafast, 300_000, 9_301_000.0, 37_000.0)] -#[case::ultrafast_at_threshold(ServiceTier::Ultrafast, 272_000, 544_500.0, 5_000.0)] -#[case::standard_above_threshold(ServiceTier::Standard, 300_000, 3_300_600.0, 13_000.0)] -#[case::priority_above_threshold(ServiceTier::Priority, 300_000, 5_701_000.0, 23_000.0)] -fn tiered_long_context_rates_are_selected_by_service_tier( - #[case] service_tier: ServiceTier, - #[case] prompt_tokens: u64, - #[case] expected_input: f64, - #[case] expected_output: f64, -) { - let standard = Rates { - cache_read: Rate::Value(3.0), - ..rates(Rate::Value(1.0), Rate::Value(2.0)) - }; - let tiers = [ - TierRates { - tier: ServiceTier::Priority, - rates: Rates { - cache_read: Rate::Value(5.0), - ..rates(Rate::Value(3.0), Rate::Value(4.0)) - }, - }, - TierRates { - tier: ServiceTier::Ultrafast, - rates: Rates { - cache_read: Rate::Value(7.0), - ..rates(Rate::Value(2.0), Rate::Value(5.0)) - }, - }, - ]; - let threshold_tiers = [ - TierRates { - tier: ServiceTier::Priority, - rates: Rates { - cache_read: Rate::Value(29.0), - ..rates(Rate::Value(19.0), Rate::Value(23.0)) - }, - }, - TierRates { - tier: ServiceTier::Ultrafast, - rates: Rates { - cache_read: Rate::Value(41.0), - ..rates(Rate::Value(31.0), Rate::Value(37.0)) - }, - }, - ]; - let thresholds = [ThresholdRates { - above_prompt_tokens: 272_000, - standard: Rates { - cache_read: Rate::Value(17.0), - ..rates(Rate::Value(11.0), Rate::Value(13.0)) - }, - tiers: &threshold_tiers, - }]; - let pricing = Pricing { - standard, - tiers: &tiers, - thresholds: &thresholds, - off_peak: None, - }; - let base = request(); - let long_context_request = Request { - usage: Usage { - prompt_tokens, - completion_tokens: 1_000, - cache_read_tokens: 100, - cache_write_tokens: 0, - ..base.usage - }, - service_tier, - ..base - }; - let cost = calculate(&pricing, &long_context_request).unwrap(); - - assert_eq!(cost.input(), expected_input); - assert_eq!(cost.output(), expected_output); -} - -#[test] -fn compile_rejects_ambiguous_rates() { - let duplicate = ThresholdRates { - above_prompt_tokens: 100, - standard: Rates::EMPTY, - tiers: &[], - }; - let specification = Pricing { - standard: rates(Rate::Value(1.0), Rate::Value(1.0)), - tiers: &[], - thresholds: &[duplicate, duplicate], - off_peak: None, - }; - assert_eq!( - compile(&specification).err(), - Some(PricingError::DuplicateThreshold) - ); - let invalid = pricing(rates(Rate::Value(f64::NAN), Rate::Value(1.0))); - assert_eq!(compile(&invalid).err(), Some(PricingError::InvalidRate)); -} - -#[test] -fn off_peak_is_one_non_wrapping_utc_window() { - let specification = Pricing { - standard: rates(Rate::Value(2.0), Rate::Value(4.0)), - tiers: &[], - thresholds: &[], - off_peak: Some(OffPeakRates { - start_utc_minute: 60, - end_utc_minute: 120, - rates: rates(Rate::Value(1.0), Rate::Value(2.0)), - }), - }; - let base = request(); - let start = Request { - billed_at_utc_minute: Some(60), - ..base - }; - let end = Request { - billed_at_utc_minute: Some(120), - ..base - }; - assert_eq!( - calculate(&specification, &base), - Err(PricingError::InvalidBillingTime) - ); - assert_eq!(calculate(&specification, &start).unwrap().input(), 100.0); - assert_eq!(calculate(&specification, &end).unwrap().input(), 200.0); -} - -#[test] -fn missing_rates_and_free_rates_remain_distinct() { - let base = request(); - let empty = Request { - usage: Usage { - prompt_tokens: 0, - completion_tokens: 0, - cache_read_tokens: 0, - cache_write_tokens: 0, - ..base.usage - }, - ..base - }; - assert_eq!( - calculate(&pricing(Rates::EMPTY), &empty), - Err(PricingError::MissingInputRate) - ); - assert_eq!( - calculate(&pricing(rates(Rate::Value(0.0), Rate::Missing)), &empty), - Err(PricingError::MissingOutputRate) - ); - assert_eq!( - calculate(&pricing(rates(Rate::Value(0.0), Rate::Value(0.0))), &empty) - .unwrap() - .total(), - 0.0 - ); -} - -#[test] -fn matches_executed_python_reference_cases() { - for row in include_str!("python_reference.tsv") - .lines() - .filter(|line| !line.starts_with('#')) - { - let fields: Vec<_> = row.split('\t').collect(); - let count = |index: usize| fields[index].parse::().unwrap(); - let number = |index: usize| fields[index].parse::().unwrap(); - let optional_rate = |index: usize| { - if fields[index].is_empty() { - Rate::Missing - } else { - Rate::Value(number(index)) - } - }; - let threshold = ThresholdRates { - above_prompt_tokens: if fields[9].is_empty() { 0 } else { count(9) }, - standard: rates(optional_rate(10), optional_rate(11)), - tiers: &[], - }; - let thresholds = if fields[9].is_empty() { - &[][..] - } else { - std::slice::from_ref(&threshold) - }; - let specification = Pricing { - standard: Rates { - cache_read: optional_rate(7), - cache_write: optional_rate(8), - ..rates(Rate::Value(number(5)), Rate::Value(number(6))) - }, - tiers: &[], - thresholds, - off_peak: None, - }; - let base = request(); - let input = Request { - usage: Usage { - prompt_tokens: count(1), - completion_tokens: count(2), - cache_read_tokens: count(3), - cache_write_tokens: count(4), - ..base.usage - }, - ..base - }; - let actual = calculate(&specification, &input).unwrap(); - assert_eq!(actual.input(), number(12), "{}", fields[0]); - assert_eq!(actual.output(), number(13), "{}", fields[0]); - } -} - -proptest::proptest! { - #[test] - fn equivalent_usage_conventions_and_breakdown_agree( - regular in 0_u64..1000, - read in 0_u64..1000, - write in 0_u64..1000, - output in 0_u64..1000, - ) { - let threshold = ThresholdRates { - above_prompt_tokens: 1000, - standard: rates(Rate::Value(5.0), Rate::Value(8.0)), - tiers: &[], - }; - let specification = Pricing { - standard: rates(Rate::Value(2.0), Rate::Value(4.0)), - tiers: &[], - thresholds: &[threshold], - off_peak: None, - }; - let base = request(); - let included = Request { - usage: Usage { - prompt_tokens: regular + read + write, - completion_tokens: output, - cache_read_tokens: read, - cache_write_tokens: write, - ..base.usage - }, - ..base - }; - let excluded = Request { - usage: Usage { - prompt_tokens: regular, - prompt_convention: PromptConvention::ExcludesCache, - ..included.usage - }, - ..included - }; - let plan = compile(&specification).unwrap(); - let left = plan.calculate(&included).unwrap(); - let right = plan.calculate(&excluded).unwrap(); - proptest::prop_assert_eq!(left, right); - proptest::prop_assert_eq!(left.total(), left.input() + left.output()); - } -} - -#[test] -fn regional_multiplier_applies_after_input_components_are_summed() { - let standard = Rates { - cache_read: Rate::Value(0.5), - cache_write: Rate::Value(3.0), - ..rates(Rate::Value(2.0), Rate::Value(4.0)) - }; - let base = request(); - let regional = Request { - region_multiplier: Some(1.1), - ..base - }; - let result = calculate(&pricing(standard), ®ional).unwrap(); - assert_eq!(result.input(), (65.0 * 2.0 + 25.0 * 0.5 + 10.0 * 3.0) * 1.1); - assert_eq!(result.output(), 20.0 * 4.0 * 1.1); - let invalid = Request { - region_multiplier: Some(f64::NAN), - ..base - }; - assert_eq!( - calculate(&pricing(standard), &invalid), - Err(PricingError::InvalidRegionMultiplier) - ); -} - -#[test] -fn compilation_sorts_thresholds_and_rejects_duplicate_tiers() { - let high = ThresholdRates { - above_prompt_tokens: 200, - standard: rates(Rate::Value(7.0), Rate::Missing), - tiers: &[], - }; - let low = ThresholdRates { - above_prompt_tokens: 100, - standard: rates(Rate::Value(5.0), Rate::Missing), - tiers: &[], - }; - let specification = Pricing { - standard: rates(Rate::Value(2.0), Rate::Value(4.0)), - tiers: &[], - thresholds: &[high, low], - off_peak: None, - }; - let base = request(); - let above_both = Request { - usage: Usage { - prompt_tokens: 201, - cache_read_tokens: 0, - cache_write_tokens: 0, - ..base.usage - }, - ..base - }; - assert_eq!( - compile(&specification) - .unwrap() - .calculate(&above_both) - .unwrap() - .rates - .input, - 7.0 - ); - let duplicate = TierRates { - tier: ServiceTier::Flex, - rates: Rates::EMPTY, - }; - let invalid = Pricing { - tiers: &[duplicate, duplicate], - thresholds: &[], - ..specification - }; - assert_eq!(compile(&invalid).err(), Some(PricingError::DuplicateTier)); -}