From 14de0c099dbfb682dc0e5ce94072cb2de7050f32 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 26 Oct 2026 16:52:41 +0000 Subject: [PATCH] feat(cost): add typed SDK non-token pricing --- litellm-rust/crates/cost/benches/calculate.rs | 4 +- litellm-rust/crates/cost/examples/charge.rs | 4 +- litellm-rust/crates/cost/src/calculate.rs | 189 +++- litellm-rust/crates/cost/src/error.rs | 4 + litellm-rust/crates/cost/src/lib.rs | 4 + litellm-rust/crates/cost/src/money.rs | 11 + litellm-rust/crates/cost/src/other.rs | 1006 +++++++++++++++++ litellm-rust/crates/cost/src/price.rs | 50 +- litellm-rust/crates/cost/src/price/meter.rs | 7 + litellm-rust/crates/cost/src/quantity.rs | 173 +++ litellm-rust/crates/cost/src/selection.rs | 83 +- litellm-rust/crates/model-catalog/src/lib.rs | 1 + .../crates/model-catalog/src/lower.rs | 12 +- .../crates/model-catalog/src/model.rs | 16 +- .../crates/model-catalog/src/other.rs | 127 +++ .../crates/model-catalog/src/wire/entry.rs | 22 + .../crates/model-catalog/src/wire/flat_key.rs | 24 + litellm-rust/crates/python-bridge/src/lib.rs | 18 +- .../python-bridge/src/model_registry.rs | 2 + .../python-bridge/src/model_registry/live.rs | 245 +++- .../python-bridge/src/model_registry/other.rs | 443 ++++++++ litellm/rust_bridge/_native.pyi | 78 +- 22 files changed, 2429 insertions(+), 94 deletions(-) create mode 100644 litellm-rust/crates/cost/src/other.rs create mode 100644 litellm-rust/crates/cost/src/quantity.rs create mode 100644 litellm-rust/crates/model-catalog/src/other.rs create mode 100644 litellm-rust/crates/python-bridge/src/model_registry/other.rs diff --git a/litellm-rust/crates/cost/benches/calculate.rs b/litellm-rust/crates/cost/benches/calculate.rs index 7c0aad16642..cf3c006a082 100644 --- a/litellm-rust/crates/cost/benches/calculate.rs +++ b/litellm-rust/crates/cost/benches/calculate.rs @@ -40,7 +40,9 @@ fn bench(c: &mut Criterion) { None, ); let usage = Usage { - quantities: [(input, 750), (output, 200), (cache, 250)].into(), + quantities: [(input, 750), (output, 200), (cache, 250)] + .map(|(meter, count)| (meter, litellm_cost::Quantity::Count(count))) + .into(), prompt_tokens: 1000, ..Usage::default() }; diff --git a/litellm-rust/crates/cost/examples/charge.rs b/litellm-rust/crates/cost/examples/charge.rs index 9eed1e3b186..efb28db4a31 100644 --- a/litellm-rust/crates/cost/examples/charge.rs +++ b/litellm-rust/crates/cost/examples/charge.rs @@ -29,7 +29,9 @@ fn main() { None, ); let usage = Usage { - quantities: [(input, 75), (cache, 25), (output, 20)].into(), + quantities: [(input, 75), (cache, 25), (output, 20)] + .map(|(meter, count)| (meter, litellm_cost::Quantity::Count(count))) + .into(), prompt_tokens: 100, ..Usage::default() }; diff --git a/litellm-rust/crates/cost/src/calculate.rs b/litellm-rust/crates/cost/src/calculate.rs index 71036462d8b..33f80e2ed08 100644 --- a/litellm-rust/crates/cost/src/calculate.rs +++ b/litellm-rust/crates/cost/src/calculate.rs @@ -2,15 +2,16 @@ use std::collections::BTreeMap; use time::OffsetDateTime; -use crate::PriceEvaluationError; use crate::money::{ScaleFactor, UnitPrice, Usd}; use crate::price::{Band, Channel, InputUnit, Meter, OutputUnit, Prices, ServiceTier}; use crate::selection::SelectionPolicy; +use crate::{PriceEvaluationError, Quantity}; -#[derive(Clone, Debug, Default, PartialEq, Eq)] +#[derive(Clone, Debug, Default, PartialEq)] pub struct Usage { - pub quantities: BTreeMap, + pub quantities: BTreeMap, pub prompt_tokens: u64, + pub completion_tokens: Option, pub inclusive_threshold: bool, pub search_results: u64, } @@ -40,16 +41,21 @@ pub struct Breakdown { #[derive(Clone, Debug, PartialEq)] pub struct Line { pub meter: Meter, - pub quantity: u64, + pub quantity: Quantity, pub unit_price: UnitPrice, pub amount: Usd, pub fallback_from: Option, + pub source_id: Option, + pub caller_default: bool, + pub report_to: Option, } #[derive(Clone, Debug, PartialEq)] pub struct SelectedRate { pub requested_meter: Meter, pub resolved: Option<(Meter, UnitPrice)>, + pub source_id: Option, + pub caller_default: bool, } #[derive(Clone, Debug, PartialEq)] @@ -69,6 +75,14 @@ pub fn calculate_with_rates( if project.iter().any(|meter| !meter.is_token()) { return Err(PriceEvaluationError::NonTokenProjection); } + for (meter, quantity) in &usage.quantities { + if let Quantity::Measured(value) = quantity { + Quantity::measured(*value, crate::QuantityUnit::Meter, meter)?; + } + } + if let Some(context) = policy.other() { + context.validate_usage(usage)?; + } let factor = ScaleFactor::combine(factors).ok_or(PriceEvaluationError::NonFiniteValue)?; let band = prices.channel_band( when.channel, @@ -79,18 +93,21 @@ pub fn calculate_with_rates( let rates = project .iter() .map(|meter| { - let resolved = policy - .resolve(prices, meter, band, usage, when) - .map(|(source, rate)| { + let charge = policy.resolve_charge(prices, meter, band, usage, when); + let resolved = charge + .as_ref() + .map(|charge| { factor - .price(rate) - .map(|rate| (source, rate)) + .price(charge.price) + .map(|rate| (charge.meter.clone(), rate)) .ok_or(PriceEvaluationError::NonFiniteValue) }) .transpose()?; Ok(SelectedRate { requested_meter: meter.clone(), resolved, + source_id: charge.as_ref().and_then(|charge| charge.source_id.clone()), + caller_default: charge.is_some_and(|charge| charge.caller_default), }) }) .collect::>()?; @@ -133,7 +150,7 @@ impl Breakdown { pub fn input(&self) -> Usd { self.lines .iter() - .filter(|line| !matches!(line.meter, Meter::Output(_))) + .filter(|line| !line.is_output()) .map(|line| line.amount) .sum() } @@ -141,12 +158,25 @@ impl Breakdown { pub fn output(&self) -> Usd { self.lines .iter() - .filter(|line| matches!(line.meter, Meter::Output(_))) + .filter(|line| line.is_output()) .map(|line| line.amount) .sum() } } +impl Line { + fn is_output(&self) -> bool { + match self.report_to { + Some(crate::ReportTo::Input) => false, + Some(crate::ReportTo::Output) => true, + None => matches!( + self.meter, + Meter::Output(_) | Meter::Feature(crate::price::Feature::ComputerUseOutputToken) + ), + } + } +} + pub fn calculate(prices: &Prices, usage: &Usage, when: &When) -> Priced { calculate_with_policy(prices, usage, when, SelectionPolicy::Generic) } @@ -166,17 +196,36 @@ pub fn calculate_with_policy( let mut lines = Vec::new(); let mut missing = Vec::new(); for (meter, &quantity) in &usage.quantities { - if quantity == 0 { + if quantity.is_zero() { continue; } - let selected = select_meter(prices, meter, quantity, usage, when); - match policy.resolve(prices, &selected, band, usage, when) { - Some((priced_meter, unit_price)) => lines.push(Line { - fallback_from: (priced_meter != selected).then(|| meter.clone()), - meter: priced_meter, - quantity, - unit_price, - amount: unit_price * quantity, + if policy + .other() + .is_some_and(|context| !context.charges(prices, meter, usage, when)) + { + continue; + } + let selected = if policy + .other() + .is_some_and(|context| matches!(context.selection, crate::OtherSelection::SearchTiers)) + { + meter.clone() + } else { + select_meter(prices, meter, quantity, usage, when) + }; + match policy.resolve_charge(prices, &selected, band, usage, when) { + Some(charge) => lines.push(Line { + fallback_from: (charge.meter != selected).then(|| meter.clone()), + meter: charge.meter, + quantity: charge.quantity.unwrap_or(quantity), + unit_price: charge.price, + amount: charge.price * charge.quantity.unwrap_or(quantity), + source_id: charge.source_id, + caller_default: charge.caller_default, + report_to: policy + .other() + .filter(|_| !meter.is_token()) + .and_then(|context| context.report_to), }), None => missing.push(meter.clone()), } @@ -198,7 +247,7 @@ pub fn calculate_with_policy( fn select_meter( prices: &Prices, meter: &Meter, - quantity: u64, + quantity: Quantity, usage: &Usage, when: &When, ) -> Meter { @@ -228,7 +277,7 @@ fn select_meter( Meter::Input(InputUnit::VideoSecond { over_seconds: Some(floor), }), - ) if quantity > floor.get() => Some((floor, &price.meter)), + ) if quantity.exceeds(floor.get()) => Some((floor, &price.meter)), _ => None, }) .max_by_key(|(floor, _)| *floor) @@ -336,7 +385,9 @@ mod tests { .chain(output.map(|rate| (OUTPUT, rate))), ); let usage = Usage { - quantities: BTreeMap::from([(INPUT, 10)]), + quantities: [(INPUT, 10)] + .map(|(meter, count)| (meter, crate::Quantity::Count(count))) + .into(), ..Usage::default() }; let evaluated = calculate_with_rates( @@ -351,6 +402,8 @@ mod tests { assert_eq!( evaluated.rates, vec![SelectedRate { + source_id: None, + caller_default: false, requested_meter: OUTPUT, resolved: output.map(|rate| (OUTPUT, UnitPrice::new(rate * factor).unwrap())) }] @@ -361,10 +414,13 @@ mod tests { total: Usd::new(10.0 * factor).unwrap(), lines: vec![Line { meter: INPUT, - quantity: 10, + quantity: 10.into(), unit_price: UnitPrice::new(factor).unwrap(), amount: Usd::new(10.0 * factor).unwrap(), - fallback_from: None + fallback_from: None, + source_id: None, + caller_default: false, + report_to: None, }], }) ); @@ -376,7 +432,9 @@ mod tests { fn projection_scales_partial_without_inventing_missing_rates(when: When, #[case] factor: f64) { let prices = prices([(INPUT, 1.0)]); let usage = Usage { - quantities: BTreeMap::from([(INPUT, 10), (OUTPUT, 5)]), + quantities: [(INPUT, 10), (OUTPUT, 5)] + .map(|(meter, count)| (meter, crate::Quantity::Count(count))) + .into(), ..Usage::default() }; let evaluated = calculate_with_rates( @@ -391,6 +449,8 @@ mod tests { assert_eq!( evaluated.rates, vec![SelectedRate { + source_id: None, + caller_default: false, requested_meter: OUTPUT, resolved: None }] @@ -401,7 +461,7 @@ mod tests { assert_eq!(missing, vec![OUTPUT]); assert_eq!(partial.total.get(), 10.0 * factor); assert_eq!(partial.lines.len(), 1); - assert_eq!(partial.lines[0].quantity, 10); + assert_eq!(partial.lines[0].quantity, Quantity::Count(10)); assert_eq!(partial.lines[0].unit_price.get(), factor); } @@ -416,7 +476,9 @@ mod tests { ) { let prices = prices([(INPUT, f64::MAX)]); let usage = Usage { - quantities: BTreeMap::from([(INPUT, quantity)]), + quantities: [(INPUT, quantity)] + .map(|(meter, count)| (meter, crate::Quantity::Count(count))) + .into(), ..Usage::default() }; let projection = if project { vec![INPUT] } else { Vec::new() }; @@ -520,6 +582,7 @@ mod tests { ), (meter.clone(), quantity), ] + .map(|(meter, count)| (meter, crate::Quantity::Count(count))) .into(), prompt_tokens: 1000, ..Usage::default() @@ -538,7 +601,7 @@ mod tests { .find(|line| line.fallback_from == Some(meter.clone())) .unwrap(); assert_eq!(fallback.meter, base); - assert_eq!(fallback.quantity, quantity); + assert_eq!(fallback.quantity, Quantity::Count(quantity)); } #[rstest] @@ -551,7 +614,9 @@ mod tests { fn an_explicit_zero_is_a_priced_line_not_a_fallback(when: When, #[case] meter: Meter) { let row = prices([(INPUT, 1.0), (OUTPUT, 2.0), (meter.clone(), 0.0)]); let usage = Usage { - quantities: [(meter.clone(), 100)].into(), + quantities: [(meter.clone(), 100)] + .map(|(meter, count)| (meter, crate::Quantity::Count(count))) + .into(), ..Usage::default() }; let result = known(calculate(&row, &usage, &when)); @@ -560,10 +625,13 @@ mod tests { result.lines, [Line { meter, - quantity: 100, + quantity: 100.into(), unit_price: UnitPrice::new(0.0).unwrap(), amount: Usd::ZERO, fallback_from: None, + source_id: None, + caller_default: false, + report_to: None, }] ); } @@ -572,7 +640,9 @@ mod tests { fn every_missing_meter_is_reported_and_priced_lines_survive(when: When) { let audio = Meter::Input(InputUnit::Token(TokenKind::Audio)); let usage = Usage { - quantities: [(INPUT, 3), (OUTPUT, 4), (audio.clone(), 2)].into(), + quantities: [(INPUT, 3), (OUTPUT, 4), (audio.clone(), 2)] + .map(|(meter, count)| (meter, crate::Quantity::Count(count))) + .into(), ..Usage::default() }; let Priced::Unpriced { missing, partial } = @@ -584,14 +654,16 @@ mod tests { assert_eq!(partial.total.get(), 8.0); assert_eq!(partial.lines.len(), 1); assert_eq!(partial.lines[0].meter, OUTPUT); - assert_eq!(partial.lines[0].quantity, 4); + assert_eq!(partial.lines[0].quantity, Quantity::Count(4)); assert_eq!(partial.output().get(), 8.0); } #[rstest] fn an_unpriced_row_cannot_fall_back_to_zero(when: When) { let usage = Usage { - quantities: [(INPUT, 1000), (OUTPUT, 1000)].into(), + quantities: [(INPUT, 1000), (OUTPUT, 1000)] + .map(|(meter, count)| (meter, crate::Quantity::Count(count))) + .into(), ..Usage::default() }; assert_eq!( @@ -609,7 +681,9 @@ mod tests { #[rstest] fn unused_meters_need_no_price(when: When) { let usage = Usage { - quantities: [(INPUT, 0), (OUTPUT, 0)].into(), + quantities: [(INPUT, 0), (OUTPUT, 0)] + .map(|(meter, count)| (meter, crate::Quantity::Count(count))) + .into(), ..Usage::default() }; assert_eq!( @@ -624,7 +698,9 @@ mod tests { #[rstest] fn one_hour_writes_fall_back_to_the_default_write_price(when: When) { let usage = Usage { - quantities: [(WRITE, 4), (HOUR, 6)].into(), + quantities: [(WRITE, 4), (HOUR, 6)] + .map(|(meter, count)| (meter, crate::Quantity::Count(count))) + .into(), ..Usage::default() }; let result = known(calculate( @@ -673,7 +749,9 @@ mod tests { None, ); let usage = Usage { - quantities: [(INPUT, 101)].into(), + quantities: [(INPUT, 101)] + .map(|(meter, count)| (meter, crate::Quantity::Count(count))) + .into(), prompt_tokens: 101, ..Usage::default() }; @@ -702,7 +780,9 @@ mod tests { #[case] meter: Meter, ) { let usage = Usage { - quantities: [(meter.clone(), 7)].into(), + quantities: [(meter.clone(), 7)] + .map(|(meter, count)| (meter, crate::Quantity::Count(count))) + .into(), ..Usage::default() }; let result = known(calculate(&prices([(meter.clone(), 0.25)]), &usage, &when)); @@ -728,7 +808,9 @@ mod tests { over_results: NonZeroU64::new(10), }); let usage = Usage { - quantities: [(query.clone(), 2)].into(), + quantities: [(query.clone(), 2)] + .map(|(meter, count)| (meter, crate::Quantity::Count(count))) + .into(), search_results: results, ..Usage::default() }; @@ -754,7 +836,9 @@ mod tests { over_seconds: NonZeroU64::new(10), }); let usage = Usage { - quantities: [(base.clone(), seconds)].into(), + quantities: [(base.clone(), seconds)] + .map(|(meter, count)| (meter, crate::Quantity::Count(count))) + .into(), ..Usage::default() }; assert_eq!( @@ -796,7 +880,9 @@ mod tests { ), ); let usage = Usage { - quantities: [(INPUT, 2), (READ, 4), (HOUR, 7), (OUTPUT, 2)].into(), + quantities: [(INPUT, 2), (READ, 4), (HOUR, 7), (OUTPUT, 2)] + .map(|(meter, count)| (meter, crate::Quantity::Count(count))) + .into(), ..Usage::default() }; let result = known(calculate( @@ -855,7 +941,9 @@ mod tests { None, ); let usage = Usage { - quantities: [(INPUT, 4), (OUTPUT, 2)].into(), + quantities: [(INPUT, 4), (OUTPUT, 2)] + .map(|(meter, count)| (meter, crate::Quantity::Count(count))) + .into(), ..Usage::default() }; assert_eq!( @@ -876,7 +964,9 @@ mod tests { #[rstest] fn absent_batch_rates_use_half_online_base_rates(when: When) { let usage = Usage { - quantities: [(INPUT, 4), (OUTPUT, 2), (READ, 6)].into(), + quantities: [(INPUT, 4), (OUTPUT, 2), (READ, 6)] + .map(|(meter, count)| (meter, crate::Quantity::Count(count))) + .into(), ..Usage::default() }; let result = known(calculate( @@ -929,7 +1019,9 @@ mod tests { None, ); let usage = Usage { - quantities: [(INPUT, 2), (READ, 3), (OUTPUT, 4)].into(), + quantities: [(INPUT, 2), (READ, 3), (OUTPUT, 4)] + .map(|(meter, count)| (meter, crate::Quantity::Count(count))) + .into(), prompt_tokens, inclusive_threshold, ..Usage::default() @@ -951,7 +1043,9 @@ mod tests { fn dedicated_modality_and_long_ttl_rates_win(when: When) { let audio = Meter::Input(InputUnit::Token(TokenKind::Audio)); let usage = Usage { - quantities: [(INPUT, 10), (audio.clone(), 5), (WRITE, 3), (HOUR, 2)].into(), + quantities: [(INPUT, 10), (audio.clone(), 5), (WRITE, 3), (HOUR, 2)] + .map(|(meter, count)| (meter, crate::Quantity::Count(count))) + .into(), ..Usage::default() }; let result = known(calculate( @@ -973,7 +1067,9 @@ mod tests { #[rstest] fn decisions_prices_are_isolated_from_online_and_batch(when: When) { let usage = Usage { - quantities: [(INPUT, 4)].into(), + quantities: [(INPUT, 4)] + .map(|(meter, count)| (meter, crate::Quantity::Count(count))) + .into(), ..Usage::default() }; assert!(matches!( @@ -1056,6 +1152,7 @@ mod tests { (READ, count(3)), (WRITE, count(4)), ] + .map(|(meter, count)| (meter, crate::Quantity::Count(count))) .into(), prompt_tokens: count(1), ..Usage::default() diff --git a/litellm-rust/crates/cost/src/error.rs b/litellm-rust/crates/cost/src/error.rs index a496923f351..7b5c1b12ce5 100644 --- a/litellm-rust/crates/cost/src/error.rs +++ b/litellm-rust/crates/cost/src/error.rs @@ -6,6 +6,10 @@ pub enum PriceEvaluationError { NonTokenProjection, #[error("pricing produced a non-finite amount or rate")] NonFiniteValue, + #[error("quantity must be finite and non-negative")] + InvalidQuantity, + #[error("measured quantity unit is not valid for this meter")] + InvalidQuantityUnit, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] diff --git a/litellm-rust/crates/cost/src/lib.rs b/litellm-rust/crates/cost/src/lib.rs index 12e997b9c0c..10569a00619 100644 --- a/litellm-rust/crates/cost/src/lib.rs +++ b/litellm-rust/crates/cost/src/lib.rs @@ -1,10 +1,14 @@ mod calculate; mod error; pub mod money; +mod other; pub mod price; +mod quantity; pub mod schedule; mod selection; pub use calculate::*; pub use error::*; +pub use other::*; +pub use quantity::*; pub use selection::*; diff --git a/litellm-rust/crates/cost/src/money.rs b/litellm-rust/crates/cost/src/money.rs index 7facfec8b00..6ff4ca3be64 100644 --- a/litellm-rust/crates/cost/src/money.rs +++ b/litellm-rust/crates/cost/src/money.rs @@ -77,6 +77,17 @@ impl Mul for UnitPrice { } } +impl Mul for UnitPrice { + type Output = Usd; + + fn mul(self, units: crate::Quantity) -> Usd { + match units { + crate::Quantity::Count(count) => self * count, + crate::Quantity::Measured(value) => Usd(self.0 * value.get()), + } + } +} + impl Mul for UnitPrice { type Output = UnitPrice; diff --git a/litellm-rust/crates/cost/src/other.rs b/litellm-rust/crates/cost/src/other.rs new file mode 100644 index 00000000000..ae5051c454c --- /dev/null +++ b/litellm-rust/crates/cost/src/other.rs @@ -0,0 +1,1006 @@ +use std::collections::BTreeSet; +use std::num::NonZeroU64; + +use crate::money::UnitPrice; +use crate::price::{ + Band, Channel, Feature, InputUnit, Meter, OutputUnit, Price, Prices, TokenKind, + When as PriceWhen, +}; +use crate::{PriceEvaluationError, SelectionPolicy, TokenPriceSources, Usage, When}; + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub enum OtherSelection { + #[default] + Additive, + PromptUnits, + FirstUnit(Vec), + SearchTiers, + Ocr, + DirectionalCharacters { + threshold: NonZeroU64, + comparison_scale: NonZeroU64, + }, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::EnumString, strum::VariantArray)] +pub enum DefaultEligibility { + #[strum(serialize = "absent")] + Absent, + #[strum(serialize = "absent_or_null")] + AbsentOrNull, +} + +#[derive( + Clone, Copy, Debug, PartialEq, Eq, strum::EnumString, strum::VariantArray, strum::IntoStaticStr, +)] +pub enum ReportTo { + #[strum(serialize = "input")] + Input, + #[strum(serialize = "output")] + Output, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct OtherPriceSource { + pub id: String, + pub declared: BTreeSet, + pub search_tiers: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SearchTier { + pub range: [u64; 2], + pub meter: Meter, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct OtherContext { + pub selection: OtherSelection, + pub sources: Vec, + pub defaults: Vec, + pub report_to: Option, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct ResolvedCharge { + pub meter: Meter, + pub price: UnitPrice, + pub source_id: Option, + pub caller_default: bool, + pub quantity: Option, +} + +impl From<(Meter, UnitPrice)> for ResolvedCharge { + fn from((meter, price): (Meter, UnitPrice)) -> Self { + Self { + meter, + price, + source_id: None, + caller_default: false, + quantity: None, + } + } +} + +impl OtherContext { + pub fn insert_default( + &mut self, + prices: &mut Prices, + meter: Meter, + price: UnitPrice, + per_units: NonZeroU64, + eligibility: DefaultEligibility, + ) { + let known = self.flat(prices, &meter, Channel::Online).is_some(); + let declared = self + .sources + .iter() + .any(|source| source.declared.contains(&meter)); + if known || (eligibility == DefaultEligibility::Absent && declared) { + return; + } + let normalized = UnitPrice::new(price.get() / per_units.get() as f64) + .expect("finite nonnegative price divided by positive units"); + prices.insert_default(self.defaults.len(), meter.clone(), normalized); + self.defaults.push(meter); + } + + pub fn validate_usage(&self, usage: &Usage) -> Result<(), PriceEvaluationError> { + if let OtherSelection::DirectionalCharacters { + threshold, + comparison_scale, + } = self.selection + { + for meter in [ + Meter::Input(InputUnit::Character), + Meter::Output(OutputUnit::Character), + ] { + if let Some(quantity) = usage.quantities.get(&meter) { + quantity.exceeds_scaled(threshold.get(), comparison_scale.get())?; + } + } + token_quantity(usage, false)?; + token_quantity(usage, true)?; + } + Ok(()) + } + + fn row_rate( + &self, + prices: &Prices, + row: usize, + meter: &Meter, + channel: Channel, + band: Band, + ) -> Option { + let price = prices.source_flat( + row, + &Price { + meter: meter.clone(), + channel, + when: PriceWhen { + band, + ..PriceWhen::default() + }, + }, + )?; + Some(ResolvedCharge { + meter: meter.clone(), + price, + source_id: Some(self.sources[row].id.clone()), + caller_default: false, + quantity: None, + }) + } + + fn default(&self, prices: &Prices, meter: &Meter) -> Option { + self.defaults + .iter() + .enumerate() + .find_map(|(index, candidate)| { + (candidate == meter) + .then(|| prices.default_rate(index, meter)) + .flatten() + .map(|price| ResolvedCharge { + meter: meter.clone(), + price, + source_id: None, + caller_default: true, + quantity: None, + }) + }) + } + + fn flat(&self, prices: &Prices, meter: &Meter, channel: Channel) -> Option { + (0..self.sources.len()) + .find_map(|row| self.row_rate(prices, row, meter, channel, Band::Base)) + .or_else(|| self.default(prices, meter)) + } + + fn first(&self, prices: &Prices, meters: &[Meter], usage: &Usage) -> Option { + (0..self.sources.len()) + .find_map(|row| { + meters + .iter() + .filter(|meter| used(usage, meter)) + .find_map(|meter| { + self.row_rate(prices, row, meter, Channel::Online, Band::Base) + }) + }) + .or_else(|| { + meters + .iter() + .filter(|meter| used(usage, meter)) + .find_map(|meter| self.default(prices, meter)) + }) + } + + fn ocr_family( + &self, + prices: &Prices, + meter: &Meter, + channel: Channel, + ) -> Option { + (0..self.sources.len()) + .find_map(|row| { + self.row_rate(prices, row, meter, channel, Band::Base) + .or_else(|| { + (channel == Channel::Batch) + .then(|| self.row_rate(prices, row, meter, Channel::Online, Band::Base)) + .flatten() + }) + }) + .or_else(|| self.default(prices, meter)) + } + + fn character_rate( + &self, + prices: &Prices, + meter: &Meter, + usage: &Usage, + ) -> Option { + let OtherSelection::DirectionalCharacters { + threshold, + comparison_scale, + } = self.selection + else { + return None; + }; + let quantity = usage.quantities.get(meter)?; + let over = quantity + .exceeds_scaled(threshold.get(), comparison_scale.get()) + .ok()?; + let band = if over { + Band::over_prompt_tokens(threshold.get()) + } else { + Band::Base + }; + (0..self.sources.len()) + .find_map(|row| self.row_rate(prices, row, meter, Channel::Online, band)) + } + + fn directional_token_threshold(&self, prices: &Prices) -> Option { + let OtherSelection::DirectionalCharacters { threshold, .. } = self.selection else { + return None; + }; + [ + Meter::TEXT_INPUT, + Meter::Output(OutputUnit::Token(TokenKind::Text)), + ] + .into_iter() + .any(|meter| { + prices + .get(&Price::online( + meter, + PriceWhen { + band: Band::OverPromptTokens(threshold), + ..Default::default() + }, + )) + .is_some() + }) + .then_some(threshold) + } + + pub(crate) fn charges( + &self, + prices: &Prices, + meter: &Meter, + usage: &Usage, + when: &When, + ) -> bool { + match &self.selection { + OtherSelection::FirstUnit(meters) if meters.contains(meter) => self + .first(prices, meters, usage) + .is_none_or(|rate| rate.meter == *meter), + OtherSelection::PromptUnits => token_alternative(meter).is_none_or(|alternative| { + !used(usage, &alternative) + || self.flat(prices, &alternative, Channel::Online).is_none() + }), + OtherSelection::Ocr => { + let credit = Meter::Feature(Feature::OcrCredit); + let credits = used(usage, &credit) + && self.ocr_family(prices, &credit, when.channel).is_some(); + match meter { + Meter::Feature(Feature::OcrPage | Feature::AnnotationPage) => !credits, + Meter::Feature(Feature::OcrCredit) => { + credits || !used(usage, &Meter::Feature(Feature::OcrPage)) + } + _ => true, + } + } + OtherSelection::DirectionalCharacters { .. } => { + if let Some(character) = direction_character(meter) { + if self.character_rate(prices, &character, usage).is_some() { + return false; + } + if self.directional_token_threshold(prices).is_some() { + return usage + .quantities + .iter() + .find(|(candidate, quantity)| { + !quantity.is_zero() + && direction_character(candidate) == Some(character.clone()) + }) + .is_some_and(|(candidate, _)| candidate == meter); + } + return true; + } + if matches!( + meter, + Meter::Input(InputUnit::Character) | Meter::Output(OutputUnit::Character) + ) { + return self.character_rate(prices, meter, usage).is_some() + || !usage + .quantities + .keys() + .any(|token| direction_character(token).as_ref() == Some(meter)); + } + true + } + _ => true, + } + } + + pub(crate) fn resolve( + &self, + prices: &Prices, + meter: &Meter, + band: Band, + usage: &Usage, + when: &When, + tokens: &TokenPriceSources, + ) -> Option { + if let OtherSelection::FirstUnit(meters) = &self.selection + && meters.contains(meter) + { + return self + .first(prices, meters, usage) + .filter(|charge| !meter.is_token() || charge.meter.is_token()); + } + if meter.is_token() { + if let Some(threshold) = self.directional_token_threshold(prices) { + let output = matches!(meter, Meter::Output(_)); + let quantity = token_quantity(usage, output).ok()?; + let source = if output { + Meter::Output(OutputUnit::Token(TokenKind::Text)) + } else { + Meter::TEXT_INPUT + }; + return crate::selection::directional_rate(prices, source, quantity, threshold) + .map(|rate| ResolvedCharge { + source_id: self.sources.first().map(|source| source.id.clone()), + quantity: Some(quantity), + ..ResolvedCharge::from(rate) + }); + } + return SelectionPolicy::SdkTokens(tokens) + .resolve(prices, meter, band, usage, when) + .map(|rate| { + let mut charge = ResolvedCharge::from(rate); + charge.source_id = self.sources.first().map(|source| source.id.clone()); + charge + }) + .or_else(|| self.flat(prices, meter, when.channel)); + } + match &self.selection { + OtherSelection::SearchTiers + if matches!(meter, Meter::Input(InputUnit::Query { .. })) => + { + self.sources + .iter() + .enumerate() + .find_map(|(row, source)| { + let tier = source + .search_tiers + .iter() + .find(|tier| { + (tier.range[0]..=tier.range[1]).contains(&usage.search_results) + }) + .or_else(|| source.search_tiers.last()); + match tier { + Some(tier) => { + self.row_rate(prices, row, &tier.meter, Channel::Online, Band::Base) + } + None => self.row_rate(prices, row, meter, Channel::Online, Band::Base), + } + }) + .or_else(|| self.default(prices, meter)) + } + OtherSelection::Ocr if matches!(meter, Meter::Feature(Feature::AnnotationPage)) => { + self.ocr_family(prices, meter, when.channel).or_else(|| { + self.ocr_family(prices, &Meter::Feature(Feature::OcrPage), when.channel) + }) + } + OtherSelection::Ocr + if matches!(meter, Meter::Feature(Feature::OcrPage | Feature::OcrCredit)) => + { + self.ocr_family(prices, meter, when.channel) + } + OtherSelection::DirectionalCharacters { .. } + if matches!( + meter, + Meter::Input(InputUnit::Character) | Meter::Output(OutputUnit::Character) + ) => + { + self.character_rate(prices, meter, usage) + } + _ => SelectionPolicy::SdkTokens(tokens) + .resolve(prices, meter, band, usage, when) + .map(|rate| ResolvedCharge { + source_id: self.sources.first().map(|source| source.id.clone()), + ..ResolvedCharge::from(rate) + }) + .or_else(|| self.flat(prices, meter, when.channel)), + } + } +} + +fn token_quantity(usage: &Usage, output: bool) -> Result { + if output && let Some(completion) = usage.completion_tokens { + return Ok(completion.into()); + } + if !output && usage.prompt_tokens > 0 { + return Ok(usage.prompt_tokens.into()); + } + let text = if output { + Meter::Output(OutputUnit::Token(TokenKind::Text)) + } else { + Meter::TEXT_INPUT + }; + if usage.quantities.iter().any(|(meter, quantity)| { + meter.is_token() + && matches!(meter, Meter::Output(_)) == output + && *meter != text + && !quantity.is_zero() + }) { + return Err(PriceEvaluationError::InvalidQuantity); + } + match usage.quantities.get(&text).copied().unwrap_or(0.into()) { + crate::Quantity::Count(count) => Ok(count.into()), + crate::Quantity::Measured(_) => Err(PriceEvaluationError::InvalidQuantity), + } +} + +fn used(usage: &Usage, meter: &Meter) -> bool { + usage + .quantities + .get(meter) + .is_some_and(|quantity| !quantity.is_zero()) +} + +fn token_alternative(meter: &Meter) -> Option { + match meter { + Meter::Input(InputUnit::Token(TokenKind::Text)) => Some(Meter::Input(InputUnit::Character)), + Meter::Input(InputUnit::Token(TokenKind::Audio)) => { + Some(Meter::Input(InputUnit::AudioSecond)) + } + Meter::Input(InputUnit::Token(TokenKind::Image)) => Some(Meter::Input(InputUnit::Image)), + Meter::Input(InputUnit::Token(TokenKind::Video)) => { + Some(Meter::Input(InputUnit::VideoSecond { over_seconds: None })) + } + _ => None, + } +} + +fn direction_character(meter: &Meter) -> Option { + match meter { + Meter::Input(InputUnit::Token(_)) | Meter::CacheRead { .. } | Meter::CacheWrite { .. } => { + Some(Meter::Input(InputUnit::Character)) + } + Meter::Output(OutputUnit::Token(_) | OutputUnit::ReasoningToken) => { + Some(Meter::Output(OutputUnit::Character)) + } + _ => None, + } +} + +#[cfg(test)] +mod tests { + const INPUT: Meter = Meter::TEXT_INPUT; + use rstest::{fixture, rstest}; + use time::OffsetDateTime; + + use super::*; + use crate::money::ScaleFactor; + use crate::price::{CacheReadMode, ServiceTier}; + use crate::{Measurement, Priced, Quantity, QuantityUnit, calculate_with_rates}; + + const INPUT_CHAR: Meter = Meter::Input(InputUnit::Character); + const OUTPUT_CHAR: Meter = Meter::Output(OutputUnit::Character); + const OUTPUT: Meter = Meter::Output(OutputUnit::Token(TokenKind::Text)); + const PAGE: Meter = Meter::Feature(Feature::OcrPage); + const ANNOTATION: Meter = Meter::Feature(Feature::AnnotationPage); + const FILE: Meter = Meter::Feature(Feature::FileSearchCall); + + #[fixture] + fn when() -> When { + When { + tier: ServiceTier::Standard, + channel: Channel::Online, + at: OffsetDateTime::UNIX_EPOCH, + } + } + + #[fixture] + fn context() -> OtherContext { + OtherContext { + sources: vec![OtherPriceSource { + id: "primary".to_owned(), + ..Default::default() + }], + ..Default::default() + } + } + + fn prices(rates: impl IntoIterator) -> Prices { + Prices::new( + rates + .into_iter() + .map(|(meter, channel, rate)| { + ( + Price { + meter, + channel, + when: Default::default(), + }, + UnitPrice::new(rate).unwrap(), + ) + }) + .collect(), + None, + ) + } + + fn evaluated( + prices: &Prices, + context: &OtherContext, + usage: &Usage, + when: &When, + factor: f64, + ) -> crate::PriceEvaluation { + calculate_with_rates( + prices, + usage, + when, + SelectionPolicy::SdkOther { + tokens: &TokenPriceSources::default(), + context, + }, + &[], + &[ScaleFactor::new(factor).unwrap()], + ) + .unwrap() + } + + #[rstest] + #[case::at_threshold(32000, 4.0)] + #[case::above_threshold(32001, 6.0)] + fn directional_character_output_bills_actual_characters( + mut context: OtherContext, + when: When, + #[case] characters: u64, + #[case] rate: f64, + #[values(1.0, 1.1)] factor: f64, + ) { + context.selection = OtherSelection::DirectionalCharacters { + threshold: NonZeroU64::new(128000).unwrap(), + comparison_scale: NonZeroU64::new(4).unwrap(), + }; + let prices = Prices::new( + [ + ( + Price::online(INPUT_CHAR, Default::default()), + UnitPrice::new(3.0).unwrap(), + ), + ( + Price::online(OUTPUT_CHAR, Default::default()), + UnitPrice::new(4.0).unwrap(), + ), + ( + Price::online( + OUTPUT_CHAR, + PriceWhen { + band: Band::over_prompt_tokens(128000), + ..Default::default() + }, + ), + UnitPrice::new(6.0).unwrap(), + ), + ( + Price::online(OUTPUT, Default::default()), + UnitPrice::new(2.0).unwrap(), + ), + ] + .into(), + None, + ); + let usage = Usage { + quantities: [ + (INPUT_CHAR, Quantity::Count(10)), + (OUTPUT_CHAR, characters.into()), + (OUTPUT, 2.into()), + ] + .into(), + ..Default::default() + }; + let Priced::Known(result) = evaluated(&prices, &context, &usage, &when, factor).pricing + else { + panic!("priced characters") + }; + assert_eq!(result.lines.len(), 2); + assert_eq!(result.lines[1].quantity, Quantity::Count(characters)); + assert_eq!(result.output().get(), characters as f64 * rate * factor); + assert_eq!(result.input().get(), 30.0 * factor); + } + + #[rstest] + fn directional_token_fallback_uses_aggregate_counts_once( + mut context: OtherContext, + mut when: When, + ) { + context.selection = OtherSelection::DirectionalCharacters { + threshold: NonZeroU64::new(128000).unwrap(), + comparison_scale: NonZeroU64::new(4).unwrap(), + }; + when.tier = ServiceTier::Priority; + let prices = Prices::new( + [ + ( + Price::online(Meter::TEXT_INPUT, Default::default()), + UnitPrice::new(1.0).unwrap(), + ), + ( + Price::online(OUTPUT, Default::default()), + UnitPrice::new(2.0).unwrap(), + ), + ( + Price::online( + OUTPUT, + PriceWhen { + band: Band::over_prompt_tokens(128000), + ..Default::default() + }, + ), + UnitPrice::new(4.0).unwrap(), + ), + ( + Price::online( + Meter::TEXT_INPUT, + PriceWhen { + tier: ServiceTier::Priority, + ..Default::default() + }, + ), + UnitPrice::new(10.0).unwrap(), + ), + ] + .into(), + None, + ); + let read = Meter::CacheRead { + kind: TokenKind::Text, + mode: CacheReadMode::Implicit, + }; + let usage = Usage { + quantities: [ + (Meter::TEXT_INPUT, 20.into()), + (read, 80.into()), + (OUTPUT, 1.into()), + (Meter::Output(OutputUnit::ReasoningToken), 200000.into()), + (OUTPUT_CHAR, 40000.into()), + ] + .into(), + prompt_tokens: 100, + completion_tokens: Some(128001), + ..Default::default() + }; + let Priced::Known(result) = evaluated(&prices, &context, &usage, &when, 1.0).pricing else { + panic!("priced aggregate fallback") + }; + assert_eq!(result.lines.len(), 2); + assert_eq!(result.input().get(), 100.0); + assert_eq!(result.output().get(), 512004.0); + assert_eq!(result.lines[1].quantity, Quantity::Count(128001)); + } + + #[rstest] + #[case::known(Some(3.0), 12.0, true)] + #[case::free(Some(0.0), 0.0, true)] + #[case::unknown(None, 10.0, false)] + fn prompt_units_suppress_only_known_alternatives( + mut context: OtherContext, + when: When, + #[case] rate: Option, + #[case] input: f64, + #[case] known: bool, + ) { + context.selection = OtherSelection::PromptUnits; + let prices = prices( + [ + (Meter::TEXT_INPUT, Channel::Online, 1.0), + (OUTPUT, Channel::Online, 2.0), + ] + .into_iter() + .chain(rate.map(|rate| (INPUT_CHAR, Channel::Online, rate))), + ); + let usage = Usage { + quantities: [ + (Meter::TEXT_INPUT, 10.into()), + (INPUT_CHAR, 4.into()), + (OUTPUT, 2.into()), + ] + .into(), + ..Default::default() + }; + let result = evaluated(&prices, &context, &usage, &when, 1.0).pricing; + assert_eq!(matches!(result, Priced::Known(_)), known); + let result = match result { + Priced::Known(result) + | Priced::Unpriced { + partial: result, .. + } => result, + }; + assert_eq!(result.input().get(), input); + assert_eq!(result.output().get(), 4.0); + } + + #[rstest] + #[case::source_before_unit(8.0, 26.0)] + #[case::zero_wins(0.0, 10.0)] + fn first_unit_keeps_row_precedence_and_genuine_tokens( + mut context: OtherContext, + when: When, + #[case] rate: f64, + #[case] total: f64, + ) { + let image = Meter::Input(InputUnit::Image); + let output_image = Meter::Output(OutputUnit::Image(None)); + context.selection = OtherSelection::FirstUnit(vec![image.clone(), output_image.clone()]); + context.sources.push(OtherPriceSource { + id: "published".to_owned(), + ..Default::default() + }); + let mut prices = prices([ + (output_image.clone(), Channel::Online, rate), + (Meter::TEXT_INPUT, Channel::Online, 1.0), + ]); + prices.append_source(1, self::prices([(image.clone(), Channel::Online, 2.0)])); + let usage = Usage { + quantities: [ + (image, 1.into()), + (output_image, 2.into()), + (Meter::TEXT_INPUT, 10.into()), + ] + .into(), + ..Default::default() + }; + let Priced::Known(result) = evaluated(&prices, &context, &usage, &when, 1.0).pricing else { + panic!("priced alternatives") + }; + assert_eq!(result.total.get(), total); + assert_eq!(result.lines.len(), 2); + assert_eq!(result.lines[1].source_id.as_deref(), Some("primary")); + } + + #[rstest] + #[case::annotation_family(Some(10.0), 16.0, "published")] + #[case::annotation_zero(Some(0.0), 6.0, "published")] + #[case::page_fallback(None, 9.0, "primary")] + fn ocr_resolves_annotation_family_before_page_fallback( + mut context: OtherContext, + mut when: When, + #[case] annotation: Option, + #[case] total: f64, + #[case] source: &str, + ) { + context.selection = OtherSelection::Ocr; + context.sources.push(OtherPriceSource { + id: "published".to_owned(), + ..Default::default() + }); + when.channel = Channel::Batch; + let mut prices = prices([(PAGE, Channel::Batch, 3.0), (PAGE, Channel::Online, 2.0)]); + prices.append_source( + 1, + self::prices(annotation.map(|rate| (ANNOTATION, Channel::Online, rate))), + ); + let usage = Usage { + quantities: [(PAGE, 2.into()), (ANNOTATION, 1.into())].into(), + ..Default::default() + }; + let Priced::Known(result) = evaluated(&prices, &context, &usage, &when, 1.0).pricing else { + panic!("priced OCR") + }; + assert_eq!(result.total.get(), total); + assert_eq!(result.lines[0].source_id.as_deref(), Some(source)); + } + + #[rstest] + #[case::absent(DefaultEligibility::Absent, false, true)] + #[case::null_stays_unknown(DefaultEligibility::Absent, true, false)] + #[case::null_eligible(DefaultEligibility::AbsentOrNull, true, true)] + fn defaults_normalize_occurrences_without_erasing_unknown( + mut context: OtherContext, + when: When, + #[case] eligibility: DefaultEligibility, + #[case] declared: bool, + #[case] known: bool, + #[values(0.0, 1.0)] factor: f64, + ) { + if declared { + context.sources[0].declared.insert(FILE); + } + let mut prices = Prices::default(); + context.insert_default( + &mut prices, + FILE, + UnitPrice::new(2.5).unwrap(), + NonZeroU64::new(1000).unwrap(), + eligibility, + ); + let usage = Usage { + quantities: [(FILE, 3.into())].into(), + ..Default::default() + }; + let result = evaluated(&prices, &context, &usage, &when, factor).pricing; + assert_eq!(matches!(result, Priced::Known(_)), known); + let result = match result { + Priced::Known(result) + | Priced::Unpriced { + partial: result, .. + } => result, + }; + assert_eq!( + result.total.get(), + if known { 0.0075 * factor } else { 0.0 } + ); + assert_eq!( + result.lines.first().map(|line| line.caller_default), + known.then_some(true) + ); + } + + #[rstest] + #[case::normalized_seconds(1500.0, QuantityUnit::Milliseconds)] + #[case::native_seconds(1.5, QuantityUnit::Meter)] + fn measured_time_uses_native_units( + context: OtherContext, + when: When, + #[case] value: f64, + #[case] unit: QuantityUnit, + ) { + let meter = Meter::WallClockSecond; + let prices = prices([(meter.clone(), Channel::Online, 0.0014)]); + let usage = Usage { + quantities: [( + meter.clone(), + Quantity::measured(Measurement::new(value).unwrap(), unit, &meter).unwrap(), + )] + .into(), + ..Default::default() + }; + let Priced::Known(result) = evaluated(&prices, &context, &usage, &when, 1.0).pricing else { + panic!("priced time") + }; + assert_eq!(result.total.get(), 0.0021); + assert_eq!( + result.lines[0].quantity, + Quantity::Measured(Measurement::new(1.5).unwrap()) + ); + } + + #[rstest] + #[case::below_first(0, 2.0)] + #[case::first_start(1, 1.0)] + #[case::first_end(25, 1.0)] + #[case::second_start(26, 2.0)] + #[case::last_end(100, 2.0)] + #[case::above_last(101, 2.0)] + fn search_tiers_use_inclusive_ranges_and_last_outside( + mut context: OtherContext, + when: When, + #[case] results: u64, + #[case] expected: f64, + ) { + let query = Meter::Input(InputUnit::Query { over_results: None }); + let above = Meter::Input(InputUnit::Query { + over_results: NonZeroU64::new(25), + }); + context.selection = OtherSelection::SearchTiers; + context.sources[0].search_tiers = vec![ + SearchTier { + range: [1, 25], + meter: query.clone(), + }, + SearchTier { + range: [26, 100], + meter: above.clone(), + }, + ]; + let prices = prices([ + (query.clone(), Channel::Online, 1.0), + (above, Channel::Online, 2.0), + ]); + let usage = Usage { + quantities: [(query, 3.into())].into(), + search_results: results, + ..Default::default() + }; + let Priced::Known(result) = evaluated(&prices, &context, &usage, &when, 1.0).pricing else { + panic!("priced search") + }; + assert_eq!(result.total.get(), expected * 3.0); + } + + #[rstest] + fn chosen_second_units_do_not_become_projected_token_prices( + mut context: OtherContext, + when: When, + ) { + let seconds = Meter::Input(InputUnit::Second); + context.selection = OtherSelection::FirstUnit(vec![seconds.clone(), INPUT]); + let prices = prices([ + (seconds.clone(), Channel::Online, 2.0), + (INPUT, Channel::Online, 1.0), + ]); + let usage = Usage { + quantities: [(seconds, 3.into()), (INPUT, 10.into())].into(), + ..Default::default() + }; + let result = calculate_with_rates( + &prices, + &usage, + &when, + SelectionPolicy::SdkOther { + tokens: &TokenPriceSources::default(), + context: &context, + }, + &[INPUT], + &[], + ) + .unwrap(); + assert!(result.rates[0].resolved.is_none()); + let Priced::Known(pricing) = result.pricing else { + panic!("priced seconds") + }; + assert_eq!(pricing.total.get(), 6.0); + assert_eq!(pricing.lines.len(), 1); + } + + #[rstest] + fn additive_token_fallbacks_share_actual_and_projection_source( + mut context: OtherContext, + when: When, + ) { + let mut primary = Prices::default(); + primary.append_source(1, prices([(INPUT, Channel::Online, 2.0)])); + context.sources.push(OtherPriceSource { + id: "published".to_owned(), + ..Default::default() + }); + let usage = Usage { + quantities: [(INPUT, 10.into())].into(), + ..Default::default() + }; + let result = calculate_with_rates( + &primary, + &usage, + &when, + SelectionPolicy::SdkOther { + tokens: &TokenPriceSources::default(), + context: &context, + }, + &[INPUT], + &[ScaleFactor::new(1.1).unwrap()], + ) + .unwrap(); + assert_eq!(result.rates[0].source_id.as_deref(), Some("published")); + assert_eq!(result.rates[0].resolved.as_ref().unwrap().1.get(), 2.2); + let Priced::Known(pricing) = result.pricing else { + panic!("priced tokens") + }; + assert_eq!(pricing.total.get(), 22.0); + assert_eq!(pricing.lines[0].source_id, result.rates[0].source_id); + } + + #[rstest] + #[case::missing_aggregate(None, None)] + #[case::provided_aggregate(Some(10), Some(10))] + #[case::zero_aggregate(Some(0), Some(0))] + fn directional_counts_never_infer_overlapping_completion_details( + #[case] completion_tokens: Option, + #[case] expected: Option, + ) { + let usage = Usage { + quantities: [ + (OUTPUT, 10.into()), + (Meter::Output(OutputUnit::ReasoningToken), 5.into()), + ] + .into(), + completion_tokens, + ..Default::default() + }; + assert_eq!(token_quantity(&usage, true).ok(), expected.map(Into::into)); + } +} diff --git a/litellm-rust/crates/cost/src/price.rs b/litellm-rust/crates/cost/src/price.rs index cb20558b757..7ad94997d17 100644 --- a/litellm-rust/crates/cost/src/price.rs +++ b/litellm-rust/crates/cost/src/price.rs @@ -58,6 +58,12 @@ pub enum PriceSource { #[default] Flat, TokenTier(u64), + RowFlat(usize), + RowTokenTier { + row: usize, + floor: u64, + }, + CallerDefault(usize), } #[derive(Clone, Debug, PartialEq)] @@ -143,6 +149,46 @@ impl Prices { self.off_peak.as_ref() } + pub fn append_source(&mut self, row: usize, prices: Self) { + self.rates + .extend(prices.rates.into_iter().map(|((price, origin), rate)| { + let origin = match origin { + PriceSource::Flat => PriceSource::RowFlat(row), + PriceSource::TokenTier(floor) => PriceSource::RowTokenTier { row, floor }, + other => other, + }; + ((price, origin), rate) + })); + } + + pub fn source_flat(&self, row: usize, price: &Price) -> Option { + self.get_from( + price, + if row == 0 { + PriceSource::Flat + } else { + PriceSource::RowFlat(row) + }, + ) + } + + pub fn insert_default(&mut self, index: usize, meter: Meter, rate: UnitPrice) { + self.rates.insert( + ( + Price::online(meter, When::default()), + PriceSource::CallerDefault(index), + ), + rate, + ); + } + + pub fn default_rate(&self, index: usize, meter: &Meter) -> Option { + self.get_from( + &Price::online(meter.clone(), When::default()), + PriceSource::CallerDefault(index), + ) + } + pub fn with_cache_read_default( mut self, ratio: ScaleFactor, @@ -212,7 +258,9 @@ impl Prices { inclusive: bool, ) -> Band { Self::find_band( - self.rates.keys().map(|(price, _)| price), + self.rates.keys().filter_map(|(price, source)| { + matches!(source, PriceSource::Flat | PriceSource::TokenTier(_)).then_some(price) + }), channel, tier, prompt_tokens, diff --git a/litellm-rust/crates/cost/src/price/meter.rs b/litellm-rust/crates/cost/src/price/meter.rs index e34b0638f45..ca546e321d2 100644 --- a/litellm-rust/crates/cost/src/price/meter.rs +++ b/litellm-rust/crates/cost/src/price/meter.rs @@ -72,6 +72,13 @@ pub enum Feature { GoogleMapsGroundingQuery, OcrCredit, OcrPage, + FileSearchCall, + FileSearchGbDay, + VectorStoreGbDay, + ComputerUseInputToken, + ComputerUseOutputToken, + SearchResult, + BrowserOpen, } #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] diff --git a/litellm-rust/crates/cost/src/quantity.rs b/litellm-rust/crates/cost/src/quantity.rs new file mode 100644 index 00000000000..8cd13928acc --- /dev/null +++ b/litellm-rust/crates/cost/src/quantity.rs @@ -0,0 +1,173 @@ +use crate::PriceEvaluationError; +use crate::price::{Feature, InputUnit, Meter, OutputUnit}; + +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum Quantity { + Count(u64), + Measured(Measurement), +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Measurement(f64); + +#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::EnumString, strum::VariantArray)] +pub enum QuantityUnit { + #[strum(serialize = "meter")] + Meter, + #[strum(serialize = "milliseconds")] + Milliseconds, +} + +impl Measurement { + pub fn new(value: f64) -> Result { + if !value.is_finite() || value < 0.0 { + return Err(PriceEvaluationError::InvalidQuantity); + } + Ok(Self(value)) + } + + pub fn get(self) -> f64 { + self.0 + } +} + +impl Quantity { + pub fn measured( + value: Measurement, + unit: QuantityUnit, + meter: &Meter, + ) -> Result { + let seconds = matches!( + meter, + Meter::WallClockSecond + | Meter::Input( + InputUnit::Second | InputUnit::AudioSecond | InputUnit::VideoSecond { .. } + ) + | Meter::Output(OutputUnit::Second(_) | OutputUnit::VideoSecond) + ); + let characters = matches!( + meter, + Meter::Input(InputUnit::Character) | Meter::Output(OutputUnit::Character) + ); + let storage = matches!( + meter, + Meter::Feature(Feature::FileSearchGbDay | Feature::VectorStoreGbDay) + ); + match unit { + QuantityUnit::Meter if seconds || characters || storage => Ok(Self::Measured(value)), + QuantityUnit::Milliseconds if seconds => { + Measurement::new(value.get() / 1000.0).map(Self::Measured) + } + _ => Err(PriceEvaluationError::InvalidQuantityUnit), + } + } + + pub fn is_zero(self) -> bool { + match self { + Self::Count(value) => value == 0, + Self::Measured(value) => value.get() == 0.0, + } + } + + pub fn exceeds(self, threshold: u64) -> bool { + match self { + Self::Count(value) => value > threshold, + Self::Measured(value) => value.get() > threshold as f64, + } + } + + pub fn exceeds_scaled(self, threshold: u64, scale: u64) -> Result { + match self { + Self::Count(value) => Ok(u128::from(value) * u128::from(scale) > u128::from(threshold)), + Self::Measured(value) => { + let scaled = Measurement::new(value.get() * scale as f64)?; + Ok(scaled.get() > threshold as f64) + } + } + } +} + +impl From for Quantity { + fn from(value: u64) -> Self { + Self::Count(value) + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + use crate::price::TokenKind; + + #[rstest] + #[case::zero(0.0)] + #[case::fraction(1.5)] + #[case::largest(f64::MAX)] + fn admits_nonnegative_finite_measurements(#[case] value: f64) { + assert_eq!(Measurement::new(value).unwrap().get(), value); + } + + #[rstest] + #[case::negative(-1.0)] + #[case::nan(f64::NAN)] + #[case::infinity(f64::INFINITY)] + #[case::negative_infinity(f64::NEG_INFINITY)] + fn rejects_invalid_measurements(#[case] value: f64) { + assert_eq!( + Measurement::new(value), + Err(PriceEvaluationError::InvalidQuantity) + ); + } + + #[rstest] + fn normalizes_milliseconds_to_the_same_duration() { + let seconds = Quantity::measured( + Measurement::new(1.5).unwrap(), + QuantityUnit::Meter, + &Meter::WallClockSecond, + ) + .unwrap(); + let millis = Quantity::measured( + Measurement::new(1500.0).unwrap(), + QuantityUnit::Milliseconds, + &Meter::WallClockSecond, + ) + .unwrap(); + assert_eq!(seconds, millis); + } + + #[rstest] + #[case::token(Meter::Input(InputUnit::Token(TokenKind::Text)), QuantityUnit::Meter)] + #[case::image(Meter::Input(InputUnit::Image), QuantityUnit::Meter)] + #[case::character_ms(Meter::Input(InputUnit::Character), QuantityUnit::Milliseconds)] + fn rejects_units_that_do_not_measure_the_meter( + #[case] meter: Meter, + #[case] unit: QuantityUnit, + ) { + assert_eq!( + Quantity::measured(Measurement::new(1.5).unwrap(), unit, &meter), + Err(PriceEvaluationError::InvalidQuantityUnit), + ); + } + + #[rstest] + #[case::past_float_precision(9_007_199_254_740_993, 9_007_199_254_740_992)] + #[case::max_count(u64::MAX, u64::MAX - 1)] + fn counts_keep_exact_thresholds(#[case] count: u64, #[case] threshold: u64) { + assert!(Quantity::Count(count).exceeds(threshold)); + assert!(!Quantity::Count(threshold).exceeds(threshold)); + assert_eq!( + Quantity::Count(count).exceeds_scaled(threshold, 1), + Ok(true) + ); + } + + #[rstest] + fn scaled_measurement_overflow_is_rejected() { + assert_eq!( + Quantity::Measured(Measurement::new(f64::MAX).unwrap()).exceeds_scaled(128_000, 4), + Err(PriceEvaluationError::InvalidQuantity), + ); + } +} diff --git a/litellm-rust/crates/cost/src/selection.rs b/litellm-rust/crates/cost/src/selection.rs index fb948a24e3d..9c0ba83ae5a 100644 --- a/litellm-rust/crates/cost/src/selection.rs +++ b/litellm-rust/crates/cost/src/selection.rs @@ -13,6 +13,10 @@ pub enum SelectionPolicy<'a> { #[default] Generic, SdkTokens(&'a TokenPriceSources), + SdkOther { + tokens: &'a TokenPriceSources, + context: &'a crate::OtherContext, + }, DirectionalTokens { threshold: NonZeroU64, }, @@ -31,7 +35,32 @@ pub struct TokenTier { pub meters: BTreeSet, } -impl SelectionPolicy<'_> { +impl<'a> SelectionPolicy<'a> { + pub(crate) fn resolve_charge( + self, + prices: &Prices, + meter: &Meter, + band: Band, + usage: &Usage, + when: &When, + ) -> Option { + match self { + Self::SdkOther { tokens, context } => { + context.resolve(prices, meter, band, usage, when, tokens) + } + _ => self + .resolve(prices, meter, band, usage, when) + .map(Into::into), + } + } + + pub(crate) fn other(self) -> Option<&'a crate::OtherContext> { + match self { + Self::SdkOther { context, .. } => Some(context), + _ => None, + } + } + pub(crate) fn resolve( self, prices: &Prices, @@ -40,6 +69,11 @@ impl SelectionPolicy<'_> { usage: &Usage, when: &When, ) -> Option<(Meter, UnitPrice)> { + if let Self::SdkOther { .. } = self { + return self + .resolve_charge(prices, meter, band, usage, when) + .map(|charge| (charge.meter, charge.price)); + } if let Self::DirectionalTokens { threshold } = self { return directional_token_rate(prices, meter, usage, threshold); } @@ -172,8 +206,27 @@ fn directional_token_rate( Meter::Output(_) => Meter::Output(OutputUnit::Token(TokenKind::Text)), _ => Meter::TEXT_INPUT, }; + directional_rate( + prices, + source.clone(), + usage + .quantities + .get(&source) + .copied() + .unwrap_or(crate::Quantity::Count(0)), + threshold, + ) +} + +pub(crate) fn directional_rate( + prices: &Prices, + source: Meter, + quantity: crate::Quantity, + threshold: NonZeroU64, +) -> Option<(Meter, UnitPrice)> { let base = Price::online(source.clone(), Default::default()); - let rate = (usage.quantities.get(&source).copied().unwrap_or(0) > threshold.get()) + let rate = quantity + .exceeds(threshold.get()) .then(|| { prices.get(&Price::online( source.clone(), @@ -573,6 +626,7 @@ mod tests { (READ, 100), (WRITE, 100), ] + .map(|(meter, count)| (meter, crate::Quantity::Count(count))) .into(), prompt_tokens: prompt, inclusive_threshold: inclusive, @@ -661,7 +715,9 @@ mod tests { overlay(None, Some(9.0)), ); let usage = Usage { - quantities: [(INPUT, input), (OUTPUT, output)].into(), + quantities: [(INPUT, input), (OUTPUT, output)] + .map(|(meter, count)| (meter, crate::Quantity::Count(count))) + .into(), prompt_tokens: 200_000, inclusive_threshold: inclusive, ..Default::default() @@ -686,18 +742,26 @@ mod tests { evaluation.rates, vec![ crate::SelectedRate { + source_id: None, + caller_default: false, requested_meter: INPUT, resolved: Some((INPUT, UnitPrice::new(expected.0).unwrap())) }, crate::SelectedRate { + source_id: None, + caller_default: false, requested_meter: OUTPUT, resolved: Some((OUTPUT, UnitPrice::new(expected.1).unwrap())) }, crate::SelectedRate { + source_id: None, + caller_default: false, requested_meter: READ, resolved: Some((INPUT, UnitPrice::new(expected.0).unwrap())) }, crate::SelectedRate { + source_id: None, + caller_default: false, requested_meter: REASONING, resolved: Some((OUTPUT, UnitPrice::new(expected.1).unwrap())) }, @@ -724,7 +788,9 @@ mod tests { fn usage(meter: Meter, prompt_tokens: u64, inclusive_threshold: bool) -> Usage { Usage { - quantities: [(meter, 5)].into(), + quantities: [(meter, 5)] + .map(|(meter, count)| (meter, crate::Quantity::Count(count))) + .into(), prompt_tokens, inclusive_threshold, ..Default::default() @@ -737,8 +803,11 @@ mod tests { total: unit_price * 5, lines: vec![Line { fallback_from: (meter != original).then_some(original), + source_id: None, + caller_default: false, + report_to: None, meter, - quantity: 5, + quantity: 5.into(), unit_price, amount: unit_price * 5, }], @@ -1076,7 +1145,9 @@ mod tests { None, ); let usage = Usage { - quantities: [(Meter::TEXT_INPUT, 5), (REASONING, 5), (OUTPUT, 0)].into(), + quantities: [(Meter::TEXT_INPUT, 5), (REASONING, 5), (OUTPUT, 0)] + .map(|(meter, count)| (meter, crate::Quantity::Count(count))) + .into(), ..Default::default() }; assert_eq!( diff --git a/litellm-rust/crates/model-catalog/src/lib.rs b/litellm-rust/crates/model-catalog/src/lib.rs index 679e69d0269..8b1c6e312f6 100644 --- a/litellm-rust/crates/model-catalog/src/lib.rs +++ b/litellm-rust/crates/model-catalog/src/lib.rs @@ -11,6 +11,7 @@ mod fallback; mod layers; mod lower; mod model; +mod other; mod resolution; mod revalidate; mod vocabulary; diff --git a/litellm-rust/crates/model-catalog/src/lower.rs b/litellm-rust/crates/model-catalog/src/lower.rs index 048cbbec8fa..70d3bc2f5e3 100644 --- a/litellm-rust/crates/model-catalog/src/lower.rs +++ b/litellm-rust/crates/model-catalog/src/lower.rs @@ -220,7 +220,7 @@ fn dedupe( { let origin = match policy { PriceLowering::Generic => PriceSource::Flat, - PriceLowering::SdkTokens => origin, + PriceLowering::SdkTokens | PriceLowering::SdkOther => origin, }; match seen.entry((price, origin)) { Entry::Occupied(first) if first.get().0 == rate => {} @@ -563,7 +563,9 @@ mod tests { entry.attributes.litellm_provider = Some("test".into()); let model = Model::lower(entry, PriceLowering::SdkTokens).unwrap(); let usage = litellm_cost::Usage { - quantities: [(meter.clone(), 5)].into(), + quantities: [(meter.clone(), 5)] + .map(|(meter, count)| (meter, litellm_cost::Quantity::Count(count))) + .into(), prompt_tokens, ..Default::default() }; @@ -646,7 +648,7 @@ mod tests { ) { let expected = match policy { PriceLowering::Generic => generic_error, - PriceLowering::SdkTokens => sdk_error, + PriceLowering::SdkTokens | PriceLowering::SdkOther => sdk_error, }; let result = lower_prices(&wire(fields), policy); match (result, expected) { @@ -678,7 +680,9 @@ mod tests { let generic = Model::try_from(wire(fields.clone())).unwrap(); let sdk = Model::lower(wire(fields), PriceLowering::SdkTokens).unwrap(); let usage = litellm_cost::Usage { - quantities: [(Meter::Input(InputUnit::Second), 10)].into(), + quantities: [(Meter::Input(InputUnit::Second), 10)] + .map(|(meter, count)| (meter, litellm_cost::Quantity::Count(count))) + .into(), prompt_tokens: prompt, inclusive_threshold: inclusive, ..Default::default() diff --git a/litellm-rust/crates/model-catalog/src/model.rs b/litellm-rust/crates/model-catalog/src/model.rs index 94017e5bf38..f6c2f7187a4 100644 --- a/litellm-rust/crates/model-catalog/src/model.rs +++ b/litellm-rust/crates/model-catalog/src/model.rs @@ -22,9 +22,19 @@ pub enum PriceLowering { #[default] Generic, SdkTokens, + SdkOther, } impl Model { + pub fn from_fields_with_other_source( + fields: &Fields, + id: String, + ) -> Result<(Self, litellm_cost::OtherPriceSource), Vec> { + let wire = EntryWire::read_other(fields); + let source = crate::other::other_price_source(fields, &wire, id); + Self::lower(wire, PriceLowering::SdkOther).map(|model| (model, source)) + } + pub fn from_fields(fields: &Fields) -> Result> { Self::from_fields_with_policy(fields, PriceLowering::Generic) } @@ -33,6 +43,10 @@ impl Model { fields: &Fields, policy: PriceLowering, ) -> Result> { - Self::lower(EntryWire::read(fields), policy) + let wire = match policy { + PriceLowering::SdkOther => EntryWire::read_other(fields), + PriceLowering::Generic | PriceLowering::SdkTokens => EntryWire::read(fields), + }; + Self::lower(wire, policy) } } diff --git a/litellm-rust/crates/model-catalog/src/other.rs b/litellm-rust/crates/model-catalog/src/other.rs new file mode 100644 index 00000000000..d9a2de36720 --- /dev/null +++ b/litellm-rust/crates/model-catalog/src/other.rs @@ -0,0 +1,127 @@ +use std::num::NonZeroU64; + +use litellm_cost::price::{InputUnit, Meter, SearchContextSize}; +use litellm_cost::{OtherPriceSource, SearchTier}; + +use crate::wire::{EntryWire, Fields, FlatKey}; + +pub(crate) fn other_price_source( + fields: &Fields, + wire: &EntryWire, + id: String, +) -> OtherPriceSource { + let declared = fields.keys().filter_map(|key| { + FlatKey::parse_other(key).and_then(|(flat, _)| { + (flat.band == litellm_cost::price::Band::Base && flat.tier.is_none()) + .then_some(flat.meter) + }) + }); + let search = fields.get("search_context_cost_per_query"); + let search_declared = [ + ("search_context_size_low", SearchContextSize::Low), + ("search_context_size_medium", SearchContextSize::Medium), + ("search_context_size_high", SearchContextSize::High), + ] + .into_iter() + .filter_map(|(key, size)| { + search + .filter(|value| value.is_null() || value.get(key).is_some()) + .map(|_| Meter::WebSearch(size)) + }); + let search_tiers = wire + .tier_rows + .iter() + .flatten() + .filter_map(|row| { + let range = row.max_results_range?; + Some(SearchTier { + range, + meter: Meter::Input(InputUnit::Query { + over_results: NonZeroU64::new(range[0].saturating_sub(1)), + }), + }) + }) + .collect(); + OtherPriceSource { + id, + declared: declared.chain(search_declared).collect(), + search_tiers, + } +} + +#[cfg(test)] +mod tests { + use litellm_cost::price::{Feature, Price}; + use rstest::rstest; + use serde_json::{Value, json}; + + use super::*; + use crate::{Model, PriceLowering}; + + #[rstest] + #[case::configured(json!(2.5), Some(0.0025))] + #[case::zero(json!(0), Some(0.0))] + #[case::null(Value::Null, None)] + fn configured_file_search_is_per_occurrence_only_in_sdk_context( + #[case] value: Value, + #[case] expected: Option, + ) { + let fields = json!({"litellm_provider":"openai", "file_search_cost_per_1k_calls":value}) + .as_object() + .unwrap() + .clone(); + let price = Price::online(Meter::Feature(Feature::FileSearchCall), Default::default()); + let sdk = Model::from_fields_with_policy(&fields, PriceLowering::SdkOther).unwrap(); + let generic = Model::from_fields(&fields).unwrap(); + assert_eq!(sdk.prices.get(&price).map(|rate| rate.get()), expected); + assert_eq!(generic.prices.get(&price), None); + } + + #[rstest] + #[case::boolean(json!(true))] + #[case::negative(json!(-1.0))] + #[case::string(json!("bad"))] + fn new_tool_rates_reject_bad_values_only_in_sdk_context(#[case] value: Value) { + let fields = json!({"litellm_provider":"openai", "file_search_cost_per_1k_calls":value}) + .as_object() + .unwrap() + .clone(); + assert!(Model::from_fields_with_policy(&fields, PriceLowering::SdkOther).is_err()); + assert!(Model::from_fields(&fields).is_ok()); + } + + #[rstest] + #[case::nested_null(json!({"search_context_size_low":null}), true)] + #[case::parent_null(Value::Null, true)] + #[case::absent_child(json!({}), false)] + fn nested_search_declarations_preserve_absent_and_null( + #[case] value: Value, + #[case] declared: bool, + ) { + let fields = json!({"search_context_cost_per_query":value}) + .as_object() + .unwrap() + .clone(); + let source = other_price_source( + &fields, + &EntryWire::read_other(&fields), + "primary".to_owned(), + ); + assert_eq!( + source + .declared + .contains(&Meter::WebSearch(SearchContextSize::Low)), + declared + ); + } + + #[rstest] + fn scalar_search_remains_rejected_in_sdk_context() { + let fields = json!({"litellm_provider":"openai", "search_context_cost_per_query":2.0}) + .as_object() + .unwrap() + .clone(); + assert!(Model::from_fields_with_policy(&fields, PriceLowering::SdkOther).is_err()); + assert!(Model::from_fields(&fields).is_err()); + } +} diff --git a/litellm-rust/crates/model-catalog/src/wire/entry.rs b/litellm-rust/crates/model-catalog/src/wire/entry.rs index 8358715ef45..ef8d5cd5f6f 100644 --- a/litellm-rust/crates/model-catalog/src/wire/entry.rs +++ b/litellm-rust/crates/model-catalog/src/wire/entry.rs @@ -29,6 +29,28 @@ pub struct EntryWire { } impl EntryWire { + pub fn read_other(fields: &Fields) -> Self { + let mut wire = Self::read(fields); + let mut reader = Reader { + problems: &mut wire.problems, + }; + for (key, value) in fields.iter().filter(|(_, value)| !value.is_null()) { + if FlatKey::parse(key).is_some() { + continue; + } + let Some((flat, per_units)) = FlatKey::parse_other(key) else { + continue; + }; + wire.unknown.remove(key); + if let Some(mut price) = reader.price(key.clone(), pointer_for_key(key), flat, value) { + price.price = UnitPrice::new(price.price.get() / per_units as f64) + .expect("finite rate divided by a positive unit basis"); + wire.prices.push(price); + } + } + wire + } + pub fn read(fields: &Fields) -> Self { let mut problems = Vec::new(); let mut wire = Self::default(); diff --git a/litellm-rust/crates/model-catalog/src/wire/flat_key.rs b/litellm-rust/crates/model-catalog/src/wire/flat_key.rs index d7c7212cdf5..0b07863b04a 100644 --- a/litellm-rust/crates/model-catalog/src/wire/flat_key.rs +++ b/litellm-rust/crates/model-catalog/src/wire/flat_key.rs @@ -21,6 +21,30 @@ pub enum TierSuffix { } impl FlatKey { + pub fn parse_other(key: &str) -> Option<(Self, u64)> { + if let Some(flat) = Self::parse(key) { + return Some((flat, 1)); + } + let (feature, per_units) = match key { + "file_search_cost_per_1k_calls" => (Feature::FileSearchCall, 1000), + "file_search_cost_per_gb_per_day" => (Feature::FileSearchGbDay, 1), + "vector_store_cost_per_gb_per_day" => (Feature::VectorStoreGbDay, 1), + "computer_use_input_cost_per_1k_tokens" => (Feature::ComputerUseInputToken, 1000), + "computer_use_output_cost_per_1k_tokens" => (Feature::ComputerUseOutputToken, 1000), + "search_cost_per_result" => (Feature::SearchResult, 1), + "browser_open_cost_per_call" => (Feature::BrowserOpen, 1), + _ => return None, + }; + Some(( + Self { + meter: Meter::Feature(feature), + band: Band::Base, + tier: None, + }, + per_units, + )) + } + pub fn parse(key: &str) -> Option { let (rest, tier) = split_tier(key); let (base, suffix) = split_above(rest)?; diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index a62f1e727ca..d461b047002 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -32,9 +32,11 @@ mod _native { use crate::logger::NativeDiagnosticProcessor; #[pymodule_export] use crate::model_registry::{ - CatalogRefresher, FetchedRegistry, InvalidProvider, ModelCostLayers, ModelInfoView, - ModelNotFound, ModelRegistry, PriceEvaluation, PriceLine, Priced, RegistrationTarget, - RegistryNotModified, RegistryUnavailable, ResolvedModel, RowError, RowErrors, SelectedRate, + Additive, CatalogRefresher, DefaultRateInput, DirectionalCharacters, FetchedRegistry, + FirstUnit, InvalidProvider, MeasuredQuantity, ModelCostLayers, ModelInfoView, + ModelNotFound, ModelRegistry, Ocr, OtherMeterInput, PriceEvaluation, PriceLine, + PriceRowInput, Priced, PromptUnits, RegistrationTarget, RegistryNotModified, + RegistryUnavailable, ResolvedModel, RowError, RowErrors, SearchTiers, SelectedRate, Unpriced, UsageInput, WhenInput, fetch_once, fetch_with_retry, has_token_prices, invalidate_model_cost_index, model_info, model_row_errors, price, price_with_rates, registration_target, resolve_model, write_rows, @@ -144,6 +146,16 @@ assert live == {"model": {"litellm_provider": "openai", "input_cost_per_token": "RowErrors", "ModelInfoView", "UsageInput", + "MeasuredQuantity", + "PriceRowInput", + "DefaultRateInput", + "Additive", + "PromptUnits", + "FirstUnit", + "SearchTiers", + "Ocr", + "DirectionalCharacters", + "OtherMeterInput", "WhenInput", "PriceLine", "Priced", diff --git a/litellm-rust/crates/python-bridge/src/model_registry.rs b/litellm-rust/crates/python-bridge/src/model_registry.rs index c355013057d..4972639f277 100644 --- a/litellm-rust/crates/python-bridge/src/model_registry.rs +++ b/litellm-rust/crates/python-bridge/src/model_registry.rs @@ -1,7 +1,9 @@ mod live; +mod other; mod refresh; pub(crate) use live::*; +pub(crate) use other::*; pub(crate) use refresh::*; use std::collections::{BTreeMap, BTreeSet, HashMap}; diff --git a/litellm-rust/crates/python-bridge/src/model_registry/live.rs b/litellm-rust/crates/python-bridge/src/model_registry/live.rs index faed82a6453..ef101002b9a 100644 --- a/litellm-rust/crates/python-bridge/src/model_registry/live.rs +++ b/litellm-rust/crates/python-bridge/src/model_registry/live.rs @@ -16,6 +16,7 @@ use pyo3::prelude::*; use pyo3::types::{PyAny, PyBool, PyDict, PyModule, PySet, PyString}; use time::OffsetDateTime; +use super::other::{LegacyCount, MeasuredQuantity, OtherMeterInput, StrictCount}; use super::{RowError, escape_pointer_segment, fields_to_dict, nonfinite_errors, row_error}; struct LowercaseIndex { @@ -325,18 +326,34 @@ pub(crate) fn registration_target( #[pyclass(frozen, get_all, module = "litellm.rust_bridge._native")] pub(crate) struct RowErrors { - errors: Vec, + pub(super) errors: Vec, } fn lower_row(key: &str, row: &Bound<'_, PyAny>) -> Result<(Fields, Model), RowErrors> { lower_row_with_policy(key, row, litellm_model_catalog::PriceLowering::Generic) } -fn lower_row_with_policy( +pub(super) fn lower_row_with_policy( key: &str, row: &Bound<'_, PyAny>, policy: litellm_model_catalog::PriceLowering, ) -> Result<(Fields, Model), RowErrors> { + let fields = row_fields(key, row)?; + let model = Model::from_fields_with_policy(&fields, policy).map_err(|errors| RowErrors { + errors: errors + .into_iter() + .map(|error| { + row_error(&Located::new( + format!("/{}{}", escape_pointer_segment(key), error.path), + error.error, + )) + }) + .collect(), + })?; + Ok((fields, model)) +} + +pub(super) fn row_fields(key: &str, row: &Bound<'_, PyAny>) -> Result { let path = format!("/{}", escape_pointer_segment(key)); let located = |error| RowErrors { errors: vec![row_error(&Located::new(&path, error))], @@ -357,14 +374,7 @@ fn lower_row_with_policy( )) }; valid.map_err(|_| invalid())?; - let fields: Fields = from_py(row).map_err(|_| invalid())?; - let model = Model::from_fields_with_policy(&fields, policy).map_err(|errors| RowErrors { - errors: errors - .into_iter() - .map(|error| row_error(&Located::new(format!("{path}{}", error.path), error.error))) - .collect(), - })?; - Ok((fields, model)) + from_py(row).map_err(|_| invalid()) } #[pyclass(frozen, module = "litellm.rust_bridge._native")] @@ -468,31 +478,64 @@ pub(crate) fn model_info(model_cost: &Bound<'_, PyDict>, key: &str) -> PyResult< #[pyclass(frozen, module = "litellm.rust_bridge._native")] pub(crate) struct UsageInput { usage: Usage, + boolean_count: bool, } #[pymethods] impl UsageInput { #[new] - #[pyo3(signature = (quantities, *, prompt_tokens=0, inclusive_threshold=false, search_results=0))] + #[pyo3(signature = (quantities, *, prompt_tokens=LegacyCount::ZERO, inclusive_threshold=false, search_results=LegacyCount::ZERO, completion_tokens=None), text_signature = "(quantities, *, prompt_tokens=0, inclusive_threshold=False, search_results=0, completion_tokens=None)")] + fn py_new( + quantities: &Bound<'_, PyDict>, + prompt_tokens: LegacyCount, + inclusive_threshold: bool, + search_results: LegacyCount, + completion_tokens: Option, + ) -> PyResult { + let mut input = Self::new( + quantities, + prompt_tokens.value, + inclusive_threshold, + search_results.value, + )?; + input.boolean_count |= prompt_tokens.boolean || search_results.boolean; + input.usage.completion_tokens = completion_tokens.map(|count| count.0); + Ok(input) + } +} + +impl UsageInput { fn new( quantities: &Bound<'_, PyDict>, prompt_tokens: u64, inclusive_threshold: bool, search_results: u64, ) -> PyResult { + let boolean_count = quantities + .iter() + .any(|(_, value)| value.is_instance_of::()); let quantities = quantities .iter() .map(|(key, quantity)| { let key: String = key.extract()?; let meter = parse_meter(&key) .ok_or_else(|| PyValueError::new_err(format!("unknown usage meter: {key}")))?; - Ok((meter, quantity.extract::()?)) + let quantity = if quantity.is_instance_of::() { + quantity + .extract::>()? + .quantity(&meter)? + } else { + litellm_cost::Quantity::Count(quantity.extract::()?) + }; + Ok((meter, quantity)) }) .collect::>()?; Ok(Self { + boolean_count, usage: Usage { quantities, prompt_tokens, + completion_tokens: None, inclusive_threshold, search_results, }, @@ -500,7 +543,7 @@ impl UsageInput { } } -fn parse_meter(key: &str) -> Option { +pub(super) fn parse_meter(key: &str) -> Option { use litellm_cost::price::{GuardrailUnit, SearchContextSize}; if let Some(counter) = key.strip_prefix("guardrail/") { return (!counter.is_empty()).then(|| Meter::Guardrail(GuardrailUnit::new(counter))); @@ -509,7 +552,8 @@ fn parse_meter(key: &str) -> Option { "web_search/low" => Some(Meter::WebSearch(SearchContextSize::Low)), "web_search/medium" => Some(Meter::WebSearch(SearchContextSize::Medium)), "web_search/high" => Some(Meter::WebSearch(SearchContextSize::High)), - key => FlatKey::parse(key) + key => FlatKey::parse_other(key) + .map(|(flat, _)| flat) .filter(|flat| !flat.has_suffix()) .map(|flat| flat.meter), } @@ -556,10 +600,29 @@ impl WhenInput { #[derive(Clone)] pub(crate) struct PriceLine { meter: String, - quantity: u64, + quantity: PythonQuantity, unit_price: f64, amount: f64, fallback_from: Option, + source_id: Option, + caller_default: bool, + quantity_unit: String, + report_to: Option, +} + +#[derive(Clone, Copy, Debug, PartialEq, IntoPyObject)] +enum PythonQuantity { + Count(u64), + Measured(f64), +} + +impl From for PythonQuantity { + fn from(quantity: litellm_cost::Quantity) -> Self { + match quantity { + litellm_cost::Quantity::Count(count) => Self::Count(count), + litellm_cost::Quantity::Measured(value) => Self::Measured(value.get()), + } + } } #[pyclass(frozen, get_all, module = "litellm.rust_bridge._native")] @@ -586,11 +649,17 @@ impl From for Priced { .lines .into_iter() .map(|line| PriceLine { + quantity_unit: quantity_unit(&line.meter).to_owned(), meter: meter_name(&line.meter), - quantity: line.quantity, + quantity: line.quantity.into(), unit_price: line.unit_price.get(), amount: line.amount.get(), fallback_from: line.fallback_from.as_ref().map(meter_name), + source_id: line.source_id, + caller_default: line.caller_default, + report_to: line + .report_to + .map(|direction| <&'static str>::from(direction).to_owned()), }) .collect(), } @@ -615,6 +684,8 @@ pub(crate) struct SelectedRate { requested_meter: String, source_meter: Option, unit_price: Option, + source_id: Option, + caller_default: bool, } #[pyclass(frozen, get_all, module = "litellm.rust_bridge._native")] @@ -645,7 +716,18 @@ enum PricePolicy { } impl PricePolicy { - fn selection(self, model: &Model, threshold: Option) -> SelectionPolicy<'_> { + fn selection<'a>( + self, + model: &'a Model, + threshold: Option, + other: Option<&'a litellm_cost::OtherContext>, + ) -> SelectionPolicy<'a> { + if let Some(context) = other { + return SelectionPolicy::SdkOther { + tokens: &model.token_price_sources, + context, + }; + } if let Some(threshold) = threshold { return SelectionPolicy::DirectionalTokens { threshold }; } @@ -742,7 +824,7 @@ fn parse_directional_threshold( let threshold = NonZeroU64::new(value.extract()?) .ok_or_else(|| PyValueError::new_err("token threshold must be a positive integer"))?; if usage.quantities.iter().any(|(meter, quantity)| { - *quantity != 0 + !quantity.is_zero() && !matches!( meter, Meter::Input(InputUnit::Token(TokenKind::Text)) @@ -775,7 +857,7 @@ pub(crate) fn has_token_prices( } #[pyfunction] -#[pyo3(signature = (row, usage, when, *, policy="generic", cache_read_default_ratio=None, directional_token_threshold=None))] +#[pyo3(signature = (row, usage, when, *, policy="generic", cache_read_default_ratio=None, directional_token_threshold=None, other_meters=None))] pub(crate) fn price( row: &Bound<'_, PyAny>, usage: &UsageInput, @@ -783,6 +865,7 @@ pub(crate) fn price( policy: &str, cache_read_default_ratio: Option<&Bound<'_, PyAny>>, directional_token_threshold: Option<&Bound<'_, PyAny>>, + other_meters: Option<&OtherMeterInput>, ) -> PyResult { let policy = pricing_policy(policy)?; let ratio = cache_read_default(policy, cache_read_default_ratio)?; @@ -793,12 +876,31 @@ pub(crate) fn price( &when.when, ratio.is_some(), )?; - let model = match policy.lower(row, when.when.channel) { + validate_other_context(policy, usage, &when.when, threshold, other_meters)?; + let (model, other) = match lower_price_context(policy, row, when.when.channel, other_meters) { Ok(model) => model, Err(errors) => return Ok(PriceResult::Errors(errors)), }; let model = policy.prepare(model, when.when.channel, ratio)?; - let policy = policy.selection(&model, threshold); + let policy = policy.selection(&model, threshold, other.as_ref()); + if other.is_some() + || usage + .usage + .quantities + .values() + .any(|quantity| matches!(quantity, litellm_cost::Quantity::Measured(_))) + { + let evaluation = litellm_cost::calculate_with_rates( + &model.prices, + &usage.usage, + &when.when, + policy, + &[], + &[], + ) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + return price_result(row.py(), evaluation.pricing); + } price_result( row.py(), litellm_cost::calculate_with_policy(&model.prices, &usage.usage, &when.when, policy), @@ -817,8 +919,8 @@ fn price_result(py: Python<'_>, priced: litellm_cost::Priced) -> PyResult>, cache_read_default_ratio: Option<&Bound<'_, PyAny>>, directional_token_threshold: Option<&Bound<'_, PyAny>>, + other_meters: Option<&OtherMeterInput>, ) -> PyResult { let policy = pricing_policy(policy)?; let ratio = cache_read_default(policy, cache_read_default_ratio)?; @@ -863,7 +966,8 @@ pub(crate) fn price_with_rates( .ok_or_else(|| PyValueError::new_err("factors must be finite non-negative numbers")) }) .collect::>>()?; - let model = match policy.lower(row, when.when.channel) { + validate_other_context(policy, usage, &when.when, threshold, other_meters)?; + let (model, other) = match lower_price_context(policy, row, when.when.channel, other_meters) { Ok(model) => model, Err(errors) => return Ok(EvaluationResult::Errors(errors)), }; @@ -872,7 +976,7 @@ pub(crate) fn price_with_rates( &model.prices, &usage.usage, &when.when, - policy.selection(&model, threshold), + policy.selection(&model, threshold, other.as_ref()), &project, &factors, ) @@ -883,6 +987,8 @@ pub(crate) fn price_with_rates( .rates .into_iter() .map(|rate| SelectedRate { + source_id: rate.source_id, + caller_default: rate.caller_default, requested_meter: meter_name(&rate.requested_meter), source_meter: rate.resolved.as_ref().map(|(meter, _)| meter_name(meter)), unit_price: rate.resolved.map(|(_, price)| price.get()), @@ -891,6 +997,71 @@ pub(crate) fn price_with_rates( })) } +fn lower_price_context( + policy: PricePolicy, + row: &Bound<'_, PyAny>, + channel: Channel, + other: Option<&OtherMeterInput>, +) -> Result<(Model, Option), RowErrors> { + match other { + Some(other) => other + .prepare(row) + .map(|(model, context)| (model, Some(context))), + None => policy.lower(row, channel).map(|model| (model, None)), + } +} + +fn validate_other_context( + policy: PricePolicy, + usage: &UsageInput, + when: &When, + threshold: Option, + other: Option<&OtherMeterInput>, +) -> PyResult<()> { + if other.is_some() && (!matches!(policy, PricePolicy::SdkTokens) || threshold.is_some()) { + return Err(PyValueError::new_err( + "other_meters requires sdk_tokens and cannot be combined with directional_token_threshold", + )); + } + if other.is_some_and(OtherMeterInput::directional) && when.channel != Channel::Online { + return Err(PyValueError::new_err( + "directional characters requires online pricing", + )); + } + if usage.boolean_count + && (other.is_some() + || usage + .usage + .quantities + .values() + .any(|quantity| matches!(quantity, litellm_cost::Quantity::Measured(_)))) + { + return Err(PyValueError::new_err( + "boolean counts are not valid in measured or other_meters usage", + )); + } + Ok(()) +} + +fn quantity_unit(meter: &Meter) -> &'static str { + use litellm_cost::price::Feature; + match meter { + Meter::WallClockSecond + | Meter::Input( + InputUnit::Second | InputUnit::AudioSecond | InputUnit::VideoSecond { .. }, + ) + | Meter::Output(OutputUnit::Second(_) | OutputUnit::VideoSecond) => "seconds", + Meter::Input(InputUnit::Character) | Meter::Output(OutputUnit::Character) => "characters", + Meter::Feature(Feature::FileSearchGbDay | Feature::VectorStoreGbDay) => "gb_days", + Meter::Feature(Feature::FileSearchCall | Feature::BrowserOpen) => "calls", + Meter::Feature(Feature::ComputerUseInputToken | Feature::ComputerUseOutputToken) => { + "tokens" + } + meter if meter.is_token() => "tokens", + _ => "count", + } +} + fn meter_name(meter: &Meter) -> String { use litellm_cost::price::*; let token = |kind: &TokenKind| match kind { @@ -957,6 +1128,13 @@ fn meter_name(meter: &Meter) -> String { Feature::GoogleMapsGroundingQuery => "google_maps_grounding_cost_per_query", Feature::OcrCredit => "ocr_cost_per_credit", Feature::OcrPage => "ocr_cost_per_page", + Feature::FileSearchCall => "file_search_cost_per_1k_calls", + Feature::FileSearchGbDay => "file_search_cost_per_gb_per_day", + Feature::VectorStoreGbDay => "vector_store_cost_per_gb_per_day", + Feature::ComputerUseInputToken => "computer_use_input_cost_per_1k_tokens", + Feature::ComputerUseOutputToken => "computer_use_output_cost_per_1k_tokens", + Feature::SearchResult => "search_cost_per_result", + Feature::BrowserOpen => "browser_open_cost_per_call", } .into(), Meter::WebSearch(context) => format!( @@ -1587,6 +1765,7 @@ assert n.model_info(models, "Model").input_cost_per_token == 2 policy, None, None, + None, ) .unwrap() else { panic!("expected invalid row") @@ -2358,6 +2537,7 @@ with pytest.raises(ValueError): policy, None, None, + None, ) .unwrap(); match result { @@ -2470,6 +2650,7 @@ assert json.dumps(row) == before "sdk_tokens", None, None, + None, ) .unwrap(); let priced = match result { @@ -2805,6 +2986,7 @@ assert isinstance(n.resolve_model(rows, model, inferred=inferred), n.ModelNotFou "generic", None, None, + None, ) .unwrap() else { panic!("not priced") @@ -2832,6 +3014,7 @@ assert isinstance(n.resolve_model(rows, model, inferred=inferred), n.ModelNotFou "generic", None, None, + None, ) .unwrap() else { panic!("not priced") @@ -2923,6 +3106,7 @@ assert seen == ["released"] "generic", None, None, + None, ) .unwrap() else { panic!("not priced: {}", fields[0]) @@ -2983,7 +3167,9 @@ assert seen == ["released"] 0, ) .unwrap(); - match price(row.as_any(), &usage, &when, "generic", None, None).unwrap() { + match price(row.as_any(), &usage, &when, "generic", None, None, None) + .unwrap() + { PriceResult::Priced(result) => assert_eq!(result.total, expected), PriceResult::Unpriced(result) => { assert_eq!(expected, 0.0); @@ -3018,7 +3204,7 @@ assert seen == ["released"] json!({"litellm_provider":"test", "input_cost_per_token": 2}), ); let PriceResult::Unpriced(result) = - price(row.as_any(), &usage, &when, "generic", None, None).unwrap() + price(row.as_any(), &usage, &when, "generic", None, None, None).unwrap() else { panic!("expected unpriced") }; @@ -3035,7 +3221,7 @@ assert seen == ["released"] row.set_item("output_cost_per_token", 0).unwrap(); row.set_item("input_cost_per_audio_token", 0).unwrap(); let PriceResult::Priced(result) = - price(row.as_any(), &usage, &when, "generic", None, None).unwrap() + price(row.as_any(), &usage, &when, "generic", None, None, None).unwrap() else { panic!("expected free") }; @@ -3064,6 +3250,7 @@ assert seen == ["released"] "generic", None, None, + None, ) .unwrap() else { panic!("not priced") @@ -3077,7 +3264,7 @@ assert seen == ["released"] result.lines[0].amount, &result.lines[0].fallback_from ), - (3, 2.0, 6.0, &None) + (PythonQuantity::Count(3), 2.0, 6.0, &None) ); }); } diff --git a/litellm-rust/crates/python-bridge/src/model_registry/other.rs b/litellm-rust/crates/python-bridge/src/model_registry/other.rs new file mode 100644 index 00000000000..c483f389c32 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/model_registry/other.rs @@ -0,0 +1,443 @@ +use std::collections::BTreeSet; +use std::num::NonZeroU64; + +use litellm_cost::money::UnitPrice; +use litellm_cost::{ + DefaultEligibility, Measurement, OtherContext, OtherSelection, Quantity, QuantityUnit, ReportTo, +}; +use litellm_model_catalog::Model; +use litellm_model_catalog::wire::Fields; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::*; +use pyo3::types::{PyAny, PyBool, PyDict, PyMapping}; + +use super::live::{RowErrors, parse_meter, row_fields}; +use super::row_error; + +#[derive(Clone, Copy)] +pub(super) struct LegacyCount { + pub value: u64, + pub boolean: bool, +} + +impl LegacyCount { + pub const ZERO: Self = Self { + value: 0, + boolean: false, + }; +} + +impl FromPyObject<'_, '_> for LegacyCount { + type Error = PyErr; + fn extract(value: Borrowed<'_, '_, PyAny>) -> PyResult { + Ok(Self { + value: value.extract()?, + boolean: value.is_instance_of::(), + }) + } +} + +#[derive(Clone, Copy)] +pub(crate) struct StrictCount(pub u64); + +impl FromPyObject<'_, '_> for StrictCount { + type Error = PyErr; + + fn extract(value: Borrowed<'_, '_, PyAny>) -> PyResult { + if value.is_instance_of::() { + return Err(PyValueError::new_err( + "count must be an integer, not a boolean", + )); + } + value.extract::().map(Self) + } +} + +fn finite(value: &Bound<'_, PyAny>) -> PyResult { + if value.is_instance_of::() { + return Err(PyValueError::new_err( + "value must be finite and non-negative, not a boolean", + )); + } + let value = value.extract::()?; + Measurement::new(value).map_err(|error| PyValueError::new_err(error.to_string()))?; + Ok(value) +} + +#[pyclass(frozen, module = "litellm.rust_bridge._native")] +pub(crate) struct MeasuredQuantity { + value: Measurement, + unit: QuantityUnit, +} + +#[pymethods] +impl MeasuredQuantity { + #[new] + #[pyo3(signature = (value, *, unit="meter"))] + fn new(value: &Bound<'_, PyAny>, unit: &str) -> PyResult { + let value = Measurement::new(finite(value)?) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + let unit = unit + .parse() + .map_err(|_| PyValueError::new_err("unit must be meter or milliseconds"))?; + Ok(Self { value, unit }) + } +} + +impl MeasuredQuantity { + pub(super) fn quantity(&self, meter: &litellm_cost::price::Meter) -> PyResult { + Quantity::measured(self.value, self.unit, meter) + .map_err(|error| PyValueError::new_err(error.to_string())) + } +} + +#[pyclass(frozen, skip_from_py_object, module = "litellm.rust_bridge._native")] +#[derive(Clone)] +pub(crate) struct PriceRowInput { + source_id: String, + fields: Fields, +} + +#[pymethods] +impl PriceRowInput { + #[new] + fn new(source_id: String, row: &Bound<'_, PyMapping>) -> PyResult { + if source_id.trim().is_empty() || source_id == "primary" { + return Err(PyValueError::new_err( + "source_id must be nonempty and cannot be primary", + )); + } + let snapshot = row.py().get_type::().call1((row,))?; + let fields = row_fields(&source_id, &snapshot).map_err(|_| { + PyValueError::new_err("fallback row must contain finite JSON-compatible values") + })?; + Ok(Self { source_id, fields }) + } +} + +#[pyclass(frozen, skip_from_py_object, module = "litellm.rust_bridge._native")] +#[derive(Clone)] +pub(crate) struct DefaultRateInput { + meter: litellm_cost::price::Meter, + rate: UnitPrice, + per_units: NonZeroU64, + eligibility: DefaultEligibility, +} + +#[pymethods] +impl DefaultRateInput { + #[new] + #[pyo3(signature = (meter, value, *, per_units=StrictCount(1), apply_when), text_signature = "(meter, value, *, per_units=1, apply_when)")] + fn new( + meter: &str, + value: &Bound<'_, PyAny>, + per_units: StrictCount, + apply_when: &str, + ) -> PyResult { + let meter = + parse_meter(meter).ok_or_else(|| PyValueError::new_err("unknown default meter"))?; + let rate = UnitPrice::new(finite(value)?) + .ok_or_else(|| PyValueError::new_err("invalid default rate"))?; + let per_units = NonZeroU64::new(per_units.0) + .ok_or_else(|| PyValueError::new_err("per_units must be positive"))?; + let eligibility = apply_when + .parse() + .map_err(|_| PyValueError::new_err("apply_when must be absent or absent_or_null"))?; + Ok(Self { + meter, + rate, + per_units, + eligibility, + }) + } +} + +#[pyclass(frozen, module = "litellm.rust_bridge._native")] +pub(crate) struct Additive; +#[pymethods] +impl Additive { + #[new] + fn new() -> Self { + Self + } +} + +#[pyclass(frozen, module = "litellm.rust_bridge._native")] +pub(crate) struct PromptUnits; +#[pymethods] +impl PromptUnits { + #[new] + fn new() -> Self { + Self + } +} + +#[pyclass(frozen, module = "litellm.rust_bridge._native")] +pub(crate) struct Ocr; +#[pymethods] +impl Ocr { + #[new] + fn new() -> Self { + Self + } +} + +#[pyclass(frozen, module = "litellm.rust_bridge._native")] +pub(crate) struct SearchTiers; +#[pymethods] +impl SearchTiers { + #[new] + #[pyo3(signature = (*, outside="last"))] + fn new(outside: &str) -> PyResult { + if outside != "last" { + return Err(PyValueError::new_err("outside must be last")); + } + Ok(Self) + } +} + +#[pyclass(frozen, module = "litellm.rust_bridge._native")] +pub(crate) struct FirstUnit { + meters: Vec, +} +#[pymethods] +impl FirstUnit { + #[new] + fn new(meters: Vec) -> PyResult { + let meters = meters + .iter() + .map(|meter| { + parse_meter(meter).ok_or_else(|| PyValueError::new_err("unknown alternative meter")) + }) + .collect::>>()?; + if meters.is_empty() || meters.iter().collect::>().len() != meters.len() { + return Err(PyValueError::new_err( + "alternative meters must be nonempty and unique", + )); + } + Ok(Self { meters }) + } +} + +#[pyclass(frozen, module = "litellm.rust_bridge._native")] +pub(crate) struct DirectionalCharacters { + threshold: NonZeroU64, + comparison_scale: NonZeroU64, +} +#[pymethods] +impl DirectionalCharacters { + #[new] + fn new(threshold: StrictCount, comparison_scale: StrictCount) -> PyResult { + let positive = |value| { + NonZeroU64::new(value).ok_or_else(|| { + PyValueError::new_err("threshold and comparison_scale must be positive") + }) + }; + Ok(Self { + threshold: positive(threshold.0)?, + comparison_scale: positive(comparison_scale.0)?, + }) + } +} + +#[derive(FromPyObject)] +enum SelectionInput<'py> { + Additive(PyRef<'py, Additive>), + PromptUnits(PyRef<'py, PromptUnits>), + FirstUnit(PyRef<'py, FirstUnit>), + SearchTiers(PyRef<'py, SearchTiers>), + Ocr(PyRef<'py, Ocr>), + DirectionalCharacters(PyRef<'py, DirectionalCharacters>), +} + +#[pyclass(frozen, module = "litellm.rust_bridge._native")] +pub(crate) struct OtherMeterInput { + selection: OtherSelection, + sources: Vec, + defaults: Vec, + report_to: Option, +} + +#[pymethods] +impl OtherMeterInput { + #[new] + #[pyo3(signature = (selection, *, fallback_sources=Vec::new(), defaults=Vec::new(), report_to=None), text_signature = "(selection, *, fallback_sources=(), defaults=(), report_to=None)")] + fn new( + selection: SelectionInput<'_>, + fallback_sources: Vec>, + defaults: Vec>, + report_to: Option<&str>, + ) -> PyResult { + let selection = match selection { + SelectionInput::Additive(value) => { + drop(value); + OtherSelection::Additive + } + SelectionInput::PromptUnits(value) => { + drop(value); + OtherSelection::PromptUnits + } + SelectionInput::FirstUnit(value) => OtherSelection::FirstUnit(value.meters.clone()), + SelectionInput::SearchTiers(value) => { + drop(value); + OtherSelection::SearchTiers + } + SelectionInput::Ocr(value) => { + drop(value); + OtherSelection::Ocr + } + SelectionInput::DirectionalCharacters(value) => OtherSelection::DirectionalCharacters { + threshold: value.threshold, + comparison_scale: value.comparison_scale, + }, + }; + if fallback_sources + .iter() + .map(|source| &source.source_id) + .collect::>() + .len() + != fallback_sources.len() + { + return Err(PyValueError::new_err("fallback source IDs must be unique")); + } + if defaults + .iter() + .map(|rate| &rate.meter) + .collect::>() + .len() + != defaults.len() + { + return Err(PyValueError::new_err("default meters must be unique")); + } + let report_to = report_to + .map(|value| { + value + .parse() + .map_err(|_| PyValueError::new_err("report_to must be input or output")) + }) + .transpose()?; + Ok(Self { + selection, + sources: fallback_sources + .into_iter() + .map(|source| (*source).clone()) + .collect(), + defaults: defaults.into_iter().map(|rate| (*rate).clone()).collect(), + report_to, + }) + } +} + +impl OtherMeterInput { + pub(super) fn directional(&self) -> bool { + matches!(self.selection, OtherSelection::DirectionalCharacters { .. }) + } + + pub(super) fn prepare( + &self, + row: &Bound<'_, PyAny>, + ) -> Result<(Model, OtherContext), RowErrors> { + let fields = row_fields("", row)?; + let errors = |errors: Vec| RowErrors { + errors: errors.iter().map(row_error).collect(), + }; + let (mut model, primary) = + Model::from_fields_with_other_source(&fields, "primary".to_owned()).map_err(errors)?; + let mut context = OtherContext { + selection: self.selection.clone(), + sources: vec![primary], + report_to: self.report_to, + ..OtherContext::default() + }; + for source in &self.sources { + let (fallback, metadata) = + Model::from_fields_with_other_source(&source.fields, source.source_id.clone()) + .map_err(errors)?; + model + .prices + .append_source(context.sources.len(), fallback.prices); + context.sources.push(metadata); + } + for default in &self.defaults { + context.insert_default( + &mut model.prices, + default.meter.clone(), + default.rate, + default.per_units, + default.eligibility, + ); + } + Ok((model, context)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rstest::rstest; + use std::ffi::CString; + + #[rstest] + #[case::zero("0", true)] + #[case::fraction("1.5", true)] + #[case::negative("-1", false)] + #[case::boolean("True", false)] + #[case::string("'1.5'", false)] + #[case::nan("float('nan')", false)] + #[case::infinite("float('inf')", false)] + #[case::none("None", false)] + fn measured_constructor_validates_python_values( + #[case] expression: &str, + #[case] accepted: bool, + ) { + Python::attach(|py| { + let value = py + .eval(&CString::new(expression).unwrap(), None, None) + .unwrap(); + assert_eq!(MeasuredQuantity::new(&value, "meter").is_ok(), accepted); + }); + } + + #[rstest] + #[case::zero("0", Some(0))] + #[case::large_exact("9007199254740993", Some(9007199254740993))] + #[case::maximum("18446744073709551615", Some(u64::MAX))] + #[case::overflow("18446744073709551616", None)] + #[case::negative("-1", None)] + #[case::fraction("1.5", None)] + #[case::boolean("True", None)] + fn new_context_counts_are_strict_and_exact( + #[case] expression: &str, + #[case] expected: Option, + ) { + Python::attach(|py| { + let value = py + .eval(&CString::new(expression).unwrap(), None, None) + .unwrap(); + assert_eq!( + value.extract::().ok().map(|count| count.0), + expected + ); + }); + } + + #[rstest] + #[case::finite("{'input_cost_per_token': 1.0}", true)] + #[case::nan("{'input_cost_per_token': float('nan')}", false)] + #[case::infinite("{'input_cost_per_token': float('inf')}", false)] + fn fallback_source_rejects_nonfinite_before_json_conversion( + #[case] expression: &str, + #[case] accepted: bool, + ) { + Python::attach(|py| { + let value = py + .eval(&CString::new(expression).unwrap(), None, None) + .unwrap(); + assert_eq!( + PriceRowInput::new("published".to_owned(), value.cast::().unwrap()) + .is_ok(), + accepted + ); + }); + } +} diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi index 06be18cd5e3..fccd2912202 100644 --- a/litellm/rust_bridge/_native.pyi +++ b/litellm/rust_bridge/_native.pyi @@ -192,17 +192,67 @@ class ModelInfoView: @property def output_cost_per_token(self) -> float | None: ... +@final +class MeasuredQuantity: + def __new__(cls, value: float, *, unit: Literal["meter", "milliseconds"] = "meter") -> MeasuredQuantity: ... + +@final +class PriceRowInput: + def __new__(cls, source_id: str, row: Mapping[str, JsonValue]) -> PriceRowInput: ... + +@final +class DefaultRateInput: + def __new__( + cls, meter: str, value: float, *, per_units: int = 1, apply_when: Literal["absent", "absent_or_null"] + ) -> DefaultRateInput: ... + +@final +class Additive: + def __new__(cls) -> Additive: ... + +@final +class PromptUnits: + def __new__(cls) -> PromptUnits: ... + +@final +class FirstUnit: + def __new__(cls, meters: Sequence[str]) -> FirstUnit: ... + +@final +class SearchTiers: + def __new__(cls, *, outside: Literal["last"] = "last") -> SearchTiers: ... + +@final +class Ocr: + def __new__(cls) -> Ocr: ... + +@final +class DirectionalCharacters: + def __new__(cls, threshold: int, comparison_scale: int) -> DirectionalCharacters: ... + +@final +class OtherMeterInput: + def __new__( + cls, + selection: Additive | PromptUnits | FirstUnit | SearchTiers | Ocr | DirectionalCharacters, + *, + fallback_sources: Sequence[PriceRowInput] = (), + defaults: Sequence[DefaultRateInput] = (), + report_to: Literal["input", "output"] | None = None, + ) -> OtherMeterInput: ... + @final class UsageInput: """Disjoint quantities keyed by unsuffixed catalog price fields, web_search/{size}, or guardrail/{counter}.""" def __new__( cls, - quantities: dict[str, int], + quantities: dict[str, int | MeasuredQuantity], *, prompt_tokens: int = 0, inclusive_threshold: bool = False, search_results: int = 0, + completion_tokens: int | None = None, ) -> UsageInput: """prompt_tokens is the full prompt size used for band selection, not the uncached input quantity.""" @@ -222,7 +272,15 @@ class PriceLine: @property def meter(self) -> str: ... @property - def quantity(self) -> int: ... + def quantity(self) -> int | float: ... + @property + def quantity_unit(self) -> str: ... + @property + def source_id(self) -> str | None: ... + @property + def caller_default(self) -> bool: ... + @property + def report_to(self) -> str | None: ... @property def unit_price(self) -> float: ... @property @@ -250,6 +308,10 @@ class Unpriced: @final class SelectedRate: + @property + def source_id(self) -> str | None: ... + @property + def caller_default(self) -> bool: ... @property def requested_meter(self) -> str: ... @property @@ -302,6 +364,7 @@ def price( policy: Literal["generic", "sdk_tokens"] = "generic", cache_read_default_ratio: float | None = None, directional_token_threshold: int | None = None, + other_meters: OtherMeterInput | None = None, ) -> Priced | Unpriced | RowErrors: ... def price_with_rates( row: dict[str, object], @@ -313,6 +376,7 @@ def price_with_rates( factors: Sequence[float] = (), cache_read_default_ratio: float | None = None, directional_token_threshold: int | None = None, + other_meters: OtherMeterInput | None = None, ) -> PriceEvaluation | RowErrors: ... def trace_encode_error(message: str) -> bytes: ... @@ -504,11 +568,16 @@ def process_state_started() -> bool: ... def reserve_process_for_forking() -> None: ... __all__ = [ + "Additive", "CatalogRefresher", + "DefaultRateInput", + "DirectionalCharacters", "FetchedRegistry", + "FirstUnit", "ForkedAfterNativeRuntimeStarted", "HuggingFaceEncoding", "InvalidProvider", + "MeasuredQuantity", "ModelCostLayers", "ModelInfoView", "ModelNotFound", @@ -516,10 +585,14 @@ __all__ = [ "NativeClickHouseSpendConfig", "NativeClickHouseSpendStorage", "NativeDiagnosticProcessor", + "Ocr", + "OtherMeterInput", "PriceEvaluation", "PriceLine", + "PriceRowInput", "Priced", "ProcessReservedForForking", + "PromptUnits", "RegistrationTarget", "RegistryNotModified", "RegistryUnavailable", @@ -530,6 +603,7 @@ __all__ = [ "RowErrors", "RustBridgeDeclined", "RustUpstreamError", + "SearchTiers", "SelectedRate", "TokenCounter", "Tokenizer",