mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
feat(cost): add typed SDK non-token pricing
This commit is contained in:
parent
7479e494b0
commit
14de0c099d
22 changed files with 2429 additions and 94 deletions
|
|
@ -40,7 +40,9 @@ fn bench(c: &mut Criterion) {
|
|||
None,
|
||||
);
|
||||
let usage = Usage {
|
||||
quantities: [(input, 750), (output, 200), (cache, 250)].into(),
|
||||
quantities: [(input, 750), (output, 200), (cache, 250)]
|
||||
.map(|(meter, count)| (meter, litellm_cost::Quantity::Count(count)))
|
||||
.into(),
|
||||
prompt_tokens: 1000,
|
||||
..Usage::default()
|
||||
};
|
||||
|
|
|
|||
|
|
@ -29,7 +29,9 @@ fn main() {
|
|||
None,
|
||||
);
|
||||
let usage = Usage {
|
||||
quantities: [(input, 75), (cache, 25), (output, 20)].into(),
|
||||
quantities: [(input, 75), (cache, 25), (output, 20)]
|
||||
.map(|(meter, count)| (meter, litellm_cost::Quantity::Count(count)))
|
||||
.into(),
|
||||
prompt_tokens: 100,
|
||||
..Usage::default()
|
||||
};
|
||||
|
|
|
|||
|
|
@ -2,15 +2,16 @@ use std::collections::BTreeMap;
|
|||
|
||||
use time::OffsetDateTime;
|
||||
|
||||
use crate::PriceEvaluationError;
|
||||
use crate::money::{ScaleFactor, UnitPrice, Usd};
|
||||
use crate::price::{Band, Channel, InputUnit, Meter, OutputUnit, Prices, ServiceTier};
|
||||
use crate::selection::SelectionPolicy;
|
||||
use crate::{PriceEvaluationError, Quantity};
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
#[derive(Clone, Debug, Default, PartialEq)]
|
||||
pub struct Usage {
|
||||
pub quantities: BTreeMap<Meter, u64>,
|
||||
pub quantities: BTreeMap<Meter, Quantity>,
|
||||
pub prompt_tokens: u64,
|
||||
pub completion_tokens: Option<u64>,
|
||||
pub inclusive_threshold: bool,
|
||||
pub search_results: u64,
|
||||
}
|
||||
|
|
@ -40,16 +41,21 @@ pub struct Breakdown {
|
|||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct Line {
|
||||
pub meter: Meter,
|
||||
pub quantity: u64,
|
||||
pub quantity: Quantity,
|
||||
pub unit_price: UnitPrice,
|
||||
pub amount: Usd,
|
||||
pub fallback_from: Option<Meter>,
|
||||
pub source_id: Option<String>,
|
||||
pub caller_default: bool,
|
||||
pub report_to: Option<crate::ReportTo>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct SelectedRate {
|
||||
pub requested_meter: Meter,
|
||||
pub resolved: Option<(Meter, UnitPrice)>,
|
||||
pub source_id: Option<String>,
|
||||
pub caller_default: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
|
|
@ -69,6 +75,14 @@ pub fn calculate_with_rates(
|
|||
if project.iter().any(|meter| !meter.is_token()) {
|
||||
return Err(PriceEvaluationError::NonTokenProjection);
|
||||
}
|
||||
for (meter, quantity) in &usage.quantities {
|
||||
if let Quantity::Measured(value) = quantity {
|
||||
Quantity::measured(*value, crate::QuantityUnit::Meter, meter)?;
|
||||
}
|
||||
}
|
||||
if let Some(context) = policy.other() {
|
||||
context.validate_usage(usage)?;
|
||||
}
|
||||
let factor = ScaleFactor::combine(factors).ok_or(PriceEvaluationError::NonFiniteValue)?;
|
||||
let band = prices.channel_band(
|
||||
when.channel,
|
||||
|
|
@ -79,18 +93,21 @@ pub fn calculate_with_rates(
|
|||
let rates = project
|
||||
.iter()
|
||||
.map(|meter| {
|
||||
let resolved = policy
|
||||
.resolve(prices, meter, band, usage, when)
|
||||
.map(|(source, rate)| {
|
||||
let charge = policy.resolve_charge(prices, meter, band, usage, when);
|
||||
let resolved = charge
|
||||
.as_ref()
|
||||
.map(|charge| {
|
||||
factor
|
||||
.price(rate)
|
||||
.map(|rate| (source, rate))
|
||||
.price(charge.price)
|
||||
.map(|rate| (charge.meter.clone(), rate))
|
||||
.ok_or(PriceEvaluationError::NonFiniteValue)
|
||||
})
|
||||
.transpose()?;
|
||||
Ok(SelectedRate {
|
||||
requested_meter: meter.clone(),
|
||||
resolved,
|
||||
source_id: charge.as_ref().and_then(|charge| charge.source_id.clone()),
|
||||
caller_default: charge.is_some_and(|charge| charge.caller_default),
|
||||
})
|
||||
})
|
||||
.collect::<Result<_, PriceEvaluationError>>()?;
|
||||
|
|
@ -133,7 +150,7 @@ impl Breakdown {
|
|||
pub fn input(&self) -> Usd {
|
||||
self.lines
|
||||
.iter()
|
||||
.filter(|line| !matches!(line.meter, Meter::Output(_)))
|
||||
.filter(|line| !line.is_output())
|
||||
.map(|line| line.amount)
|
||||
.sum()
|
||||
}
|
||||
|
|
@ -141,12 +158,25 @@ impl Breakdown {
|
|||
pub fn output(&self) -> Usd {
|
||||
self.lines
|
||||
.iter()
|
||||
.filter(|line| matches!(line.meter, Meter::Output(_)))
|
||||
.filter(|line| line.is_output())
|
||||
.map(|line| line.amount)
|
||||
.sum()
|
||||
}
|
||||
}
|
||||
|
||||
impl Line {
|
||||
fn is_output(&self) -> bool {
|
||||
match self.report_to {
|
||||
Some(crate::ReportTo::Input) => false,
|
||||
Some(crate::ReportTo::Output) => true,
|
||||
None => matches!(
|
||||
self.meter,
|
||||
Meter::Output(_) | Meter::Feature(crate::price::Feature::ComputerUseOutputToken)
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn calculate(prices: &Prices, usage: &Usage, when: &When) -> Priced {
|
||||
calculate_with_policy(prices, usage, when, SelectionPolicy::Generic)
|
||||
}
|
||||
|
|
@ -166,17 +196,36 @@ pub fn calculate_with_policy(
|
|||
let mut lines = Vec::new();
|
||||
let mut missing = Vec::new();
|
||||
for (meter, &quantity) in &usage.quantities {
|
||||
if quantity == 0 {
|
||||
if quantity.is_zero() {
|
||||
continue;
|
||||
}
|
||||
let selected = select_meter(prices, meter, quantity, usage, when);
|
||||
match policy.resolve(prices, &selected, band, usage, when) {
|
||||
Some((priced_meter, unit_price)) => lines.push(Line {
|
||||
fallback_from: (priced_meter != selected).then(|| meter.clone()),
|
||||
meter: priced_meter,
|
||||
quantity,
|
||||
unit_price,
|
||||
amount: unit_price * quantity,
|
||||
if policy
|
||||
.other()
|
||||
.is_some_and(|context| !context.charges(prices, meter, usage, when))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let selected = if policy
|
||||
.other()
|
||||
.is_some_and(|context| matches!(context.selection, crate::OtherSelection::SearchTiers))
|
||||
{
|
||||
meter.clone()
|
||||
} else {
|
||||
select_meter(prices, meter, quantity, usage, when)
|
||||
};
|
||||
match policy.resolve_charge(prices, &selected, band, usage, when) {
|
||||
Some(charge) => lines.push(Line {
|
||||
fallback_from: (charge.meter != selected).then(|| meter.clone()),
|
||||
meter: charge.meter,
|
||||
quantity: charge.quantity.unwrap_or(quantity),
|
||||
unit_price: charge.price,
|
||||
amount: charge.price * charge.quantity.unwrap_or(quantity),
|
||||
source_id: charge.source_id,
|
||||
caller_default: charge.caller_default,
|
||||
report_to: policy
|
||||
.other()
|
||||
.filter(|_| !meter.is_token())
|
||||
.and_then(|context| context.report_to),
|
||||
}),
|
||||
None => missing.push(meter.clone()),
|
||||
}
|
||||
|
|
@ -198,7 +247,7 @@ pub fn calculate_with_policy(
|
|||
fn select_meter(
|
||||
prices: &Prices,
|
||||
meter: &Meter,
|
||||
quantity: u64,
|
||||
quantity: Quantity,
|
||||
usage: &Usage,
|
||||
when: &When,
|
||||
) -> Meter {
|
||||
|
|
@ -228,7 +277,7 @@ fn select_meter(
|
|||
Meter::Input(InputUnit::VideoSecond {
|
||||
over_seconds: Some(floor),
|
||||
}),
|
||||
) if quantity > floor.get() => Some((floor, &price.meter)),
|
||||
) if quantity.exceeds(floor.get()) => Some((floor, &price.meter)),
|
||||
_ => None,
|
||||
})
|
||||
.max_by_key(|(floor, _)| *floor)
|
||||
|
|
@ -336,7 +385,9 @@ mod tests {
|
|||
.chain(output.map(|rate| (OUTPUT, rate))),
|
||||
);
|
||||
let usage = Usage {
|
||||
quantities: BTreeMap::from([(INPUT, 10)]),
|
||||
quantities: [(INPUT, 10)]
|
||||
.map(|(meter, count)| (meter, crate::Quantity::Count(count)))
|
||||
.into(),
|
||||
..Usage::default()
|
||||
};
|
||||
let evaluated = calculate_with_rates(
|
||||
|
|
@ -351,6 +402,8 @@ mod tests {
|
|||
assert_eq!(
|
||||
evaluated.rates,
|
||||
vec![SelectedRate {
|
||||
source_id: None,
|
||||
caller_default: false,
|
||||
requested_meter: OUTPUT,
|
||||
resolved: output.map(|rate| (OUTPUT, UnitPrice::new(rate * factor).unwrap()))
|
||||
}]
|
||||
|
|
@ -361,10 +414,13 @@ mod tests {
|
|||
total: Usd::new(10.0 * factor).unwrap(),
|
||||
lines: vec![Line {
|
||||
meter: INPUT,
|
||||
quantity: 10,
|
||||
quantity: 10.into(),
|
||||
unit_price: UnitPrice::new(factor).unwrap(),
|
||||
amount: Usd::new(10.0 * factor).unwrap(),
|
||||
fallback_from: None
|
||||
fallback_from: None,
|
||||
source_id: None,
|
||||
caller_default: false,
|
||||
report_to: None,
|
||||
}],
|
||||
})
|
||||
);
|
||||
|
|
@ -376,7 +432,9 @@ mod tests {
|
|||
fn projection_scales_partial_without_inventing_missing_rates(when: When, #[case] factor: f64) {
|
||||
let prices = prices([(INPUT, 1.0)]);
|
||||
let usage = Usage {
|
||||
quantities: BTreeMap::from([(INPUT, 10), (OUTPUT, 5)]),
|
||||
quantities: [(INPUT, 10), (OUTPUT, 5)]
|
||||
.map(|(meter, count)| (meter, crate::Quantity::Count(count)))
|
||||
.into(),
|
||||
..Usage::default()
|
||||
};
|
||||
let evaluated = calculate_with_rates(
|
||||
|
|
@ -391,6 +449,8 @@ mod tests {
|
|||
assert_eq!(
|
||||
evaluated.rates,
|
||||
vec![SelectedRate {
|
||||
source_id: None,
|
||||
caller_default: false,
|
||||
requested_meter: OUTPUT,
|
||||
resolved: None
|
||||
}]
|
||||
|
|
@ -401,7 +461,7 @@ mod tests {
|
|||
assert_eq!(missing, vec![OUTPUT]);
|
||||
assert_eq!(partial.total.get(), 10.0 * factor);
|
||||
assert_eq!(partial.lines.len(), 1);
|
||||
assert_eq!(partial.lines[0].quantity, 10);
|
||||
assert_eq!(partial.lines[0].quantity, Quantity::Count(10));
|
||||
assert_eq!(partial.lines[0].unit_price.get(), factor);
|
||||
}
|
||||
|
||||
|
|
@ -416,7 +476,9 @@ mod tests {
|
|||
) {
|
||||
let prices = prices([(INPUT, f64::MAX)]);
|
||||
let usage = Usage {
|
||||
quantities: BTreeMap::from([(INPUT, quantity)]),
|
||||
quantities: [(INPUT, quantity)]
|
||||
.map(|(meter, count)| (meter, crate::Quantity::Count(count)))
|
||||
.into(),
|
||||
..Usage::default()
|
||||
};
|
||||
let projection = if project { vec![INPUT] } else { Vec::new() };
|
||||
|
|
@ -520,6 +582,7 @@ mod tests {
|
|||
),
|
||||
(meter.clone(), quantity),
|
||||
]
|
||||
.map(|(meter, count)| (meter, crate::Quantity::Count(count)))
|
||||
.into(),
|
||||
prompt_tokens: 1000,
|
||||
..Usage::default()
|
||||
|
|
@ -538,7 +601,7 @@ mod tests {
|
|||
.find(|line| line.fallback_from == Some(meter.clone()))
|
||||
.unwrap();
|
||||
assert_eq!(fallback.meter, base);
|
||||
assert_eq!(fallback.quantity, quantity);
|
||||
assert_eq!(fallback.quantity, Quantity::Count(quantity));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
|
|
@ -551,7 +614,9 @@ mod tests {
|
|||
fn an_explicit_zero_is_a_priced_line_not_a_fallback(when: When, #[case] meter: Meter) {
|
||||
let row = prices([(INPUT, 1.0), (OUTPUT, 2.0), (meter.clone(), 0.0)]);
|
||||
let usage = Usage {
|
||||
quantities: [(meter.clone(), 100)].into(),
|
||||
quantities: [(meter.clone(), 100)]
|
||||
.map(|(meter, count)| (meter, crate::Quantity::Count(count)))
|
||||
.into(),
|
||||
..Usage::default()
|
||||
};
|
||||
let result = known(calculate(&row, &usage, &when));
|
||||
|
|
@ -560,10 +625,13 @@ mod tests {
|
|||
result.lines,
|
||||
[Line {
|
||||
meter,
|
||||
quantity: 100,
|
||||
quantity: 100.into(),
|
||||
unit_price: UnitPrice::new(0.0).unwrap(),
|
||||
amount: Usd::ZERO,
|
||||
fallback_from: None,
|
||||
source_id: None,
|
||||
caller_default: false,
|
||||
report_to: None,
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
|
@ -572,7 +640,9 @@ mod tests {
|
|||
fn every_missing_meter_is_reported_and_priced_lines_survive(when: When) {
|
||||
let audio = Meter::Input(InputUnit::Token(TokenKind::Audio));
|
||||
let usage = Usage {
|
||||
quantities: [(INPUT, 3), (OUTPUT, 4), (audio.clone(), 2)].into(),
|
||||
quantities: [(INPUT, 3), (OUTPUT, 4), (audio.clone(), 2)]
|
||||
.map(|(meter, count)| (meter, crate::Quantity::Count(count)))
|
||||
.into(),
|
||||
..Usage::default()
|
||||
};
|
||||
let Priced::Unpriced { missing, partial } =
|
||||
|
|
@ -584,14 +654,16 @@ mod tests {
|
|||
assert_eq!(partial.total.get(), 8.0);
|
||||
assert_eq!(partial.lines.len(), 1);
|
||||
assert_eq!(partial.lines[0].meter, OUTPUT);
|
||||
assert_eq!(partial.lines[0].quantity, 4);
|
||||
assert_eq!(partial.lines[0].quantity, Quantity::Count(4));
|
||||
assert_eq!(partial.output().get(), 8.0);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn an_unpriced_row_cannot_fall_back_to_zero(when: When) {
|
||||
let usage = Usage {
|
||||
quantities: [(INPUT, 1000), (OUTPUT, 1000)].into(),
|
||||
quantities: [(INPUT, 1000), (OUTPUT, 1000)]
|
||||
.map(|(meter, count)| (meter, crate::Quantity::Count(count)))
|
||||
.into(),
|
||||
..Usage::default()
|
||||
};
|
||||
assert_eq!(
|
||||
|
|
@ -609,7 +681,9 @@ mod tests {
|
|||
#[rstest]
|
||||
fn unused_meters_need_no_price(when: When) {
|
||||
let usage = Usage {
|
||||
quantities: [(INPUT, 0), (OUTPUT, 0)].into(),
|
||||
quantities: [(INPUT, 0), (OUTPUT, 0)]
|
||||
.map(|(meter, count)| (meter, crate::Quantity::Count(count)))
|
||||
.into(),
|
||||
..Usage::default()
|
||||
};
|
||||
assert_eq!(
|
||||
|
|
@ -624,7 +698,9 @@ mod tests {
|
|||
#[rstest]
|
||||
fn one_hour_writes_fall_back_to_the_default_write_price(when: When) {
|
||||
let usage = Usage {
|
||||
quantities: [(WRITE, 4), (HOUR, 6)].into(),
|
||||
quantities: [(WRITE, 4), (HOUR, 6)]
|
||||
.map(|(meter, count)| (meter, crate::Quantity::Count(count)))
|
||||
.into(),
|
||||
..Usage::default()
|
||||
};
|
||||
let result = known(calculate(
|
||||
|
|
@ -673,7 +749,9 @@ mod tests {
|
|||
None,
|
||||
);
|
||||
let usage = Usage {
|
||||
quantities: [(INPUT, 101)].into(),
|
||||
quantities: [(INPUT, 101)]
|
||||
.map(|(meter, count)| (meter, crate::Quantity::Count(count)))
|
||||
.into(),
|
||||
prompt_tokens: 101,
|
||||
..Usage::default()
|
||||
};
|
||||
|
|
@ -702,7 +780,9 @@ mod tests {
|
|||
#[case] meter: Meter,
|
||||
) {
|
||||
let usage = Usage {
|
||||
quantities: [(meter.clone(), 7)].into(),
|
||||
quantities: [(meter.clone(), 7)]
|
||||
.map(|(meter, count)| (meter, crate::Quantity::Count(count)))
|
||||
.into(),
|
||||
..Usage::default()
|
||||
};
|
||||
let result = known(calculate(&prices([(meter.clone(), 0.25)]), &usage, &when));
|
||||
|
|
@ -728,7 +808,9 @@ mod tests {
|
|||
over_results: NonZeroU64::new(10),
|
||||
});
|
||||
let usage = Usage {
|
||||
quantities: [(query.clone(), 2)].into(),
|
||||
quantities: [(query.clone(), 2)]
|
||||
.map(|(meter, count)| (meter, crate::Quantity::Count(count)))
|
||||
.into(),
|
||||
search_results: results,
|
||||
..Usage::default()
|
||||
};
|
||||
|
|
@ -754,7 +836,9 @@ mod tests {
|
|||
over_seconds: NonZeroU64::new(10),
|
||||
});
|
||||
let usage = Usage {
|
||||
quantities: [(base.clone(), seconds)].into(),
|
||||
quantities: [(base.clone(), seconds)]
|
||||
.map(|(meter, count)| (meter, crate::Quantity::Count(count)))
|
||||
.into(),
|
||||
..Usage::default()
|
||||
};
|
||||
assert_eq!(
|
||||
|
|
@ -796,7 +880,9 @@ mod tests {
|
|||
),
|
||||
);
|
||||
let usage = Usage {
|
||||
quantities: [(INPUT, 2), (READ, 4), (HOUR, 7), (OUTPUT, 2)].into(),
|
||||
quantities: [(INPUT, 2), (READ, 4), (HOUR, 7), (OUTPUT, 2)]
|
||||
.map(|(meter, count)| (meter, crate::Quantity::Count(count)))
|
||||
.into(),
|
||||
..Usage::default()
|
||||
};
|
||||
let result = known(calculate(
|
||||
|
|
@ -855,7 +941,9 @@ mod tests {
|
|||
None,
|
||||
);
|
||||
let usage = Usage {
|
||||
quantities: [(INPUT, 4), (OUTPUT, 2)].into(),
|
||||
quantities: [(INPUT, 4), (OUTPUT, 2)]
|
||||
.map(|(meter, count)| (meter, crate::Quantity::Count(count)))
|
||||
.into(),
|
||||
..Usage::default()
|
||||
};
|
||||
assert_eq!(
|
||||
|
|
@ -876,7 +964,9 @@ mod tests {
|
|||
#[rstest]
|
||||
fn absent_batch_rates_use_half_online_base_rates(when: When) {
|
||||
let usage = Usage {
|
||||
quantities: [(INPUT, 4), (OUTPUT, 2), (READ, 6)].into(),
|
||||
quantities: [(INPUT, 4), (OUTPUT, 2), (READ, 6)]
|
||||
.map(|(meter, count)| (meter, crate::Quantity::Count(count)))
|
||||
.into(),
|
||||
..Usage::default()
|
||||
};
|
||||
let result = known(calculate(
|
||||
|
|
@ -929,7 +1019,9 @@ mod tests {
|
|||
None,
|
||||
);
|
||||
let usage = Usage {
|
||||
quantities: [(INPUT, 2), (READ, 3), (OUTPUT, 4)].into(),
|
||||
quantities: [(INPUT, 2), (READ, 3), (OUTPUT, 4)]
|
||||
.map(|(meter, count)| (meter, crate::Quantity::Count(count)))
|
||||
.into(),
|
||||
prompt_tokens,
|
||||
inclusive_threshold,
|
||||
..Usage::default()
|
||||
|
|
@ -951,7 +1043,9 @@ mod tests {
|
|||
fn dedicated_modality_and_long_ttl_rates_win(when: When) {
|
||||
let audio = Meter::Input(InputUnit::Token(TokenKind::Audio));
|
||||
let usage = Usage {
|
||||
quantities: [(INPUT, 10), (audio.clone(), 5), (WRITE, 3), (HOUR, 2)].into(),
|
||||
quantities: [(INPUT, 10), (audio.clone(), 5), (WRITE, 3), (HOUR, 2)]
|
||||
.map(|(meter, count)| (meter, crate::Quantity::Count(count)))
|
||||
.into(),
|
||||
..Usage::default()
|
||||
};
|
||||
let result = known(calculate(
|
||||
|
|
@ -973,7 +1067,9 @@ mod tests {
|
|||
#[rstest]
|
||||
fn decisions_prices_are_isolated_from_online_and_batch(when: When) {
|
||||
let usage = Usage {
|
||||
quantities: [(INPUT, 4)].into(),
|
||||
quantities: [(INPUT, 4)]
|
||||
.map(|(meter, count)| (meter, crate::Quantity::Count(count)))
|
||||
.into(),
|
||||
..Usage::default()
|
||||
};
|
||||
assert!(matches!(
|
||||
|
|
@ -1056,6 +1152,7 @@ mod tests {
|
|||
(READ, count(3)),
|
||||
(WRITE, count(4)),
|
||||
]
|
||||
.map(|(meter, count)| (meter, crate::Quantity::Count(count)))
|
||||
.into(),
|
||||
prompt_tokens: count(1),
|
||||
..Usage::default()
|
||||
|
|
|
|||
|
|
@ -6,6 +6,10 @@ pub enum PriceEvaluationError {
|
|||
NonTokenProjection,
|
||||
#[error("pricing produced a non-finite amount or rate")]
|
||||
NonFiniteValue,
|
||||
#[error("quantity must be finite and non-negative")]
|
||||
InvalidQuantity,
|
||||
#[error("measured quantity unit is not valid for this meter")]
|
||||
InvalidQuantityUnit,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
|
|
|
|||
|
|
@ -1,10 +1,14 @@
|
|||
mod calculate;
|
||||
mod error;
|
||||
pub mod money;
|
||||
mod other;
|
||||
pub mod price;
|
||||
mod quantity;
|
||||
pub mod schedule;
|
||||
mod selection;
|
||||
|
||||
pub use calculate::*;
|
||||
pub use error::*;
|
||||
pub use other::*;
|
||||
pub use quantity::*;
|
||||
pub use selection::*;
|
||||
|
|
|
|||
|
|
@ -77,6 +77,17 @@ impl Mul<u64> for UnitPrice {
|
|||
}
|
||||
}
|
||||
|
||||
impl Mul<crate::Quantity> for UnitPrice {
|
||||
type Output = Usd;
|
||||
|
||||
fn mul(self, units: crate::Quantity) -> Usd {
|
||||
match units {
|
||||
crate::Quantity::Count(count) => self * count,
|
||||
crate::Quantity::Measured(value) => Usd(self.0 * value.get()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Mul<Multiplier> for UnitPrice {
|
||||
type Output = UnitPrice;
|
||||
|
||||
|
|
|
|||
1006
litellm-rust/crates/cost/src/other.rs
Normal file
1006
litellm-rust/crates/cost/src/other.rs
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -58,6 +58,12 @@ pub enum PriceSource {
|
|||
#[default]
|
||||
Flat,
|
||||
TokenTier(u64),
|
||||
RowFlat(usize),
|
||||
RowTokenTier {
|
||||
row: usize,
|
||||
floor: u64,
|
||||
},
|
||||
CallerDefault(usize),
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
|
|
@ -143,6 +149,46 @@ impl Prices {
|
|||
self.off_peak.as_ref()
|
||||
}
|
||||
|
||||
pub fn append_source(&mut self, row: usize, prices: Self) {
|
||||
self.rates
|
||||
.extend(prices.rates.into_iter().map(|((price, origin), rate)| {
|
||||
let origin = match origin {
|
||||
PriceSource::Flat => PriceSource::RowFlat(row),
|
||||
PriceSource::TokenTier(floor) => PriceSource::RowTokenTier { row, floor },
|
||||
other => other,
|
||||
};
|
||||
((price, origin), rate)
|
||||
}));
|
||||
}
|
||||
|
||||
pub fn source_flat(&self, row: usize, price: &Price) -> Option<UnitPrice> {
|
||||
self.get_from(
|
||||
price,
|
||||
if row == 0 {
|
||||
PriceSource::Flat
|
||||
} else {
|
||||
PriceSource::RowFlat(row)
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
pub fn insert_default(&mut self, index: usize, meter: Meter, rate: UnitPrice) {
|
||||
self.rates.insert(
|
||||
(
|
||||
Price::online(meter, When::default()),
|
||||
PriceSource::CallerDefault(index),
|
||||
),
|
||||
rate,
|
||||
);
|
||||
}
|
||||
|
||||
pub fn default_rate(&self, index: usize, meter: &Meter) -> Option<UnitPrice> {
|
||||
self.get_from(
|
||||
&Price::online(meter.clone(), When::default()),
|
||||
PriceSource::CallerDefault(index),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn with_cache_read_default(
|
||||
mut self,
|
||||
ratio: ScaleFactor,
|
||||
|
|
@ -212,7 +258,9 @@ impl Prices {
|
|||
inclusive: bool,
|
||||
) -> Band {
|
||||
Self::find_band(
|
||||
self.rates.keys().map(|(price, _)| price),
|
||||
self.rates.keys().filter_map(|(price, source)| {
|
||||
matches!(source, PriceSource::Flat | PriceSource::TokenTier(_)).then_some(price)
|
||||
}),
|
||||
channel,
|
||||
tier,
|
||||
prompt_tokens,
|
||||
|
|
|
|||
|
|
@ -72,6 +72,13 @@ pub enum Feature {
|
|||
GoogleMapsGroundingQuery,
|
||||
OcrCredit,
|
||||
OcrPage,
|
||||
FileSearchCall,
|
||||
FileSearchGbDay,
|
||||
VectorStoreGbDay,
|
||||
ComputerUseInputToken,
|
||||
ComputerUseOutputToken,
|
||||
SearchResult,
|
||||
BrowserOpen,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
|
|
|
|||
173
litellm-rust/crates/cost/src/quantity.rs
Normal file
173
litellm-rust/crates/cost/src/quantity.rs
Normal file
|
|
@ -0,0 +1,173 @@
|
|||
use crate::PriceEvaluationError;
|
||||
use crate::price::{Feature, InputUnit, Meter, OutputUnit};
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub enum Quantity {
|
||||
Count(u64),
|
||||
Measured(Measurement),
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct Measurement(f64);
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::EnumString, strum::VariantArray)]
|
||||
pub enum QuantityUnit {
|
||||
#[strum(serialize = "meter")]
|
||||
Meter,
|
||||
#[strum(serialize = "milliseconds")]
|
||||
Milliseconds,
|
||||
}
|
||||
|
||||
impl Measurement {
|
||||
pub fn new(value: f64) -> Result<Self, PriceEvaluationError> {
|
||||
if !value.is_finite() || value < 0.0 {
|
||||
return Err(PriceEvaluationError::InvalidQuantity);
|
||||
}
|
||||
Ok(Self(value))
|
||||
}
|
||||
|
||||
pub fn get(self) -> f64 {
|
||||
self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl Quantity {
|
||||
pub fn measured(
|
||||
value: Measurement,
|
||||
unit: QuantityUnit,
|
||||
meter: &Meter,
|
||||
) -> Result<Self, PriceEvaluationError> {
|
||||
let seconds = matches!(
|
||||
meter,
|
||||
Meter::WallClockSecond
|
||||
| Meter::Input(
|
||||
InputUnit::Second | InputUnit::AudioSecond | InputUnit::VideoSecond { .. }
|
||||
)
|
||||
| Meter::Output(OutputUnit::Second(_) | OutputUnit::VideoSecond)
|
||||
);
|
||||
let characters = matches!(
|
||||
meter,
|
||||
Meter::Input(InputUnit::Character) | Meter::Output(OutputUnit::Character)
|
||||
);
|
||||
let storage = matches!(
|
||||
meter,
|
||||
Meter::Feature(Feature::FileSearchGbDay | Feature::VectorStoreGbDay)
|
||||
);
|
||||
match unit {
|
||||
QuantityUnit::Meter if seconds || characters || storage => Ok(Self::Measured(value)),
|
||||
QuantityUnit::Milliseconds if seconds => {
|
||||
Measurement::new(value.get() / 1000.0).map(Self::Measured)
|
||||
}
|
||||
_ => Err(PriceEvaluationError::InvalidQuantityUnit),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_zero(self) -> bool {
|
||||
match self {
|
||||
Self::Count(value) => value == 0,
|
||||
Self::Measured(value) => value.get() == 0.0,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn exceeds(self, threshold: u64) -> bool {
|
||||
match self {
|
||||
Self::Count(value) => value > threshold,
|
||||
Self::Measured(value) => value.get() > threshold as f64,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn exceeds_scaled(self, threshold: u64, scale: u64) -> Result<bool, PriceEvaluationError> {
|
||||
match self {
|
||||
Self::Count(value) => Ok(u128::from(value) * u128::from(scale) > u128::from(threshold)),
|
||||
Self::Measured(value) => {
|
||||
let scaled = Measurement::new(value.get() * scale as f64)?;
|
||||
Ok(scaled.get() > threshold as f64)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<u64> for Quantity {
|
||||
fn from(value: u64) -> Self {
|
||||
Self::Count(value)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use super::*;
|
||||
use crate::price::TokenKind;
|
||||
|
||||
#[rstest]
|
||||
#[case::zero(0.0)]
|
||||
#[case::fraction(1.5)]
|
||||
#[case::largest(f64::MAX)]
|
||||
fn admits_nonnegative_finite_measurements(#[case] value: f64) {
|
||||
assert_eq!(Measurement::new(value).unwrap().get(), value);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::negative(-1.0)]
|
||||
#[case::nan(f64::NAN)]
|
||||
#[case::infinity(f64::INFINITY)]
|
||||
#[case::negative_infinity(f64::NEG_INFINITY)]
|
||||
fn rejects_invalid_measurements(#[case] value: f64) {
|
||||
assert_eq!(
|
||||
Measurement::new(value),
|
||||
Err(PriceEvaluationError::InvalidQuantity)
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn normalizes_milliseconds_to_the_same_duration() {
|
||||
let seconds = Quantity::measured(
|
||||
Measurement::new(1.5).unwrap(),
|
||||
QuantityUnit::Meter,
|
||||
&Meter::WallClockSecond,
|
||||
)
|
||||
.unwrap();
|
||||
let millis = Quantity::measured(
|
||||
Measurement::new(1500.0).unwrap(),
|
||||
QuantityUnit::Milliseconds,
|
||||
&Meter::WallClockSecond,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(seconds, millis);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::token(Meter::Input(InputUnit::Token(TokenKind::Text)), QuantityUnit::Meter)]
|
||||
#[case::image(Meter::Input(InputUnit::Image), QuantityUnit::Meter)]
|
||||
#[case::character_ms(Meter::Input(InputUnit::Character), QuantityUnit::Milliseconds)]
|
||||
fn rejects_units_that_do_not_measure_the_meter(
|
||||
#[case] meter: Meter,
|
||||
#[case] unit: QuantityUnit,
|
||||
) {
|
||||
assert_eq!(
|
||||
Quantity::measured(Measurement::new(1.5).unwrap(), unit, &meter),
|
||||
Err(PriceEvaluationError::InvalidQuantityUnit),
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::past_float_precision(9_007_199_254_740_993, 9_007_199_254_740_992)]
|
||||
#[case::max_count(u64::MAX, u64::MAX - 1)]
|
||||
fn counts_keep_exact_thresholds(#[case] count: u64, #[case] threshold: u64) {
|
||||
assert!(Quantity::Count(count).exceeds(threshold));
|
||||
assert!(!Quantity::Count(threshold).exceeds(threshold));
|
||||
assert_eq!(
|
||||
Quantity::Count(count).exceeds_scaled(threshold, 1),
|
||||
Ok(true)
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn scaled_measurement_overflow_is_rejected() {
|
||||
assert_eq!(
|
||||
Quantity::Measured(Measurement::new(f64::MAX).unwrap()).exceeds_scaled(128_000, 4),
|
||||
Err(PriceEvaluationError::InvalidQuantity),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -13,6 +13,10 @@ pub enum SelectionPolicy<'a> {
|
|||
#[default]
|
||||
Generic,
|
||||
SdkTokens(&'a TokenPriceSources),
|
||||
SdkOther {
|
||||
tokens: &'a TokenPriceSources,
|
||||
context: &'a crate::OtherContext,
|
||||
},
|
||||
DirectionalTokens {
|
||||
threshold: NonZeroU64,
|
||||
},
|
||||
|
|
@ -31,7 +35,32 @@ pub struct TokenTier {
|
|||
pub meters: BTreeSet<Meter>,
|
||||
}
|
||||
|
||||
impl SelectionPolicy<'_> {
|
||||
impl<'a> SelectionPolicy<'a> {
|
||||
pub(crate) fn resolve_charge(
|
||||
self,
|
||||
prices: &Prices,
|
||||
meter: &Meter,
|
||||
band: Band,
|
||||
usage: &Usage,
|
||||
when: &When,
|
||||
) -> Option<crate::ResolvedCharge> {
|
||||
match self {
|
||||
Self::SdkOther { tokens, context } => {
|
||||
context.resolve(prices, meter, band, usage, when, tokens)
|
||||
}
|
||||
_ => self
|
||||
.resolve(prices, meter, band, usage, when)
|
||||
.map(Into::into),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn other(self) -> Option<&'a crate::OtherContext> {
|
||||
match self {
|
||||
Self::SdkOther { context, .. } => Some(context),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn resolve(
|
||||
self,
|
||||
prices: &Prices,
|
||||
|
|
@ -40,6 +69,11 @@ impl SelectionPolicy<'_> {
|
|||
usage: &Usage,
|
||||
when: &When,
|
||||
) -> Option<(Meter, UnitPrice)> {
|
||||
if let Self::SdkOther { .. } = self {
|
||||
return self
|
||||
.resolve_charge(prices, meter, band, usage, when)
|
||||
.map(|charge| (charge.meter, charge.price));
|
||||
}
|
||||
if let Self::DirectionalTokens { threshold } = self {
|
||||
return directional_token_rate(prices, meter, usage, threshold);
|
||||
}
|
||||
|
|
@ -172,8 +206,27 @@ fn directional_token_rate(
|
|||
Meter::Output(_) => Meter::Output(OutputUnit::Token(TokenKind::Text)),
|
||||
_ => Meter::TEXT_INPUT,
|
||||
};
|
||||
directional_rate(
|
||||
prices,
|
||||
source.clone(),
|
||||
usage
|
||||
.quantities
|
||||
.get(&source)
|
||||
.copied()
|
||||
.unwrap_or(crate::Quantity::Count(0)),
|
||||
threshold,
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn directional_rate(
|
||||
prices: &Prices,
|
||||
source: Meter,
|
||||
quantity: crate::Quantity,
|
||||
threshold: NonZeroU64,
|
||||
) -> Option<(Meter, UnitPrice)> {
|
||||
let base = Price::online(source.clone(), Default::default());
|
||||
let rate = (usage.quantities.get(&source).copied().unwrap_or(0) > threshold.get())
|
||||
let rate = quantity
|
||||
.exceeds(threshold.get())
|
||||
.then(|| {
|
||||
prices.get(&Price::online(
|
||||
source.clone(),
|
||||
|
|
@ -573,6 +626,7 @@ mod tests {
|
|||
(READ, 100),
|
||||
(WRITE, 100),
|
||||
]
|
||||
.map(|(meter, count)| (meter, crate::Quantity::Count(count)))
|
||||
.into(),
|
||||
prompt_tokens: prompt,
|
||||
inclusive_threshold: inclusive,
|
||||
|
|
@ -661,7 +715,9 @@ mod tests {
|
|||
overlay(None, Some(9.0)),
|
||||
);
|
||||
let usage = Usage {
|
||||
quantities: [(INPUT, input), (OUTPUT, output)].into(),
|
||||
quantities: [(INPUT, input), (OUTPUT, output)]
|
||||
.map(|(meter, count)| (meter, crate::Quantity::Count(count)))
|
||||
.into(),
|
||||
prompt_tokens: 200_000,
|
||||
inclusive_threshold: inclusive,
|
||||
..Default::default()
|
||||
|
|
@ -686,18 +742,26 @@ mod tests {
|
|||
evaluation.rates,
|
||||
vec![
|
||||
crate::SelectedRate {
|
||||
source_id: None,
|
||||
caller_default: false,
|
||||
requested_meter: INPUT,
|
||||
resolved: Some((INPUT, UnitPrice::new(expected.0).unwrap()))
|
||||
},
|
||||
crate::SelectedRate {
|
||||
source_id: None,
|
||||
caller_default: false,
|
||||
requested_meter: OUTPUT,
|
||||
resolved: Some((OUTPUT, UnitPrice::new(expected.1).unwrap()))
|
||||
},
|
||||
crate::SelectedRate {
|
||||
source_id: None,
|
||||
caller_default: false,
|
||||
requested_meter: READ,
|
||||
resolved: Some((INPUT, UnitPrice::new(expected.0).unwrap()))
|
||||
},
|
||||
crate::SelectedRate {
|
||||
source_id: None,
|
||||
caller_default: false,
|
||||
requested_meter: REASONING,
|
||||
resolved: Some((OUTPUT, UnitPrice::new(expected.1).unwrap()))
|
||||
},
|
||||
|
|
@ -724,7 +788,9 @@ mod tests {
|
|||
|
||||
fn usage(meter: Meter, prompt_tokens: u64, inclusive_threshold: bool) -> Usage {
|
||||
Usage {
|
||||
quantities: [(meter, 5)].into(),
|
||||
quantities: [(meter, 5)]
|
||||
.map(|(meter, count)| (meter, crate::Quantity::Count(count)))
|
||||
.into(),
|
||||
prompt_tokens,
|
||||
inclusive_threshold,
|
||||
..Default::default()
|
||||
|
|
@ -737,8 +803,11 @@ mod tests {
|
|||
total: unit_price * 5,
|
||||
lines: vec![Line {
|
||||
fallback_from: (meter != original).then_some(original),
|
||||
source_id: None,
|
||||
caller_default: false,
|
||||
report_to: None,
|
||||
meter,
|
||||
quantity: 5,
|
||||
quantity: 5.into(),
|
||||
unit_price,
|
||||
amount: unit_price * 5,
|
||||
}],
|
||||
|
|
@ -1076,7 +1145,9 @@ mod tests {
|
|||
None,
|
||||
);
|
||||
let usage = Usage {
|
||||
quantities: [(Meter::TEXT_INPUT, 5), (REASONING, 5), (OUTPUT, 0)].into(),
|
||||
quantities: [(Meter::TEXT_INPUT, 5), (REASONING, 5), (OUTPUT, 0)]
|
||||
.map(|(meter, count)| (meter, crate::Quantity::Count(count)))
|
||||
.into(),
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ mod fallback;
|
|||
mod layers;
|
||||
mod lower;
|
||||
mod model;
|
||||
mod other;
|
||||
mod resolution;
|
||||
mod revalidate;
|
||||
mod vocabulary;
|
||||
|
|
|
|||
|
|
@ -220,7 +220,7 @@ fn dedupe(
|
|||
{
|
||||
let origin = match policy {
|
||||
PriceLowering::Generic => PriceSource::Flat,
|
||||
PriceLowering::SdkTokens => origin,
|
||||
PriceLowering::SdkTokens | PriceLowering::SdkOther => origin,
|
||||
};
|
||||
match seen.entry((price, origin)) {
|
||||
Entry::Occupied(first) if first.get().0 == rate => {}
|
||||
|
|
@ -563,7 +563,9 @@ mod tests {
|
|||
entry.attributes.litellm_provider = Some("test".into());
|
||||
let model = Model::lower(entry, PriceLowering::SdkTokens).unwrap();
|
||||
let usage = litellm_cost::Usage {
|
||||
quantities: [(meter.clone(), 5)].into(),
|
||||
quantities: [(meter.clone(), 5)]
|
||||
.map(|(meter, count)| (meter, litellm_cost::Quantity::Count(count)))
|
||||
.into(),
|
||||
prompt_tokens,
|
||||
..Default::default()
|
||||
};
|
||||
|
|
@ -646,7 +648,7 @@ mod tests {
|
|||
) {
|
||||
let expected = match policy {
|
||||
PriceLowering::Generic => generic_error,
|
||||
PriceLowering::SdkTokens => sdk_error,
|
||||
PriceLowering::SdkTokens | PriceLowering::SdkOther => sdk_error,
|
||||
};
|
||||
let result = lower_prices(&wire(fields), policy);
|
||||
match (result, expected) {
|
||||
|
|
@ -678,7 +680,9 @@ mod tests {
|
|||
let generic = Model::try_from(wire(fields.clone())).unwrap();
|
||||
let sdk = Model::lower(wire(fields), PriceLowering::SdkTokens).unwrap();
|
||||
let usage = litellm_cost::Usage {
|
||||
quantities: [(Meter::Input(InputUnit::Second), 10)].into(),
|
||||
quantities: [(Meter::Input(InputUnit::Second), 10)]
|
||||
.map(|(meter, count)| (meter, litellm_cost::Quantity::Count(count)))
|
||||
.into(),
|
||||
prompt_tokens: prompt,
|
||||
inclusive_threshold: inclusive,
|
||||
..Default::default()
|
||||
|
|
|
|||
|
|
@ -22,9 +22,19 @@ pub enum PriceLowering {
|
|||
#[default]
|
||||
Generic,
|
||||
SdkTokens,
|
||||
SdkOther,
|
||||
}
|
||||
|
||||
impl Model {
|
||||
pub fn from_fields_with_other_source(
|
||||
fields: &Fields,
|
||||
id: String,
|
||||
) -> Result<(Self, litellm_cost::OtherPriceSource), Vec<Located>> {
|
||||
let wire = EntryWire::read_other(fields);
|
||||
let source = crate::other::other_price_source(fields, &wire, id);
|
||||
Self::lower(wire, PriceLowering::SdkOther).map(|model| (model, source))
|
||||
}
|
||||
|
||||
pub fn from_fields(fields: &Fields) -> Result<Self, Vec<Located>> {
|
||||
Self::from_fields_with_policy(fields, PriceLowering::Generic)
|
||||
}
|
||||
|
|
@ -33,6 +43,10 @@ impl Model {
|
|||
fields: &Fields,
|
||||
policy: PriceLowering,
|
||||
) -> Result<Self, Vec<Located>> {
|
||||
Self::lower(EntryWire::read(fields), policy)
|
||||
let wire = match policy {
|
||||
PriceLowering::SdkOther => EntryWire::read_other(fields),
|
||||
PriceLowering::Generic | PriceLowering::SdkTokens => EntryWire::read(fields),
|
||||
};
|
||||
Self::lower(wire, policy)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
127
litellm-rust/crates/model-catalog/src/other.rs
Normal file
127
litellm-rust/crates/model-catalog/src/other.rs
Normal file
|
|
@ -0,0 +1,127 @@
|
|||
use std::num::NonZeroU64;
|
||||
|
||||
use litellm_cost::price::{InputUnit, Meter, SearchContextSize};
|
||||
use litellm_cost::{OtherPriceSource, SearchTier};
|
||||
|
||||
use crate::wire::{EntryWire, Fields, FlatKey};
|
||||
|
||||
pub(crate) fn other_price_source(
|
||||
fields: &Fields,
|
||||
wire: &EntryWire,
|
||||
id: String,
|
||||
) -> OtherPriceSource {
|
||||
let declared = fields.keys().filter_map(|key| {
|
||||
FlatKey::parse_other(key).and_then(|(flat, _)| {
|
||||
(flat.band == litellm_cost::price::Band::Base && flat.tier.is_none())
|
||||
.then_some(flat.meter)
|
||||
})
|
||||
});
|
||||
let search = fields.get("search_context_cost_per_query");
|
||||
let search_declared = [
|
||||
("search_context_size_low", SearchContextSize::Low),
|
||||
("search_context_size_medium", SearchContextSize::Medium),
|
||||
("search_context_size_high", SearchContextSize::High),
|
||||
]
|
||||
.into_iter()
|
||||
.filter_map(|(key, size)| {
|
||||
search
|
||||
.filter(|value| value.is_null() || value.get(key).is_some())
|
||||
.map(|_| Meter::WebSearch(size))
|
||||
});
|
||||
let search_tiers = wire
|
||||
.tier_rows
|
||||
.iter()
|
||||
.flatten()
|
||||
.filter_map(|row| {
|
||||
let range = row.max_results_range?;
|
||||
Some(SearchTier {
|
||||
range,
|
||||
meter: Meter::Input(InputUnit::Query {
|
||||
over_results: NonZeroU64::new(range[0].saturating_sub(1)),
|
||||
}),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
OtherPriceSource {
|
||||
id,
|
||||
declared: declared.chain(search_declared).collect(),
|
||||
search_tiers,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use litellm_cost::price::{Feature, Price};
|
||||
use rstest::rstest;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use super::*;
|
||||
use crate::{Model, PriceLowering};
|
||||
|
||||
#[rstest]
|
||||
#[case::configured(json!(2.5), Some(0.0025))]
|
||||
#[case::zero(json!(0), Some(0.0))]
|
||||
#[case::null(Value::Null, None)]
|
||||
fn configured_file_search_is_per_occurrence_only_in_sdk_context(
|
||||
#[case] value: Value,
|
||||
#[case] expected: Option<f64>,
|
||||
) {
|
||||
let fields = json!({"litellm_provider":"openai", "file_search_cost_per_1k_calls":value})
|
||||
.as_object()
|
||||
.unwrap()
|
||||
.clone();
|
||||
let price = Price::online(Meter::Feature(Feature::FileSearchCall), Default::default());
|
||||
let sdk = Model::from_fields_with_policy(&fields, PriceLowering::SdkOther).unwrap();
|
||||
let generic = Model::from_fields(&fields).unwrap();
|
||||
assert_eq!(sdk.prices.get(&price).map(|rate| rate.get()), expected);
|
||||
assert_eq!(generic.prices.get(&price), None);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::boolean(json!(true))]
|
||||
#[case::negative(json!(-1.0))]
|
||||
#[case::string(json!("bad"))]
|
||||
fn new_tool_rates_reject_bad_values_only_in_sdk_context(#[case] value: Value) {
|
||||
let fields = json!({"litellm_provider":"openai", "file_search_cost_per_1k_calls":value})
|
||||
.as_object()
|
||||
.unwrap()
|
||||
.clone();
|
||||
assert!(Model::from_fields_with_policy(&fields, PriceLowering::SdkOther).is_err());
|
||||
assert!(Model::from_fields(&fields).is_ok());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::nested_null(json!({"search_context_size_low":null}), true)]
|
||||
#[case::parent_null(Value::Null, true)]
|
||||
#[case::absent_child(json!({}), false)]
|
||||
fn nested_search_declarations_preserve_absent_and_null(
|
||||
#[case] value: Value,
|
||||
#[case] declared: bool,
|
||||
) {
|
||||
let fields = json!({"search_context_cost_per_query":value})
|
||||
.as_object()
|
||||
.unwrap()
|
||||
.clone();
|
||||
let source = other_price_source(
|
||||
&fields,
|
||||
&EntryWire::read_other(&fields),
|
||||
"primary".to_owned(),
|
||||
);
|
||||
assert_eq!(
|
||||
source
|
||||
.declared
|
||||
.contains(&Meter::WebSearch(SearchContextSize::Low)),
|
||||
declared
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn scalar_search_remains_rejected_in_sdk_context() {
|
||||
let fields = json!({"litellm_provider":"openai", "search_context_cost_per_query":2.0})
|
||||
.as_object()
|
||||
.unwrap()
|
||||
.clone();
|
||||
assert!(Model::from_fields_with_policy(&fields, PriceLowering::SdkOther).is_err());
|
||||
assert!(Model::from_fields(&fields).is_err());
|
||||
}
|
||||
}
|
||||
|
|
@ -29,6 +29,28 @@ pub struct EntryWire {
|
|||
}
|
||||
|
||||
impl EntryWire {
|
||||
pub fn read_other(fields: &Fields) -> Self {
|
||||
let mut wire = Self::read(fields);
|
||||
let mut reader = Reader {
|
||||
problems: &mut wire.problems,
|
||||
};
|
||||
for (key, value) in fields.iter().filter(|(_, value)| !value.is_null()) {
|
||||
if FlatKey::parse(key).is_some() {
|
||||
continue;
|
||||
}
|
||||
let Some((flat, per_units)) = FlatKey::parse_other(key) else {
|
||||
continue;
|
||||
};
|
||||
wire.unknown.remove(key);
|
||||
if let Some(mut price) = reader.price(key.clone(), pointer_for_key(key), flat, value) {
|
||||
price.price = UnitPrice::new(price.price.get() / per_units as f64)
|
||||
.expect("finite rate divided by a positive unit basis");
|
||||
wire.prices.push(price);
|
||||
}
|
||||
}
|
||||
wire
|
||||
}
|
||||
|
||||
pub fn read(fields: &Fields) -> Self {
|
||||
let mut problems = Vec::new();
|
||||
let mut wire = Self::default();
|
||||
|
|
|
|||
|
|
@ -21,6 +21,30 @@ pub enum TierSuffix {
|
|||
}
|
||||
|
||||
impl FlatKey {
|
||||
pub fn parse_other(key: &str) -> Option<(Self, u64)> {
|
||||
if let Some(flat) = Self::parse(key) {
|
||||
return Some((flat, 1));
|
||||
}
|
||||
let (feature, per_units) = match key {
|
||||
"file_search_cost_per_1k_calls" => (Feature::FileSearchCall, 1000),
|
||||
"file_search_cost_per_gb_per_day" => (Feature::FileSearchGbDay, 1),
|
||||
"vector_store_cost_per_gb_per_day" => (Feature::VectorStoreGbDay, 1),
|
||||
"computer_use_input_cost_per_1k_tokens" => (Feature::ComputerUseInputToken, 1000),
|
||||
"computer_use_output_cost_per_1k_tokens" => (Feature::ComputerUseOutputToken, 1000),
|
||||
"search_cost_per_result" => (Feature::SearchResult, 1),
|
||||
"browser_open_cost_per_call" => (Feature::BrowserOpen, 1),
|
||||
_ => return None,
|
||||
};
|
||||
Some((
|
||||
Self {
|
||||
meter: Meter::Feature(feature),
|
||||
band: Band::Base,
|
||||
tier: None,
|
||||
},
|
||||
per_units,
|
||||
))
|
||||
}
|
||||
|
||||
pub fn parse(key: &str) -> Option<Self> {
|
||||
let (rest, tier) = split_tier(key);
|
||||
let (base, suffix) = split_above(rest)?;
|
||||
|
|
|
|||
|
|
@ -32,9 +32,11 @@ mod _native {
|
|||
use crate::logger::NativeDiagnosticProcessor;
|
||||
#[pymodule_export]
|
||||
use crate::model_registry::{
|
||||
CatalogRefresher, FetchedRegistry, InvalidProvider, ModelCostLayers, ModelInfoView,
|
||||
ModelNotFound, ModelRegistry, PriceEvaluation, PriceLine, Priced, RegistrationTarget,
|
||||
RegistryNotModified, RegistryUnavailable, ResolvedModel, RowError, RowErrors, SelectedRate,
|
||||
Additive, CatalogRefresher, DefaultRateInput, DirectionalCharacters, FetchedRegistry,
|
||||
FirstUnit, InvalidProvider, MeasuredQuantity, ModelCostLayers, ModelInfoView,
|
||||
ModelNotFound, ModelRegistry, Ocr, OtherMeterInput, PriceEvaluation, PriceLine,
|
||||
PriceRowInput, Priced, PromptUnits, RegistrationTarget, RegistryNotModified,
|
||||
RegistryUnavailable, ResolvedModel, RowError, RowErrors, SearchTiers, SelectedRate,
|
||||
Unpriced, UsageInput, WhenInput, fetch_once, fetch_with_retry, has_token_prices,
|
||||
invalidate_model_cost_index, model_info, model_row_errors, price, price_with_rates,
|
||||
registration_target, resolve_model, write_rows,
|
||||
|
|
@ -144,6 +146,16 @@ assert live == {"model": {"litellm_provider": "openai", "input_cost_per_token":
|
|||
"RowErrors",
|
||||
"ModelInfoView",
|
||||
"UsageInput",
|
||||
"MeasuredQuantity",
|
||||
"PriceRowInput",
|
||||
"DefaultRateInput",
|
||||
"Additive",
|
||||
"PromptUnits",
|
||||
"FirstUnit",
|
||||
"SearchTiers",
|
||||
"Ocr",
|
||||
"DirectionalCharacters",
|
||||
"OtherMeterInput",
|
||||
"WhenInput",
|
||||
"PriceLine",
|
||||
"Priced",
|
||||
|
|
|
|||
|
|
@ -1,7 +1,9 @@
|
|||
mod live;
|
||||
mod other;
|
||||
mod refresh;
|
||||
|
||||
pub(crate) use live::*;
|
||||
pub(crate) use other::*;
|
||||
pub(crate) use refresh::*;
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet, HashMap};
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ use pyo3::prelude::*;
|
|||
use pyo3::types::{PyAny, PyBool, PyDict, PyModule, PySet, PyString};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
use super::other::{LegacyCount, MeasuredQuantity, OtherMeterInput, StrictCount};
|
||||
use super::{RowError, escape_pointer_segment, fields_to_dict, nonfinite_errors, row_error};
|
||||
|
||||
struct LowercaseIndex {
|
||||
|
|
@ -325,18 +326,34 @@ pub(crate) fn registration_target(
|
|||
|
||||
#[pyclass(frozen, get_all, module = "litellm.rust_bridge._native")]
|
||||
pub(crate) struct RowErrors {
|
||||
errors: Vec<RowError>,
|
||||
pub(super) errors: Vec<RowError>,
|
||||
}
|
||||
|
||||
fn lower_row(key: &str, row: &Bound<'_, PyAny>) -> Result<(Fields, Model), RowErrors> {
|
||||
lower_row_with_policy(key, row, litellm_model_catalog::PriceLowering::Generic)
|
||||
}
|
||||
|
||||
fn lower_row_with_policy(
|
||||
pub(super) fn lower_row_with_policy(
|
||||
key: &str,
|
||||
row: &Bound<'_, PyAny>,
|
||||
policy: litellm_model_catalog::PriceLowering,
|
||||
) -> Result<(Fields, Model), RowErrors> {
|
||||
let fields = row_fields(key, row)?;
|
||||
let model = Model::from_fields_with_policy(&fields, policy).map_err(|errors| RowErrors {
|
||||
errors: errors
|
||||
.into_iter()
|
||||
.map(|error| {
|
||||
row_error(&Located::new(
|
||||
format!("/{}{}", escape_pointer_segment(key), error.path),
|
||||
error.error,
|
||||
))
|
||||
})
|
||||
.collect(),
|
||||
})?;
|
||||
Ok((fields, model))
|
||||
}
|
||||
|
||||
pub(super) fn row_fields(key: &str, row: &Bound<'_, PyAny>) -> Result<Fields, RowErrors> {
|
||||
let path = format!("/{}", escape_pointer_segment(key));
|
||||
let located = |error| RowErrors {
|
||||
errors: vec![row_error(&Located::new(&path, error))],
|
||||
|
|
@ -357,14 +374,7 @@ fn lower_row_with_policy(
|
|||
))
|
||||
};
|
||||
valid.map_err(|_| invalid())?;
|
||||
let fields: Fields = from_py(row).map_err(|_| invalid())?;
|
||||
let model = Model::from_fields_with_policy(&fields, policy).map_err(|errors| RowErrors {
|
||||
errors: errors
|
||||
.into_iter()
|
||||
.map(|error| row_error(&Located::new(format!("{path}{}", error.path), error.error)))
|
||||
.collect(),
|
||||
})?;
|
||||
Ok((fields, model))
|
||||
from_py(row).map_err(|_| invalid())
|
||||
}
|
||||
|
||||
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
|
||||
|
|
@ -468,31 +478,64 @@ pub(crate) fn model_info(model_cost: &Bound<'_, PyDict>, key: &str) -> PyResult<
|
|||
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
|
||||
pub(crate) struct UsageInput {
|
||||
usage: Usage,
|
||||
boolean_count: bool,
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl UsageInput {
|
||||
#[new]
|
||||
#[pyo3(signature = (quantities, *, prompt_tokens=0, inclusive_threshold=false, search_results=0))]
|
||||
#[pyo3(signature = (quantities, *, prompt_tokens=LegacyCount::ZERO, inclusive_threshold=false, search_results=LegacyCount::ZERO, completion_tokens=None), text_signature = "(quantities, *, prompt_tokens=0, inclusive_threshold=False, search_results=0, completion_tokens=None)")]
|
||||
fn py_new(
|
||||
quantities: &Bound<'_, PyDict>,
|
||||
prompt_tokens: LegacyCount,
|
||||
inclusive_threshold: bool,
|
||||
search_results: LegacyCount,
|
||||
completion_tokens: Option<StrictCount>,
|
||||
) -> PyResult<Self> {
|
||||
let mut input = Self::new(
|
||||
quantities,
|
||||
prompt_tokens.value,
|
||||
inclusive_threshold,
|
||||
search_results.value,
|
||||
)?;
|
||||
input.boolean_count |= prompt_tokens.boolean || search_results.boolean;
|
||||
input.usage.completion_tokens = completion_tokens.map(|count| count.0);
|
||||
Ok(input)
|
||||
}
|
||||
}
|
||||
|
||||
impl UsageInput {
|
||||
fn new(
|
||||
quantities: &Bound<'_, PyDict>,
|
||||
prompt_tokens: u64,
|
||||
inclusive_threshold: bool,
|
||||
search_results: u64,
|
||||
) -> PyResult<Self> {
|
||||
let boolean_count = quantities
|
||||
.iter()
|
||||
.any(|(_, value)| value.is_instance_of::<PyBool>());
|
||||
let quantities = quantities
|
||||
.iter()
|
||||
.map(|(key, quantity)| {
|
||||
let key: String = key.extract()?;
|
||||
let meter = parse_meter(&key)
|
||||
.ok_or_else(|| PyValueError::new_err(format!("unknown usage meter: {key}")))?;
|
||||
Ok((meter, quantity.extract::<u64>()?))
|
||||
let quantity = if quantity.is_instance_of::<MeasuredQuantity>() {
|
||||
quantity
|
||||
.extract::<PyRef<'_, MeasuredQuantity>>()?
|
||||
.quantity(&meter)?
|
||||
} else {
|
||||
litellm_cost::Quantity::Count(quantity.extract::<u64>()?)
|
||||
};
|
||||
Ok((meter, quantity))
|
||||
})
|
||||
.collect::<PyResult<_>>()?;
|
||||
Ok(Self {
|
||||
boolean_count,
|
||||
usage: Usage {
|
||||
quantities,
|
||||
prompt_tokens,
|
||||
completion_tokens: None,
|
||||
inclusive_threshold,
|
||||
search_results,
|
||||
},
|
||||
|
|
@ -500,7 +543,7 @@ impl UsageInput {
|
|||
}
|
||||
}
|
||||
|
||||
fn parse_meter(key: &str) -> Option<Meter> {
|
||||
pub(super) fn parse_meter(key: &str) -> Option<Meter> {
|
||||
use litellm_cost::price::{GuardrailUnit, SearchContextSize};
|
||||
if let Some(counter) = key.strip_prefix("guardrail/") {
|
||||
return (!counter.is_empty()).then(|| Meter::Guardrail(GuardrailUnit::new(counter)));
|
||||
|
|
@ -509,7 +552,8 @@ fn parse_meter(key: &str) -> Option<Meter> {
|
|||
"web_search/low" => Some(Meter::WebSearch(SearchContextSize::Low)),
|
||||
"web_search/medium" => Some(Meter::WebSearch(SearchContextSize::Medium)),
|
||||
"web_search/high" => Some(Meter::WebSearch(SearchContextSize::High)),
|
||||
key => FlatKey::parse(key)
|
||||
key => FlatKey::parse_other(key)
|
||||
.map(|(flat, _)| flat)
|
||||
.filter(|flat| !flat.has_suffix())
|
||||
.map(|flat| flat.meter),
|
||||
}
|
||||
|
|
@ -556,10 +600,29 @@ impl WhenInput {
|
|||
#[derive(Clone)]
|
||||
pub(crate) struct PriceLine {
|
||||
meter: String,
|
||||
quantity: u64,
|
||||
quantity: PythonQuantity,
|
||||
unit_price: f64,
|
||||
amount: f64,
|
||||
fallback_from: Option<String>,
|
||||
source_id: Option<String>,
|
||||
caller_default: bool,
|
||||
quantity_unit: String,
|
||||
report_to: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, IntoPyObject)]
|
||||
enum PythonQuantity {
|
||||
Count(u64),
|
||||
Measured(f64),
|
||||
}
|
||||
|
||||
impl From<litellm_cost::Quantity> for PythonQuantity {
|
||||
fn from(quantity: litellm_cost::Quantity) -> Self {
|
||||
match quantity {
|
||||
litellm_cost::Quantity::Count(count) => Self::Count(count),
|
||||
litellm_cost::Quantity::Measured(value) => Self::Measured(value.get()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[pyclass(frozen, get_all, module = "litellm.rust_bridge._native")]
|
||||
|
|
@ -586,11 +649,17 @@ impl From<litellm_cost::Breakdown> for Priced {
|
|||
.lines
|
||||
.into_iter()
|
||||
.map(|line| PriceLine {
|
||||
quantity_unit: quantity_unit(&line.meter).to_owned(),
|
||||
meter: meter_name(&line.meter),
|
||||
quantity: line.quantity,
|
||||
quantity: line.quantity.into(),
|
||||
unit_price: line.unit_price.get(),
|
||||
amount: line.amount.get(),
|
||||
fallback_from: line.fallback_from.as_ref().map(meter_name),
|
||||
source_id: line.source_id,
|
||||
caller_default: line.caller_default,
|
||||
report_to: line
|
||||
.report_to
|
||||
.map(|direction| <&'static str>::from(direction).to_owned()),
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
|
|
@ -615,6 +684,8 @@ pub(crate) struct SelectedRate {
|
|||
requested_meter: String,
|
||||
source_meter: Option<String>,
|
||||
unit_price: Option<f64>,
|
||||
source_id: Option<String>,
|
||||
caller_default: bool,
|
||||
}
|
||||
|
||||
#[pyclass(frozen, get_all, module = "litellm.rust_bridge._native")]
|
||||
|
|
@ -645,7 +716,18 @@ enum PricePolicy {
|
|||
}
|
||||
|
||||
impl PricePolicy {
|
||||
fn selection(self, model: &Model, threshold: Option<NonZeroU64>) -> SelectionPolicy<'_> {
|
||||
fn selection<'a>(
|
||||
self,
|
||||
model: &'a Model,
|
||||
threshold: Option<NonZeroU64>,
|
||||
other: Option<&'a litellm_cost::OtherContext>,
|
||||
) -> SelectionPolicy<'a> {
|
||||
if let Some(context) = other {
|
||||
return SelectionPolicy::SdkOther {
|
||||
tokens: &model.token_price_sources,
|
||||
context,
|
||||
};
|
||||
}
|
||||
if let Some(threshold) = threshold {
|
||||
return SelectionPolicy::DirectionalTokens { threshold };
|
||||
}
|
||||
|
|
@ -742,7 +824,7 @@ fn parse_directional_threshold(
|
|||
let threshold = NonZeroU64::new(value.extract()?)
|
||||
.ok_or_else(|| PyValueError::new_err("token threshold must be a positive integer"))?;
|
||||
if usage.quantities.iter().any(|(meter, quantity)| {
|
||||
*quantity != 0
|
||||
!quantity.is_zero()
|
||||
&& !matches!(
|
||||
meter,
|
||||
Meter::Input(InputUnit::Token(TokenKind::Text))
|
||||
|
|
@ -775,7 +857,7 @@ pub(crate) fn has_token_prices(
|
|||
}
|
||||
|
||||
#[pyfunction]
|
||||
#[pyo3(signature = (row, usage, when, *, policy="generic", cache_read_default_ratio=None, directional_token_threshold=None))]
|
||||
#[pyo3(signature = (row, usage, when, *, policy="generic", cache_read_default_ratio=None, directional_token_threshold=None, other_meters=None))]
|
||||
pub(crate) fn price(
|
||||
row: &Bound<'_, PyAny>,
|
||||
usage: &UsageInput,
|
||||
|
|
@ -783,6 +865,7 @@ pub(crate) fn price(
|
|||
policy: &str,
|
||||
cache_read_default_ratio: Option<&Bound<'_, PyAny>>,
|
||||
directional_token_threshold: Option<&Bound<'_, PyAny>>,
|
||||
other_meters: Option<&OtherMeterInput>,
|
||||
) -> PyResult<PriceResult> {
|
||||
let policy = pricing_policy(policy)?;
|
||||
let ratio = cache_read_default(policy, cache_read_default_ratio)?;
|
||||
|
|
@ -793,12 +876,31 @@ pub(crate) fn price(
|
|||
&when.when,
|
||||
ratio.is_some(),
|
||||
)?;
|
||||
let model = match policy.lower(row, when.when.channel) {
|
||||
validate_other_context(policy, usage, &when.when, threshold, other_meters)?;
|
||||
let (model, other) = match lower_price_context(policy, row, when.when.channel, other_meters) {
|
||||
Ok(model) => model,
|
||||
Err(errors) => return Ok(PriceResult::Errors(errors)),
|
||||
};
|
||||
let model = policy.prepare(model, when.when.channel, ratio)?;
|
||||
let policy = policy.selection(&model, threshold);
|
||||
let policy = policy.selection(&model, threshold, other.as_ref());
|
||||
if other.is_some()
|
||||
|| usage
|
||||
.usage
|
||||
.quantities
|
||||
.values()
|
||||
.any(|quantity| matches!(quantity, litellm_cost::Quantity::Measured(_)))
|
||||
{
|
||||
let evaluation = litellm_cost::calculate_with_rates(
|
||||
&model.prices,
|
||||
&usage.usage,
|
||||
&when.when,
|
||||
policy,
|
||||
&[],
|
||||
&[],
|
||||
)
|
||||
.map_err(|error| PyValueError::new_err(error.to_string()))?;
|
||||
return price_result(row.py(), evaluation.pricing);
|
||||
}
|
||||
price_result(
|
||||
row.py(),
|
||||
litellm_cost::calculate_with_policy(&model.prices, &usage.usage, &when.when, policy),
|
||||
|
|
@ -817,8 +919,8 @@ fn price_result(py: Python<'_>, priced: litellm_cost::Priced) -> PyResult<PriceR
|
|||
|
||||
#[pyfunction]
|
||||
#[pyo3(
|
||||
signature = (row, usage, when, *, policy="generic", project=Vec::new(), factors=Vec::new(), cache_read_default_ratio=None, directional_token_threshold=None),
|
||||
text_signature = "(row, usage, when, *, policy='generic', project=(), factors=(), cache_read_default_ratio=None, directional_token_threshold=None)"
|
||||
signature = (row, usage, when, *, policy="generic", project=Vec::new(), factors=Vec::new(), cache_read_default_ratio=None, directional_token_threshold=None, other_meters=None),
|
||||
text_signature = "(row, usage, when, *, policy='generic', project=(), factors=(), cache_read_default_ratio=None, directional_token_threshold=None, other_meters=None)"
|
||||
)]
|
||||
#[expect(
|
||||
clippy::too_many_arguments,
|
||||
|
|
@ -833,6 +935,7 @@ pub(crate) fn price_with_rates(
|
|||
factors: Vec<Bound<'_, PyAny>>,
|
||||
cache_read_default_ratio: Option<&Bound<'_, PyAny>>,
|
||||
directional_token_threshold: Option<&Bound<'_, PyAny>>,
|
||||
other_meters: Option<&OtherMeterInput>,
|
||||
) -> PyResult<EvaluationResult> {
|
||||
let policy = pricing_policy(policy)?;
|
||||
let ratio = cache_read_default(policy, cache_read_default_ratio)?;
|
||||
|
|
@ -863,7 +966,8 @@ pub(crate) fn price_with_rates(
|
|||
.ok_or_else(|| PyValueError::new_err("factors must be finite non-negative numbers"))
|
||||
})
|
||||
.collect::<PyResult<Vec<_>>>()?;
|
||||
let model = match policy.lower(row, when.when.channel) {
|
||||
validate_other_context(policy, usage, &when.when, threshold, other_meters)?;
|
||||
let (model, other) = match lower_price_context(policy, row, when.when.channel, other_meters) {
|
||||
Ok(model) => model,
|
||||
Err(errors) => return Ok(EvaluationResult::Errors(errors)),
|
||||
};
|
||||
|
|
@ -872,7 +976,7 @@ pub(crate) fn price_with_rates(
|
|||
&model.prices,
|
||||
&usage.usage,
|
||||
&when.when,
|
||||
policy.selection(&model, threshold),
|
||||
policy.selection(&model, threshold, other.as_ref()),
|
||||
&project,
|
||||
&factors,
|
||||
)
|
||||
|
|
@ -883,6 +987,8 @@ pub(crate) fn price_with_rates(
|
|||
.rates
|
||||
.into_iter()
|
||||
.map(|rate| SelectedRate {
|
||||
source_id: rate.source_id,
|
||||
caller_default: rate.caller_default,
|
||||
requested_meter: meter_name(&rate.requested_meter),
|
||||
source_meter: rate.resolved.as_ref().map(|(meter, _)| meter_name(meter)),
|
||||
unit_price: rate.resolved.map(|(_, price)| price.get()),
|
||||
|
|
@ -891,6 +997,71 @@ pub(crate) fn price_with_rates(
|
|||
}))
|
||||
}
|
||||
|
||||
fn lower_price_context(
|
||||
policy: PricePolicy,
|
||||
row: &Bound<'_, PyAny>,
|
||||
channel: Channel,
|
||||
other: Option<&OtherMeterInput>,
|
||||
) -> Result<(Model, Option<litellm_cost::OtherContext>), RowErrors> {
|
||||
match other {
|
||||
Some(other) => other
|
||||
.prepare(row)
|
||||
.map(|(model, context)| (model, Some(context))),
|
||||
None => policy.lower(row, channel).map(|model| (model, None)),
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_other_context(
|
||||
policy: PricePolicy,
|
||||
usage: &UsageInput,
|
||||
when: &When,
|
||||
threshold: Option<NonZeroU64>,
|
||||
other: Option<&OtherMeterInput>,
|
||||
) -> PyResult<()> {
|
||||
if other.is_some() && (!matches!(policy, PricePolicy::SdkTokens) || threshold.is_some()) {
|
||||
return Err(PyValueError::new_err(
|
||||
"other_meters requires sdk_tokens and cannot be combined with directional_token_threshold",
|
||||
));
|
||||
}
|
||||
if other.is_some_and(OtherMeterInput::directional) && when.channel != Channel::Online {
|
||||
return Err(PyValueError::new_err(
|
||||
"directional characters requires online pricing",
|
||||
));
|
||||
}
|
||||
if usage.boolean_count
|
||||
&& (other.is_some()
|
||||
|| usage
|
||||
.usage
|
||||
.quantities
|
||||
.values()
|
||||
.any(|quantity| matches!(quantity, litellm_cost::Quantity::Measured(_))))
|
||||
{
|
||||
return Err(PyValueError::new_err(
|
||||
"boolean counts are not valid in measured or other_meters usage",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn quantity_unit(meter: &Meter) -> &'static str {
|
||||
use litellm_cost::price::Feature;
|
||||
match meter {
|
||||
Meter::WallClockSecond
|
||||
| Meter::Input(
|
||||
InputUnit::Second | InputUnit::AudioSecond | InputUnit::VideoSecond { .. },
|
||||
)
|
||||
| Meter::Output(OutputUnit::Second(_) | OutputUnit::VideoSecond) => "seconds",
|
||||
Meter::Input(InputUnit::Character) | Meter::Output(OutputUnit::Character) => "characters",
|
||||
Meter::Feature(Feature::FileSearchGbDay | Feature::VectorStoreGbDay) => "gb_days",
|
||||
Meter::Feature(Feature::FileSearchCall | Feature::BrowserOpen) => "calls",
|
||||
Meter::Feature(Feature::ComputerUseInputToken | Feature::ComputerUseOutputToken) => {
|
||||
"tokens"
|
||||
}
|
||||
meter if meter.is_token() => "tokens",
|
||||
_ => "count",
|
||||
}
|
||||
}
|
||||
|
||||
fn meter_name(meter: &Meter) -> String {
|
||||
use litellm_cost::price::*;
|
||||
let token = |kind: &TokenKind| match kind {
|
||||
|
|
@ -957,6 +1128,13 @@ fn meter_name(meter: &Meter) -> String {
|
|||
Feature::GoogleMapsGroundingQuery => "google_maps_grounding_cost_per_query",
|
||||
Feature::OcrCredit => "ocr_cost_per_credit",
|
||||
Feature::OcrPage => "ocr_cost_per_page",
|
||||
Feature::FileSearchCall => "file_search_cost_per_1k_calls",
|
||||
Feature::FileSearchGbDay => "file_search_cost_per_gb_per_day",
|
||||
Feature::VectorStoreGbDay => "vector_store_cost_per_gb_per_day",
|
||||
Feature::ComputerUseInputToken => "computer_use_input_cost_per_1k_tokens",
|
||||
Feature::ComputerUseOutputToken => "computer_use_output_cost_per_1k_tokens",
|
||||
Feature::SearchResult => "search_cost_per_result",
|
||||
Feature::BrowserOpen => "browser_open_cost_per_call",
|
||||
}
|
||||
.into(),
|
||||
Meter::WebSearch(context) => format!(
|
||||
|
|
@ -1587,6 +1765,7 @@ assert n.model_info(models, "Model").input_cost_per_token == 2
|
|||
policy,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.unwrap() else {
|
||||
panic!("expected invalid row")
|
||||
|
|
@ -2358,6 +2537,7 @@ with pytest.raises(ValueError):
|
|||
policy,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
match result {
|
||||
|
|
@ -2470,6 +2650,7 @@ assert json.dumps(row) == before
|
|||
"sdk_tokens",
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
let priced = match result {
|
||||
|
|
@ -2805,6 +2986,7 @@ assert isinstance(n.resolve_model(rows, model, inferred=inferred), n.ModelNotFou
|
|||
"generic",
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.unwrap() else {
|
||||
panic!("not priced")
|
||||
|
|
@ -2832,6 +3014,7 @@ assert isinstance(n.resolve_model(rows, model, inferred=inferred), n.ModelNotFou
|
|||
"generic",
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.unwrap() else {
|
||||
panic!("not priced")
|
||||
|
|
@ -2923,6 +3106,7 @@ assert seen == ["released"]
|
|||
"generic",
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.unwrap() else {
|
||||
panic!("not priced: {}", fields[0])
|
||||
|
|
@ -2983,7 +3167,9 @@ assert seen == ["released"]
|
|||
0,
|
||||
)
|
||||
.unwrap();
|
||||
match price(row.as_any(), &usage, &when, "generic", None, None).unwrap() {
|
||||
match price(row.as_any(), &usage, &when, "generic", None, None, None)
|
||||
.unwrap()
|
||||
{
|
||||
PriceResult::Priced(result) => assert_eq!(result.total, expected),
|
||||
PriceResult::Unpriced(result) => {
|
||||
assert_eq!(expected, 0.0);
|
||||
|
|
@ -3018,7 +3204,7 @@ assert seen == ["released"]
|
|||
json!({"litellm_provider":"test", "input_cost_per_token": 2}),
|
||||
);
|
||||
let PriceResult::Unpriced(result) =
|
||||
price(row.as_any(), &usage, &when, "generic", None, None).unwrap()
|
||||
price(row.as_any(), &usage, &when, "generic", None, None, None).unwrap()
|
||||
else {
|
||||
panic!("expected unpriced")
|
||||
};
|
||||
|
|
@ -3035,7 +3221,7 @@ assert seen == ["released"]
|
|||
row.set_item("output_cost_per_token", 0).unwrap();
|
||||
row.set_item("input_cost_per_audio_token", 0).unwrap();
|
||||
let PriceResult::Priced(result) =
|
||||
price(row.as_any(), &usage, &when, "generic", None, None).unwrap()
|
||||
price(row.as_any(), &usage, &when, "generic", None, None, None).unwrap()
|
||||
else {
|
||||
panic!("expected free")
|
||||
};
|
||||
|
|
@ -3064,6 +3250,7 @@ assert seen == ["released"]
|
|||
"generic",
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.unwrap() else {
|
||||
panic!("not priced")
|
||||
|
|
@ -3077,7 +3264,7 @@ assert seen == ["released"]
|
|||
result.lines[0].amount,
|
||||
&result.lines[0].fallback_from
|
||||
),
|
||||
(3, 2.0, 6.0, &None)
|
||||
(PythonQuantity::Count(3), 2.0, 6.0, &None)
|
||||
);
|
||||
});
|
||||
}
|
||||
|
|
|
|||
443
litellm-rust/crates/python-bridge/src/model_registry/other.rs
Normal file
443
litellm-rust/crates/python-bridge/src/model_registry/other.rs
Normal file
|
|
@ -0,0 +1,443 @@
|
|||
use std::collections::BTreeSet;
|
||||
use std::num::NonZeroU64;
|
||||
|
||||
use litellm_cost::money::UnitPrice;
|
||||
use litellm_cost::{
|
||||
DefaultEligibility, Measurement, OtherContext, OtherSelection, Quantity, QuantityUnit, ReportTo,
|
||||
};
|
||||
use litellm_model_catalog::Model;
|
||||
use litellm_model_catalog::wire::Fields;
|
||||
use pyo3::exceptions::PyValueError;
|
||||
use pyo3::prelude::*;
|
||||
use pyo3::types::{PyAny, PyBool, PyDict, PyMapping};
|
||||
|
||||
use super::live::{RowErrors, parse_meter, row_fields};
|
||||
use super::row_error;
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub(super) struct LegacyCount {
|
||||
pub value: u64,
|
||||
pub boolean: bool,
|
||||
}
|
||||
|
||||
impl LegacyCount {
|
||||
pub const ZERO: Self = Self {
|
||||
value: 0,
|
||||
boolean: false,
|
||||
};
|
||||
}
|
||||
|
||||
impl FromPyObject<'_, '_> for LegacyCount {
|
||||
type Error = PyErr;
|
||||
fn extract(value: Borrowed<'_, '_, PyAny>) -> PyResult<Self> {
|
||||
Ok(Self {
|
||||
value: value.extract()?,
|
||||
boolean: value.is_instance_of::<PyBool>(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub(crate) struct StrictCount(pub u64);
|
||||
|
||||
impl FromPyObject<'_, '_> for StrictCount {
|
||||
type Error = PyErr;
|
||||
|
||||
fn extract(value: Borrowed<'_, '_, PyAny>) -> PyResult<Self> {
|
||||
if value.is_instance_of::<PyBool>() {
|
||||
return Err(PyValueError::new_err(
|
||||
"count must be an integer, not a boolean",
|
||||
));
|
||||
}
|
||||
value.extract::<u64>().map(Self)
|
||||
}
|
||||
}
|
||||
|
||||
fn finite(value: &Bound<'_, PyAny>) -> PyResult<f64> {
|
||||
if value.is_instance_of::<PyBool>() {
|
||||
return Err(PyValueError::new_err(
|
||||
"value must be finite and non-negative, not a boolean",
|
||||
));
|
||||
}
|
||||
let value = value.extract::<f64>()?;
|
||||
Measurement::new(value).map_err(|error| PyValueError::new_err(error.to_string()))?;
|
||||
Ok(value)
|
||||
}
|
||||
|
||||
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
|
||||
pub(crate) struct MeasuredQuantity {
|
||||
value: Measurement,
|
||||
unit: QuantityUnit,
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl MeasuredQuantity {
|
||||
#[new]
|
||||
#[pyo3(signature = (value, *, unit="meter"))]
|
||||
fn new(value: &Bound<'_, PyAny>, unit: &str) -> PyResult<Self> {
|
||||
let value = Measurement::new(finite(value)?)
|
||||
.map_err(|error| PyValueError::new_err(error.to_string()))?;
|
||||
let unit = unit
|
||||
.parse()
|
||||
.map_err(|_| PyValueError::new_err("unit must be meter or milliseconds"))?;
|
||||
Ok(Self { value, unit })
|
||||
}
|
||||
}
|
||||
|
||||
impl MeasuredQuantity {
|
||||
pub(super) fn quantity(&self, meter: &litellm_cost::price::Meter) -> PyResult<Quantity> {
|
||||
Quantity::measured(self.value, self.unit, meter)
|
||||
.map_err(|error| PyValueError::new_err(error.to_string()))
|
||||
}
|
||||
}
|
||||
|
||||
#[pyclass(frozen, skip_from_py_object, module = "litellm.rust_bridge._native")]
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct PriceRowInput {
|
||||
source_id: String,
|
||||
fields: Fields,
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl PriceRowInput {
|
||||
#[new]
|
||||
fn new(source_id: String, row: &Bound<'_, PyMapping>) -> PyResult<Self> {
|
||||
if source_id.trim().is_empty() || source_id == "primary" {
|
||||
return Err(PyValueError::new_err(
|
||||
"source_id must be nonempty and cannot be primary",
|
||||
));
|
||||
}
|
||||
let snapshot = row.py().get_type::<PyDict>().call1((row,))?;
|
||||
let fields = row_fields(&source_id, &snapshot).map_err(|_| {
|
||||
PyValueError::new_err("fallback row must contain finite JSON-compatible values")
|
||||
})?;
|
||||
Ok(Self { source_id, fields })
|
||||
}
|
||||
}
|
||||
|
||||
#[pyclass(frozen, skip_from_py_object, module = "litellm.rust_bridge._native")]
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct DefaultRateInput {
|
||||
meter: litellm_cost::price::Meter,
|
||||
rate: UnitPrice,
|
||||
per_units: NonZeroU64,
|
||||
eligibility: DefaultEligibility,
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl DefaultRateInput {
|
||||
#[new]
|
||||
#[pyo3(signature = (meter, value, *, per_units=StrictCount(1), apply_when), text_signature = "(meter, value, *, per_units=1, apply_when)")]
|
||||
fn new(
|
||||
meter: &str,
|
||||
value: &Bound<'_, PyAny>,
|
||||
per_units: StrictCount,
|
||||
apply_when: &str,
|
||||
) -> PyResult<Self> {
|
||||
let meter =
|
||||
parse_meter(meter).ok_or_else(|| PyValueError::new_err("unknown default meter"))?;
|
||||
let rate = UnitPrice::new(finite(value)?)
|
||||
.ok_or_else(|| PyValueError::new_err("invalid default rate"))?;
|
||||
let per_units = NonZeroU64::new(per_units.0)
|
||||
.ok_or_else(|| PyValueError::new_err("per_units must be positive"))?;
|
||||
let eligibility = apply_when
|
||||
.parse()
|
||||
.map_err(|_| PyValueError::new_err("apply_when must be absent or absent_or_null"))?;
|
||||
Ok(Self {
|
||||
meter,
|
||||
rate,
|
||||
per_units,
|
||||
eligibility,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
|
||||
pub(crate) struct Additive;
|
||||
#[pymethods]
|
||||
impl Additive {
|
||||
#[new]
|
||||
fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
}
|
||||
|
||||
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
|
||||
pub(crate) struct PromptUnits;
|
||||
#[pymethods]
|
||||
impl PromptUnits {
|
||||
#[new]
|
||||
fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
}
|
||||
|
||||
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
|
||||
pub(crate) struct Ocr;
|
||||
#[pymethods]
|
||||
impl Ocr {
|
||||
#[new]
|
||||
fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
}
|
||||
|
||||
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
|
||||
pub(crate) struct SearchTiers;
|
||||
#[pymethods]
|
||||
impl SearchTiers {
|
||||
#[new]
|
||||
#[pyo3(signature = (*, outside="last"))]
|
||||
fn new(outside: &str) -> PyResult<Self> {
|
||||
if outside != "last" {
|
||||
return Err(PyValueError::new_err("outside must be last"));
|
||||
}
|
||||
Ok(Self)
|
||||
}
|
||||
}
|
||||
|
||||
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
|
||||
pub(crate) struct FirstUnit {
|
||||
meters: Vec<litellm_cost::price::Meter>,
|
||||
}
|
||||
#[pymethods]
|
||||
impl FirstUnit {
|
||||
#[new]
|
||||
fn new(meters: Vec<String>) -> PyResult<Self> {
|
||||
let meters = meters
|
||||
.iter()
|
||||
.map(|meter| {
|
||||
parse_meter(meter).ok_or_else(|| PyValueError::new_err("unknown alternative meter"))
|
||||
})
|
||||
.collect::<PyResult<Vec<_>>>()?;
|
||||
if meters.is_empty() || meters.iter().collect::<BTreeSet<_>>().len() != meters.len() {
|
||||
return Err(PyValueError::new_err(
|
||||
"alternative meters must be nonempty and unique",
|
||||
));
|
||||
}
|
||||
Ok(Self { meters })
|
||||
}
|
||||
}
|
||||
|
||||
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
|
||||
pub(crate) struct DirectionalCharacters {
|
||||
threshold: NonZeroU64,
|
||||
comparison_scale: NonZeroU64,
|
||||
}
|
||||
#[pymethods]
|
||||
impl DirectionalCharacters {
|
||||
#[new]
|
||||
fn new(threshold: StrictCount, comparison_scale: StrictCount) -> PyResult<Self> {
|
||||
let positive = |value| {
|
||||
NonZeroU64::new(value).ok_or_else(|| {
|
||||
PyValueError::new_err("threshold and comparison_scale must be positive")
|
||||
})
|
||||
};
|
||||
Ok(Self {
|
||||
threshold: positive(threshold.0)?,
|
||||
comparison_scale: positive(comparison_scale.0)?,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(FromPyObject)]
|
||||
enum SelectionInput<'py> {
|
||||
Additive(PyRef<'py, Additive>),
|
||||
PromptUnits(PyRef<'py, PromptUnits>),
|
||||
FirstUnit(PyRef<'py, FirstUnit>),
|
||||
SearchTiers(PyRef<'py, SearchTiers>),
|
||||
Ocr(PyRef<'py, Ocr>),
|
||||
DirectionalCharacters(PyRef<'py, DirectionalCharacters>),
|
||||
}
|
||||
|
||||
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
|
||||
pub(crate) struct OtherMeterInput {
|
||||
selection: OtherSelection,
|
||||
sources: Vec<PriceRowInput>,
|
||||
defaults: Vec<DefaultRateInput>,
|
||||
report_to: Option<ReportTo>,
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl OtherMeterInput {
|
||||
#[new]
|
||||
#[pyo3(signature = (selection, *, fallback_sources=Vec::new(), defaults=Vec::new(), report_to=None), text_signature = "(selection, *, fallback_sources=(), defaults=(), report_to=None)")]
|
||||
fn new(
|
||||
selection: SelectionInput<'_>,
|
||||
fallback_sources: Vec<PyRef<'_, PriceRowInput>>,
|
||||
defaults: Vec<PyRef<'_, DefaultRateInput>>,
|
||||
report_to: Option<&str>,
|
||||
) -> PyResult<Self> {
|
||||
let selection = match selection {
|
||||
SelectionInput::Additive(value) => {
|
||||
drop(value);
|
||||
OtherSelection::Additive
|
||||
}
|
||||
SelectionInput::PromptUnits(value) => {
|
||||
drop(value);
|
||||
OtherSelection::PromptUnits
|
||||
}
|
||||
SelectionInput::FirstUnit(value) => OtherSelection::FirstUnit(value.meters.clone()),
|
||||
SelectionInput::SearchTiers(value) => {
|
||||
drop(value);
|
||||
OtherSelection::SearchTiers
|
||||
}
|
||||
SelectionInput::Ocr(value) => {
|
||||
drop(value);
|
||||
OtherSelection::Ocr
|
||||
}
|
||||
SelectionInput::DirectionalCharacters(value) => OtherSelection::DirectionalCharacters {
|
||||
threshold: value.threshold,
|
||||
comparison_scale: value.comparison_scale,
|
||||
},
|
||||
};
|
||||
if fallback_sources
|
||||
.iter()
|
||||
.map(|source| &source.source_id)
|
||||
.collect::<BTreeSet<_>>()
|
||||
.len()
|
||||
!= fallback_sources.len()
|
||||
{
|
||||
return Err(PyValueError::new_err("fallback source IDs must be unique"));
|
||||
}
|
||||
if defaults
|
||||
.iter()
|
||||
.map(|rate| &rate.meter)
|
||||
.collect::<BTreeSet<_>>()
|
||||
.len()
|
||||
!= defaults.len()
|
||||
{
|
||||
return Err(PyValueError::new_err("default meters must be unique"));
|
||||
}
|
||||
let report_to = report_to
|
||||
.map(|value| {
|
||||
value
|
||||
.parse()
|
||||
.map_err(|_| PyValueError::new_err("report_to must be input or output"))
|
||||
})
|
||||
.transpose()?;
|
||||
Ok(Self {
|
||||
selection,
|
||||
sources: fallback_sources
|
||||
.into_iter()
|
||||
.map(|source| (*source).clone())
|
||||
.collect(),
|
||||
defaults: defaults.into_iter().map(|rate| (*rate).clone()).collect(),
|
||||
report_to,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl OtherMeterInput {
|
||||
pub(super) fn directional(&self) -> bool {
|
||||
matches!(self.selection, OtherSelection::DirectionalCharacters { .. })
|
||||
}
|
||||
|
||||
pub(super) fn prepare(
|
||||
&self,
|
||||
row: &Bound<'_, PyAny>,
|
||||
) -> Result<(Model, OtherContext), RowErrors> {
|
||||
let fields = row_fields("", row)?;
|
||||
let errors = |errors: Vec<litellm_model_catalog::Located>| RowErrors {
|
||||
errors: errors.iter().map(row_error).collect(),
|
||||
};
|
||||
let (mut model, primary) =
|
||||
Model::from_fields_with_other_source(&fields, "primary".to_owned()).map_err(errors)?;
|
||||
let mut context = OtherContext {
|
||||
selection: self.selection.clone(),
|
||||
sources: vec![primary],
|
||||
report_to: self.report_to,
|
||||
..OtherContext::default()
|
||||
};
|
||||
for source in &self.sources {
|
||||
let (fallback, metadata) =
|
||||
Model::from_fields_with_other_source(&source.fields, source.source_id.clone())
|
||||
.map_err(errors)?;
|
||||
model
|
||||
.prices
|
||||
.append_source(context.sources.len(), fallback.prices);
|
||||
context.sources.push(metadata);
|
||||
}
|
||||
for default in &self.defaults {
|
||||
context.insert_default(
|
||||
&mut model.prices,
|
||||
default.meter.clone(),
|
||||
default.rate,
|
||||
default.per_units,
|
||||
default.eligibility,
|
||||
);
|
||||
}
|
||||
Ok((model, context))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rstest::rstest;
|
||||
use std::ffi::CString;
|
||||
|
||||
#[rstest]
|
||||
#[case::zero("0", true)]
|
||||
#[case::fraction("1.5", true)]
|
||||
#[case::negative("-1", false)]
|
||||
#[case::boolean("True", false)]
|
||||
#[case::string("'1.5'", false)]
|
||||
#[case::nan("float('nan')", false)]
|
||||
#[case::infinite("float('inf')", false)]
|
||||
#[case::none("None", false)]
|
||||
fn measured_constructor_validates_python_values(
|
||||
#[case] expression: &str,
|
||||
#[case] accepted: bool,
|
||||
) {
|
||||
Python::attach(|py| {
|
||||
let value = py
|
||||
.eval(&CString::new(expression).unwrap(), None, None)
|
||||
.unwrap();
|
||||
assert_eq!(MeasuredQuantity::new(&value, "meter").is_ok(), accepted);
|
||||
});
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::zero("0", Some(0))]
|
||||
#[case::large_exact("9007199254740993", Some(9007199254740993))]
|
||||
#[case::maximum("18446744073709551615", Some(u64::MAX))]
|
||||
#[case::overflow("18446744073709551616", None)]
|
||||
#[case::negative("-1", None)]
|
||||
#[case::fraction("1.5", None)]
|
||||
#[case::boolean("True", None)]
|
||||
fn new_context_counts_are_strict_and_exact(
|
||||
#[case] expression: &str,
|
||||
#[case] expected: Option<u64>,
|
||||
) {
|
||||
Python::attach(|py| {
|
||||
let value = py
|
||||
.eval(&CString::new(expression).unwrap(), None, None)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
value.extract::<StrictCount>().ok().map(|count| count.0),
|
||||
expected
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::finite("{'input_cost_per_token': 1.0}", true)]
|
||||
#[case::nan("{'input_cost_per_token': float('nan')}", false)]
|
||||
#[case::infinite("{'input_cost_per_token': float('inf')}", false)]
|
||||
fn fallback_source_rejects_nonfinite_before_json_conversion(
|
||||
#[case] expression: &str,
|
||||
#[case] accepted: bool,
|
||||
) {
|
||||
Python::attach(|py| {
|
||||
let value = py
|
||||
.eval(&CString::new(expression).unwrap(), None, None)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
PriceRowInput::new("published".to_owned(), value.cast::<PyMapping>().unwrap())
|
||||
.is_ok(),
|
||||
accepted
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
@ -192,17 +192,67 @@ class ModelInfoView:
|
|||
@property
|
||||
def output_cost_per_token(self) -> float | None: ...
|
||||
|
||||
@final
|
||||
class MeasuredQuantity:
|
||||
def __new__(cls, value: float, *, unit: Literal["meter", "milliseconds"] = "meter") -> MeasuredQuantity: ...
|
||||
|
||||
@final
|
||||
class PriceRowInput:
|
||||
def __new__(cls, source_id: str, row: Mapping[str, JsonValue]) -> PriceRowInput: ...
|
||||
|
||||
@final
|
||||
class DefaultRateInput:
|
||||
def __new__(
|
||||
cls, meter: str, value: float, *, per_units: int = 1, apply_when: Literal["absent", "absent_or_null"]
|
||||
) -> DefaultRateInput: ...
|
||||
|
||||
@final
|
||||
class Additive:
|
||||
def __new__(cls) -> Additive: ...
|
||||
|
||||
@final
|
||||
class PromptUnits:
|
||||
def __new__(cls) -> PromptUnits: ...
|
||||
|
||||
@final
|
||||
class FirstUnit:
|
||||
def __new__(cls, meters: Sequence[str]) -> FirstUnit: ...
|
||||
|
||||
@final
|
||||
class SearchTiers:
|
||||
def __new__(cls, *, outside: Literal["last"] = "last") -> SearchTiers: ...
|
||||
|
||||
@final
|
||||
class Ocr:
|
||||
def __new__(cls) -> Ocr: ...
|
||||
|
||||
@final
|
||||
class DirectionalCharacters:
|
||||
def __new__(cls, threshold: int, comparison_scale: int) -> DirectionalCharacters: ...
|
||||
|
||||
@final
|
||||
class OtherMeterInput:
|
||||
def __new__(
|
||||
cls,
|
||||
selection: Additive | PromptUnits | FirstUnit | SearchTiers | Ocr | DirectionalCharacters,
|
||||
*,
|
||||
fallback_sources: Sequence[PriceRowInput] = (),
|
||||
defaults: Sequence[DefaultRateInput] = (),
|
||||
report_to: Literal["input", "output"] | None = None,
|
||||
) -> OtherMeterInput: ...
|
||||
|
||||
@final
|
||||
class UsageInput:
|
||||
"""Disjoint quantities keyed by unsuffixed catalog price fields, web_search/{size}, or guardrail/{counter}."""
|
||||
|
||||
def __new__(
|
||||
cls,
|
||||
quantities: dict[str, int],
|
||||
quantities: dict[str, int | MeasuredQuantity],
|
||||
*,
|
||||
prompt_tokens: int = 0,
|
||||
inclusive_threshold: bool = False,
|
||||
search_results: int = 0,
|
||||
completion_tokens: int | None = None,
|
||||
) -> UsageInput:
|
||||
"""prompt_tokens is the full prompt size used for band selection, not the uncached input quantity."""
|
||||
|
||||
|
|
@ -222,7 +272,15 @@ class PriceLine:
|
|||
@property
|
||||
def meter(self) -> str: ...
|
||||
@property
|
||||
def quantity(self) -> int: ...
|
||||
def quantity(self) -> int | float: ...
|
||||
@property
|
||||
def quantity_unit(self) -> str: ...
|
||||
@property
|
||||
def source_id(self) -> str | None: ...
|
||||
@property
|
||||
def caller_default(self) -> bool: ...
|
||||
@property
|
||||
def report_to(self) -> str | None: ...
|
||||
@property
|
||||
def unit_price(self) -> float: ...
|
||||
@property
|
||||
|
|
@ -250,6 +308,10 @@ class Unpriced:
|
|||
|
||||
@final
|
||||
class SelectedRate:
|
||||
@property
|
||||
def source_id(self) -> str | None: ...
|
||||
@property
|
||||
def caller_default(self) -> bool: ...
|
||||
@property
|
||||
def requested_meter(self) -> str: ...
|
||||
@property
|
||||
|
|
@ -302,6 +364,7 @@ def price(
|
|||
policy: Literal["generic", "sdk_tokens"] = "generic",
|
||||
cache_read_default_ratio: float | None = None,
|
||||
directional_token_threshold: int | None = None,
|
||||
other_meters: OtherMeterInput | None = None,
|
||||
) -> Priced | Unpriced | RowErrors: ...
|
||||
def price_with_rates(
|
||||
row: dict[str, object],
|
||||
|
|
@ -313,6 +376,7 @@ def price_with_rates(
|
|||
factors: Sequence[float] = (),
|
||||
cache_read_default_ratio: float | None = None,
|
||||
directional_token_threshold: int | None = None,
|
||||
other_meters: OtherMeterInput | None = None,
|
||||
) -> PriceEvaluation | RowErrors: ...
|
||||
def trace_encode_error(message: str) -> bytes: ...
|
||||
|
||||
|
|
@ -504,11 +568,16 @@ def process_state_started() -> bool: ...
|
|||
def reserve_process_for_forking() -> None: ...
|
||||
|
||||
__all__ = [
|
||||
"Additive",
|
||||
"CatalogRefresher",
|
||||
"DefaultRateInput",
|
||||
"DirectionalCharacters",
|
||||
"FetchedRegistry",
|
||||
"FirstUnit",
|
||||
"ForkedAfterNativeRuntimeStarted",
|
||||
"HuggingFaceEncoding",
|
||||
"InvalidProvider",
|
||||
"MeasuredQuantity",
|
||||
"ModelCostLayers",
|
||||
"ModelInfoView",
|
||||
"ModelNotFound",
|
||||
|
|
@ -516,10 +585,14 @@ __all__ = [
|
|||
"NativeClickHouseSpendConfig",
|
||||
"NativeClickHouseSpendStorage",
|
||||
"NativeDiagnosticProcessor",
|
||||
"Ocr",
|
||||
"OtherMeterInput",
|
||||
"PriceEvaluation",
|
||||
"PriceLine",
|
||||
"PriceRowInput",
|
||||
"Priced",
|
||||
"ProcessReservedForForking",
|
||||
"PromptUnits",
|
||||
"RegistrationTarget",
|
||||
"RegistryNotModified",
|
||||
"RegistryUnavailable",
|
||||
|
|
@ -530,6 +603,7 @@ __all__ = [
|
|||
"RowErrors",
|
||||
"RustBridgeDeclined",
|
||||
"RustUpstreamError",
|
||||
"SearchTiers",
|
||||
"SelectedRate",
|
||||
"TokenCounter",
|
||||
"Tokenizer",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue