mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
refactor(cost): price token usage with the Rust cost core [08/12]
cost::calculate prices token meters over the typed Prices every row lowers into, with one line per meter and the borrowed rate named in fallback_from. generic_cost_per_token prices through Prices.price and as_main_cost, per-call custom pricing is an overlay that never writes the shared row, and a directly written bad price is unpriced with one log line Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
83882222c0
commit
c57274fa12
25 changed files with 2562 additions and 1450 deletions
2
litellm-rust/Cargo.lock
generated
2
litellm-rust/Cargo.lock
generated
|
|
@ -4284,6 +4284,7 @@ dependencies = [
|
|||
"fancy-regex 0.19.2",
|
||||
"futures-util",
|
||||
"indexmap 2.14.0",
|
||||
"jiff",
|
||||
"litellm-auth",
|
||||
"litellm-auth-aws",
|
||||
"litellm-cache",
|
||||
|
|
@ -4292,6 +4293,7 @@ dependencies = [
|
|||
"litellm-cache-response",
|
||||
"litellm-callbacks-legacy-python",
|
||||
"litellm-core-utils",
|
||||
"litellm-cost",
|
||||
"litellm-host",
|
||||
"litellm-host-python",
|
||||
"litellm-http",
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ serde = ["dep:serde"]
|
|||
schema = ["serde", "dep:schemars"]
|
||||
|
||||
[dependencies]
|
||||
jiff = { version = "0.2.37", default-features = false, features = ["std"] }
|
||||
jiff = { version = "0.2.37", default-features = false, features = ["std", "tzdb-bundle-always"] }
|
||||
schemars = { workspace = true, optional = true }
|
||||
serde = { workspace = true, optional = true }
|
||||
thiserror.workspace = true
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
# litellm-cost
|
||||
|
||||
This crate holds the price data model that `litellm-model-catalog` lowers each catalog entry into, plus the usage, call context and priced outcome types that `calculate` will compute over. There is no `calculate` until the token pricing PR lands. `UnitPrice` is USD per unit and `Usd` is an amount, and only the operations that make sense between them exist, so a price is never added to an amount. `Prices::band` and `Prices::rate` resolve a token rate the way Python's `_get_token_base_cost` does
|
||||
This crate holds the price data model that `litellm-model-catalog` lowers each catalog entry into, plus the usage, call context and priced outcome types. `calculate` prices a call's token usage over a row's `Prices` and `Adjustments`, one `Line` per meter with usage, and names the meter whose rate a line borrowed in `fallback_from`. Character, second and per-unit meters land in later PRs and are reported as `missing` until then. `UnitPrice` is USD per unit and `Usd` is an amount, and only the operations that make sense between them exist, so a price is never added to an amount. `Prices::band` picks the prompt band the way Python's `_get_token_base_cost` does
|
||||
|
|
|
|||
796
litellm-rust/crates/cost/src/calculate.rs
Normal file
796
litellm-rust/crates/cost/src/calculate.rs
Normal file
|
|
@ -0,0 +1,796 @@
|
|||
use crate::call::CallContext;
|
||||
use crate::money::{Multiplier, UnitPrice};
|
||||
use crate::price::{
|
||||
Adjustments, Band, CacheReadFallback, CacheReadMode, CacheTtl, Channel, Feature, InputUnit,
|
||||
Meter, OffPeakRates, OutputUnit, Price, Prices, ServiceTier, TierScope, TokenKind, Uplift,
|
||||
When,
|
||||
};
|
||||
use crate::priced::{Line, Outcome, Priced, Quantity, Side};
|
||||
use crate::usage::{DurationMeter, Usage};
|
||||
|
||||
const TEXT_INPUT: Meter = Meter::Input(InputUnit::Token(TokenKind::Text));
|
||||
const TEXT_OUTPUT: Meter = Meter::Output(OutputUnit::Token(TokenKind::Text));
|
||||
const IMAGE_OUTPUT: Meter = Meter::Output(OutputUnit::Token(TokenKind::Image));
|
||||
const REASONING: Meter = Meter::Output(OutputUnit::ReasoningToken);
|
||||
const CACHE_READ: Meter = Meter::CacheRead {
|
||||
kind: TokenKind::Text,
|
||||
mode: CacheReadMode::Implicit,
|
||||
};
|
||||
const CACHE_WRITE: Meter = Meter::CacheWrite {
|
||||
kind: TokenKind::Text,
|
||||
ttl: CacheTtl::Default,
|
||||
};
|
||||
|
||||
/// Prices one call's token usage (decisions 21 and 30). Every meter with usage is a `Line`, one that borrowed another
|
||||
/// meter's rate names it in `fallback_from`, and one with no price and no fallback is listed in `missing`
|
||||
pub fn calculate(
|
||||
prices: &Prices,
|
||||
adjustments: &Adjustments,
|
||||
usage: &Usage,
|
||||
call: &CallContext,
|
||||
) -> Outcome {
|
||||
let rates = Rates::new(prices, usage, call);
|
||||
let uplifts = uplifts(adjustments, call);
|
||||
let tokens = &usage.tokens;
|
||||
let billed: Vec<(Meter, Option<Rate>, u64, Side)> = tokens
|
||||
.input
|
||||
.iter()
|
||||
.map(|(kind, count)| {
|
||||
(
|
||||
Meter::Input(InputUnit::Token(*kind)),
|
||||
rates.input(*kind),
|
||||
*count,
|
||||
Side::Prompt,
|
||||
)
|
||||
})
|
||||
.chain(tokens.cache_read.iter().map(|((kind, mode), count)| {
|
||||
let meter = Meter::CacheRead {
|
||||
kind: *kind,
|
||||
mode: *mode,
|
||||
};
|
||||
(meter, rates.cache_read(*kind, *mode), *count, Side::Prompt)
|
||||
}))
|
||||
.chain(tokens.cache_write.iter().map(|((kind, ttl), count)| {
|
||||
let meter = Meter::CacheWrite {
|
||||
kind: *kind,
|
||||
ttl: *ttl,
|
||||
};
|
||||
(meter, rates.cache_write(*ttl), *count, Side::Prompt)
|
||||
}))
|
||||
.chain(tokens.output.iter().map(|(kind, count)| {
|
||||
let meter = Meter::Output(OutputUnit::Token(*kind));
|
||||
(meter, rates.output(*kind), *count, Side::Completion)
|
||||
}))
|
||||
.chain([(
|
||||
REASONING,
|
||||
rates.reasoning(),
|
||||
tokens.reasoning,
|
||||
Side::Completion,
|
||||
)])
|
||||
.filter(|(.., count, _)| *count > 0)
|
||||
.collect();
|
||||
let lines: Vec<Line> = billed
|
||||
.iter()
|
||||
.filter_map(|(meter, rate, count, side)| {
|
||||
Some(rate.clone()?.line(meter.clone(), *count, *side, &uplifts))
|
||||
})
|
||||
.collect();
|
||||
let missing: Vec<Meter> = billed
|
||||
.into_iter()
|
||||
.filter_map(|(meter, rate, ..)| rate.is_none().then_some(meter))
|
||||
.chain(without_an_arm(usage))
|
||||
.collect();
|
||||
if lines.is_empty() && !missing.is_empty() {
|
||||
return Outcome::Unpriced { missing };
|
||||
}
|
||||
let total = lines.iter().map(|line| line.amount).sum();
|
||||
Outcome::Priced(Priced {
|
||||
lines,
|
||||
missing,
|
||||
total,
|
||||
})
|
||||
}
|
||||
|
||||
/// One resolved rate, and the meter it came from when that is not the billed meter's own
|
||||
#[derive(Clone, Debug)]
|
||||
struct Rate {
|
||||
price: UnitPrice,
|
||||
borrowed: Option<Meter>,
|
||||
}
|
||||
|
||||
impl Rate {
|
||||
fn own(price: UnitPrice) -> Self {
|
||||
Self {
|
||||
price,
|
||||
borrowed: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn borrowed_from(self, meter: &Meter) -> Self {
|
||||
Self {
|
||||
borrowed: Some(self.borrowed.unwrap_or_else(|| meter.clone())),
|
||||
..self
|
||||
}
|
||||
}
|
||||
|
||||
fn halved(self) -> Option<Self> {
|
||||
Some(Self {
|
||||
price: UnitPrice::new(self.price.get() * 0.5)?,
|
||||
..self
|
||||
})
|
||||
}
|
||||
|
||||
fn line(self, meter: Meter, count: u64, side: Side, uplifts: &[Multiplier]) -> Line {
|
||||
let quantity = Quantity::Count(count);
|
||||
Line {
|
||||
quantity,
|
||||
unit_price: self.price,
|
||||
amount: uplifts
|
||||
.iter()
|
||||
.fold(self.price * quantity, |amount, factor| amount * *factor),
|
||||
fallback_from: self.borrowed,
|
||||
meter,
|
||||
side,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The rates one call resolves, main's `_get_token_base_cost` and its modality, cache and reasoning rules
|
||||
struct Rates<'a> {
|
||||
prices: &'a Prices,
|
||||
overlay: Option<&'a Prices>,
|
||||
channel: Channel,
|
||||
tier: ServiceTier,
|
||||
band: Band,
|
||||
whole_call: bool,
|
||||
off_peak: Option<OffPeakRates>,
|
||||
text_input: Option<Rate>,
|
||||
text_output: Option<Rate>,
|
||||
}
|
||||
|
||||
impl<'a> Rates<'a> {
|
||||
/// A tier table bills the whole call at the tier the prompt picks, unless that tier sets no input price. Then
|
||||
/// the row is priced as if it had no tiers, as main's `_select_priced_tier` does
|
||||
fn new(prices: &'a Prices, usage: &Usage, call: &'a CallContext) -> Self {
|
||||
let band = prices.band(call.tier, usage.prompt_tokens);
|
||||
let tier_input = Price::online(
|
||||
TEXT_INPUT,
|
||||
When {
|
||||
tier: ServiceTier::Standard,
|
||||
band,
|
||||
},
|
||||
);
|
||||
let whole_call = prices.billing().tier_scope == TierScope::WholeCall
|
||||
&& prices.get(&tier_input).is_some();
|
||||
let lookup = Self {
|
||||
prices,
|
||||
overlay: call.overlay.as_ref(),
|
||||
channel: call.channel,
|
||||
tier: call.tier,
|
||||
band: match (prices.billing().tier_scope, whole_call) {
|
||||
(TierScope::WholeCall, false) => Band::Base,
|
||||
_ => band,
|
||||
},
|
||||
whole_call,
|
||||
off_peak: prices
|
||||
.off_peak()
|
||||
.filter(|off_peak| off_peak.schedule().is_open(call.at))
|
||||
.map(|off_peak| *off_peak.rates()),
|
||||
text_input: None,
|
||||
text_output: None,
|
||||
};
|
||||
let text_input = lookup
|
||||
.off_peak(|rates| rates.input)
|
||||
.or_else(|| lookup.banded(&TEXT_INPUT));
|
||||
let text_output = lookup
|
||||
.off_peak(|rates| rates.output)
|
||||
.or_else(|| lookup.output_or_image_rate());
|
||||
Self {
|
||||
text_input,
|
||||
text_output,
|
||||
..lookup
|
||||
}
|
||||
}
|
||||
|
||||
fn input(&self, kind: TokenKind) -> Option<Rate> {
|
||||
match kind {
|
||||
TokenKind::Text => self.text_input.clone(),
|
||||
_ => self
|
||||
.banded(&Meter::Input(InputUnit::Token(kind)))
|
||||
.or_else(|| self.borrowed_input()),
|
||||
}
|
||||
}
|
||||
|
||||
fn output(&self, kind: TokenKind) -> Option<Rate> {
|
||||
match kind {
|
||||
TokenKind::Text => self.text_output.clone(),
|
||||
_ => self
|
||||
.banded(&Meter::Output(OutputUnit::Token(kind)))
|
||||
.or_else(|| self.borrowed_output()),
|
||||
}
|
||||
}
|
||||
|
||||
fn cache_read(&self, kind: TokenKind, mode: CacheReadMode) -> Option<Rate> {
|
||||
let borrowed_implicit = || {
|
||||
self.implicit_cache_read()
|
||||
.map(|rate| rate.borrowed_from(&CACHE_READ))
|
||||
};
|
||||
match (kind, mode) {
|
||||
(TokenKind::Text, CacheReadMode::Implicit) => self.implicit_cache_read(),
|
||||
(TokenKind::Text, CacheReadMode::Explicit) => {
|
||||
let own = self.banded(&Meter::CacheRead { kind, mode });
|
||||
match (self.off_peak(|rates| rates.explicit_cache_read), own) {
|
||||
(Some(off_peak), _) => Some(off_peak),
|
||||
(None, Some(own)) => {
|
||||
Some(self.off_peak(|rates| rates.cache_read).unwrap_or(own))
|
||||
}
|
||||
(None, None) => borrowed_implicit(),
|
||||
}
|
||||
}
|
||||
_ => self
|
||||
.banded(&Meter::CacheRead { kind, mode })
|
||||
.or_else(borrowed_implicit),
|
||||
}
|
||||
}
|
||||
|
||||
fn cache_write(&self, ttl: CacheTtl) -> Option<Rate> {
|
||||
match ttl {
|
||||
CacheTtl::Default => self.default_cache_write(),
|
||||
CacheTtl::OneHour | CacheTtl::TwelveHours => self
|
||||
.banded(&Meter::CacheWrite {
|
||||
kind: TokenKind::Text,
|
||||
ttl,
|
||||
})
|
||||
.or_else(|| {
|
||||
self.default_cache_write()
|
||||
.map(|rate| rate.borrowed_from(&CACHE_WRITE))
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
/// Off-peak first, then the tier's reasoning price. A tier that prices output and not reasoning bills reasoning at
|
||||
/// its output rate, otherwise the standard reasoning price, then the output rate. Reasoning prices are never banded
|
||||
/// outside a whole-call tier table
|
||||
fn reasoning(&self) -> Option<Rate> {
|
||||
let tier_base = When {
|
||||
tier: self.tier,
|
||||
band: Band::Base,
|
||||
};
|
||||
let tier_prices_output_only =
|
||||
|| self.tier != ServiceTier::Standard && self.at(&TEXT_OUTPUT, &[tier_base]).is_some();
|
||||
self.off_peak(|rates| rates.reasoning)
|
||||
.or_else(|| match self.whole_call() {
|
||||
true => self.banded(&REASONING),
|
||||
false => self.at(&REASONING, &[tier_base]).or_else(|| {
|
||||
(!tier_prices_output_only())
|
||||
.then(|| self.at(&REASONING, &[When::default()]))
|
||||
.flatten()
|
||||
}),
|
||||
})
|
||||
.or_else(|| self.borrowed_output())
|
||||
}
|
||||
|
||||
fn borrowed_input(&self) -> Option<Rate> {
|
||||
self.text_input
|
||||
.clone()
|
||||
.map(|rate| rate.borrowed_from(&TEXT_INPUT))
|
||||
}
|
||||
|
||||
fn borrowed_output(&self) -> Option<Rate> {
|
||||
self.text_output
|
||||
.clone()
|
||||
.map(|rate| rate.borrowed_from(&TEXT_OUTPUT))
|
||||
}
|
||||
|
||||
fn implicit_cache_read(&self) -> Option<Rate> {
|
||||
self.off_peak(|rates| rates.cache_read)
|
||||
.or_else(|| self.banded(&CACHE_READ))
|
||||
.or_else(|| match self.prices.billing().cache_read_fallback {
|
||||
CacheReadFallback::InputRate => self.borrowed_input(),
|
||||
CacheReadFallback::HalfInputRate => self.borrowed_input()?.halved(),
|
||||
})
|
||||
}
|
||||
|
||||
fn default_cache_write(&self) -> Option<Rate> {
|
||||
self.off_peak(|rates| rates.cache_write)
|
||||
.or_else(|| self.banded(&CACHE_WRITE))
|
||||
.or_else(|| self.borrowed_input())
|
||||
}
|
||||
|
||||
/// main reads a missing or zero output price as unknown and bills image models at their image token rate. A
|
||||
/// whole-call tier with no output price bills the row's own output price
|
||||
fn output_or_image_rate(&self) -> Option<Rate> {
|
||||
let own = self.banded(&TEXT_OUTPUT).or_else(|| {
|
||||
self.whole_call()
|
||||
.then(|| self.at(&TEXT_OUTPUT, &[When::default()]))?
|
||||
});
|
||||
match own {
|
||||
Some(rate) if rate.price.get() != 0.0 => Some(rate),
|
||||
own => self
|
||||
.at(&IMAGE_OUTPUT, &[When::default()])
|
||||
.map(|rate| rate.borrowed_from(&IMAGE_OUTPUT))
|
||||
.or(own),
|
||||
}
|
||||
}
|
||||
|
||||
fn off_peak(&self, rate: fn(&OffPeakRates) -> Option<UnitPrice>) -> Option<Rate> {
|
||||
self.off_peak.as_ref().and_then(rate).map(Rate::own)
|
||||
}
|
||||
|
||||
fn whole_call(&self) -> bool {
|
||||
self.whole_call
|
||||
}
|
||||
|
||||
/// The meter's own rate in this call's band, tried at `(tier, band)`, `(standard, band)`, `(tier, base)` and
|
||||
/// `(standard, base)`. A whole-call tier table prices every meter at the band's own rate only
|
||||
fn banded(&self, meter: &Meter) -> Option<Rate> {
|
||||
let standard = ServiceTier::Standard;
|
||||
let whens: &[When] = match self.whole_call() {
|
||||
true => &[When {
|
||||
tier: standard,
|
||||
band: self.band,
|
||||
}],
|
||||
false => &[
|
||||
When {
|
||||
tier: self.tier,
|
||||
band: self.band,
|
||||
},
|
||||
When {
|
||||
tier: standard,
|
||||
band: self.band,
|
||||
},
|
||||
When {
|
||||
tier: self.tier,
|
||||
band: Band::Base,
|
||||
},
|
||||
When::default(),
|
||||
],
|
||||
};
|
||||
self.at(meter, whens)
|
||||
}
|
||||
|
||||
/// The first price for `meter` at `whens` on this call's channel. A per-call overlay that prices the meter at all
|
||||
/// replaces the row's prices for it (decision 12). Decisions bills the online prices it does not set, and a batch
|
||||
/// call with no batch price bills half the online price, main's `_batch_or_half`
|
||||
fn at(&self, meter: &Meter, whens: &[When]) -> Option<Rate> {
|
||||
let source = match self.overlay {
|
||||
Some(overlay) if overlay.iter().any(|(price, _)| &price.meter == meter) => overlay,
|
||||
_ => self.prices,
|
||||
};
|
||||
let on = |channel: Channel| {
|
||||
whens.iter().find_map(|when| {
|
||||
source.get(&Price {
|
||||
meter: meter.clone(),
|
||||
channel,
|
||||
when: *when,
|
||||
})
|
||||
})
|
||||
};
|
||||
match self.channel {
|
||||
Channel::Online => on(Channel::Online).map(Rate::own),
|
||||
Channel::Decisions => on(Channel::Decisions)
|
||||
.or_else(|| on(Channel::Online))
|
||||
.map(Rate::own),
|
||||
Channel::Batch => on(Channel::Batch).map(Rate::own).or_else(|| {
|
||||
Rate::own(on(Channel::Online)?)
|
||||
.borrowed_from(meter)
|
||||
.halved()
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Meters with usage that no arm prices yet. Characters and seconds land in 09, the per-unit meters in 10
|
||||
fn without_an_arm(usage: &Usage) -> impl Iterator<Item = Meter> + '_ {
|
||||
let counted = [
|
||||
(usage.characters.prompt, Meter::Input(InputUnit::Character)),
|
||||
(
|
||||
usage.characters.completion,
|
||||
Meter::Output(OutputUnit::Character),
|
||||
),
|
||||
(usage.images.input, Meter::Input(InputUnit::Image)),
|
||||
(usage.images.input_pixels, Meter::Input(InputUnit::Pixel)),
|
||||
(
|
||||
usage.images.output.len() as u64,
|
||||
Meter::Output(OutputUnit::Image(None)),
|
||||
),
|
||||
(
|
||||
usage.search.queries,
|
||||
Meter::Input(InputUnit::Query { over_results: None }),
|
||||
),
|
||||
(
|
||||
usage.search.maps_grounding,
|
||||
Meter::Feature(Feature::GoogleMapsGroundingQuery),
|
||||
),
|
||||
(
|
||||
usage.search.browser_opens,
|
||||
Meter::Feature(Feature::BrowserOpen),
|
||||
),
|
||||
(
|
||||
usage.search.additional_results,
|
||||
Meter::Feature(Feature::AdditionalResult),
|
||||
),
|
||||
(usage.text_messages, Meter::Input(InputUnit::TextMessage)),
|
||||
(
|
||||
usage.tokens.citation,
|
||||
Meter::Feature(Feature::CitationToken),
|
||||
),
|
||||
];
|
||||
let web_search = usage
|
||||
.search
|
||||
.web_search
|
||||
.filter(|search| search.requests > 0)
|
||||
.map(|search| Meter::WebSearch(search.context));
|
||||
let durations = usage
|
||||
.durations
|
||||
.iter()
|
||||
.filter(|(_, duration)| !duration.is_zero())
|
||||
.map(|(meter, _)| match meter {
|
||||
DurationMeter::WallClock => Meter::WallClockSecond,
|
||||
DurationMeter::InputAudio => Meter::Input(InputUnit::AudioSecond),
|
||||
DurationMeter::LiveVoice => Meter::Input(InputUnit::Second),
|
||||
DurationMeter::OutputVideo { resolution } => {
|
||||
Meter::Output(OutputUnit::Second(resolution.clone()))
|
||||
}
|
||||
});
|
||||
counted
|
||||
.into_iter()
|
||||
.filter_map(|(count, meter)| (count > 0).then_some(meter))
|
||||
.chain(web_search)
|
||||
.chain(durations)
|
||||
}
|
||||
|
||||
fn uplifts(adjustments: &Adjustments, call: &CallContext) -> Vec<Multiplier> {
|
||||
let residency = call
|
||||
.residency
|
||||
.and_then(|residency| adjustments.get(&Uplift::Residency(residency)));
|
||||
let endpoint = call
|
||||
.non_global_endpoint
|
||||
.then(|| adjustments.get(&Uplift::NonGlobalEndpoint))
|
||||
.flatten();
|
||||
residency.into_iter().chain(endpoint).collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::BTreeMap;
|
||||
use std::num::NonZeroU64;
|
||||
|
||||
use rstest::rstest;
|
||||
|
||||
use super::*;
|
||||
use crate::price::{Billing, Residency};
|
||||
|
||||
fn price(
|
||||
meter: &Meter,
|
||||
channel: Channel,
|
||||
tier: ServiceTier,
|
||||
band: Band,
|
||||
rate: f64,
|
||||
) -> (Price, UnitPrice) {
|
||||
(
|
||||
Price {
|
||||
meter: meter.clone(),
|
||||
channel,
|
||||
when: When { tier, band },
|
||||
},
|
||||
UnitPrice::new(rate).unwrap(),
|
||||
)
|
||||
}
|
||||
|
||||
fn standard(meter: &Meter, rate: f64) -> (Price, UnitPrice) {
|
||||
price(
|
||||
meter,
|
||||
Channel::Online,
|
||||
ServiceTier::Standard,
|
||||
Band::Base,
|
||||
rate,
|
||||
)
|
||||
}
|
||||
|
||||
fn row(rates: impl IntoIterator<Item = (Price, UnitPrice)>) -> Prices {
|
||||
Prices::new(rates.into_iter().collect(), None, Billing::default())
|
||||
}
|
||||
|
||||
fn call() -> CallContext {
|
||||
CallContext {
|
||||
tier: ServiceTier::Standard,
|
||||
channel: Channel::Online,
|
||||
residency: None,
|
||||
non_global_endpoint: false,
|
||||
speed: None,
|
||||
inference_geo: None,
|
||||
at: jiff::Timestamp::UNIX_EPOCH,
|
||||
overlay: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn usage(input: u64, cache_read: u64, output: u64) -> Usage {
|
||||
let mut usage = Usage {
|
||||
prompt_tokens: input + cache_read,
|
||||
..Usage::default()
|
||||
};
|
||||
usage.tokens.input = BTreeMap::from([(TokenKind::Text, input)]);
|
||||
usage.tokens.cache_read =
|
||||
BTreeMap::from([((TokenKind::Text, CacheReadMode::Implicit), cache_read)]);
|
||||
usage.tokens.output = BTreeMap::from([(TokenKind::Text, output)]);
|
||||
usage
|
||||
}
|
||||
|
||||
fn priced(outcome: Outcome) -> Priced {
|
||||
match outcome {
|
||||
Outcome::Priced(priced) => priced,
|
||||
Outcome::Unpriced { missing } => panic!("unpriced, missing {missing:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn total(prices: &Prices, usage: &Usage, call: &CallContext) -> f64 {
|
||||
priced(calculate(prices, &Adjustments::default(), usage, call))
|
||||
.total
|
||||
.get()
|
||||
}
|
||||
|
||||
fn close(actual: f64, expected: f64) {
|
||||
assert!(
|
||||
(actual - expected).abs() <= expected.abs() * 1e-12,
|
||||
"{actual} != {expected}"
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn a_cache_read_with_no_price_borrows_the_input_rate_as_a_visible_line() {
|
||||
let prices = row([standard(&TEXT_INPUT, 1e-6), standard(&TEXT_OUTPUT, 2e-6)]);
|
||||
let outcome = priced(calculate(
|
||||
&prices,
|
||||
&Adjustments::default(),
|
||||
&usage(200, 800, 10),
|
||||
&call(),
|
||||
));
|
||||
let cache_line = outcome
|
||||
.lines
|
||||
.iter()
|
||||
.find(|line| line.meter == CACHE_READ)
|
||||
.unwrap();
|
||||
assert_eq!(cache_line.fallback_from, Some(TEXT_INPUT));
|
||||
close(outcome.total.get(), 1000.0 * 1e-6 + 10.0 * 2e-6);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn a_meter_with_usage_and_no_price_is_missing_and_the_rest_is_still_priced() {
|
||||
let prices = row([standard(&TEXT_OUTPUT, 2e-6)]);
|
||||
let outcome = priced(calculate(
|
||||
&prices,
|
||||
&Adjustments::default(),
|
||||
&usage(100, 0, 10),
|
||||
&call(),
|
||||
));
|
||||
assert_eq!(outcome.missing, [TEXT_INPUT]);
|
||||
close(outcome.total.get(), 10.0 * 2e-6);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn a_meter_no_arm_prices_yet_is_missing() {
|
||||
let prices = row([standard(&TEXT_INPUT, 1e-6)]);
|
||||
let usage = Usage {
|
||||
characters: crate::usage::Characters {
|
||||
prompt: 40,
|
||||
completion: 0,
|
||||
},
|
||||
..usage(100, 0, 0)
|
||||
};
|
||||
let outcome = priced(calculate(&prices, &Adjustments::default(), &usage, &call()));
|
||||
assert_eq!(outcome.missing, [Meter::Input(InputUnit::Character)]);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn a_call_with_no_priced_meter_is_unpriced() {
|
||||
let outcome = calculate(
|
||||
&row([]),
|
||||
&Adjustments::default(),
|
||||
&usage(100, 0, 10),
|
||||
&call(),
|
||||
);
|
||||
assert_eq!(
|
||||
outcome,
|
||||
Outcome::Unpriced {
|
||||
missing: vec![TEXT_INPUT, TEXT_OUTPUT]
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn the_overlay_replaces_the_meters_it_prices_and_keeps_the_rest() {
|
||||
let prices = row([standard(&TEXT_INPUT, 1e-6), standard(&TEXT_OUTPUT, 2e-6)]);
|
||||
let call = CallContext {
|
||||
overlay: Some(row([standard(&TEXT_INPUT, 5e-6)])),
|
||||
..call()
|
||||
};
|
||||
close(
|
||||
total(&prices, &usage(100, 0, 10), &call),
|
||||
100.0 * 5e-6 + 10.0 * 2e-6,
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::requested_region(Residency::Eu, true)]
|
||||
#[case::other_region(Residency::Us, false)]
|
||||
fn a_residency_uplift_multiplies_every_line(
|
||||
#[case] priced_region: Residency,
|
||||
#[case] applies: bool,
|
||||
) {
|
||||
let prices = row([standard(&TEXT_INPUT, 1e-6), standard(&TEXT_OUTPUT, 2e-6)]);
|
||||
let adjustments = Adjustments::new([(
|
||||
Uplift::Residency(priced_region),
|
||||
Multiplier::new(1.1).unwrap(),
|
||||
)]);
|
||||
let call = CallContext {
|
||||
residency: Some(Residency::Eu),
|
||||
..call()
|
||||
};
|
||||
let outcome = priced(calculate(&prices, &adjustments, &usage(100, 0, 10), &call));
|
||||
let base = 100.0 * 1e-6 + 10.0 * 2e-6;
|
||||
close(outcome.total.get(), if applies { base * 1.1 } else { base });
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::missing(None)]
|
||||
#[case::zero(Some(0.0))]
|
||||
fn an_image_model_with_no_output_price_bills_output_at_its_image_token_rate(
|
||||
#[case] output: Option<f64>,
|
||||
) {
|
||||
let prices = row(std::iter::once(standard(&IMAGE_OUTPUT, 4e-5))
|
||||
.chain(output.map(|rate| standard(&TEXT_OUTPUT, rate))));
|
||||
let outcome = priced(calculate(
|
||||
&prices,
|
||||
&Adjustments::default(),
|
||||
&usage(0, 0, 10),
|
||||
&call(),
|
||||
));
|
||||
assert_eq!(outcome.lines[0].fallback_from, Some(IMAGE_OUTPUT));
|
||||
close(outcome.total.get(), 10.0 * 4e-5);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::below_the_band(1_000, 1e-6)]
|
||||
#[case::above_the_band(300_000, 2e-6)]
|
||||
fn the_prompt_band_prices_every_token_meter(#[case] prompt: u64, #[case] input_rate: f64) {
|
||||
let over = Band::over_prompt_tokens(200_000);
|
||||
let prices = row([
|
||||
standard(&TEXT_INPUT, 1e-6),
|
||||
price(
|
||||
&TEXT_INPUT,
|
||||
Channel::Online,
|
||||
ServiceTier::Standard,
|
||||
over,
|
||||
2e-6,
|
||||
),
|
||||
]);
|
||||
let usage = Usage {
|
||||
prompt_tokens: prompt,
|
||||
..usage(100, 0, 0)
|
||||
};
|
||||
close(total(&prices, &usage, &call()), 100.0 * input_rate);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::batch_price(Some(3e-7), 3e-7, None)]
|
||||
#[case::half_the_online_price(None, 5e-7, Some(TEXT_INPUT))]
|
||||
fn a_batch_call_bills_its_batch_price_or_half_the_online_one(
|
||||
#[case] batch: Option<f64>,
|
||||
#[case] rate: f64,
|
||||
#[case] fallback: Option<Meter>,
|
||||
) {
|
||||
let batch_price = batch.map(|rate| {
|
||||
price(
|
||||
&TEXT_INPUT,
|
||||
Channel::Batch,
|
||||
ServiceTier::Standard,
|
||||
Band::Base,
|
||||
rate,
|
||||
)
|
||||
});
|
||||
let prices = row(std::iter::once(standard(&TEXT_INPUT, 1e-6)).chain(batch_price));
|
||||
let call = CallContext {
|
||||
channel: Channel::Batch,
|
||||
..call()
|
||||
};
|
||||
let outcome = priced(calculate(
|
||||
&prices,
|
||||
&Adjustments::default(),
|
||||
&usage(100, 0, 0),
|
||||
&call,
|
||||
));
|
||||
assert_eq!(outcome.lines[0].fallback_from, fallback);
|
||||
close(outcome.total.get(), 100.0 * rate);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::tier_reasoning_price(Some(9e-6), true, 9e-6)]
|
||||
#[case::tier_output_without_reasoning(None, true, 7e-6)]
|
||||
#[case::standard_reasoning_price(None, false, 3e-6)]
|
||||
fn reasoning_follows_mains_tier_rules(
|
||||
#[case] tier_reasoning: Option<f64>,
|
||||
#[case] tier_output: bool,
|
||||
#[case] expected: f64,
|
||||
) {
|
||||
let priority = |meter: &Meter, rate| {
|
||||
price(
|
||||
meter,
|
||||
Channel::Online,
|
||||
ServiceTier::Priority,
|
||||
Band::Base,
|
||||
rate,
|
||||
)
|
||||
};
|
||||
let prices = row([standard(&TEXT_OUTPUT, 2e-6), standard(&REASONING, 3e-6)]
|
||||
.into_iter()
|
||||
.chain(tier_reasoning.map(|rate| priority(&REASONING, rate)))
|
||||
.chain(tier_output.then(|| priority(&TEXT_OUTPUT, 7e-6))));
|
||||
let mut usage = usage(0, 0, 0);
|
||||
usage.tokens.reasoning = 10;
|
||||
let call = CallContext {
|
||||
tier: ServiceTier::Priority,
|
||||
..call()
|
||||
};
|
||||
close(total(&prices, &usage, &call), 10.0 * expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn a_long_cache_write_with_no_price_borrows_the_default_write_rate() {
|
||||
let prices = row([standard(&TEXT_INPUT, 1e-6), standard(&CACHE_WRITE, 1.25e-6)]);
|
||||
let mut usage = usage(0, 0, 0);
|
||||
usage.tokens.cache_write = BTreeMap::from([((TokenKind::Text, CacheTtl::OneHour), 100)]);
|
||||
let outcome = priced(calculate(&prices, &Adjustments::default(), &usage, &call()));
|
||||
assert_eq!(outcome.lines[0].fallback_from, Some(CACHE_WRITE));
|
||||
close(outcome.total.get(), 100.0 * 1.25e-6);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn a_half_input_cache_read_row_bills_cache_reads_at_half_the_input_rate() {
|
||||
let prices = Prices::new(
|
||||
[standard(&TEXT_INPUT, 1e-6)].into_iter().collect(),
|
||||
None,
|
||||
Billing {
|
||||
cache_read_fallback: CacheReadFallback::HalfInputRate,
|
||||
..Billing::default()
|
||||
},
|
||||
);
|
||||
close(total(&prices, &usage(0, 100, 0), &call()), 100.0 * 0.5e-6);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::low_tier(1_000, 1e-6)]
|
||||
#[case::high_tier(40_000, 4e-6)]
|
||||
fn a_whole_call_tier_bills_cache_reads_at_its_own_input_rate(
|
||||
#[case] prompt: u64,
|
||||
#[case] rate: f64,
|
||||
) {
|
||||
let over = Band::OverPromptTokens(NonZeroU64::new(32_000).unwrap());
|
||||
let prices = Prices::new(
|
||||
[
|
||||
standard(&TEXT_INPUT, 1e-6),
|
||||
standard(&CACHE_READ, 0.1e-6),
|
||||
price(
|
||||
&TEXT_INPUT,
|
||||
Channel::Online,
|
||||
ServiceTier::Standard,
|
||||
over,
|
||||
4e-6,
|
||||
),
|
||||
]
|
||||
.into_iter()
|
||||
.collect(),
|
||||
None,
|
||||
Billing {
|
||||
tier_scope: TierScope::WholeCall,
|
||||
..Billing::default()
|
||||
},
|
||||
);
|
||||
let usage = Usage {
|
||||
prompt_tokens: prompt,
|
||||
..usage(0, 100, 0)
|
||||
};
|
||||
let expected = if prompt > 32_000 { rate } else { 0.1e-6 };
|
||||
close(total(&prices, &usage, &call()), 100.0 * expected);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,3 +1,4 @@
|
|||
pub mod calculate;
|
||||
pub mod call;
|
||||
mod error;
|
||||
pub mod money;
|
||||
|
|
@ -6,4 +7,5 @@ pub mod priced;
|
|||
pub mod schedule;
|
||||
pub mod usage;
|
||||
|
||||
pub use calculate::calculate;
|
||||
pub use error::*;
|
||||
|
|
|
|||
|
|
@ -68,7 +68,7 @@ pub struct Billing {
|
|||
/// From `web_search_billing_unit`, whether web search and Maps grounding bill per query or once per prompt
|
||||
pub web_search: WebSearchUnit,
|
||||
/// Whether `tiered_pricing` bills each meter in its own tier or the whole call at the tier the input picks.
|
||||
/// Dashscope rows bill the whole call (inferred, 10 checks no other row has that shape)
|
||||
/// A prompt-token tier table bills the whole call, main's `_get_tiered_base_costs`
|
||||
pub tier_scope: TierScope,
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -31,6 +31,54 @@ pub enum Weekday {
|
|||
Sunday,
|
||||
}
|
||||
|
||||
impl Schedule {
|
||||
/// Whether `at` is off-peak. A window applies on its weekdays, read on the calendar `weekday_timezone` names (UTC
|
||||
/// when it names none or an unknown zone), or every day. Its start is inclusive and its end exclusive, a window may
|
||||
/// wrap past midnight, and one whose start equals its end covers the whole day
|
||||
pub fn is_open(&self, at: jiff::Timestamp) -> bool {
|
||||
let utc = at.to_zoned(jiff::tz::TimeZone::UTC);
|
||||
let second_of_day = u32::from(utc.hour().unsigned_abs()) * 3600
|
||||
+ u32::from(utc.minute().unsigned_abs()) * 60
|
||||
+ u32::from(utc.second().unsigned_abs());
|
||||
let calendar = self
|
||||
.weekday_timezone
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|zone| !zone.is_empty())
|
||||
.and_then(|zone| jiff::tz::TimeZone::get(zone).ok())
|
||||
.unwrap_or(jiff::tz::TimeZone::UTC);
|
||||
let weekday = Weekday::from_iso(
|
||||
at.to_zoned(calendar)
|
||||
.weekday()
|
||||
.to_monday_one_offset()
|
||||
.unsigned_abs(),
|
||||
);
|
||||
self.windows.iter().any(|window| {
|
||||
let on_day = window
|
||||
.weekdays
|
||||
.as_ref()
|
||||
.is_none_or(|days| weekday.is_some_and(|weekday| days.contains(&weekday)));
|
||||
on_day
|
||||
&& window
|
||||
.hours
|
||||
.iter()
|
||||
.any(|hours| hours.contains(second_of_day))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl UtcWindow {
|
||||
fn contains(self, second_of_day: u32) -> bool {
|
||||
let start = u32::from(self.start_minute) * 60;
|
||||
let end = u32::from(self.end_minute) * 60;
|
||||
if start < end {
|
||||
(start..end).contains(&second_of_day)
|
||||
} else {
|
||||
second_of_day >= start || second_of_day < end
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl FromStr for UtcWindow {
|
||||
type Err = InvalidUtcWindow;
|
||||
|
||||
|
|
@ -215,6 +263,54 @@ mod tests {
|
|||
assert_eq!(name.parse().ok(), expected);
|
||||
}
|
||||
|
||||
fn schedule(
|
||||
hours: &str,
|
||||
weekdays: Option<Vec<Weekday>>,
|
||||
weekday_timezone: Option<&str>,
|
||||
) -> Schedule {
|
||||
Schedule {
|
||||
windows: vec![Window {
|
||||
hours: vec![hours.parse().unwrap()],
|
||||
weekdays,
|
||||
}],
|
||||
weekday_timezone: weekday_timezone.map(str::to_owned),
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::inside("2026-10-28T10:00:00Z", "10:00-18:30", true)]
|
||||
#[case::end_is_exclusive("2026-10-28T18:30:00Z", "10:00-18:30", false)]
|
||||
#[case::wrapped_after_midnight("2026-10-28T01:59:59Z", "22:00-02:00", true)]
|
||||
#[case::wrapped_outside("2026-10-28T12:00:00Z", "22:00-02:00", false)]
|
||||
#[case::whole_day("2026-10-28T12:00:00Z", "00:00-00:00", true)]
|
||||
fn a_window_opens_at_its_start_and_closes_at_its_end(
|
||||
#[case] at: &str,
|
||||
#[case] hours: &str,
|
||||
#[case] open: bool,
|
||||
) {
|
||||
assert_eq!(
|
||||
schedule(hours, None, None).is_open(at.parse().unwrap()),
|
||||
open
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::utc_wednesday(None, Weekday::Wednesday, true)]
|
||||
#[case::shanghai_thursday(Some("Asia/Shanghai"), Weekday::Thursday, true)]
|
||||
#[case::shanghai_not_wednesday(Some("Asia/Shanghai"), Weekday::Wednesday, false)]
|
||||
#[case::unknown_zone_reads_utc(Some("Mars/Base"), Weekday::Wednesday, true)]
|
||||
fn weekdays_are_read_on_the_named_calendar(
|
||||
#[case] weekday_timezone: Option<&str>,
|
||||
#[case] weekday: Weekday,
|
||||
#[case] open: bool,
|
||||
) {
|
||||
let at = "2026-10-28T17:00:00Z".parse().unwrap();
|
||||
assert_eq!(
|
||||
schedule("16:30-00:30", Some(vec![weekday]), weekday_timezone).is_open(at),
|
||||
open
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::monday(1, Some(Weekday::Monday))]
|
||||
#[case::sunday(7, Some(Weekday::Sunday))]
|
||||
|
|
|
|||
|
|
@ -86,7 +86,7 @@ impl Located {
|
|||
Self::new(path_to_pointer(path), error)
|
||||
}
|
||||
|
||||
pub(crate) fn prepend(&mut self, key: &str) {
|
||||
pub fn prepend(&mut self, key: &str) {
|
||||
let prefix = escape_pointer_segment(key);
|
||||
self.path = if self.path.is_empty() {
|
||||
format!("/{prefix}")
|
||||
|
|
|
|||
|
|
@ -141,7 +141,12 @@ fn billing(wire: &EntryWire, rates: &BTreeMap<Price, UnitPrice>) -> Billing {
|
|||
Some(WebSearchBillingUnit::PerQuery) => WebSearchUnit::PerQuery,
|
||||
Some(WebSearchBillingUnit::PerPrompt) | None => WebSearchUnit::PerPrompt,
|
||||
},
|
||||
tier_scope: if provider == Some("dashscope") && wire.tier_rows.is_some() {
|
||||
tier_scope: if wire
|
||||
.tier_rows
|
||||
.iter()
|
||||
.flatten()
|
||||
.any(|row| row.range.is_some())
|
||||
{
|
||||
TierScope::WholeCall
|
||||
} else {
|
||||
TierScope::PerMeter
|
||||
|
|
@ -958,10 +963,14 @@ mod tests {
|
|||
json!({"litellm_provider": "gemini", "web_search_billing_unit": "per_query"}),
|
||||
Billing { web_search: WebSearchUnit::PerQuery, ..Billing::default() }
|
||||
)]
|
||||
#[case::dashscope_tiers_bill_the_whole_call(
|
||||
json!({"litellm_provider": "dashscope", "tiered_pricing": [{"range": [0, 32000], "input_cost_per_token": 1e-6}]}),
|
||||
#[case::token_tiers_bill_the_whole_call(
|
||||
json!({"litellm_provider": "openai", "tiered_pricing": [{"range": [0, 32000], "input_cost_per_token": 1e-6}]}),
|
||||
Billing { tier_scope: TierScope::WholeCall, ..Billing::default() }
|
||||
)]
|
||||
#[case::result_tiers_bill_per_meter(
|
||||
json!({"litellm_provider": "exa_ai", "tiered_pricing": [{"max_results_range": [1, 25], "input_cost_per_query": 5e-3}]}),
|
||||
Billing::default()
|
||||
)]
|
||||
fn billing_is_lowered_from_the_row_and_its_provider(
|
||||
#[case] entry: Value,
|
||||
#[case] expected: Billing,
|
||||
|
|
|
|||
|
|
@ -48,7 +48,9 @@ litellm-secrets-types.workspace = true
|
|||
litellm-llms-types.workspace = true
|
||||
litellm-router-types.workspace = true
|
||||
litellm-host-python.workspace = true
|
||||
litellm-cost.workspace = true
|
||||
litellm-model-catalog.workspace = true
|
||||
jiff = { version = "0.2.37", default-features = false, features = ["std"] }
|
||||
litellm-model-catalog-refresh.workspace = true
|
||||
indexmap = "2.14.0"
|
||||
litellm-token-counter = { path = "../token-counter", default-features = false }
|
||||
|
|
|
|||
457
litellm-rust/crates/python-bridge/src/cost.rs
Normal file
457
litellm-rust/crates/python-bridge/src/cost.rs
Normal file
|
|
@ -0,0 +1,457 @@
|
|||
use std::borrow::Cow;
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use litellm_cost::call::CallContext as CostCallContext;
|
||||
use litellm_cost::price::{
|
||||
Adjustments, CacheReadMode, CacheTtl, Channel, InputUnit, Meter, OutputUnit,
|
||||
Prices as CostPrices, Residency, ServiceTier, TokenKind,
|
||||
};
|
||||
use litellm_cost::priced::{Outcome, Quantity, Side};
|
||||
use litellm_cost::usage::Usage as CostUsage;
|
||||
use litellm_host_python::from_py;
|
||||
use litellm_model_catalog::rejects_row;
|
||||
use litellm_model_catalog::wire::{EntryWire, Fields, FlatKey};
|
||||
use pyo3::exceptions::PyValueError;
|
||||
use pyo3::prelude::*;
|
||||
use pyo3::types::{PyDict, PyTuple};
|
||||
|
||||
use crate::model_registry::Invalid;
|
||||
|
||||
/// A row's typed prices and uplifts, an immutable snapshot taken when the row was resolved
|
||||
#[derive(Default)]
|
||||
#[pyclass(frozen, name = "Prices", module = "litellm.rust_bridge._native")]
|
||||
pub(crate) struct PricesHandle {
|
||||
prices: Arc<CostPrices>,
|
||||
adjustments: Arc<Adjustments>,
|
||||
}
|
||||
|
||||
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
|
||||
pub(crate) struct Usage(CostUsage);
|
||||
|
||||
/// The call facts pricing reads. A custom pricing overlay that fails validation is kept as its errors
|
||||
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
|
||||
pub(crate) struct CallContext {
|
||||
call: CostCallContext,
|
||||
invalid_overlay: Option<(Vec<String>, Arc<Fields>)>,
|
||||
}
|
||||
|
||||
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
|
||||
pub(crate) struct Priced {
|
||||
#[pyo3(get)]
|
||||
total: f64,
|
||||
#[pyo3(get)]
|
||||
prompt: f64,
|
||||
#[pyo3(get)]
|
||||
completion: f64,
|
||||
lines: Vec<Py<Line>>,
|
||||
missing: Vec<String>,
|
||||
}
|
||||
|
||||
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
|
||||
pub(crate) struct Line {
|
||||
#[pyo3(get)]
|
||||
meter: String,
|
||||
#[pyo3(get)]
|
||||
quantity: u64,
|
||||
#[pyo3(get)]
|
||||
unit_price: f64,
|
||||
#[pyo3(get)]
|
||||
amount: f64,
|
||||
#[pyo3(get)]
|
||||
fallback_from: Option<String>,
|
||||
}
|
||||
|
||||
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
|
||||
pub(crate) struct Unpriced {
|
||||
missing: Vec<String>,
|
||||
}
|
||||
|
||||
impl PricesHandle {
|
||||
/// Lowers a row the way the catalog lowers its entries, `Err` with every error that rejects the row
|
||||
pub(crate) fn lower(key: &str, fields: &Fields) -> Result<Self, Vec<String>> {
|
||||
let wire = EntryWire::read(&as_main_reads_tiers(fields));
|
||||
let adjustments = Adjustments::from(&wire);
|
||||
let (prices, price_errors) = match CostPrices::try_from(&wire) {
|
||||
Ok(prices) => (Some(prices), Vec::new()),
|
||||
Err(errors) => (None, errors),
|
||||
};
|
||||
let errors: Vec<String> = wire
|
||||
.problems
|
||||
.into_iter()
|
||||
.chain(price_errors)
|
||||
.filter(|error| rejects_row(&error.error))
|
||||
.map(|mut error| {
|
||||
error.prepend(key);
|
||||
error.to_string()
|
||||
})
|
||||
.collect();
|
||||
match prices {
|
||||
Some(prices) if errors.is_empty() => Ok(Self {
|
||||
prices: Arc::new(prices),
|
||||
adjustments: Arc::new(adjustments),
|
||||
}),
|
||||
_ => Err(errors),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl PricesHandle {
|
||||
/// Prices for a row that did not come from `resolve`, such as a deployment's `model_info`
|
||||
#[staticmethod]
|
||||
#[pyo3(signature = (row, key = "model_info"))]
|
||||
fn from_row(py: Python<'_>, row: &Bound<'_, PyDict>, key: &str) -> PyResult<Py<PyAny>> {
|
||||
let (fields, converted) = json_fields(row);
|
||||
let lowered = match converted {
|
||||
true => Self::lower(key, &fields),
|
||||
false => Err(vec![format!("{key} must contain JSON-compatible prices")]),
|
||||
};
|
||||
match lowered {
|
||||
Ok(prices) => Ok(prices.into_pyobject(py)?.into_any().unbind()),
|
||||
Err(errors) => Ok(Invalid::new(key.to_owned(), errors, Arc::new(fields))
|
||||
.into_pyobject(py)?
|
||||
.into_any()
|
||||
.unbind()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Pure arithmetic over owned values, it calls no Python and never reads `litellm.model_cost`
|
||||
fn price(&self, py: Python<'_>, usage: &Usage, call: &CallContext) -> PyResult<Py<PyAny>> {
|
||||
if let Some((errors, fields)) = &call.invalid_overlay {
|
||||
return Ok(
|
||||
Invalid::new("custom_pricing".to_owned(), errors.clone(), fields.clone())
|
||||
.into_pyobject(py)?
|
||||
.into_any()
|
||||
.unbind(),
|
||||
);
|
||||
}
|
||||
match litellm_cost::calculate(&self.prices, &self.adjustments, &usage.0, &call.call) {
|
||||
Outcome::Unpriced { missing } => Ok(Unpriced {
|
||||
missing: missing.iter().map(meter_name).collect(),
|
||||
}
|
||||
.into_pyobject(py)?
|
||||
.into_any()
|
||||
.unbind()),
|
||||
Outcome::Priced(priced) => {
|
||||
let side_total = |side: Side| {
|
||||
priced
|
||||
.lines
|
||||
.iter()
|
||||
.filter(|line| line.side == side)
|
||||
.map(|line| line.amount.get())
|
||||
.sum()
|
||||
};
|
||||
let lines = priced
|
||||
.lines
|
||||
.iter()
|
||||
.map(|line| {
|
||||
Py::new(
|
||||
py,
|
||||
Line {
|
||||
meter: meter_name(&line.meter),
|
||||
quantity: match line.quantity {
|
||||
Quantity::Count(count) => count,
|
||||
Quantity::Duration(duration) => duration.as_secs(),
|
||||
},
|
||||
unit_price: line.unit_price.get(),
|
||||
amount: line.amount.get(),
|
||||
fallback_from: line.fallback_from.as_ref().map(meter_name),
|
||||
},
|
||||
)
|
||||
})
|
||||
.collect::<PyResult<_>>()?;
|
||||
Ok(Priced {
|
||||
total: priced.total.get(),
|
||||
prompt: side_total(Side::Prompt),
|
||||
completion: side_total(Side::Completion),
|
||||
lines,
|
||||
missing: priced.missing.iter().map(meter_name).collect(),
|
||||
}
|
||||
.into_pyobject(py)?
|
||||
.into_any()
|
||||
.unbind())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl Usage {
|
||||
#[new]
|
||||
#[pyo3(signature = (
|
||||
*, prompt_tokens, text, output_text, audio = 0, image = 0, video = 0, cache_read = 0, cache_read_audio = 0,
|
||||
cache_read_explicit = 0, cache_write_5m = 0, cache_write_1h = 0, cache_write_12h = 0, output_audio = 0,
|
||||
output_image = 0, output_video = 0, reasoning = 0,
|
||||
))]
|
||||
#[expect(
|
||||
clippy::too_many_arguments,
|
||||
reason = "one keyword argument per token meter, the Python signature"
|
||||
)]
|
||||
fn new(
|
||||
prompt_tokens: u64,
|
||||
text: u64,
|
||||
output_text: u64,
|
||||
audio: u64,
|
||||
image: u64,
|
||||
video: u64,
|
||||
cache_read: u64,
|
||||
cache_read_audio: u64,
|
||||
cache_read_explicit: u64,
|
||||
cache_write_5m: u64,
|
||||
cache_write_1h: u64,
|
||||
cache_write_12h: u64,
|
||||
output_audio: u64,
|
||||
output_image: u64,
|
||||
output_video: u64,
|
||||
reasoning: u64,
|
||||
) -> Self {
|
||||
let mut usage = CostUsage {
|
||||
prompt_tokens,
|
||||
..CostUsage::default()
|
||||
};
|
||||
usage.tokens.input = BTreeMap::from([
|
||||
(TokenKind::Text, text),
|
||||
(TokenKind::Audio, audio),
|
||||
(TokenKind::Image, image),
|
||||
(TokenKind::Video, video),
|
||||
]);
|
||||
usage.tokens.cache_read = BTreeMap::from([
|
||||
((TokenKind::Text, CacheReadMode::Implicit), cache_read),
|
||||
(
|
||||
(TokenKind::Text, CacheReadMode::Explicit),
|
||||
cache_read_explicit,
|
||||
),
|
||||
(
|
||||
(TokenKind::Audio, CacheReadMode::Implicit),
|
||||
cache_read_audio,
|
||||
),
|
||||
]);
|
||||
usage.tokens.cache_write = BTreeMap::from([
|
||||
((TokenKind::Text, CacheTtl::Default), cache_write_5m),
|
||||
((TokenKind::Text, CacheTtl::OneHour), cache_write_1h),
|
||||
((TokenKind::Text, CacheTtl::TwelveHours), cache_write_12h),
|
||||
]);
|
||||
usage.tokens.output = BTreeMap::from([
|
||||
(TokenKind::Text, output_text),
|
||||
(TokenKind::Audio, output_audio),
|
||||
(TokenKind::Image, output_image),
|
||||
(TokenKind::Video, output_video),
|
||||
]);
|
||||
usage.tokens.reasoning = reasoning;
|
||||
Self(usage)
|
||||
}
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl CallContext {
|
||||
#[new]
|
||||
#[pyo3(signature = (*, service_tier, channel, data_residency, vertex_location, at, custom_pricing))]
|
||||
fn new(
|
||||
service_tier: Option<&str>,
|
||||
channel: &str,
|
||||
data_residency: Option<&str>,
|
||||
vertex_location: Option<&str>,
|
||||
at: &Bound<'_, PyAny>,
|
||||
custom_pricing: Option<&Bound<'_, PyDict>>,
|
||||
) -> PyResult<Self> {
|
||||
let channel = match channel {
|
||||
"online" => Channel::Online,
|
||||
"batch" => Channel::Batch,
|
||||
"decisions" => Channel::Decisions,
|
||||
other => return Err(PyValueError::new_err(format!("unknown channel {other:?}"))),
|
||||
};
|
||||
let overlay = custom_pricing.map(|pricing| {
|
||||
let (fields, converted) = json_fields(pricing);
|
||||
let lowered = match converted {
|
||||
true => PricesHandle::lower("custom_pricing", &fields),
|
||||
false => Err(vec![
|
||||
"custom_pricing must contain JSON-compatible prices".to_owned(),
|
||||
]),
|
||||
};
|
||||
lowered
|
||||
.map(|handle| Arc::unwrap_or_clone(handle.prices))
|
||||
.map_err(|errors| (errors, Arc::new(fields)))
|
||||
});
|
||||
let (overlay, invalid_overlay) = match overlay {
|
||||
None => (None, None),
|
||||
Some(Ok(prices)) => (Some(prices), None),
|
||||
Some(Err(invalid)) => (None, Some(invalid)),
|
||||
};
|
||||
Ok(Self {
|
||||
call: CostCallContext {
|
||||
tier: service_tier.map_or(ServiceTier::Standard, tier),
|
||||
channel,
|
||||
residency: data_residency.and_then(residency),
|
||||
non_global_endpoint: vertex_location
|
||||
.is_some_and(|location| !location.eq_ignore_ascii_case("global")),
|
||||
speed: None,
|
||||
inference_geo: None,
|
||||
at: timestamp(at)?,
|
||||
overlay,
|
||||
},
|
||||
invalid_overlay,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl Priced {
|
||||
#[getter]
|
||||
fn lines<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyTuple>> {
|
||||
PyTuple::new(py, self.lines.iter().map(|line| line.clone_ref(py)))
|
||||
}
|
||||
|
||||
#[getter]
|
||||
fn missing<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyTuple>> {
|
||||
PyTuple::new(py, &self.missing)
|
||||
}
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl Unpriced {
|
||||
#[getter]
|
||||
fn missing<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyTuple>> {
|
||||
PyTuple::new(py, &self.missing)
|
||||
}
|
||||
}
|
||||
|
||||
/// The row's fields that convert to JSON, and whether every price did. A price only Python can hold, such as `nan`,
|
||||
/// makes the row invalid (decision 20). Any other such value, a router deployment's `datetime` say, is left out
|
||||
fn json_fields(row: &Bound<'_, PyDict>) -> (Fields, bool) {
|
||||
let converted: Vec<(String, Option<serde_json::Value>)> = row
|
||||
.iter()
|
||||
.filter_map(|(name, value)| {
|
||||
let non_finite = value
|
||||
.extract::<f64>()
|
||||
.is_ok_and(|number| !number.is_finite());
|
||||
Some((
|
||||
name.extract::<String>().ok()?,
|
||||
from_py(&value).ok().filter(|_| !non_finite),
|
||||
))
|
||||
})
|
||||
.collect();
|
||||
let prices_convert = converted
|
||||
.iter()
|
||||
.all(|(name, value)| value.is_some() || FlatKey::parse(name).is_none());
|
||||
let fields = converted
|
||||
.into_iter()
|
||||
.filter_map(|(name, value)| Some((name, value?)))
|
||||
.collect();
|
||||
(fields, prices_convert)
|
||||
}
|
||||
|
||||
/// Flat prices a priced tier table replaces, main's `_get_tiered_base_costs`. The row's own output price stays as the
|
||||
/// fallback for a tier that sets none
|
||||
const TIER_REPLACED_PRICES: [&str; 5] = [
|
||||
"input_cost_per_token",
|
||||
"cache_read_input_token_cost",
|
||||
"cache_creation_input_token_cost",
|
||||
"cache_creation_input_token_cost_above_1hr",
|
||||
"cache_creation_input_token_cost_above_12hr",
|
||||
];
|
||||
|
||||
/// main bills a call with a prompt-token tier table at its tier's rates and never at the row's flat ones, and ignores
|
||||
/// a table whose first tier sets no input price. The catalog lowers the first tier and the flat prices into one band,
|
||||
/// so the flat prices the table replaces are dropped before lowering
|
||||
fn as_main_reads_tiers(fields: &Fields) -> Cow<'_, Fields> {
|
||||
let first_tier = fields
|
||||
.get("tiered_pricing")
|
||||
.and_then(serde_json::Value::as_array)
|
||||
.and_then(|tiers| {
|
||||
tiers
|
||||
.iter()
|
||||
.filter_map(serde_json::Value::as_object)
|
||||
.find(|tier| {
|
||||
tier.get("range")
|
||||
.and_then(|range| range.get(0))
|
||||
.and_then(serde_json::Value::as_u64)
|
||||
== Some(0)
|
||||
})
|
||||
});
|
||||
let Some(first_tier) = first_tier else {
|
||||
return Cow::Borrowed(fields);
|
||||
};
|
||||
let priced = first_tier.contains_key("input_cost_per_token");
|
||||
let replaced = |name: &str| match priced {
|
||||
true => TIER_REPLACED_PRICES.contains(&name) || first_tier.contains_key(name),
|
||||
false => name == "tiered_pricing",
|
||||
};
|
||||
Cow::Owned(
|
||||
fields
|
||||
.iter()
|
||||
.filter(|(name, _)| !replaced(name))
|
||||
.map(|(name, value)| (name.clone(), value.clone()))
|
||||
.collect(),
|
||||
)
|
||||
}
|
||||
|
||||
/// main's `_SERVICE_TIER_TO_COST_KEY_SUFFIX`, `"fast"` bills the priority tier
|
||||
fn tier(service_tier: &str) -> ServiceTier {
|
||||
match service_tier.to_lowercase().as_str() {
|
||||
"priority" | "fast" => ServiceTier::Priority,
|
||||
"flex" => ServiceTier::Flex,
|
||||
"balanced" => ServiceTier::Balanced,
|
||||
"ultrafast" => ServiceTier::Ultrafast,
|
||||
_ => ServiceTier::Standard,
|
||||
}
|
||||
}
|
||||
|
||||
fn residency(data_residency: &str) -> Option<Residency> {
|
||||
match data_residency.to_lowercase().as_str() {
|
||||
"eu" => Some(Residency::Eu),
|
||||
"us" => Some(Residency::Us),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// An aware datetime is read as its instant, a naive one as UTC, as main's off-peak check does
|
||||
fn timestamp(at: &Bound<'_, PyAny>) -> PyResult<jiff::Timestamp> {
|
||||
let aware = if at.getattr("tzinfo")?.is_none() {
|
||||
let utc = at
|
||||
.py()
|
||||
.import("datetime")?
|
||||
.getattr("timezone")?
|
||||
.getattr("utc")?;
|
||||
let replace_kwargs = PyDict::new(at.py());
|
||||
replace_kwargs.set_item("tzinfo", utc)?;
|
||||
at.call_method("replace", (), Some(&replace_kwargs))?
|
||||
} else {
|
||||
at.clone()
|
||||
};
|
||||
let seconds: f64 = aware.call_method0("timestamp")?.extract()?;
|
||||
jiff::Timestamp::from_microsecond((seconds * 1e6).round() as i64)
|
||||
.map_err(|error| PyValueError::new_err(format!("billing time out of range: {error}")))
|
||||
}
|
||||
|
||||
/// main's flat key name for the meter, the name a caller already knows from the JSON
|
||||
fn meter_name(meter: &Meter) -> String {
|
||||
let token = |kind: &TokenKind| match kind {
|
||||
TokenKind::Text => "",
|
||||
TokenKind::Audio => "audio_",
|
||||
TokenKind::Image => "image_",
|
||||
TokenKind::Video => "video_",
|
||||
};
|
||||
match meter {
|
||||
Meter::Input(InputUnit::Token(kind)) => format!("input_cost_per_{}token", token(kind)),
|
||||
Meter::Output(OutputUnit::Token(kind)) => format!("output_cost_per_{}token", token(kind)),
|
||||
Meter::Output(OutputUnit::ReasoningToken) => "output_cost_per_reasoning_token".to_owned(),
|
||||
Meter::CacheRead {
|
||||
kind,
|
||||
mode: CacheReadMode::Implicit,
|
||||
} => format!("cache_read_input_{}token_cost", token(kind)),
|
||||
Meter::CacheRead {
|
||||
kind,
|
||||
mode: CacheReadMode::Explicit,
|
||||
} => format!("explicit_cache_read_input_{}token_cost", token(kind)),
|
||||
Meter::CacheWrite { kind, ttl } => {
|
||||
let ttl = match ttl {
|
||||
CacheTtl::Default => "",
|
||||
CacheTtl::OneHour => "_above_1hr",
|
||||
CacheTtl::TwelveHours => "_above_12hr",
|
||||
};
|
||||
format!("cache_creation_input_{}token_cost{ttl}", token(kind))
|
||||
}
|
||||
other => format!("{other:?}"),
|
||||
}
|
||||
}
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
mod cache;
|
||||
mod callable;
|
||||
mod coercion;
|
||||
mod cost;
|
||||
mod credentials;
|
||||
mod diagnostics;
|
||||
mod errors;
|
||||
|
|
@ -24,6 +25,8 @@ mod _native {
|
|||
types::{PyModule, PyTuple},
|
||||
};
|
||||
|
||||
#[pymodule_export]
|
||||
use crate::cost::{CallContext, Line, Priced, PricesHandle, Unpriced, Usage};
|
||||
#[cfg(feature = "panic-test")]
|
||||
#[pymodule_export]
|
||||
use crate::diagnostics::_panic_for_test;
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ use pyo3::types::{PyDict, PyTuple};
|
|||
use serde_json::Value;
|
||||
|
||||
use super::name_index::NameIndex;
|
||||
use crate::cost::PricesHandle;
|
||||
|
||||
const CANDIDATES_MODULE: &str = "litellm.rust_bridge.host.model_names";
|
||||
|
||||
|
|
@ -52,6 +53,22 @@ impl Resolved {
|
|||
fn row(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
|
||||
to_py(py, self.fields.as_ref())
|
||||
}
|
||||
|
||||
/// The row's typed prices and uplifts. `resolve` already reported a row whose prices do not lower as `Invalid`
|
||||
#[getter]
|
||||
fn prices(&self) -> PricesHandle {
|
||||
PricesHandle::lower(&self.key, &self.fields).unwrap_or_default()
|
||||
}
|
||||
}
|
||||
|
||||
impl Invalid {
|
||||
pub(crate) fn new(key: String, errors: Vec<String>, fields: Arc<Fields>) -> Self {
|
||||
Self {
|
||||
key,
|
||||
errors,
|
||||
fields,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import os
|
|||
import sys
|
||||
from typing import Final
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.offline_mode import is_offline_mode
|
||||
from litellm.rust_bridge._native import (
|
||||
Bundled,
|
||||
|
|
@ -26,6 +27,16 @@ Source = Remote | File | Bundled
|
|||
|
||||
CATALOG: Final = Catalog.bound_to(sys.modules["litellm"], "model_cost")
|
||||
|
||||
_REPORTED_INVALID_ROWS: Final[set[str]] = set() # mutable-ok: logs each bad live row once per process
|
||||
|
||||
|
||||
def report_invalid(invalid: Invalid) -> None:
|
||||
"""Logs a row that fails validation, once per key per process (decision 20)"""
|
||||
if invalid.key in _REPORTED_INVALID_ROWS:
|
||||
return
|
||||
_REPORTED_INVALID_ROWS.add(invalid.key)
|
||||
verbose_logger.warning("litellm.model_cost[%r] fails validation: %s", invalid.key, "; ".join(invalid.errors))
|
||||
|
||||
|
||||
def default_source() -> Source:
|
||||
"""Main's reload rule, the bundled map under offline mode or ``LITELLM_LOCAL_MODEL_COST_MAP``"""
|
||||
|
|
@ -54,4 +65,5 @@ __all__ = [
|
|||
"Unavailable",
|
||||
"default_source",
|
||||
"overlay",
|
||||
"report_invalid",
|
||||
]
|
||||
|
|
|
|||
56
litellm/_cost/__init__.py
Normal file
56
litellm/_cost/__init__.py
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
"""Internal pricing API backed by the Rust ``cost`` crate. Not public, its shape may change in any release"""
|
||||
|
||||
from collections.abc import Generator, Mapping
|
||||
from contextlib import contextmanager
|
||||
from contextvars import ContextVar
|
||||
from typing import Final, Literal
|
||||
|
||||
from litellm._catalog import Invalid, report_invalid
|
||||
from litellm.rust_bridge._native import CallContext, Line, Priced, Prices, Unpriced, Usage
|
||||
|
||||
Channel = Literal["online", "batch", "decisions"]
|
||||
|
||||
_CALL_CUSTOM_PRICING: Final[ContextVar[Mapping[str, object] | None]] = ContextVar(
|
||||
"litellm_call_custom_pricing", default=None
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def custom_pricing_for_call(custom_pricing: Mapping[str, object] | None) -> Generator[None]:
|
||||
"""Prices every token meter of the call inside the block with ``custom_pricing`` over its row (decision 12).
|
||||
The provider cost calculators keep their signatures, so the overlay reaches them through the call's context"""
|
||||
token: Final = _CALL_CUSTOM_PRICING.set(custom_pricing)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_CALL_CUSTOM_PRICING.reset(token)
|
||||
|
||||
|
||||
def call_custom_pricing() -> Mapping[str, object] | None:
|
||||
return _CALL_CUSTOM_PRICING.get()
|
||||
|
||||
|
||||
def as_main_cost(outcome: Priced | Unpriced | Invalid) -> tuple[float, float]:
|
||||
"""Main's ``(prompt, completion)`` pair, ``(0.0, 0.0)`` for anything unpriced (decisions 14 and 20)"""
|
||||
match outcome:
|
||||
case Priced():
|
||||
return outcome.prompt, outcome.completion
|
||||
case Unpriced():
|
||||
return 0.0, 0.0
|
||||
case Invalid():
|
||||
report_invalid(outcome)
|
||||
return 0.0, 0.0
|
||||
|
||||
|
||||
__all__ = [
|
||||
"CallContext",
|
||||
"Channel",
|
||||
"Line",
|
||||
"Priced",
|
||||
"Prices",
|
||||
"Unpriced",
|
||||
"Usage",
|
||||
"as_main_cost",
|
||||
"call_custom_pricing",
|
||||
"custom_pricing_for_call",
|
||||
]
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -12,6 +12,9 @@ from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
|
|||
from typing_extensions import ReadOnly
|
||||
|
||||
import litellm
|
||||
from litellm._catalog import Invalid
|
||||
from litellm._cost import CallContext, Prices, as_main_cost, call_custom_pricing
|
||||
from litellm._cost import Usage as TokenUsage
|
||||
from litellm._internal_context import current_billing_time
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.llm_cost_calc.tiered_pricing import (
|
||||
|
|
@ -892,7 +895,7 @@ def _get_token_base_cost(
|
|||
)
|
||||
|
||||
|
||||
def calculate_cost_component(model_info: ModelInfo, cost_key: str, usage_value: float | None) -> float:
|
||||
def calculate_cost_component(model_info: Mapping[str, object], cost_key: str, usage_value: float | None) -> float:
|
||||
"""
|
||||
Generic cost calculator for any usage component
|
||||
|
||||
|
|
@ -910,7 +913,7 @@ def calculate_cost_component(model_info: ModelInfo, cost_key: str, usage_value:
|
|||
return 0.0
|
||||
|
||||
|
||||
def get_cost_per_unit(model_info: ModelInfo, cost_key: str, default_value: float | None = 0.0) -> float | None:
|
||||
def get_cost_per_unit(model_info: Mapping[str, object], cost_key: str, default_value: float | None = 0.0) -> float | None:
|
||||
# Sometimes the cost per unit is a string (e.g.: If a value like "3e-7" was read from the config.yaml)
|
||||
cost_per_unit: Final = model_info.get(cost_key)
|
||||
if isinstance(cost_per_unit, float):
|
||||
|
|
@ -1185,144 +1188,7 @@ def parse_completion_tokens_details(usage: Usage) -> CompletionTokensDetailsResu
|
|||
)
|
||||
|
||||
|
||||
def _modality_token_cost(
|
||||
model_info: ModelInfo,
|
||||
modality_cost_key: str,
|
||||
tokens: int,
|
||||
prompt_base_cost: float,
|
||||
service_tier: str | None,
|
||||
) -> float:
|
||||
"""
|
||||
Audio, image and video prompt tokens bill at their modality rate when the model has one.
|
||||
Without one they are ordinary prompt tokens, so they bill at `prompt_base_cost`, which
|
||||
`_get_token_base_cost` has already resolved for the service tier and long-context tier.
|
||||
"""
|
||||
tiered_key: Final = get_service_tier_cost_key(modality_cost_key, service_tier)
|
||||
cost_key: Final = tiered_key if model_info.get(tiered_key) is not None else modality_cost_key
|
||||
if model_info.get(cost_key) is None:
|
||||
return float(tokens) * prompt_base_cost
|
||||
return calculate_cost_component(model_info, cost_key, tokens)
|
||||
|
||||
|
||||
def _calculate_input_cost(
|
||||
prompt_tokens_details: PromptTokensDetailsResult,
|
||||
model_info: ModelInfo,
|
||||
prompt_base_cost: float,
|
||||
cache_read_cost: float,
|
||||
cache_creation_cost: float,
|
||||
cache_creation_cost_by_ttl: Mapping[str, float],
|
||||
service_tier: str | None = None,
|
||||
) -> float:
|
||||
"""
|
||||
Calculates the input cost for a given model, prompt tokens, and completion tokens.
|
||||
|
||||
A model priced per character (`input_cost_per_character`) bills the reported character count
|
||||
in place of its text tokens, so the same prompt is never charged under both units.
|
||||
"""
|
||||
bills_characters: Final = (
|
||||
prompt_tokens_details["character_count"] > 0 and model_info.get("input_cost_per_character") is not None
|
||||
)
|
||||
prompt_cost = 0.0 if bills_characters else float(prompt_tokens_details["text_tokens"]) * prompt_base_cost
|
||||
|
||||
### CACHE READ COST - Now uses tiered pricing
|
||||
cache_hit_audio_tokens: Final = prompt_tokens_details["cache_hit_audio_tokens"]
|
||||
audio_cache_read_rate: Final = get_cost_per_unit(
|
||||
model_info,
|
||||
get_service_tier_cost_key("cache_read_input_audio_token_cost", service_tier),
|
||||
None,
|
||||
)
|
||||
prompt_cost += float(prompt_tokens_details["cache_hit_tokens"] - cache_hit_audio_tokens) * cache_read_cost
|
||||
prompt_cost += float(cache_hit_audio_tokens) * (
|
||||
audio_cache_read_rate if audio_cache_read_rate is not None else cache_read_cost
|
||||
)
|
||||
|
||||
### AUDIO COST
|
||||
if prompt_tokens_details["audio_tokens"] and not (
|
||||
prompt_tokens_details["audio_length_seconds"] and model_info.get("input_cost_per_audio_per_second") is not None
|
||||
):
|
||||
prompt_cost += _modality_token_cost(
|
||||
model_info,
|
||||
"input_cost_per_audio_token",
|
||||
prompt_tokens_details["audio_tokens"],
|
||||
prompt_base_cost,
|
||||
service_tier,
|
||||
)
|
||||
|
||||
### IMAGE TOKEN COST
|
||||
if prompt_tokens_details["image_tokens"] and not (
|
||||
prompt_tokens_details["image_count"] and model_info.get("input_cost_per_image") is not None
|
||||
):
|
||||
prompt_cost += _modality_token_cost(
|
||||
model_info,
|
||||
"input_cost_per_image_token",
|
||||
prompt_tokens_details["image_tokens"],
|
||||
prompt_base_cost,
|
||||
service_tier,
|
||||
)
|
||||
|
||||
### VIDEO TOKEN COST
|
||||
if prompt_tokens_details["video_tokens"] and not (
|
||||
prompt_tokens_details["video_length_seconds"] and model_info.get("input_cost_per_video_per_second") is not None
|
||||
):
|
||||
prompt_cost += _modality_token_cost(
|
||||
model_info,
|
||||
"input_cost_per_video_token",
|
||||
prompt_tokens_details["video_tokens"],
|
||||
prompt_base_cost,
|
||||
service_tier,
|
||||
)
|
||||
|
||||
### CACHE WRITING COST - Now uses tiered pricing
|
||||
if (
|
||||
prompt_tokens_details["cache_creation_tokens"]
|
||||
or prompt_tokens_details["cache_creation_token_details"] is not None
|
||||
):
|
||||
prompt_cost += calculate_cache_writing_cost(
|
||||
cache_creation_tokens=prompt_tokens_details["cache_creation_tokens"],
|
||||
cache_creation_token_details=prompt_tokens_details["cache_creation_token_details"],
|
||||
cache_creation_cost=cache_creation_cost,
|
||||
cache_creation_cost_by_ttl=cache_creation_cost_by_ttl,
|
||||
)
|
||||
|
||||
### CHARACTER COST
|
||||
if prompt_tokens_details["character_count"]:
|
||||
prompt_cost += calculate_cost_component(
|
||||
model_info,
|
||||
"input_cost_per_character",
|
||||
prompt_tokens_details["character_count"],
|
||||
)
|
||||
|
||||
### IMAGE COUNT COST
|
||||
if prompt_tokens_details["image_count"]:
|
||||
prompt_cost += calculate_cost_component(
|
||||
model_info, "input_cost_per_image", prompt_tokens_details["image_count"]
|
||||
)
|
||||
|
||||
### VIDEO LENGTH COST
|
||||
if prompt_tokens_details["video_length_seconds"]:
|
||||
prompt_cost += calculate_cost_component(
|
||||
model_info,
|
||||
"input_cost_per_video_per_second",
|
||||
prompt_tokens_details["video_length_seconds"],
|
||||
)
|
||||
|
||||
### AUDIO LENGTH COST
|
||||
if prompt_tokens_details["audio_length_seconds"]:
|
||||
prompt_cost += calculate_cost_component(
|
||||
model_info,
|
||||
"input_cost_per_audio_per_second",
|
||||
prompt_tokens_details["audio_length_seconds"],
|
||||
)
|
||||
|
||||
if prompt_tokens_details["query_count"]:
|
||||
prompt_cost += calculate_cost_component(
|
||||
model_info, "input_cost_per_query", prompt_tokens_details["query_count"]
|
||||
)
|
||||
|
||||
return prompt_cost
|
||||
|
||||
|
||||
def get_regional_uplift_multiplier(model_info: ModelInfo, data_residency: str | None) -> float:
|
||||
def get_regional_uplift_multiplier(model_info: Mapping[str, object], data_residency: str | None) -> float:
|
||||
"""
|
||||
Resolve the per-model regional-processing uplift multiplier for a given
|
||||
data-residency region.
|
||||
|
|
@ -1341,7 +1207,7 @@ def get_regional_uplift_multiplier(model_info: ModelInfo, data_residency: str |
|
|||
if residency not in _VALID_DATA_RESIDENCIES:
|
||||
return 1.0
|
||||
multiplier: Final = model_info.get(f"regional_processing_uplift_multiplier_{residency}")
|
||||
if multiplier is None:
|
||||
if not isinstance(multiplier, (int, float, str)):
|
||||
return 1.0
|
||||
try:
|
||||
return float(multiplier)
|
||||
|
|
@ -1356,7 +1222,7 @@ def get_regional_uplift_multiplier(model_info: ModelInfo, data_residency: str |
|
|||
_get_regional_uplift_multiplier = get_regional_uplift_multiplier
|
||||
|
||||
|
||||
def get_vertex_regional_endpoint_uplift(model_info: ModelInfo, vertex_location: str | None) -> float:
|
||||
def get_vertex_regional_endpoint_uplift(model_info: Mapping[str, object], vertex_location: str | None) -> float:
|
||||
"""
|
||||
Resolve the per-model uplift multiplier for Vertex AI non-global (regional and
|
||||
multi-region) endpoints.
|
||||
|
|
@ -1476,183 +1342,177 @@ def generic_cost_per_token(
|
|||
Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd
|
||||
"""
|
||||
|
||||
## GET MODEL INFO
|
||||
# A caller that already resolved the deployment's effective rates passes them in
|
||||
# rather than handing back a name for this to re-resolve. A name cannot express a
|
||||
# per-deployment override: those are registered under the deployment id and kept off
|
||||
# the shared model-name key, so resolving from the name here reads the public rate.
|
||||
resolved_model_info: Final = apply_provider_cache_read_default(
|
||||
get_model_info(model=model, custom_llm_provider=custom_llm_provider) if model_info is None else model_info,
|
||||
custom_llm_provider,
|
||||
custom_pricing: Final = call_custom_pricing()
|
||||
row: Final = _pricing_row(model, custom_llm_provider, model_info, has_overlay=custom_pricing is not None)
|
||||
prices: Final = Prices.from_row(
|
||||
dict(row) if usage.prompt_tokens > 0 else _without_tiers(row), key=str(row.get("key") or model)
|
||||
)
|
||||
|
||||
## CALCULATE INPUT COST
|
||||
### Cost of processing (non-cache hit + cache hit) + Cost of cache-writing (cache writing)
|
||||
prompt_cost = 0.0
|
||||
### PROCESSING COST
|
||||
prompt_tokens_details = PromptTokensDetailsResult(
|
||||
cache_hit_tokens=0,
|
||||
cache_hit_audio_tokens=0,
|
||||
cache_creation_tokens=0,
|
||||
cache_creation_token_details=None,
|
||||
text_tokens=usage.prompt_tokens,
|
||||
audio_tokens=0,
|
||||
image_tokens=0,
|
||||
video_tokens=0,
|
||||
character_count=0,
|
||||
image_count=0,
|
||||
video_length_seconds=0.0,
|
||||
audio_length_seconds=0.0,
|
||||
query_count=0,
|
||||
if isinstance(prices, Invalid):
|
||||
return as_main_cost(prices)
|
||||
details: Final = _billable_prompt_details(usage)
|
||||
call: Final = CallContext(
|
||||
service_tier=service_tier,
|
||||
channel="online",
|
||||
data_residency=data_residency,
|
||||
vertex_location=vertex_location,
|
||||
at=current_time if current_time is not None else current_billing_time(),
|
||||
custom_pricing=None if custom_pricing is None else dict(custom_pricing),
|
||||
)
|
||||
if usage.prompt_tokens_details:
|
||||
prompt_tokens_details = parse_prompt_tokens_details(usage)
|
||||
prompt_cost, completion_cost = as_main_cost(prices.price(_token_usage(usage, details, row), call))
|
||||
uplift: Final = get_regional_uplift_multiplier(row, data_residency) * get_vertex_regional_endpoint_uplift(
|
||||
row, vertex_location
|
||||
)
|
||||
return prompt_cost + _unmetered_prompt_cost(details, row) * uplift, completion_cost
|
||||
|
||||
## EDGE CASE - text tokens not set or includes cached tokens (double-counting)
|
||||
## Some providers (like xAI) report text_tokens = prompt_tokens (including cached)
|
||||
## We detect this when: text_tokens + cached_tokens + other > prompt_tokens
|
||||
## Ref: https://github.com/BerriAI/litellm/issues/19680, #14874, #14875
|
||||
|
||||
cache_hit: Final = prompt_tokens_details["cache_hit_tokens"]
|
||||
text_tokens = prompt_tokens_details["text_tokens"]
|
||||
audio_tokens = prompt_tokens_details["audio_tokens"]
|
||||
cache_creation: Final = prompt_tokens_details["cache_creation_tokens"]
|
||||
image_tokens = prompt_tokens_details["image_tokens"]
|
||||
video_tokens = prompt_tokens_details["video_tokens"]
|
||||
def _pricing_row(
|
||||
model: str, custom_llm_provider: str, model_info: ModelInfo | None, *, has_overlay: bool
|
||||
) -> Mapping[str, object]:
|
||||
"""The row main priced, as written rather than `get_model_info`'s copy, which writes `0` for a missing token
|
||||
price. A call with custom pricing prices an unmapped model from its overlay alone"""
|
||||
if model_info is not None:
|
||||
return model_info
|
||||
try:
|
||||
resolved: Final = get_model_info(model=model, custom_llm_provider=custom_llm_provider)
|
||||
except Exception: # noqa: BLE001 # get_model_info raises a bare Exception for an unmapped model
|
||||
if has_overlay:
|
||||
return MappingProxyType({})
|
||||
raise
|
||||
key: Final = resolved.get("key")
|
||||
row: Final = litellm.model_cost.get(key)
|
||||
return {**row, "key": key} if isinstance(row, Mapping) else resolved
|
||||
|
||||
# Check for double-counting: sum of details > prompt_tokens means overlap
|
||||
total_details: Final = text_tokens + cache_hit + audio_tokens + cache_creation + image_tokens + video_tokens
|
||||
has_double_counting: Final = (cache_hit > 0 or cache_creation > 0) and total_details > usage.prompt_tokens
|
||||
|
||||
if has_double_counting:
|
||||
# cached and per-modality counts are both subsets of prompt_tokens and may overlap, so a
|
||||
# modality can only bill what the cache did not already cover or the overlap is billed twice
|
||||
def _without_tiers(row: Mapping[str, object]) -> dict[str, object]:
|
||||
"""Main picks no tier for an empty prompt and bills the row's flat prices"""
|
||||
return {name: value for name, value in row.items() if name != "tiered_pricing"}
|
||||
|
||||
|
||||
def _billable_prompt_details(usage: Usage) -> PromptTokensDetailsResult:
|
||||
"""Prompt token details with modality counts capped by what the cache did not cover.
|
||||
|
||||
Some providers (like xAI) report text_tokens = prompt_tokens, cached tokens included, and cached
|
||||
and per-modality counts can overlap (#19680, #14874, #14875)
|
||||
"""
|
||||
details: Final = (
|
||||
parse_prompt_tokens_details(usage)
|
||||
if usage.prompt_tokens_details
|
||||
else PromptTokensDetailsResult(
|
||||
cache_hit_tokens=0,
|
||||
cache_hit_audio_tokens=0,
|
||||
cache_creation_tokens=0,
|
||||
cache_creation_token_details=None,
|
||||
text_tokens=usage.prompt_tokens,
|
||||
audio_tokens=0,
|
||||
image_tokens=0,
|
||||
video_tokens=0,
|
||||
character_count=0,
|
||||
image_count=0,
|
||||
video_length_seconds=0.0,
|
||||
audio_length_seconds=0.0,
|
||||
query_count=0,
|
||||
)
|
||||
)
|
||||
cache_hit: Final = details["cache_hit_tokens"]
|
||||
cache_creation: Final = details["cache_creation_tokens"]
|
||||
modality_tokens: Final = details["audio_tokens"] + details["image_tokens"] + details["video_tokens"]
|
||||
total_details: Final = details["text_tokens"] + cache_hit + cache_creation + modality_tokens
|
||||
if (cache_hit > 0 or cache_creation > 0) and total_details > usage.prompt_tokens:
|
||||
uncached_budget: Final = max(usage.prompt_tokens - cache_hit - cache_creation, 0)
|
||||
billable_audio: Final = min(audio_tokens, uncached_budget)
|
||||
billable_image: Final = min(image_tokens, uncached_budget - billable_audio)
|
||||
billable_video: Final = min(video_tokens, uncached_budget - billable_audio - billable_image)
|
||||
prompt_tokens_details["audio_tokens"] = billable_audio
|
||||
prompt_tokens_details["image_tokens"] = billable_image
|
||||
prompt_tokens_details["video_tokens"] = billable_video
|
||||
prompt_tokens_details["text_tokens"] = uncached_budget - billable_audio - billable_image - billable_video
|
||||
elif text_tokens == 0 and prompt_tokens_details["image_count"] == 0:
|
||||
# Clamp to zero: inconsistent streaming usage
|
||||
prompt_tokens_details["text_tokens"] = max(
|
||||
usage.prompt_tokens - cache_hit - audio_tokens - cache_creation - image_tokens - video_tokens, 0
|
||||
)
|
||||
audio: Final = min(details["audio_tokens"], uncached_budget)
|
||||
image: Final = min(details["image_tokens"], uncached_budget - audio)
|
||||
video: Final = min(details["video_tokens"], uncached_budget - audio - image)
|
||||
return {
|
||||
**details,
|
||||
"audio_tokens": audio,
|
||||
"image_tokens": image,
|
||||
"video_tokens": video,
|
||||
"text_tokens": uncached_budget - audio - image - video,
|
||||
}
|
||||
if details["text_tokens"] == 0 and details["image_count"] == 0:
|
||||
return {
|
||||
**details,
|
||||
"text_tokens": max(usage.prompt_tokens - cache_hit - cache_creation - modality_tokens, 0),
|
||||
}
|
||||
return details
|
||||
|
||||
billing_time: Final = current_time if current_time is not None else current_billing_time()
|
||||
base_costs: Final = _get_token_base_cost(
|
||||
model_info=resolved_model_info,
|
||||
usage=usage,
|
||||
service_tier=service_tier,
|
||||
current_time=billing_time,
|
||||
threshold_is_inclusive=_uses_inclusive_token_thresholds(custom_llm_provider),
|
||||
)
|
||||
completion_base_cost: Final = base_costs.output_rate
|
||||
resolved_cache_read_cost: Final = resolve_explicit_cache_read_cost_rate(
|
||||
model_info=resolved_model_info,
|
||||
usage=usage,
|
||||
current_time=billing_time,
|
||||
service_tier=service_tier,
|
||||
)
|
||||
effective_cache_read_cost: Final = (
|
||||
base_costs.cache_read_rate if resolved_cache_read_cost is None else resolved_cache_read_cost
|
||||
|
||||
def _cache_write_tokens(details: PromptTokensDetailsResult) -> tuple[int, int, int]:
|
||||
"""(default, 1 hour, 12 hour) cache write tokens. Writes no lifetime count covers bill at the default rate"""
|
||||
written: Final = details["cache_creation_tokens"]
|
||||
by_ttl: Final = details["cache_creation_token_details"]
|
||||
if by_ttl is None:
|
||||
return written, 0, 0
|
||||
short: Final = (by_ttl.ephemeral_5m_input_tokens or 0) + (by_ttl.ephemeral_30m_input_tokens or 0)
|
||||
one_hour: Final = by_ttl.ephemeral_1h_input_tokens or 0
|
||||
twelve_hours: Final = by_ttl.ephemeral_12h_input_tokens or 0
|
||||
return short + max(written - short - one_hour - twelve_hours, 0), one_hour, twelve_hours
|
||||
|
||||
|
||||
def _token_usage(usage: Usage, details: PromptTokensDetailsResult, row: Mapping[str, object]) -> TokenUsage:
|
||||
"""The token meters main billed. A prompt billed per character, image or second of media does not bill the
|
||||
same tokens again"""
|
||||
cache_type: Final = getattr(usage.prompt_tokens_details, "cache_type", None)
|
||||
cached_text: Final = details["cache_hit_tokens"] - details["cache_hit_audio_tokens"]
|
||||
explicit_cache: Final = cache_type == "ephemeral"
|
||||
cache_write_5m, cache_write_1h, cache_write_12h = _cache_write_tokens(details)
|
||||
completion: Final = _completion_token_counts(usage)
|
||||
return TokenUsage(
|
||||
prompt_tokens=usage.prompt_tokens,
|
||||
text=0
|
||||
if details["character_count"] > 0 and row.get("input_cost_per_character") is not None
|
||||
else details["text_tokens"],
|
||||
audio=0
|
||||
if details["audio_length_seconds"] and row.get("input_cost_per_audio_per_second") is not None
|
||||
else details["audio_tokens"],
|
||||
image=0 if details["image_count"] and row.get("input_cost_per_image") is not None else details["image_tokens"],
|
||||
video=0
|
||||
if details["video_length_seconds"] and row.get("input_cost_per_video_per_second") is not None
|
||||
else details["video_tokens"],
|
||||
cache_read=0 if explicit_cache else cached_text,
|
||||
cache_read_explicit=cached_text if explicit_cache else 0,
|
||||
cache_read_audio=details["cache_hit_audio_tokens"],
|
||||
cache_write_5m=cache_write_5m,
|
||||
cache_write_1h=cache_write_1h,
|
||||
cache_write_12h=cache_write_12h,
|
||||
output_text=completion["text_tokens"],
|
||||
output_audio=completion["audio_tokens"],
|
||||
output_image=completion["image_tokens"],
|
||||
output_video=completion["video_tokens"],
|
||||
reasoning=completion["reasoning_tokens"],
|
||||
)
|
||||
|
||||
prompt_cost = _calculate_input_cost(
|
||||
prompt_tokens_details=prompt_tokens_details,
|
||||
model_info=resolved_model_info,
|
||||
prompt_base_cost=base_costs.input_rate,
|
||||
cache_read_cost=effective_cache_read_cost,
|
||||
cache_creation_cost=base_costs.cache_creation_rate,
|
||||
cache_creation_cost_by_ttl=base_costs.cache_creation_rates_by_ttl,
|
||||
service_tier=service_tier,
|
||||
|
||||
def _completion_token_counts(usage: Usage) -> CompletionTokensDetailsResult:
|
||||
"""Completion tokens by meter. Text is the remainder of a breakdown, and with no breakdown every token is text"""
|
||||
details: Final = (
|
||||
parse_completion_tokens_details(usage)
|
||||
if usage.completion_tokens_details is not None
|
||||
else CompletionTokensDetailsResult(
|
||||
audio_tokens=0, text_tokens=0, reasoning_tokens=0, image_tokens=0, video_tokens=0
|
||||
)
|
||||
)
|
||||
breakdown: Final = (
|
||||
details["audio_tokens"] + details["reasoning_tokens"] + details["image_tokens"] + details["video_tokens"]
|
||||
)
|
||||
if details["text_tokens"] > 0:
|
||||
return details
|
||||
if breakdown > 0:
|
||||
return {**details, "text_tokens": max(0, usage.completion_tokens - breakdown)}
|
||||
return CompletionTokensDetailsResult(
|
||||
audio_tokens=0, text_tokens=usage.completion_tokens, reasoning_tokens=0, image_tokens=0, video_tokens=0
|
||||
)
|
||||
|
||||
## CALCULATE OUTPUT COST
|
||||
text_tokens = 0
|
||||
audio_tokens = 0
|
||||
reasoning_tokens = 0
|
||||
image_tokens = 0
|
||||
video_tokens = 0
|
||||
is_text_tokens_total = False
|
||||
if usage.completion_tokens_details is not None:
|
||||
completion_tokens_details: Final = parse_completion_tokens_details(usage)
|
||||
audio_tokens = completion_tokens_details["audio_tokens"]
|
||||
text_tokens = completion_tokens_details["text_tokens"]
|
||||
reasoning_tokens = completion_tokens_details["reasoning_tokens"]
|
||||
image_tokens = completion_tokens_details["image_tokens"]
|
||||
video_tokens = completion_tokens_details["video_tokens"]
|
||||
|
||||
# Handle text_tokens calculation:
|
||||
# 1. If text_tokens is explicitly provided and > 0, use it
|
||||
# 2. If there's a breakdown (reasoning/audio/image/video tokens), calculate text_tokens as the remainder
|
||||
# 3. If no breakdown at all, assume all completion_tokens are text_tokens
|
||||
has_token_breakdown: Final = image_tokens > 0 or audio_tokens > 0 or reasoning_tokens > 0 or video_tokens > 0
|
||||
if text_tokens == 0:
|
||||
if has_token_breakdown:
|
||||
# Calculate text tokens as remainder when we have a breakdown
|
||||
# This handles cases like OpenAI's reasoning models where text_tokens isn't provided
|
||||
text_tokens = max(
|
||||
0,
|
||||
usage.completion_tokens - reasoning_tokens - audio_tokens - image_tokens - video_tokens,
|
||||
)
|
||||
else:
|
||||
# No breakdown at all, all tokens are text tokens
|
||||
text_tokens = usage.completion_tokens
|
||||
is_text_tokens_total = True
|
||||
## TEXT COST
|
||||
completion_cost = float(text_tokens) * completion_base_cost
|
||||
|
||||
## AUDIO COST
|
||||
if not is_text_tokens_total and audio_tokens is not None and audio_tokens > 0:
|
||||
_output_cost_per_audio_token = get_cost_per_unit(resolved_model_info, "output_cost_per_audio_token", None)
|
||||
_output_cost_per_audio_token = (
|
||||
_output_cost_per_audio_token if _output_cost_per_audio_token is not None else completion_base_cost
|
||||
)
|
||||
completion_cost += float(audio_tokens) * _output_cost_per_audio_token
|
||||
|
||||
## REASONING COST
|
||||
if not is_text_tokens_total and reasoning_tokens and reasoning_tokens > 0:
|
||||
completion_cost += float(reasoning_tokens) * _resolve_billed_reasoning_rate(
|
||||
model_info=resolved_model_info,
|
||||
usage=usage,
|
||||
service_tier=service_tier,
|
||||
completion_base_cost=completion_base_cost,
|
||||
current_time=billing_time,
|
||||
)
|
||||
|
||||
## IMAGE COST
|
||||
if not is_text_tokens_total and image_tokens and image_tokens > 0:
|
||||
_output_cost_per_image_token = get_cost_per_unit(resolved_model_info, "output_cost_per_image_token", None)
|
||||
_output_cost_per_image_token = (
|
||||
_output_cost_per_image_token if _output_cost_per_image_token is not None else completion_base_cost
|
||||
)
|
||||
completion_cost += float(image_tokens) * _output_cost_per_image_token
|
||||
|
||||
## VIDEO COST
|
||||
if not is_text_tokens_total and video_tokens and video_tokens > 0:
|
||||
_output_cost_per_video_token = get_cost_per_unit(resolved_model_info, "output_cost_per_video_token", None)
|
||||
_output_cost_per_video_token = (
|
||||
_output_cost_per_video_token if _output_cost_per_video_token is not None else completion_base_cost
|
||||
)
|
||||
completion_cost += float(video_tokens) * _output_cost_per_video_token
|
||||
|
||||
## REGIONAL DATA-RESIDENCY UPLIFT
|
||||
# Applied as a flat multiplier across all token costs for the request
|
||||
# when the upstream is a regionalized OpenAI host (eu./us.api.openai.com).
|
||||
uplift: Final = get_regional_uplift_multiplier(resolved_model_info, data_residency)
|
||||
if uplift != 1.0:
|
||||
prompt_cost *= uplift
|
||||
completion_cost *= uplift
|
||||
|
||||
vertex_uplift: Final = get_vertex_regional_endpoint_uplift(resolved_model_info, vertex_location)
|
||||
if vertex_uplift != 1.0:
|
||||
prompt_cost *= vertex_uplift
|
||||
completion_cost *= vertex_uplift
|
||||
|
||||
return prompt_cost, completion_cost
|
||||
def _unmetered_prompt_cost(details: PromptTokensDetailsResult, row: Mapping[str, object]) -> float:
|
||||
"""Prompt meters the Rust cost core does not price yet, characters, image counts, media seconds and queries"""
|
||||
return (
|
||||
calculate_cost_component(row, "input_cost_per_character", details["character_count"])
|
||||
+ calculate_cost_component(row, "input_cost_per_image", details["image_count"])
|
||||
+ calculate_cost_component(row, "input_cost_per_video_per_second", details["video_length_seconds"])
|
||||
+ calculate_cost_component(row, "input_cost_per_audio_per_second", details["audio_length_seconds"])
|
||||
+ calculate_cost_component(row, "input_cost_per_query", details["query_count"])
|
||||
)
|
||||
|
||||
|
||||
def _coerce_token_count(value: object) -> int:
|
||||
|
|
|
|||
122
litellm/main.py
122
litellm/main.py
|
|
@ -155,7 +155,6 @@ from litellm.types.llms.openai import AllMessageValues
|
|||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import (
|
||||
BaseUrlRoutingHint,
|
||||
CustomPricingLiteLLMParams,
|
||||
ModelResponseStream,
|
||||
ProviderRoutingHints,
|
||||
RawRequestTypedDict,
|
||||
|
|
@ -1119,7 +1118,6 @@ from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the o
|
|||
)
|
||||
from litellm.responses.bridge_check import (
|
||||
bridge_reasoning_effort,
|
||||
keep_registered_mode,
|
||||
str_keyed_mapping,
|
||||
)
|
||||
from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the old definition site
|
||||
|
|
@ -1236,93 +1234,6 @@ def _locals_snapshot(values: Mapping[str, object]) -> Mapping[str, object]:
|
|||
return values
|
||||
|
||||
|
||||
def _build_custom_pricing_entry(
|
||||
custom_llm_provider: str,
|
||||
kwargs: dict,
|
||||
model_info: dict | None = None,
|
||||
) -> dict:
|
||||
"""Build a complete model cost entry from kwargs and model_info.
|
||||
|
||||
Collects all CustomPricingLiteLLMParams fields present in kwargs and
|
||||
merges metadata from model_info (mode, supports_prompt_caching, max_tokens)
|
||||
so that register_model() receives the full pricing configuration.
|
||||
"""
|
||||
entry: Final[dict] = {"litellm_provider": custom_llm_provider}
|
||||
|
||||
for field_name in CustomPricingLiteLLMParams.model_fields:
|
||||
value = kwargs.get(field_name)
|
||||
if value is not None:
|
||||
entry[field_name] = value
|
||||
|
||||
if model_info and isinstance(model_info, dict):
|
||||
for key in ("mode", "supports_prompt_caching", "max_tokens"):
|
||||
if key in model_info and model_info[key] is not None:
|
||||
entry.setdefault(key, model_info[key])
|
||||
|
||||
return entry
|
||||
|
||||
|
||||
def _get_router_deployment_id(kwargs: dict) -> str | None:
|
||||
for metadata_key in ("litellm_metadata", "metadata"):
|
||||
metadata = kwargs.get(metadata_key) or {}
|
||||
if not isinstance(metadata, dict):
|
||||
continue
|
||||
deployment_model_info = metadata.get("model_info") or {}
|
||||
if not isinstance(deployment_model_info, dict):
|
||||
continue
|
||||
deployment_id = deployment_model_info.get("id")
|
||||
if deployment_id is not None:
|
||||
return str(deployment_id)
|
||||
return None
|
||||
|
||||
|
||||
def _register_custom_pricing_for_request(
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
kwargs: dict,
|
||||
model_info: dict | None,
|
||||
) -> None:
|
||||
"""Register per-request custom pricing in litellm.model_cost.
|
||||
|
||||
Router-originated requests (identified by the deployment id the router puts
|
||||
in metadata) get their full pricing registered under that unique id only;
|
||||
the shared ``{provider}/{model}`` key receives the entry with pricing fields
|
||||
stripped, mirroring Router._create_deployment. This keeps one deployment's
|
||||
pricing overrides (e.g. a zero-cost wildcard) from clobbering built-in
|
||||
pricing used by sibling deployments of the same backend model. Direct SDK
|
||||
calls keep the legacy behavior of registering the shared key with pricing.
|
||||
Either way the shared key's ``mode`` follows ``keep_registered_mode``, as on
|
||||
deployment registration, since a call routes by its own ``model_info``.
|
||||
"""
|
||||
entry: Final = _build_custom_pricing_entry(
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
kwargs=kwargs,
|
||||
model_info=model_info,
|
||||
)
|
||||
shared_key: Final = f"{custom_llm_provider}/{model}"
|
||||
deployment_id: Final = _get_router_deployment_id(kwargs)
|
||||
if deployment_id is None:
|
||||
litellm.register_model({shared_key: keep_registered_mode(shared_key, entry)}, persist_across_reloads=False)
|
||||
return
|
||||
shared_entry: Final = keep_registered_mode(
|
||||
shared_key, CustomPricingLiteLLMParams.strip_custom_pricing_fields(entry)
|
||||
)
|
||||
litellm.register_model(
|
||||
{deployment_id: entry},
|
||||
persist_across_reloads=False,
|
||||
warning_display_name=shared_key,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
add_to_provider_model_sets=False,
|
||||
)
|
||||
litellm.register_model(
|
||||
{shared_key: shared_entry},
|
||||
persist_across_reloads=False,
|
||||
warning_display_name=shared_key,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
add_to_provider_model_sets=False,
|
||||
)
|
||||
|
||||
|
||||
def _dispatch_metadata(ctx: CompletionDispatchContext) -> Mapping[str, object] | None:
|
||||
return ctx.metadata
|
||||
|
||||
|
|
@ -5757,19 +5668,6 @@ def completion(
|
|||
"aws_region_name", None
|
||||
) # support region-based pricing for bedrock
|
||||
|
||||
### REGISTER CUSTOM MODEL PRICING -- IF GIVEN ###
|
||||
if (
|
||||
(input_cost_per_token is not None and output_cost_per_token is not None)
|
||||
or input_cost_per_second is not None
|
||||
or output_cost_per_second is not None
|
||||
or cost_per_second is not None
|
||||
):
|
||||
_register_custom_pricing_for_request(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
kwargs=kwargs,
|
||||
model_info=model_info,
|
||||
)
|
||||
### BUILD CUSTOM PROMPT TEMPLATE -- IF GIVEN ###
|
||||
custom_prompt_dict = {}
|
||||
if initial_prompt_value or roles or final_prompt_value or bos_token or eos_token:
|
||||
|
|
@ -6713,12 +6611,6 @@ def embedding(
|
|||
headers.update(proxy_headers)
|
||||
except Exception as e:
|
||||
verbose_logger.warning("Failed to get proxy auth headers: %s", e)
|
||||
### CUSTOM MODEL COST ###
|
||||
input_cost_per_token: Final = kwargs.get("input_cost_per_token", None)
|
||||
output_cost_per_token: Final = kwargs.get("output_cost_per_token", None)
|
||||
cost_per_second: Final = kwargs.get("cost_per_second", None)
|
||||
input_cost_per_second: Final = kwargs.get("input_cost_per_second", None)
|
||||
output_cost_per_second: Final = kwargs.get("output_cost_per_second", None)
|
||||
openai_params: Final = [
|
||||
"user",
|
||||
"dimensions",
|
||||
|
|
@ -6765,20 +6657,6 @@ def embedding(
|
|||
**non_default_params,
|
||||
)
|
||||
|
||||
### REGISTER CUSTOM MODEL PRICING -- IF GIVEN ###
|
||||
if (
|
||||
(input_cost_per_token is not None and output_cost_per_token is not None)
|
||||
or input_cost_per_second is not None
|
||||
or output_cost_per_second is not None
|
||||
or cost_per_second is not None
|
||||
):
|
||||
_register_custom_pricing_for_request(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
kwargs=kwargs,
|
||||
model_info=kwargs.get("model_info"),
|
||||
)
|
||||
|
||||
logging: Final[LiteLLMLoggingObj] = litellm_logging_obj
|
||||
logging.update_environment_variables(
|
||||
model=model,
|
||||
|
|
|
|||
|
|
@ -103,6 +103,7 @@ class Provenance:
|
|||
@final
|
||||
class Resolved:
|
||||
key: str
|
||||
prices: Prices
|
||||
def model_info(self, provider: str | None = None) -> dict[str, JsonValue]: ...
|
||||
def supports(self, capability: str) -> bool: ...
|
||||
def row(self) -> dict[str, JsonValue]: ...
|
||||
|
|
@ -118,6 +119,69 @@ class Invalid:
|
|||
def model_info(self, provider: str | None = None) -> dict[str, JsonValue]: ...
|
||||
def row(self) -> dict[str, JsonValue]: ...
|
||||
|
||||
@final
|
||||
class Prices:
|
||||
@staticmethod
|
||||
def from_row(row: Mapping[str, object], key: str = "model_info") -> Prices | Invalid: ...
|
||||
def price(self, usage: Usage, call: CallContext) -> Priced | Unpriced | Invalid: ...
|
||||
|
||||
@final
|
||||
class Usage:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
prompt_tokens: int,
|
||||
text: int,
|
||||
output_text: int,
|
||||
audio: int = 0,
|
||||
image: int = 0,
|
||||
video: int = 0,
|
||||
cache_read: int = 0,
|
||||
cache_read_audio: int = 0,
|
||||
cache_read_explicit: int = 0,
|
||||
cache_write_5m: int = 0,
|
||||
cache_write_1h: int = 0,
|
||||
cache_write_12h: int = 0,
|
||||
output_audio: int = 0,
|
||||
output_image: int = 0,
|
||||
output_video: int = 0,
|
||||
reasoning: int = 0,
|
||||
) -> None: ...
|
||||
|
||||
|
||||
@final
|
||||
class CallContext:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
service_tier: str | None,
|
||||
channel: Literal["online", "batch", "decisions"],
|
||||
data_residency: str | None,
|
||||
vertex_location: str | None,
|
||||
at: datetime,
|
||||
custom_pricing: Mapping[str, object] | None,
|
||||
) -> None: ...
|
||||
|
||||
@final
|
||||
class Line:
|
||||
meter: str
|
||||
quantity: int
|
||||
unit_price: float
|
||||
amount: float
|
||||
fallback_from: str | None
|
||||
|
||||
@final
|
||||
class Priced:
|
||||
total: float
|
||||
prompt: float
|
||||
completion: float
|
||||
lines: tuple[Line, ...]
|
||||
missing: tuple[str, ...]
|
||||
|
||||
@final
|
||||
class Unpriced:
|
||||
missing: tuple[str, ...]
|
||||
|
||||
@final
|
||||
class Registered:
|
||||
written: list[str]
|
||||
|
|
|
|||
|
|
@ -6081,9 +6081,6 @@ def get_provider_info(model: str, custom_llm_provider: str | None) -> ProviderSp
|
|||
return json_provider.provider_info if json_provider is not None and json_provider.provider_info else None
|
||||
|
||||
|
||||
_REPORTED_INVALID_ROWS: Final[set[str]] = set() # mutable-ok: logs each bad live row once per process
|
||||
|
||||
|
||||
def _model_not_mapped_message(model: str, custom_llm_provider: str | None) -> str:
|
||||
return (
|
||||
f"This model isn't mapped yet. model={model}, custom_llm_provider={custom_llm_provider}. "
|
||||
|
|
@ -6192,15 +6189,12 @@ def get_model_info_helper(
|
|||
and strip it. Tried last so no model that already resolves through 1-6 can change.
|
||||
"""
|
||||
|
||||
from litellm._catalog import CATALOG, Invalid, Missing
|
||||
from litellm._catalog import CATALOG, Invalid, Missing, report_invalid
|
||||
|
||||
resolution: Final = CATALOG.resolve(model, requested_provider)
|
||||
key: str | None = None if isinstance(resolution, Missing) else resolution.key
|
||||
if isinstance(resolution, Invalid) and resolution.key not in _REPORTED_INVALID_ROWS:
|
||||
_REPORTED_INVALID_ROWS.add(resolution.key)
|
||||
verbose_logger.warning(
|
||||
"litellm.model_cost[%r] fails validation: %s", resolution.key, "; ".join(resolution.errors)
|
||||
)
|
||||
if isinstance(resolution, Invalid):
|
||||
report_invalid(resolution)
|
||||
_model_info: dict[str, Any] | None = None if isinstance(resolution, Missing) else resolution.row()
|
||||
|
||||
if _model_info is not None and key is not None and (_model_info.get("mode") or "chat") in _BACKFILL_MODES:
|
||||
|
|
|
|||
|
|
@ -10,9 +10,7 @@ from litellm._internal_context import pinned_billing_time
|
|||
from litellm.litellm_core_utils.llm_cost_calc.utils import (
|
||||
BilledTokenRates,
|
||||
CostCalculatorUtils,
|
||||
PromptTokensDetailsResult,
|
||||
TokenRates,
|
||||
_calculate_input_cost,
|
||||
_get_token_base_cost,
|
||||
_is_off_peak,
|
||||
_is_within_off_peak_window,
|
||||
|
|
@ -1919,7 +1917,7 @@ def test_calculate_cost_component_with_string_values():
|
|||
|
||||
|
||||
def test_string_cost_values_edge_cases():
|
||||
"""Test edge cases for string cost value handling."""
|
||||
"""A numeric string is a price, a word is not, and a row with one bad price is unpriced (decision 20)"""
|
||||
from unittest.mock import patch
|
||||
|
||||
# Test with mixed string and float cost values
|
||||
|
|
@ -1954,14 +1952,7 @@ def test_string_cost_values_edge_cases():
|
|||
model="test-model", usage=usage, custom_llm_provider="test-provider"
|
||||
)
|
||||
|
||||
# Expected costs:
|
||||
# Prompt: 1000 * 1e-6 + 100 * 0 (invalid string becomes 0)
|
||||
# Completion: 500 * 2e-6 + 50 * 2e-6 (audio tokens fall back to base cost when output_cost_per_audio_token is None)
|
||||
expected_prompt_cost = 1000 * 1e-6
|
||||
expected_completion_cost = 500 * 2e-6 + 50 * 2e-6
|
||||
|
||||
assert round(prompt_cost, 12) == round(expected_prompt_cost, 12)
|
||||
assert round(completion_cost, 12) == round(expected_completion_cost, 12)
|
||||
assert (prompt_cost, completion_cost) == (0.0, 0.0)
|
||||
|
||||
|
||||
def test_string_cost_values_with_threshold():
|
||||
|
|
@ -2225,46 +2216,38 @@ def test_cache_writing_cost_with_zero_creation_tokens_and_ephemeral_details():
|
|||
"""
|
||||
Regression test: when cache_creation_tokens is 0 but cache_creation_token_details
|
||||
has non-zero ephemeral tokens, the cost must still be calculated.
|
||||
This ensures the guard in _calculate_input_cost doesn't skip
|
||||
calculate_cache_writing_cost when only ephemeral token details are present.
|
||||
"""
|
||||
cache_creation_cost = 3.75e-06
|
||||
cache_creation_cost_above_1hr = 6e-06
|
||||
|
||||
prompt_tokens_details: PromptTokensDetailsResult = {
|
||||
"cache_hit_tokens": 0,
|
||||
"cache_hit_audio_tokens": 0,
|
||||
"cache_creation_tokens": 0,
|
||||
"cache_creation_token_details": CacheCreationTokenDetails(
|
||||
ephemeral_5m_input_tokens=100,
|
||||
ephemeral_1h_input_tokens=200,
|
||||
usage = Usage(
|
||||
prompt_tokens=300,
|
||||
completion_tokens=0,
|
||||
total_tokens=300,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(
|
||||
text_tokens=0,
|
||||
cache_creation_tokens=0,
|
||||
cache_creation_token_details=CacheCreationTokenDetails(
|
||||
ephemeral_5m_input_tokens=100,
|
||||
ephemeral_1h_input_tokens=200,
|
||||
),
|
||||
),
|
||||
"text_tokens": 0,
|
||||
"audio_tokens": 0,
|
||||
"image_tokens": 0,
|
||||
"video_tokens": 0,
|
||||
"character_count": 0,
|
||||
"image_count": 0,
|
||||
"video_length_seconds": 0.0,
|
||||
"audio_length_seconds": 0.0,
|
||||
"query_count": 0,
|
||||
}
|
||||
|
||||
model_info: ModelInfo = {}
|
||||
|
||||
result = _calculate_input_cost(
|
||||
prompt_tokens_details=prompt_tokens_details,
|
||||
model_info=model_info,
|
||||
prompt_base_cost=0.0,
|
||||
cache_read_cost=0.0,
|
||||
cache_creation_cost=cache_creation_cost,
|
||||
cache_creation_cost_by_ttl={"1hr": cache_creation_cost_above_1hr, "12hr": cache_creation_cost_above_1hr},
|
||||
)
|
||||
|
||||
# Expected: (100 * 3.75e-06) + (200 * 6e-06) = 0.000375 + 0.0012 = 0.001575
|
||||
expected = (100 * cache_creation_cost) + (200 * cache_creation_cost_above_1hr)
|
||||
assert result > 0, "Cost should not be zero when ephemeral token details are present"
|
||||
assert round(result, 6) == round(expected, 6)
|
||||
prompt_cost, _ = generic_cost_per_token(
|
||||
model="litellm-test-ephemeral-cache-writes",
|
||||
usage=usage,
|
||||
custom_llm_provider="anthropic",
|
||||
model_info=cast(
|
||||
ModelInfo,
|
||||
{
|
||||
"input_cost_per_token": 0.0,
|
||||
"cache_creation_input_token_cost": cache_creation_cost,
|
||||
"cache_creation_input_token_cost_above_1hr": cache_creation_cost_above_1hr,
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
assert prompt_cost == pytest.approx(100 * cache_creation_cost + 200 * cache_creation_cost_above_1hr)
|
||||
|
||||
|
||||
def test_service_tier_ultrafast_pricing():
|
||||
|
|
|
|||
|
|
@ -528,7 +528,8 @@ async def test_completion_sagemaker(
|
|||
|
||||
assert route.called
|
||||
assert response.choices[0].message.content == "SageMaker reply"
|
||||
assert litellm.completion_cost(completion_response=response) == pytest.approx(rate)
|
||||
assert litellm.completion_cost(completion_response=response, custom_cost_per_second=rate) == pytest.approx(rate)
|
||||
assert "sagemaker/jumpstart-dft-hf-textgeneration1-mp-20240815-185614" not in litellm.model_cost
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -10590,3 +10590,46 @@ def test_bedrock_registered_cache_prices_preserve_dynamic_threshold_rates(
|
|||
assert entry is not None
|
||||
assert entry[1]["input_cost_per_token_above_300k_tokens"] == 0.03
|
||||
assert entry[1][tier_field] == (tier_override if tier_override is not None else 0.007)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"bad_price",
|
||||
[pytest.param("abc", id="word"), pytest.param(-1.0, id="negative"), pytest.param(float("nan"), id="nan")],
|
||||
)
|
||||
def test_a_directly_written_bad_price_is_unpriced_and_logged_once(
|
||||
bad_price: object, caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""Decision 20. On main the same 1000 prompt tokens gave a TypeError, -0.001 or nan"""
|
||||
import logging
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
|
||||
key: Final = f"litellm-test-bad-price-{type(bad_price).__name__}-{bad_price!r}"
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
key,
|
||||
{"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": bad_price, "output_cost_per_token": 1e-6},
|
||||
)
|
||||
response: Final = litellm.ModelResponse(
|
||||
model=key, usage=litellm.Usage(prompt_tokens=1000, completion_tokens=0, total_tokens=1000)
|
||||
)
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger=verbose_logger.name):
|
||||
per_token: Final = cost_per_token(model=key, prompt_tokens=1000, completion_tokens=0, custom_llm_provider="openai")
|
||||
total: Final = completion_cost(completion_response=response, model=key, custom_llm_provider="openai")
|
||||
|
||||
assert per_token == (0.0, 0.0)
|
||||
assert total == 0.0
|
||||
assert [record for record in caplog.records if key in record.getMessage() and "fails validation" in record.getMessage()] != []
|
||||
assert len([record for record in caplog.records if key in record.getMessage()]) == 1
|
||||
|
||||
|
||||
def test_a_directly_written_null_price_is_unpriced_like_a_missing_one(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
key: Final = "litellm-test-null-price"
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
key,
|
||||
{"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": None, "output_cost_per_token": 1e-6},
|
||||
)
|
||||
|
||||
assert cost_per_token(model=key, prompt_tokens=1000, completion_tokens=0, custom_llm_provider="openai") == (0.0, 0.0)
|
||||
|
|
|
|||
|
|
@ -1,12 +1,6 @@
|
|||
"""
|
||||
Test that register_model() in completion() and embedding() passes all
|
||||
custom pricing fields from kwargs and model_info, not just the base
|
||||
input/output costs.
|
||||
|
||||
Previously, only input_cost_per_token, output_cost_per_token, and
|
||||
litellm_provider were forwarded. Fields like cache_read_input_token_cost,
|
||||
mode, and supports_prompt_caching were dropped, causing incorrect cost
|
||||
calculations for DB-sourced models with prompt caching pricing.
|
||||
Custom pricing through register_model() and per call. A router deployment's prices are registered under its id, and a
|
||||
direct SDK call's prices price that call only, never the shared row (decision 12)
|
||||
"""
|
||||
|
||||
import copy
|
||||
|
|
@ -21,7 +15,7 @@ import pytest
|
|||
import litellm
|
||||
from litellm._catalog import Fetched, File
|
||||
from litellm.litellm_core_utils.get_model_cost_map import reload_model_cost_map
|
||||
from litellm.main import _build_custom_pricing_entry
|
||||
from litellm.types.utils import EmbeddingResponse, ModelResponse
|
||||
from litellm.utils import _invalidate_model_cost_lowercase_map
|
||||
|
||||
|
||||
|
|
@ -44,131 +38,6 @@ def _restore_model_cost_entries(original_entries):
|
|||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
|
||||
def test_build_custom_pricing_entry_includes_all_kwargs_fields():
|
||||
"""All CustomPricingLiteLLMParams fields present in kwargs should be
|
||||
included in the resulting entry dict."""
|
||||
kwargs = {
|
||||
"input_cost_per_token": 0.001,
|
||||
"output_cost_per_token": 0.002,
|
||||
"cache_read_input_token_cost": 0.00025,
|
||||
"cache_creation_input_token_cost": 0.005,
|
||||
"output_cost_per_reasoning_token": 0.01,
|
||||
"input_cost_per_audio_token": 0.003,
|
||||
"unrelated_kwarg": "should_be_ignored",
|
||||
}
|
||||
|
||||
entry = _build_custom_pricing_entry(
|
||||
custom_llm_provider="openai",
|
||||
kwargs=kwargs,
|
||||
)
|
||||
|
||||
assert entry["litellm_provider"] == "openai"
|
||||
assert entry["input_cost_per_token"] == 0.001
|
||||
assert entry["output_cost_per_token"] == 0.002
|
||||
assert entry["cache_read_input_token_cost"] == 0.00025
|
||||
assert entry["cache_creation_input_token_cost"] == 0.005
|
||||
assert entry["output_cost_per_reasoning_token"] == 0.01
|
||||
assert entry["input_cost_per_audio_token"] == 0.003
|
||||
assert "unrelated_kwarg" not in entry
|
||||
|
||||
|
||||
def test_build_custom_pricing_entry_merges_model_info_metadata():
|
||||
"""Fields from model_info (mode, supports_prompt_caching, max_tokens)
|
||||
should be merged into the entry when present."""
|
||||
kwargs = {
|
||||
"input_cost_per_token": 0.001,
|
||||
"output_cost_per_token": 0.002,
|
||||
}
|
||||
model_info = {
|
||||
"id": "deployment-123",
|
||||
"mode": "chat",
|
||||
"supports_prompt_caching": True,
|
||||
"max_tokens": 128000,
|
||||
}
|
||||
|
||||
entry = _build_custom_pricing_entry(
|
||||
custom_llm_provider="openai",
|
||||
kwargs=kwargs,
|
||||
model_info=model_info,
|
||||
)
|
||||
|
||||
assert entry["mode"] == "chat"
|
||||
assert entry["supports_prompt_caching"] is True
|
||||
assert entry["max_tokens"] == 128000
|
||||
|
||||
|
||||
def test_build_custom_pricing_entry_setdefault_does_not_override_existing():
|
||||
"""model_info uses setdefault, so it should not override a key that is
|
||||
already present in the entry dict. Currently CustomPricingLiteLLMParams
|
||||
and the model_info keys (mode, supports_prompt_caching, max_tokens) do
|
||||
not overlap, but if they ever do, setdefault ensures the kwargs-sourced
|
||||
value wins."""
|
||||
kwargs = {
|
||||
"input_cost_per_token": 0.001,
|
||||
"output_cost_per_token": 0.002,
|
||||
}
|
||||
model_info = {
|
||||
"mode": "chat",
|
||||
"supports_prompt_caching": True,
|
||||
"max_tokens": 128000,
|
||||
}
|
||||
|
||||
entry = _build_custom_pricing_entry(
|
||||
custom_llm_provider="openai",
|
||||
kwargs=kwargs,
|
||||
model_info=model_info,
|
||||
)
|
||||
|
||||
assert entry["mode"] == "chat"
|
||||
assert entry["supports_prompt_caching"] is True
|
||||
assert entry["max_tokens"] == 128000
|
||||
|
||||
# Verify setdefault behavior: if a model_info key already exists in
|
||||
# the entry (e.g. from a future CustomPricingLiteLLMParams addition),
|
||||
# setdefault must not overwrite it.
|
||||
entry["mode"] = "embedding" # simulate pre-existing value
|
||||
# Re-apply setdefault the same way _build_custom_pricing_entry does
|
||||
entry.setdefault("mode", model_info["mode"])
|
||||
assert entry["mode"] == "embedding" # must NOT revert to "chat"
|
||||
|
||||
|
||||
def test_build_custom_pricing_entry_skips_none_values():
|
||||
"""Fields with None values in kwargs should not be included."""
|
||||
kwargs = {
|
||||
"input_cost_per_token": 0.001,
|
||||
"output_cost_per_token": None, # explicitly None
|
||||
"cache_read_input_token_cost": None,
|
||||
}
|
||||
|
||||
entry = _build_custom_pricing_entry(
|
||||
custom_llm_provider="openai",
|
||||
kwargs=kwargs,
|
||||
)
|
||||
|
||||
assert entry["input_cost_per_token"] == 0.001
|
||||
assert "output_cost_per_token" not in entry
|
||||
assert "cache_read_input_token_cost" not in entry
|
||||
|
||||
|
||||
def test_build_custom_pricing_entry_handles_no_model_info():
|
||||
"""Should work correctly when model_info is None."""
|
||||
kwargs = {
|
||||
"input_cost_per_token": 0.001,
|
||||
"output_cost_per_token": 0.002,
|
||||
}
|
||||
|
||||
entry = _build_custom_pricing_entry(
|
||||
custom_llm_provider="openai",
|
||||
kwargs=kwargs,
|
||||
model_info=None,
|
||||
)
|
||||
|
||||
assert entry["litellm_provider"] == "openai"
|
||||
assert entry["input_cost_per_token"] == 0.001
|
||||
assert entry["output_cost_per_token"] == 0.002
|
||||
assert "mode" not in entry
|
||||
|
||||
|
||||
def test_register_model_receives_cache_pricing_fields():
|
||||
"""End-to-end: when register_model is called with a full pricing entry,
|
||||
the cache pricing fields should be present in litellm.model_cost."""
|
||||
|
|
@ -199,23 +68,6 @@ def test_register_model_receives_cache_pricing_fields():
|
|||
litellm.model_cost.pop(model_key, None)
|
||||
|
||||
|
||||
def test_build_custom_pricing_entry_time_based():
|
||||
"""Time-based pricing fields should be included correctly."""
|
||||
kwargs = {
|
||||
"input_cost_per_second": 0.01,
|
||||
"output_cost_per_second": 0.02,
|
||||
}
|
||||
|
||||
entry = _build_custom_pricing_entry(
|
||||
custom_llm_provider="openai",
|
||||
kwargs=kwargs,
|
||||
)
|
||||
|
||||
assert entry["litellm_provider"] == "openai"
|
||||
assert entry["input_cost_per_second"] == 0.01
|
||||
assert entry["output_cost_per_second"] == 0.02
|
||||
|
||||
|
||||
def test_register_model_strips_none_litellm_provider():
|
||||
"""``get_model_info`` returns ``litellm_provider: None`` for deployments
|
||||
registered without a provider (e.g. ``Router.add_deployment`` flows).
|
||||
|
|
@ -603,8 +455,7 @@ def test_embedding_router_zero_pricing_does_not_clobber_builtin_pricing():
|
|||
litellm.get_model_info(model=shared_key)["input_cost_per_token"]
|
||||
== builtin_input_cost
|
||||
), "wildcard deployment's zero pricing leaked into the shared model_cost key"
|
||||
assert litellm.model_cost[deployment_id]["input_cost_per_token"] == 0.0
|
||||
assert litellm.model_cost[deployment_id]["output_cost_per_token"] == 0.0
|
||||
assert deployment_id not in litellm.model_cost
|
||||
|
||||
sibling_response = litellm.embedding(
|
||||
model=shared_key,
|
||||
|
|
@ -620,20 +471,15 @@ def test_embedding_router_zero_pricing_does_not_clobber_builtin_pricing():
|
|||
_restore_model_cost_entries(snapshot)
|
||||
|
||||
|
||||
def test_embedding_router_custom_pricing_costs_request_via_deployment_id():
|
||||
"""The request that carries custom pricing must still be costed with that
|
||||
pricing (via its deployment id entry), while the shared backend key keeps
|
||||
the built-in rate for siblings.
|
||||
def test_embedding_router_custom_pricing_costs_request_with_its_own_prices():
|
||||
"""The request that carries custom pricing is costed with that pricing, while the shared backend key keeps the
|
||||
built-in rate for siblings
|
||||
"""
|
||||
shared_key = "openai/text-embedding-3-small"
|
||||
deployment_id = "lit3991-wildcard-embed-custom"
|
||||
override_input_cost = 5e-05
|
||||
snapshot = _snapshot_model_cost_entries(
|
||||
[shared_key, "text-embedding-3-small", deployment_id]
|
||||
)
|
||||
builtin_input_cost = litellm.get_model_info(model=shared_key)[
|
||||
"input_cost_per_token"
|
||||
]
|
||||
snapshot = _snapshot_model_cost_entries([shared_key, "text-embedding-3-small", deployment_id])
|
||||
builtin_input_cost = litellm.get_model_info(model=shared_key)["input_cost_per_token"]
|
||||
assert builtin_input_cost != override_input_cost
|
||||
|
||||
try:
|
||||
|
|
@ -648,19 +494,8 @@ def test_embedding_router_custom_pricing_costs_request_via_deployment_id():
|
|||
mock_response=[0.1, 0.2],
|
||||
)
|
||||
|
||||
request_cost = litellm.completion_cost(
|
||||
completion_response=response,
|
||||
model=shared_key,
|
||||
custom_llm_provider="openai",
|
||||
call_type="embedding",
|
||||
custom_pricing=True,
|
||||
router_model_id=deployment_id,
|
||||
)
|
||||
assert request_cost == pytest.approx(10 * override_input_cost)
|
||||
assert (
|
||||
litellm.get_model_info(model=shared_key)["input_cost_per_token"]
|
||||
== builtin_input_cost
|
||||
)
|
||||
assert _response_cost(response) == pytest.approx(10 * override_input_cost)
|
||||
assert litellm.get_model_info(model=shared_key)["input_cost_per_token"] == builtin_input_cost
|
||||
finally:
|
||||
_restore_model_cost_entries(snapshot)
|
||||
|
||||
|
|
@ -693,43 +528,80 @@ def test_completion_router_zero_pricing_does_not_clobber_builtin_pricing():
|
|||
litellm.get_model_info(model=shared_key)["input_cost_per_token"]
|
||||
== builtin_input_cost
|
||||
), "wildcard deployment's zero pricing leaked into the shared model_cost key"
|
||||
assert litellm.model_cost[deployment_id]["input_cost_per_token"] == 0.0
|
||||
assert deployment_id not in litellm.model_cost
|
||||
finally:
|
||||
_restore_model_cost_entries(snapshot)
|
||||
|
||||
|
||||
def test_embedding_direct_sdk_custom_pricing_still_registers_shared_key():
|
||||
"""Direct SDK calls (no router deployment id in metadata) keep the legacy
|
||||
behavior: custom pricing is registered under ``{provider}/{model}`` and the
|
||||
request is costed with it.
|
||||
"""
|
||||
model_key = "openai/lit3991-direct-sdk-embed-model"
|
||||
override_input_cost = 3e-05
|
||||
try:
|
||||
response = litellm.embedding(
|
||||
model=model_key,
|
||||
input=["hello"],
|
||||
api_key="fake-key",
|
||||
input_cost_per_token=override_input_cost,
|
||||
output_cost_per_token=override_input_cost * 2,
|
||||
mock_response=[0.1, 0.2],
|
||||
)
|
||||
def test_embedding_direct_sdk_custom_pricing_prices_the_call_without_registering_the_shared_key():
|
||||
model_key: Final = "openai/lit3991-direct-sdk-embed-model"
|
||||
override_input_cost: Final = 3e-05
|
||||
response: Final = litellm.embedding(
|
||||
model=model_key,
|
||||
input=["hello"],
|
||||
api_key="fake-key",
|
||||
input_cost_per_token=override_input_cost,
|
||||
output_cost_per_token=override_input_cost * 2,
|
||||
mock_response=[0.1, 0.2],
|
||||
)
|
||||
|
||||
assert (
|
||||
litellm.model_cost[model_key]["input_cost_per_token"]
|
||||
== override_input_cost
|
||||
)
|
||||
cost = litellm.completion_cost(
|
||||
completion_response=response,
|
||||
model=model_key,
|
||||
custom_llm_provider="openai",
|
||||
call_type="embedding",
|
||||
custom_pricing=True,
|
||||
)
|
||||
assert cost == pytest.approx(10 * override_input_cost)
|
||||
finally:
|
||||
litellm.model_cost.pop(model_key, None)
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
assert model_key not in litellm.model_cost
|
||||
assert isinstance(response, EmbeddingResponse)
|
||||
assert _response_cost(response) == pytest.approx(response.usage.prompt_tokens * override_input_cost)
|
||||
|
||||
|
||||
def _response_cost(response: object) -> float:
|
||||
assert isinstance(response, (ModelResponse, EmbeddingResponse))
|
||||
cost: Final = response._hidden_params["response_cost"] # pyright: ignore[reportPrivateUsage] # the SDK reports a call's cost only here
|
||||
assert isinstance(cost, float)
|
||||
return cost
|
||||
|
||||
|
||||
def _tenant_call_cost(**custom_pricing: float) -> tuple[float, litellm.Usage]:
|
||||
response: Final = litellm.completion(
|
||||
model="openai/gpt-4o",
|
||||
messages=[{"role": "user", "content": "price this call"}],
|
||||
api_key="fake-key",
|
||||
mock_response="priced",
|
||||
**custom_pricing,
|
||||
)
|
||||
assert isinstance(response, ModelResponse)
|
||||
usage: Final = response.usage
|
||||
assert isinstance(usage, litellm.Usage)
|
||||
return _response_cost(response), usage
|
||||
|
||||
|
||||
def test_one_tenants_custom_pricing_never_reprices_another_tenants_calls():
|
||||
"""Two tenants in one SDK process. Tenant A's negotiated price prices only A's call"""
|
||||
catalog_row_before: Final = copy.deepcopy(litellm.model_cost["gpt-4o"])
|
||||
shared_key_before: Final = copy.deepcopy(litellm.model_cost.get("openai/gpt-4o"))
|
||||
tenant_b_before, usage = _tenant_call_cost()
|
||||
tenant_a, tenant_a_usage = _tenant_call_cost(input_cost_per_token=1e-06, output_cost_per_token=4e-06)
|
||||
tenant_b_after, _ = _tenant_call_cost()
|
||||
|
||||
input_rate, output_rate = catalog_row_before["input_cost_per_token"], catalog_row_before["output_cost_per_token"]
|
||||
assert isinstance(input_rate, float) and isinstance(output_rate, float)
|
||||
catalog_cost: Final = usage.prompt_tokens * input_rate + usage.completion_tokens * output_rate
|
||||
assert tenant_b_before == pytest.approx(catalog_cost)
|
||||
assert tenant_a == pytest.approx(tenant_a_usage.prompt_tokens * 1e-06 + tenant_a_usage.completion_tokens * 4e-06)
|
||||
assert tenant_b_after == pytest.approx(catalog_cost)
|
||||
assert litellm.model_cost["gpt-4o"] == catalog_row_before
|
||||
assert litellm.model_cost.get("openai/gpt-4o") == shared_key_before
|
||||
|
||||
|
||||
def test_custom_pricing_never_writes_the_calls_model_info_onto_the_shared_row():
|
||||
model_key: Final = "openai/litellm-test-custom-priced-responses-mode"
|
||||
litellm.completion(
|
||||
model=model_key,
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
api_key="fake-key",
|
||||
input_cost_per_token=1e-06,
|
||||
output_cost_per_token=2e-06,
|
||||
model_info={"mode": "responses"},
|
||||
mock_response="hello back",
|
||||
)
|
||||
|
||||
assert model_key not in litellm.model_cost
|
||||
|
||||
|
||||
@pytest.mark.parametrize("route", ["", "converse/", "invoke/"])
|
||||
|
|
@ -1468,22 +1340,18 @@ def test_completion_cost_applies_off_peak_only_deployment_pricing():
|
|||
del router
|
||||
|
||||
|
||||
def test_completion_registers_cost_per_second_pricing():
|
||||
def test_completion_prices_per_second_custom_pricing_without_registering_it():
|
||||
model_key: Final = "openai/test-cost-per-second-registration"
|
||||
original_entries: Final = _snapshot_model_cost_entries([model_key])
|
||||
response: Final = litellm.completion(
|
||||
model=model_key,
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
api_key="fake-key",
|
||||
cost_per_second=0.02,
|
||||
mock_response="hello back",
|
||||
)
|
||||
|
||||
try:
|
||||
litellm.completion(
|
||||
model=model_key,
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
api_key="fake-key",
|
||||
cost_per_second=0.02,
|
||||
mock_response="hello back",
|
||||
)
|
||||
|
||||
assert litellm.model_cost[model_key]["cost_per_second"] == 0.02
|
||||
finally:
|
||||
_restore_model_cost_entries(original_entries)
|
||||
assert model_key not in litellm.model_cost
|
||||
assert _response_cost(response) >= 0
|
||||
|
||||
|
||||
def test_update_model_cost():
|
||||
|
|
@ -1533,33 +1401,3 @@ def test_register_model_scopes_builtin_match_to_the_given_provider():
|
|||
assert litellm.model_cost[builtin_key] == builtin_row_before
|
||||
finally:
|
||||
_restore_model_cost_entries(model_cost_entries)
|
||||
|
||||
|
||||
def test_per_request_custom_pricing_scopes_a_colliding_deployment_id_to_its_provider():
|
||||
from litellm.main import _register_custom_pricing_for_request
|
||||
|
||||
deployment_id: Final = "baseten/zai-org/glm-5.2"
|
||||
builtin_key: Final = "baseten/zai-org/GLM-5.2"
|
||||
shared_key: Final = "openai/zai-org/GLM-5.2"
|
||||
model_cost_entries: Final = _snapshot_model_cost_entries((deployment_id, builtin_key, shared_key))
|
||||
builtin_row_before: Final = copy.deepcopy(litellm.model_cost[builtin_key])
|
||||
openai_models_before: Final = frozenset(litellm.open_ai_chat_completion_models)
|
||||
try:
|
||||
_register_custom_pricing_for_request(
|
||||
model="zai-org/GLM-5.2",
|
||||
custom_llm_provider="openai",
|
||||
kwargs={
|
||||
"input_cost_per_token": 0.00000096,
|
||||
"output_cost_per_token": 0.00000302,
|
||||
"metadata": {"model_info": {"id": deployment_id}},
|
||||
},
|
||||
model_info={"mode": "chat"},
|
||||
)
|
||||
|
||||
entry: Final = litellm.model_cost[deployment_id]
|
||||
assert entry["input_cost_per_token"] == 0.00000096
|
||||
assert entry["output_cost_per_token"] == 0.00000302
|
||||
assert litellm.model_cost[builtin_key] == builtin_row_before
|
||||
finally:
|
||||
_restore_model_cost_entries(model_cost_entries)
|
||||
litellm.open_ai_chat_completion_models.intersection_update(openai_models_before)
|
||||
|
|
|
|||
|
|
@ -2243,7 +2243,7 @@ def test_register_model_price_is_used_for_completion_cost() -> None:
|
|||
|
||||
@respx.mock
|
||||
@pytest.mark.usefixtures("isolated_openai_model_sets", "local_model_cost_map")
|
||||
def test_completion_registers_test_owned_model_prices() -> None:
|
||||
def test_completion_prices_per_call_prices_without_registering_them() -> None:
|
||||
model: Final = "migration-owned-pricing-model"
|
||||
messages: Final = [{"role": "user", "content": "price control"}]
|
||||
litellm.register_model(
|
||||
|
|
@ -2286,8 +2286,7 @@ def test_completion_registers_test_owned_model_prices() -> None:
|
|||
)
|
||||
|
||||
assert json.loads(route.calls[0].request.content) == {"model": model, "messages": messages}
|
||||
assert litellm.model_cost[f"openai/{model}"]["input_cost_per_token"] == 0.001
|
||||
assert litellm.model_cost[f"openai/{model}"]["output_cost_per_token"] == 0.002
|
||||
assert f"openai/{model}" not in litellm.model_cost
|
||||
assert litellm.model_cost[model]["input_cost_per_token"] == 0.0001
|
||||
assert response._hidden_params["response_cost"] == pytest.approx(0.018)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue