refactor(cost): price token usage with the Rust cost core [08/12]

cost::calculate prices token meters over the typed Prices every row lowers into, with one line per meter and the
borrowed rate named in fallback_from. generic_cost_per_token prices through Prices.price and as_main_cost, per-call
custom pricing is an overlay that never writes the shared row, and a directly written bad price is unpriced with one
log line

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Yujong Lee 2026-10-28 01:55:35 -07:00
parent 83882222c0
commit c57274fa12
25 changed files with 2562 additions and 1450 deletions

View file

@ -4284,6 +4284,7 @@ dependencies = [
"fancy-regex 0.19.2",
"futures-util",
"indexmap 2.14.0",
"jiff",
"litellm-auth",
"litellm-auth-aws",
"litellm-cache",
@ -4292,6 +4293,7 @@ dependencies = [
"litellm-cache-response",
"litellm-callbacks-legacy-python",
"litellm-core-utils",
"litellm-cost",
"litellm-host",
"litellm-host-python",
"litellm-http",

View file

@ -10,7 +10,7 @@ serde = ["dep:serde"]
schema = ["serde", "dep:schemars"]
[dependencies]
jiff = { version = "0.2.37", default-features = false, features = ["std"] }
jiff = { version = "0.2.37", default-features = false, features = ["std", "tzdb-bundle-always"] }
schemars = { workspace = true, optional = true }
serde = { workspace = true, optional = true }
thiserror.workspace = true

View file

@ -1,3 +1,3 @@
# litellm-cost
This crate holds the price data model that `litellm-model-catalog` lowers each catalog entry into, plus the usage, call context and priced outcome types that `calculate` will compute over. There is no `calculate` until the token pricing PR lands. `UnitPrice` is USD per unit and `Usd` is an amount, and only the operations that make sense between them exist, so a price is never added to an amount. `Prices::band` and `Prices::rate` resolve a token rate the way Python's `_get_token_base_cost` does
This crate holds the price data model that `litellm-model-catalog` lowers each catalog entry into, plus the usage, call context and priced outcome types. `calculate` prices a call's token usage over a row's `Prices` and `Adjustments`, one `Line` per meter with usage, and names the meter whose rate a line borrowed in `fallback_from`. Character, second and per-unit meters land in later PRs and are reported as `missing` until then. `UnitPrice` is USD per unit and `Usd` is an amount, and only the operations that make sense between them exist, so a price is never added to an amount. `Prices::band` picks the prompt band the way Python's `_get_token_base_cost` does

View file

@ -0,0 +1,796 @@
use crate::call::CallContext;
use crate::money::{Multiplier, UnitPrice};
use crate::price::{
Adjustments, Band, CacheReadFallback, CacheReadMode, CacheTtl, Channel, Feature, InputUnit,
Meter, OffPeakRates, OutputUnit, Price, Prices, ServiceTier, TierScope, TokenKind, Uplift,
When,
};
use crate::priced::{Line, Outcome, Priced, Quantity, Side};
use crate::usage::{DurationMeter, Usage};
const TEXT_INPUT: Meter = Meter::Input(InputUnit::Token(TokenKind::Text));
const TEXT_OUTPUT: Meter = Meter::Output(OutputUnit::Token(TokenKind::Text));
const IMAGE_OUTPUT: Meter = Meter::Output(OutputUnit::Token(TokenKind::Image));
const REASONING: Meter = Meter::Output(OutputUnit::ReasoningToken);
const CACHE_READ: Meter = Meter::CacheRead {
kind: TokenKind::Text,
mode: CacheReadMode::Implicit,
};
const CACHE_WRITE: Meter = Meter::CacheWrite {
kind: TokenKind::Text,
ttl: CacheTtl::Default,
};
/// Prices one call's token usage (decisions 21 and 30). Every meter with usage is a `Line`, one that borrowed another
/// meter's rate names it in `fallback_from`, and one with no price and no fallback is listed in `missing`
pub fn calculate(
prices: &Prices,
adjustments: &Adjustments,
usage: &Usage,
call: &CallContext,
) -> Outcome {
let rates = Rates::new(prices, usage, call);
let uplifts = uplifts(adjustments, call);
let tokens = &usage.tokens;
let billed: Vec<(Meter, Option<Rate>, u64, Side)> = tokens
.input
.iter()
.map(|(kind, count)| {
(
Meter::Input(InputUnit::Token(*kind)),
rates.input(*kind),
*count,
Side::Prompt,
)
})
.chain(tokens.cache_read.iter().map(|((kind, mode), count)| {
let meter = Meter::CacheRead {
kind: *kind,
mode: *mode,
};
(meter, rates.cache_read(*kind, *mode), *count, Side::Prompt)
}))
.chain(tokens.cache_write.iter().map(|((kind, ttl), count)| {
let meter = Meter::CacheWrite {
kind: *kind,
ttl: *ttl,
};
(meter, rates.cache_write(*ttl), *count, Side::Prompt)
}))
.chain(tokens.output.iter().map(|(kind, count)| {
let meter = Meter::Output(OutputUnit::Token(*kind));
(meter, rates.output(*kind), *count, Side::Completion)
}))
.chain([(
REASONING,
rates.reasoning(),
tokens.reasoning,
Side::Completion,
)])
.filter(|(.., count, _)| *count > 0)
.collect();
let lines: Vec<Line> = billed
.iter()
.filter_map(|(meter, rate, count, side)| {
Some(rate.clone()?.line(meter.clone(), *count, *side, &uplifts))
})
.collect();
let missing: Vec<Meter> = billed
.into_iter()
.filter_map(|(meter, rate, ..)| rate.is_none().then_some(meter))
.chain(without_an_arm(usage))
.collect();
if lines.is_empty() && !missing.is_empty() {
return Outcome::Unpriced { missing };
}
let total = lines.iter().map(|line| line.amount).sum();
Outcome::Priced(Priced {
lines,
missing,
total,
})
}
/// One resolved rate, and the meter it came from when that is not the billed meter's own
#[derive(Clone, Debug)]
struct Rate {
price: UnitPrice,
borrowed: Option<Meter>,
}
impl Rate {
fn own(price: UnitPrice) -> Self {
Self {
price,
borrowed: None,
}
}
fn borrowed_from(self, meter: &Meter) -> Self {
Self {
borrowed: Some(self.borrowed.unwrap_or_else(|| meter.clone())),
..self
}
}
fn halved(self) -> Option<Self> {
Some(Self {
price: UnitPrice::new(self.price.get() * 0.5)?,
..self
})
}
fn line(self, meter: Meter, count: u64, side: Side, uplifts: &[Multiplier]) -> Line {
let quantity = Quantity::Count(count);
Line {
quantity,
unit_price: self.price,
amount: uplifts
.iter()
.fold(self.price * quantity, |amount, factor| amount * *factor),
fallback_from: self.borrowed,
meter,
side,
}
}
}
/// The rates one call resolves, main's `_get_token_base_cost` and its modality, cache and reasoning rules
struct Rates<'a> {
prices: &'a Prices,
overlay: Option<&'a Prices>,
channel: Channel,
tier: ServiceTier,
band: Band,
whole_call: bool,
off_peak: Option<OffPeakRates>,
text_input: Option<Rate>,
text_output: Option<Rate>,
}
impl<'a> Rates<'a> {
/// A tier table bills the whole call at the tier the prompt picks, unless that tier sets no input price. Then
/// the row is priced as if it had no tiers, as main's `_select_priced_tier` does
fn new(prices: &'a Prices, usage: &Usage, call: &'a CallContext) -> Self {
let band = prices.band(call.tier, usage.prompt_tokens);
let tier_input = Price::online(
TEXT_INPUT,
When {
tier: ServiceTier::Standard,
band,
},
);
let whole_call = prices.billing().tier_scope == TierScope::WholeCall
&& prices.get(&tier_input).is_some();
let lookup = Self {
prices,
overlay: call.overlay.as_ref(),
channel: call.channel,
tier: call.tier,
band: match (prices.billing().tier_scope, whole_call) {
(TierScope::WholeCall, false) => Band::Base,
_ => band,
},
whole_call,
off_peak: prices
.off_peak()
.filter(|off_peak| off_peak.schedule().is_open(call.at))
.map(|off_peak| *off_peak.rates()),
text_input: None,
text_output: None,
};
let text_input = lookup
.off_peak(|rates| rates.input)
.or_else(|| lookup.banded(&TEXT_INPUT));
let text_output = lookup
.off_peak(|rates| rates.output)
.or_else(|| lookup.output_or_image_rate());
Self {
text_input,
text_output,
..lookup
}
}
fn input(&self, kind: TokenKind) -> Option<Rate> {
match kind {
TokenKind::Text => self.text_input.clone(),
_ => self
.banded(&Meter::Input(InputUnit::Token(kind)))
.or_else(|| self.borrowed_input()),
}
}
fn output(&self, kind: TokenKind) -> Option<Rate> {
match kind {
TokenKind::Text => self.text_output.clone(),
_ => self
.banded(&Meter::Output(OutputUnit::Token(kind)))
.or_else(|| self.borrowed_output()),
}
}
fn cache_read(&self, kind: TokenKind, mode: CacheReadMode) -> Option<Rate> {
let borrowed_implicit = || {
self.implicit_cache_read()
.map(|rate| rate.borrowed_from(&CACHE_READ))
};
match (kind, mode) {
(TokenKind::Text, CacheReadMode::Implicit) => self.implicit_cache_read(),
(TokenKind::Text, CacheReadMode::Explicit) => {
let own = self.banded(&Meter::CacheRead { kind, mode });
match (self.off_peak(|rates| rates.explicit_cache_read), own) {
(Some(off_peak), _) => Some(off_peak),
(None, Some(own)) => {
Some(self.off_peak(|rates| rates.cache_read).unwrap_or(own))
}
(None, None) => borrowed_implicit(),
}
}
_ => self
.banded(&Meter::CacheRead { kind, mode })
.or_else(borrowed_implicit),
}
}
fn cache_write(&self, ttl: CacheTtl) -> Option<Rate> {
match ttl {
CacheTtl::Default => self.default_cache_write(),
CacheTtl::OneHour | CacheTtl::TwelveHours => self
.banded(&Meter::CacheWrite {
kind: TokenKind::Text,
ttl,
})
.or_else(|| {
self.default_cache_write()
.map(|rate| rate.borrowed_from(&CACHE_WRITE))
}),
}
}
/// Off-peak first, then the tier's reasoning price. A tier that prices output and not reasoning bills reasoning at
/// its output rate, otherwise the standard reasoning price, then the output rate. Reasoning prices are never banded
/// outside a whole-call tier table
fn reasoning(&self) -> Option<Rate> {
let tier_base = When {
tier: self.tier,
band: Band::Base,
};
let tier_prices_output_only =
|| self.tier != ServiceTier::Standard && self.at(&TEXT_OUTPUT, &[tier_base]).is_some();
self.off_peak(|rates| rates.reasoning)
.or_else(|| match self.whole_call() {
true => self.banded(&REASONING),
false => self.at(&REASONING, &[tier_base]).or_else(|| {
(!tier_prices_output_only())
.then(|| self.at(&REASONING, &[When::default()]))
.flatten()
}),
})
.or_else(|| self.borrowed_output())
}
fn borrowed_input(&self) -> Option<Rate> {
self.text_input
.clone()
.map(|rate| rate.borrowed_from(&TEXT_INPUT))
}
fn borrowed_output(&self) -> Option<Rate> {
self.text_output
.clone()
.map(|rate| rate.borrowed_from(&TEXT_OUTPUT))
}
fn implicit_cache_read(&self) -> Option<Rate> {
self.off_peak(|rates| rates.cache_read)
.or_else(|| self.banded(&CACHE_READ))
.or_else(|| match self.prices.billing().cache_read_fallback {
CacheReadFallback::InputRate => self.borrowed_input(),
CacheReadFallback::HalfInputRate => self.borrowed_input()?.halved(),
})
}
fn default_cache_write(&self) -> Option<Rate> {
self.off_peak(|rates| rates.cache_write)
.or_else(|| self.banded(&CACHE_WRITE))
.or_else(|| self.borrowed_input())
}
/// main reads a missing or zero output price as unknown and bills image models at their image token rate. A
/// whole-call tier with no output price bills the row's own output price
fn output_or_image_rate(&self) -> Option<Rate> {
let own = self.banded(&TEXT_OUTPUT).or_else(|| {
self.whole_call()
.then(|| self.at(&TEXT_OUTPUT, &[When::default()]))?
});
match own {
Some(rate) if rate.price.get() != 0.0 => Some(rate),
own => self
.at(&IMAGE_OUTPUT, &[When::default()])
.map(|rate| rate.borrowed_from(&IMAGE_OUTPUT))
.or(own),
}
}
fn off_peak(&self, rate: fn(&OffPeakRates) -> Option<UnitPrice>) -> Option<Rate> {
self.off_peak.as_ref().and_then(rate).map(Rate::own)
}
fn whole_call(&self) -> bool {
self.whole_call
}
/// The meter's own rate in this call's band, tried at `(tier, band)`, `(standard, band)`, `(tier, base)` and
/// `(standard, base)`. A whole-call tier table prices every meter at the band's own rate only
fn banded(&self, meter: &Meter) -> Option<Rate> {
let standard = ServiceTier::Standard;
let whens: &[When] = match self.whole_call() {
true => &[When {
tier: standard,
band: self.band,
}],
false => &[
When {
tier: self.tier,
band: self.band,
},
When {
tier: standard,
band: self.band,
},
When {
tier: self.tier,
band: Band::Base,
},
When::default(),
],
};
self.at(meter, whens)
}
/// The first price for `meter` at `whens` on this call's channel. A per-call overlay that prices the meter at all
/// replaces the row's prices for it (decision 12). Decisions bills the online prices it does not set, and a batch
/// call with no batch price bills half the online price, main's `_batch_or_half`
fn at(&self, meter: &Meter, whens: &[When]) -> Option<Rate> {
let source = match self.overlay {
Some(overlay) if overlay.iter().any(|(price, _)| &price.meter == meter) => overlay,
_ => self.prices,
};
let on = |channel: Channel| {
whens.iter().find_map(|when| {
source.get(&Price {
meter: meter.clone(),
channel,
when: *when,
})
})
};
match self.channel {
Channel::Online => on(Channel::Online).map(Rate::own),
Channel::Decisions => on(Channel::Decisions)
.or_else(|| on(Channel::Online))
.map(Rate::own),
Channel::Batch => on(Channel::Batch).map(Rate::own).or_else(|| {
Rate::own(on(Channel::Online)?)
.borrowed_from(meter)
.halved()
}),
}
}
}
/// Meters with usage that no arm prices yet. Characters and seconds land in 09, the per-unit meters in 10
fn without_an_arm(usage: &Usage) -> impl Iterator<Item = Meter> + '_ {
let counted = [
(usage.characters.prompt, Meter::Input(InputUnit::Character)),
(
usage.characters.completion,
Meter::Output(OutputUnit::Character),
),
(usage.images.input, Meter::Input(InputUnit::Image)),
(usage.images.input_pixels, Meter::Input(InputUnit::Pixel)),
(
usage.images.output.len() as u64,
Meter::Output(OutputUnit::Image(None)),
),
(
usage.search.queries,
Meter::Input(InputUnit::Query { over_results: None }),
),
(
usage.search.maps_grounding,
Meter::Feature(Feature::GoogleMapsGroundingQuery),
),
(
usage.search.browser_opens,
Meter::Feature(Feature::BrowserOpen),
),
(
usage.search.additional_results,
Meter::Feature(Feature::AdditionalResult),
),
(usage.text_messages, Meter::Input(InputUnit::TextMessage)),
(
usage.tokens.citation,
Meter::Feature(Feature::CitationToken),
),
];
let web_search = usage
.search
.web_search
.filter(|search| search.requests > 0)
.map(|search| Meter::WebSearch(search.context));
let durations = usage
.durations
.iter()
.filter(|(_, duration)| !duration.is_zero())
.map(|(meter, _)| match meter {
DurationMeter::WallClock => Meter::WallClockSecond,
DurationMeter::InputAudio => Meter::Input(InputUnit::AudioSecond),
DurationMeter::LiveVoice => Meter::Input(InputUnit::Second),
DurationMeter::OutputVideo { resolution } => {
Meter::Output(OutputUnit::Second(resolution.clone()))
}
});
counted
.into_iter()
.filter_map(|(count, meter)| (count > 0).then_some(meter))
.chain(web_search)
.chain(durations)
}
fn uplifts(adjustments: &Adjustments, call: &CallContext) -> Vec<Multiplier> {
let residency = call
.residency
.and_then(|residency| adjustments.get(&Uplift::Residency(residency)));
let endpoint = call
.non_global_endpoint
.then(|| adjustments.get(&Uplift::NonGlobalEndpoint))
.flatten();
residency.into_iter().chain(endpoint).collect()
}
#[cfg(test)]
mod tests {
use std::collections::BTreeMap;
use std::num::NonZeroU64;
use rstest::rstest;
use super::*;
use crate::price::{Billing, Residency};
fn price(
meter: &Meter,
channel: Channel,
tier: ServiceTier,
band: Band,
rate: f64,
) -> (Price, UnitPrice) {
(
Price {
meter: meter.clone(),
channel,
when: When { tier, band },
},
UnitPrice::new(rate).unwrap(),
)
}
fn standard(meter: &Meter, rate: f64) -> (Price, UnitPrice) {
price(
meter,
Channel::Online,
ServiceTier::Standard,
Band::Base,
rate,
)
}
fn row(rates: impl IntoIterator<Item = (Price, UnitPrice)>) -> Prices {
Prices::new(rates.into_iter().collect(), None, Billing::default())
}
fn call() -> CallContext {
CallContext {
tier: ServiceTier::Standard,
channel: Channel::Online,
residency: None,
non_global_endpoint: false,
speed: None,
inference_geo: None,
at: jiff::Timestamp::UNIX_EPOCH,
overlay: None,
}
}
fn usage(input: u64, cache_read: u64, output: u64) -> Usage {
let mut usage = Usage {
prompt_tokens: input + cache_read,
..Usage::default()
};
usage.tokens.input = BTreeMap::from([(TokenKind::Text, input)]);
usage.tokens.cache_read =
BTreeMap::from([((TokenKind::Text, CacheReadMode::Implicit), cache_read)]);
usage.tokens.output = BTreeMap::from([(TokenKind::Text, output)]);
usage
}
fn priced(outcome: Outcome) -> Priced {
match outcome {
Outcome::Priced(priced) => priced,
Outcome::Unpriced { missing } => panic!("unpriced, missing {missing:?}"),
}
}
fn total(prices: &Prices, usage: &Usage, call: &CallContext) -> f64 {
priced(calculate(prices, &Adjustments::default(), usage, call))
.total
.get()
}
fn close(actual: f64, expected: f64) {
assert!(
(actual - expected).abs() <= expected.abs() * 1e-12,
"{actual} != {expected}"
);
}
#[rstest]
fn a_cache_read_with_no_price_borrows_the_input_rate_as_a_visible_line() {
let prices = row([standard(&TEXT_INPUT, 1e-6), standard(&TEXT_OUTPUT, 2e-6)]);
let outcome = priced(calculate(
&prices,
&Adjustments::default(),
&usage(200, 800, 10),
&call(),
));
let cache_line = outcome
.lines
.iter()
.find(|line| line.meter == CACHE_READ)
.unwrap();
assert_eq!(cache_line.fallback_from, Some(TEXT_INPUT));
close(outcome.total.get(), 1000.0 * 1e-6 + 10.0 * 2e-6);
}
#[rstest]
fn a_meter_with_usage_and_no_price_is_missing_and_the_rest_is_still_priced() {
let prices = row([standard(&TEXT_OUTPUT, 2e-6)]);
let outcome = priced(calculate(
&prices,
&Adjustments::default(),
&usage(100, 0, 10),
&call(),
));
assert_eq!(outcome.missing, [TEXT_INPUT]);
close(outcome.total.get(), 10.0 * 2e-6);
}
#[rstest]
fn a_meter_no_arm_prices_yet_is_missing() {
let prices = row([standard(&TEXT_INPUT, 1e-6)]);
let usage = Usage {
characters: crate::usage::Characters {
prompt: 40,
completion: 0,
},
..usage(100, 0, 0)
};
let outcome = priced(calculate(&prices, &Adjustments::default(), &usage, &call()));
assert_eq!(outcome.missing, [Meter::Input(InputUnit::Character)]);
}
#[rstest]
fn a_call_with_no_priced_meter_is_unpriced() {
let outcome = calculate(
&row([]),
&Adjustments::default(),
&usage(100, 0, 10),
&call(),
);
assert_eq!(
outcome,
Outcome::Unpriced {
missing: vec![TEXT_INPUT, TEXT_OUTPUT]
}
);
}
#[rstest]
fn the_overlay_replaces_the_meters_it_prices_and_keeps_the_rest() {
let prices = row([standard(&TEXT_INPUT, 1e-6), standard(&TEXT_OUTPUT, 2e-6)]);
let call = CallContext {
overlay: Some(row([standard(&TEXT_INPUT, 5e-6)])),
..call()
};
close(
total(&prices, &usage(100, 0, 10), &call),
100.0 * 5e-6 + 10.0 * 2e-6,
);
}
#[rstest]
#[case::requested_region(Residency::Eu, true)]
#[case::other_region(Residency::Us, false)]
fn a_residency_uplift_multiplies_every_line(
#[case] priced_region: Residency,
#[case] applies: bool,
) {
let prices = row([standard(&TEXT_INPUT, 1e-6), standard(&TEXT_OUTPUT, 2e-6)]);
let adjustments = Adjustments::new([(
Uplift::Residency(priced_region),
Multiplier::new(1.1).unwrap(),
)]);
let call = CallContext {
residency: Some(Residency::Eu),
..call()
};
let outcome = priced(calculate(&prices, &adjustments, &usage(100, 0, 10), &call));
let base = 100.0 * 1e-6 + 10.0 * 2e-6;
close(outcome.total.get(), if applies { base * 1.1 } else { base });
}
#[rstest]
#[case::missing(None)]
#[case::zero(Some(0.0))]
fn an_image_model_with_no_output_price_bills_output_at_its_image_token_rate(
#[case] output: Option<f64>,
) {
let prices = row(std::iter::once(standard(&IMAGE_OUTPUT, 4e-5))
.chain(output.map(|rate| standard(&TEXT_OUTPUT, rate))));
let outcome = priced(calculate(
&prices,
&Adjustments::default(),
&usage(0, 0, 10),
&call(),
));
assert_eq!(outcome.lines[0].fallback_from, Some(IMAGE_OUTPUT));
close(outcome.total.get(), 10.0 * 4e-5);
}
#[rstest]
#[case::below_the_band(1_000, 1e-6)]
#[case::above_the_band(300_000, 2e-6)]
fn the_prompt_band_prices_every_token_meter(#[case] prompt: u64, #[case] input_rate: f64) {
let over = Band::over_prompt_tokens(200_000);
let prices = row([
standard(&TEXT_INPUT, 1e-6),
price(
&TEXT_INPUT,
Channel::Online,
ServiceTier::Standard,
over,
2e-6,
),
]);
let usage = Usage {
prompt_tokens: prompt,
..usage(100, 0, 0)
};
close(total(&prices, &usage, &call()), 100.0 * input_rate);
}
#[rstest]
#[case::batch_price(Some(3e-7), 3e-7, None)]
#[case::half_the_online_price(None, 5e-7, Some(TEXT_INPUT))]
fn a_batch_call_bills_its_batch_price_or_half_the_online_one(
#[case] batch: Option<f64>,
#[case] rate: f64,
#[case] fallback: Option<Meter>,
) {
let batch_price = batch.map(|rate| {
price(
&TEXT_INPUT,
Channel::Batch,
ServiceTier::Standard,
Band::Base,
rate,
)
});
let prices = row(std::iter::once(standard(&TEXT_INPUT, 1e-6)).chain(batch_price));
let call = CallContext {
channel: Channel::Batch,
..call()
};
let outcome = priced(calculate(
&prices,
&Adjustments::default(),
&usage(100, 0, 0),
&call,
));
assert_eq!(outcome.lines[0].fallback_from, fallback);
close(outcome.total.get(), 100.0 * rate);
}
#[rstest]
#[case::tier_reasoning_price(Some(9e-6), true, 9e-6)]
#[case::tier_output_without_reasoning(None, true, 7e-6)]
#[case::standard_reasoning_price(None, false, 3e-6)]
fn reasoning_follows_mains_tier_rules(
#[case] tier_reasoning: Option<f64>,
#[case] tier_output: bool,
#[case] expected: f64,
) {
let priority = |meter: &Meter, rate| {
price(
meter,
Channel::Online,
ServiceTier::Priority,
Band::Base,
rate,
)
};
let prices = row([standard(&TEXT_OUTPUT, 2e-6), standard(&REASONING, 3e-6)]
.into_iter()
.chain(tier_reasoning.map(|rate| priority(&REASONING, rate)))
.chain(tier_output.then(|| priority(&TEXT_OUTPUT, 7e-6))));
let mut usage = usage(0, 0, 0);
usage.tokens.reasoning = 10;
let call = CallContext {
tier: ServiceTier::Priority,
..call()
};
close(total(&prices, &usage, &call), 10.0 * expected);
}
#[rstest]
fn a_long_cache_write_with_no_price_borrows_the_default_write_rate() {
let prices = row([standard(&TEXT_INPUT, 1e-6), standard(&CACHE_WRITE, 1.25e-6)]);
let mut usage = usage(0, 0, 0);
usage.tokens.cache_write = BTreeMap::from([((TokenKind::Text, CacheTtl::OneHour), 100)]);
let outcome = priced(calculate(&prices, &Adjustments::default(), &usage, &call()));
assert_eq!(outcome.lines[0].fallback_from, Some(CACHE_WRITE));
close(outcome.total.get(), 100.0 * 1.25e-6);
}
#[rstest]
fn a_half_input_cache_read_row_bills_cache_reads_at_half_the_input_rate() {
let prices = Prices::new(
[standard(&TEXT_INPUT, 1e-6)].into_iter().collect(),
None,
Billing {
cache_read_fallback: CacheReadFallback::HalfInputRate,
..Billing::default()
},
);
close(total(&prices, &usage(0, 100, 0), &call()), 100.0 * 0.5e-6);
}
#[rstest]
#[case::low_tier(1_000, 1e-6)]
#[case::high_tier(40_000, 4e-6)]
fn a_whole_call_tier_bills_cache_reads_at_its_own_input_rate(
#[case] prompt: u64,
#[case] rate: f64,
) {
let over = Band::OverPromptTokens(NonZeroU64::new(32_000).unwrap());
let prices = Prices::new(
[
standard(&TEXT_INPUT, 1e-6),
standard(&CACHE_READ, 0.1e-6),
price(
&TEXT_INPUT,
Channel::Online,
ServiceTier::Standard,
over,
4e-6,
),
]
.into_iter()
.collect(),
None,
Billing {
tier_scope: TierScope::WholeCall,
..Billing::default()
},
);
let usage = Usage {
prompt_tokens: prompt,
..usage(0, 100, 0)
};
let expected = if prompt > 32_000 { rate } else { 0.1e-6 };
close(total(&prices, &usage, &call()), 100.0 * expected);
}
}

View file

@ -1,3 +1,4 @@
pub mod calculate;
pub mod call;
mod error;
pub mod money;
@ -6,4 +7,5 @@ pub mod priced;
pub mod schedule;
pub mod usage;
pub use calculate::calculate;
pub use error::*;

View file

@ -68,7 +68,7 @@ pub struct Billing {
/// From `web_search_billing_unit`, whether web search and Maps grounding bill per query or once per prompt
pub web_search: WebSearchUnit,
/// Whether `tiered_pricing` bills each meter in its own tier or the whole call at the tier the input picks.
/// Dashscope rows bill the whole call (inferred, 10 checks no other row has that shape)
/// A prompt-token tier table bills the whole call, main's `_get_tiered_base_costs`
pub tier_scope: TierScope,
}

View file

@ -31,6 +31,54 @@ pub enum Weekday {
Sunday,
}
impl Schedule {
/// Whether `at` is off-peak. A window applies on its weekdays, read on the calendar `weekday_timezone` names (UTC
/// when it names none or an unknown zone), or every day. Its start is inclusive and its end exclusive, a window may
/// wrap past midnight, and one whose start equals its end covers the whole day
pub fn is_open(&self, at: jiff::Timestamp) -> bool {
let utc = at.to_zoned(jiff::tz::TimeZone::UTC);
let second_of_day = u32::from(utc.hour().unsigned_abs()) * 3600
+ u32::from(utc.minute().unsigned_abs()) * 60
+ u32::from(utc.second().unsigned_abs());
let calendar = self
.weekday_timezone
.as_deref()
.map(str::trim)
.filter(|zone| !zone.is_empty())
.and_then(|zone| jiff::tz::TimeZone::get(zone).ok())
.unwrap_or(jiff::tz::TimeZone::UTC);
let weekday = Weekday::from_iso(
at.to_zoned(calendar)
.weekday()
.to_monday_one_offset()
.unsigned_abs(),
);
self.windows.iter().any(|window| {
let on_day = window
.weekdays
.as_ref()
.is_none_or(|days| weekday.is_some_and(|weekday| days.contains(&weekday)));
on_day
&& window
.hours
.iter()
.any(|hours| hours.contains(second_of_day))
})
}
}
impl UtcWindow {
fn contains(self, second_of_day: u32) -> bool {
let start = u32::from(self.start_minute) * 60;
let end = u32::from(self.end_minute) * 60;
if start < end {
(start..end).contains(&second_of_day)
} else {
second_of_day >= start || second_of_day < end
}
}
}
impl FromStr for UtcWindow {
type Err = InvalidUtcWindow;
@ -215,6 +263,54 @@ mod tests {
assert_eq!(name.parse().ok(), expected);
}
fn schedule(
hours: &str,
weekdays: Option<Vec<Weekday>>,
weekday_timezone: Option<&str>,
) -> Schedule {
Schedule {
windows: vec![Window {
hours: vec![hours.parse().unwrap()],
weekdays,
}],
weekday_timezone: weekday_timezone.map(str::to_owned),
}
}
#[rstest]
#[case::inside("2026-10-28T10:00:00Z", "10:00-18:30", true)]
#[case::end_is_exclusive("2026-10-28T18:30:00Z", "10:00-18:30", false)]
#[case::wrapped_after_midnight("2026-10-28T01:59:59Z", "22:00-02:00", true)]
#[case::wrapped_outside("2026-10-28T12:00:00Z", "22:00-02:00", false)]
#[case::whole_day("2026-10-28T12:00:00Z", "00:00-00:00", true)]
fn a_window_opens_at_its_start_and_closes_at_its_end(
#[case] at: &str,
#[case] hours: &str,
#[case] open: bool,
) {
assert_eq!(
schedule(hours, None, None).is_open(at.parse().unwrap()),
open
);
}
#[rstest]
#[case::utc_wednesday(None, Weekday::Wednesday, true)]
#[case::shanghai_thursday(Some("Asia/Shanghai"), Weekday::Thursday, true)]
#[case::shanghai_not_wednesday(Some("Asia/Shanghai"), Weekday::Wednesday, false)]
#[case::unknown_zone_reads_utc(Some("Mars/Base"), Weekday::Wednesday, true)]
fn weekdays_are_read_on_the_named_calendar(
#[case] weekday_timezone: Option<&str>,
#[case] weekday: Weekday,
#[case] open: bool,
) {
let at = "2026-10-28T17:00:00Z".parse().unwrap();
assert_eq!(
schedule("16:30-00:30", Some(vec![weekday]), weekday_timezone).is_open(at),
open
);
}
#[rstest]
#[case::monday(1, Some(Weekday::Monday))]
#[case::sunday(7, Some(Weekday::Sunday))]

View file

@ -86,7 +86,7 @@ impl Located {
Self::new(path_to_pointer(path), error)
}
pub(crate) fn prepend(&mut self, key: &str) {
pub fn prepend(&mut self, key: &str) {
let prefix = escape_pointer_segment(key);
self.path = if self.path.is_empty() {
format!("/{prefix}")

View file

@ -141,7 +141,12 @@ fn billing(wire: &EntryWire, rates: &BTreeMap<Price, UnitPrice>) -> Billing {
Some(WebSearchBillingUnit::PerQuery) => WebSearchUnit::PerQuery,
Some(WebSearchBillingUnit::PerPrompt) | None => WebSearchUnit::PerPrompt,
},
tier_scope: if provider == Some("dashscope") && wire.tier_rows.is_some() {
tier_scope: if wire
.tier_rows
.iter()
.flatten()
.any(|row| row.range.is_some())
{
TierScope::WholeCall
} else {
TierScope::PerMeter
@ -958,10 +963,14 @@ mod tests {
json!({"litellm_provider": "gemini", "web_search_billing_unit": "per_query"}),
Billing { web_search: WebSearchUnit::PerQuery, ..Billing::default() }
)]
#[case::dashscope_tiers_bill_the_whole_call(
json!({"litellm_provider": "dashscope", "tiered_pricing": [{"range": [0, 32000], "input_cost_per_token": 1e-6}]}),
#[case::token_tiers_bill_the_whole_call(
json!({"litellm_provider": "openai", "tiered_pricing": [{"range": [0, 32000], "input_cost_per_token": 1e-6}]}),
Billing { tier_scope: TierScope::WholeCall, ..Billing::default() }
)]
#[case::result_tiers_bill_per_meter(
json!({"litellm_provider": "exa_ai", "tiered_pricing": [{"max_results_range": [1, 25], "input_cost_per_query": 5e-3}]}),
Billing::default()
)]
fn billing_is_lowered_from_the_row_and_its_provider(
#[case] entry: Value,
#[case] expected: Billing,

View file

@ -48,7 +48,9 @@ litellm-secrets-types.workspace = true
litellm-llms-types.workspace = true
litellm-router-types.workspace = true
litellm-host-python.workspace = true
litellm-cost.workspace = true
litellm-model-catalog.workspace = true
jiff = { version = "0.2.37", default-features = false, features = ["std"] }
litellm-model-catalog-refresh.workspace = true
indexmap = "2.14.0"
litellm-token-counter = { path = "../token-counter", default-features = false }

View file

@ -0,0 +1,457 @@
use std::borrow::Cow;
use std::collections::BTreeMap;
use std::sync::Arc;
use litellm_cost::call::CallContext as CostCallContext;
use litellm_cost::price::{
Adjustments, CacheReadMode, CacheTtl, Channel, InputUnit, Meter, OutputUnit,
Prices as CostPrices, Residency, ServiceTier, TokenKind,
};
use litellm_cost::priced::{Outcome, Quantity, Side};
use litellm_cost::usage::Usage as CostUsage;
use litellm_host_python::from_py;
use litellm_model_catalog::rejects_row;
use litellm_model_catalog::wire::{EntryWire, Fields, FlatKey};
use pyo3::exceptions::PyValueError;
use pyo3::prelude::*;
use pyo3::types::{PyDict, PyTuple};
use crate::model_registry::Invalid;
/// A row's typed prices and uplifts, an immutable snapshot taken when the row was resolved
#[derive(Default)]
#[pyclass(frozen, name = "Prices", module = "litellm.rust_bridge._native")]
pub(crate) struct PricesHandle {
prices: Arc<CostPrices>,
adjustments: Arc<Adjustments>,
}
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
pub(crate) struct Usage(CostUsage);
/// The call facts pricing reads. A custom pricing overlay that fails validation is kept as its errors
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
pub(crate) struct CallContext {
call: CostCallContext,
invalid_overlay: Option<(Vec<String>, Arc<Fields>)>,
}
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
pub(crate) struct Priced {
#[pyo3(get)]
total: f64,
#[pyo3(get)]
prompt: f64,
#[pyo3(get)]
completion: f64,
lines: Vec<Py<Line>>,
missing: Vec<String>,
}
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
pub(crate) struct Line {
#[pyo3(get)]
meter: String,
#[pyo3(get)]
quantity: u64,
#[pyo3(get)]
unit_price: f64,
#[pyo3(get)]
amount: f64,
#[pyo3(get)]
fallback_from: Option<String>,
}
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
pub(crate) struct Unpriced {
missing: Vec<String>,
}
impl PricesHandle {
/// Lowers a row the way the catalog lowers its entries, `Err` with every error that rejects the row
pub(crate) fn lower(key: &str, fields: &Fields) -> Result<Self, Vec<String>> {
let wire = EntryWire::read(&as_main_reads_tiers(fields));
let adjustments = Adjustments::from(&wire);
let (prices, price_errors) = match CostPrices::try_from(&wire) {
Ok(prices) => (Some(prices), Vec::new()),
Err(errors) => (None, errors),
};
let errors: Vec<String> = wire
.problems
.into_iter()
.chain(price_errors)
.filter(|error| rejects_row(&error.error))
.map(|mut error| {
error.prepend(key);
error.to_string()
})
.collect();
match prices {
Some(prices) if errors.is_empty() => Ok(Self {
prices: Arc::new(prices),
adjustments: Arc::new(adjustments),
}),
_ => Err(errors),
}
}
}
#[pymethods]
impl PricesHandle {
/// Prices for a row that did not come from `resolve`, such as a deployment's `model_info`
#[staticmethod]
#[pyo3(signature = (row, key = "model_info"))]
fn from_row(py: Python<'_>, row: &Bound<'_, PyDict>, key: &str) -> PyResult<Py<PyAny>> {
let (fields, converted) = json_fields(row);
let lowered = match converted {
true => Self::lower(key, &fields),
false => Err(vec![format!("{key} must contain JSON-compatible prices")]),
};
match lowered {
Ok(prices) => Ok(prices.into_pyobject(py)?.into_any().unbind()),
Err(errors) => Ok(Invalid::new(key.to_owned(), errors, Arc::new(fields))
.into_pyobject(py)?
.into_any()
.unbind()),
}
}
/// Pure arithmetic over owned values, it calls no Python and never reads `litellm.model_cost`
fn price(&self, py: Python<'_>, usage: &Usage, call: &CallContext) -> PyResult<Py<PyAny>> {
if let Some((errors, fields)) = &call.invalid_overlay {
return Ok(
Invalid::new("custom_pricing".to_owned(), errors.clone(), fields.clone())
.into_pyobject(py)?
.into_any()
.unbind(),
);
}
match litellm_cost::calculate(&self.prices, &self.adjustments, &usage.0, &call.call) {
Outcome::Unpriced { missing } => Ok(Unpriced {
missing: missing.iter().map(meter_name).collect(),
}
.into_pyobject(py)?
.into_any()
.unbind()),
Outcome::Priced(priced) => {
let side_total = |side: Side| {
priced
.lines
.iter()
.filter(|line| line.side == side)
.map(|line| line.amount.get())
.sum()
};
let lines = priced
.lines
.iter()
.map(|line| {
Py::new(
py,
Line {
meter: meter_name(&line.meter),
quantity: match line.quantity {
Quantity::Count(count) => count,
Quantity::Duration(duration) => duration.as_secs(),
},
unit_price: line.unit_price.get(),
amount: line.amount.get(),
fallback_from: line.fallback_from.as_ref().map(meter_name),
},
)
})
.collect::<PyResult<_>>()?;
Ok(Priced {
total: priced.total.get(),
prompt: side_total(Side::Prompt),
completion: side_total(Side::Completion),
lines,
missing: priced.missing.iter().map(meter_name).collect(),
}
.into_pyobject(py)?
.into_any()
.unbind())
}
}
}
}
#[pymethods]
impl Usage {
#[new]
#[pyo3(signature = (
*, prompt_tokens, text, output_text, audio = 0, image = 0, video = 0, cache_read = 0, cache_read_audio = 0,
cache_read_explicit = 0, cache_write_5m = 0, cache_write_1h = 0, cache_write_12h = 0, output_audio = 0,
output_image = 0, output_video = 0, reasoning = 0,
))]
#[expect(
clippy::too_many_arguments,
reason = "one keyword argument per token meter, the Python signature"
)]
fn new(
prompt_tokens: u64,
text: u64,
output_text: u64,
audio: u64,
image: u64,
video: u64,
cache_read: u64,
cache_read_audio: u64,
cache_read_explicit: u64,
cache_write_5m: u64,
cache_write_1h: u64,
cache_write_12h: u64,
output_audio: u64,
output_image: u64,
output_video: u64,
reasoning: u64,
) -> Self {
let mut usage = CostUsage {
prompt_tokens,
..CostUsage::default()
};
usage.tokens.input = BTreeMap::from([
(TokenKind::Text, text),
(TokenKind::Audio, audio),
(TokenKind::Image, image),
(TokenKind::Video, video),
]);
usage.tokens.cache_read = BTreeMap::from([
((TokenKind::Text, CacheReadMode::Implicit), cache_read),
(
(TokenKind::Text, CacheReadMode::Explicit),
cache_read_explicit,
),
(
(TokenKind::Audio, CacheReadMode::Implicit),
cache_read_audio,
),
]);
usage.tokens.cache_write = BTreeMap::from([
((TokenKind::Text, CacheTtl::Default), cache_write_5m),
((TokenKind::Text, CacheTtl::OneHour), cache_write_1h),
((TokenKind::Text, CacheTtl::TwelveHours), cache_write_12h),
]);
usage.tokens.output = BTreeMap::from([
(TokenKind::Text, output_text),
(TokenKind::Audio, output_audio),
(TokenKind::Image, output_image),
(TokenKind::Video, output_video),
]);
usage.tokens.reasoning = reasoning;
Self(usage)
}
}
#[pymethods]
impl CallContext {
#[new]
#[pyo3(signature = (*, service_tier, channel, data_residency, vertex_location, at, custom_pricing))]
fn new(
service_tier: Option<&str>,
channel: &str,
data_residency: Option<&str>,
vertex_location: Option<&str>,
at: &Bound<'_, PyAny>,
custom_pricing: Option<&Bound<'_, PyDict>>,
) -> PyResult<Self> {
let channel = match channel {
"online" => Channel::Online,
"batch" => Channel::Batch,
"decisions" => Channel::Decisions,
other => return Err(PyValueError::new_err(format!("unknown channel {other:?}"))),
};
let overlay = custom_pricing.map(|pricing| {
let (fields, converted) = json_fields(pricing);
let lowered = match converted {
true => PricesHandle::lower("custom_pricing", &fields),
false => Err(vec![
"custom_pricing must contain JSON-compatible prices".to_owned(),
]),
};
lowered
.map(|handle| Arc::unwrap_or_clone(handle.prices))
.map_err(|errors| (errors, Arc::new(fields)))
});
let (overlay, invalid_overlay) = match overlay {
None => (None, None),
Some(Ok(prices)) => (Some(prices), None),
Some(Err(invalid)) => (None, Some(invalid)),
};
Ok(Self {
call: CostCallContext {
tier: service_tier.map_or(ServiceTier::Standard, tier),
channel,
residency: data_residency.and_then(residency),
non_global_endpoint: vertex_location
.is_some_and(|location| !location.eq_ignore_ascii_case("global")),
speed: None,
inference_geo: None,
at: timestamp(at)?,
overlay,
},
invalid_overlay,
})
}
}
#[pymethods]
impl Priced {
#[getter]
fn lines<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyTuple>> {
PyTuple::new(py, self.lines.iter().map(|line| line.clone_ref(py)))
}
#[getter]
fn missing<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyTuple>> {
PyTuple::new(py, &self.missing)
}
}
#[pymethods]
impl Unpriced {
#[getter]
fn missing<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyTuple>> {
PyTuple::new(py, &self.missing)
}
}
/// The row's fields that convert to JSON, and whether every price did. A price only Python can hold, such as `nan`,
/// makes the row invalid (decision 20). Any other such value, a router deployment's `datetime` say, is left out
fn json_fields(row: &Bound<'_, PyDict>) -> (Fields, bool) {
let converted: Vec<(String, Option<serde_json::Value>)> = row
.iter()
.filter_map(|(name, value)| {
let non_finite = value
.extract::<f64>()
.is_ok_and(|number| !number.is_finite());
Some((
name.extract::<String>().ok()?,
from_py(&value).ok().filter(|_| !non_finite),
))
})
.collect();
let prices_convert = converted
.iter()
.all(|(name, value)| value.is_some() || FlatKey::parse(name).is_none());
let fields = converted
.into_iter()
.filter_map(|(name, value)| Some((name, value?)))
.collect();
(fields, prices_convert)
}
/// Flat prices a priced tier table replaces, main's `_get_tiered_base_costs`. The row's own output price stays as the
/// fallback for a tier that sets none
const TIER_REPLACED_PRICES: [&str; 5] = [
"input_cost_per_token",
"cache_read_input_token_cost",
"cache_creation_input_token_cost",
"cache_creation_input_token_cost_above_1hr",
"cache_creation_input_token_cost_above_12hr",
];
/// main bills a call with a prompt-token tier table at its tier's rates and never at the row's flat ones, and ignores
/// a table whose first tier sets no input price. The catalog lowers the first tier and the flat prices into one band,
/// so the flat prices the table replaces are dropped before lowering
fn as_main_reads_tiers(fields: &Fields) -> Cow<'_, Fields> {
let first_tier = fields
.get("tiered_pricing")
.and_then(serde_json::Value::as_array)
.and_then(|tiers| {
tiers
.iter()
.filter_map(serde_json::Value::as_object)
.find(|tier| {
tier.get("range")
.and_then(|range| range.get(0))
.and_then(serde_json::Value::as_u64)
== Some(0)
})
});
let Some(first_tier) = first_tier else {
return Cow::Borrowed(fields);
};
let priced = first_tier.contains_key("input_cost_per_token");
let replaced = |name: &str| match priced {
true => TIER_REPLACED_PRICES.contains(&name) || first_tier.contains_key(name),
false => name == "tiered_pricing",
};
Cow::Owned(
fields
.iter()
.filter(|(name, _)| !replaced(name))
.map(|(name, value)| (name.clone(), value.clone()))
.collect(),
)
}
/// main's `_SERVICE_TIER_TO_COST_KEY_SUFFIX`, `"fast"` bills the priority tier
fn tier(service_tier: &str) -> ServiceTier {
match service_tier.to_lowercase().as_str() {
"priority" | "fast" => ServiceTier::Priority,
"flex" => ServiceTier::Flex,
"balanced" => ServiceTier::Balanced,
"ultrafast" => ServiceTier::Ultrafast,
_ => ServiceTier::Standard,
}
}
fn residency(data_residency: &str) -> Option<Residency> {
match data_residency.to_lowercase().as_str() {
"eu" => Some(Residency::Eu),
"us" => Some(Residency::Us),
_ => None,
}
}
/// An aware datetime is read as its instant, a naive one as UTC, as main's off-peak check does
fn timestamp(at: &Bound<'_, PyAny>) -> PyResult<jiff::Timestamp> {
let aware = if at.getattr("tzinfo")?.is_none() {
let utc = at
.py()
.import("datetime")?
.getattr("timezone")?
.getattr("utc")?;
let replace_kwargs = PyDict::new(at.py());
replace_kwargs.set_item("tzinfo", utc)?;
at.call_method("replace", (), Some(&replace_kwargs))?
} else {
at.clone()
};
let seconds: f64 = aware.call_method0("timestamp")?.extract()?;
jiff::Timestamp::from_microsecond((seconds * 1e6).round() as i64)
.map_err(|error| PyValueError::new_err(format!("billing time out of range: {error}")))
}
/// main's flat key name for the meter, the name a caller already knows from the JSON
fn meter_name(meter: &Meter) -> String {
let token = |kind: &TokenKind| match kind {
TokenKind::Text => "",
TokenKind::Audio => "audio_",
TokenKind::Image => "image_",
TokenKind::Video => "video_",
};
match meter {
Meter::Input(InputUnit::Token(kind)) => format!("input_cost_per_{}token", token(kind)),
Meter::Output(OutputUnit::Token(kind)) => format!("output_cost_per_{}token", token(kind)),
Meter::Output(OutputUnit::ReasoningToken) => "output_cost_per_reasoning_token".to_owned(),
Meter::CacheRead {
kind,
mode: CacheReadMode::Implicit,
} => format!("cache_read_input_{}token_cost", token(kind)),
Meter::CacheRead {
kind,
mode: CacheReadMode::Explicit,
} => format!("explicit_cache_read_input_{}token_cost", token(kind)),
Meter::CacheWrite { kind, ttl } => {
let ttl = match ttl {
CacheTtl::Default => "",
CacheTtl::OneHour => "_above_1hr",
CacheTtl::TwelveHours => "_above_12hr",
};
format!("cache_creation_input_{}token_cost{ttl}", token(kind))
}
other => format!("{other:?}"),
}
}

View file

@ -1,6 +1,7 @@
mod cache;
mod callable;
mod coercion;
mod cost;
mod credentials;
mod diagnostics;
mod errors;
@ -24,6 +25,8 @@ mod _native {
types::{PyModule, PyTuple},
};
#[pymodule_export]
use crate::cost::{CallContext, Line, Priced, PricesHandle, Unpriced, Usage};
#[cfg(feature = "panic-test")]
#[pymodule_export]
use crate::diagnostics::_panic_for_test;

View file

@ -11,6 +11,7 @@ use pyo3::types::{PyDict, PyTuple};
use serde_json::Value;
use super::name_index::NameIndex;
use crate::cost::PricesHandle;
const CANDIDATES_MODULE: &str = "litellm.rust_bridge.host.model_names";
@ -52,6 +53,22 @@ impl Resolved {
fn row(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
to_py(py, self.fields.as_ref())
}
/// The row's typed prices and uplifts. `resolve` already reported a row whose prices do not lower as `Invalid`
#[getter]
fn prices(&self) -> PricesHandle {
PricesHandle::lower(&self.key, &self.fields).unwrap_or_default()
}
}
impl Invalid {
pub(crate) fn new(key: String, errors: Vec<String>, fields: Arc<Fields>) -> Self {
Self {
key,
errors,
fields,
}
}
}
#[pymethods]

View file

@ -4,6 +4,7 @@ import os
import sys
from typing import Final
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.offline_mode import is_offline_mode
from litellm.rust_bridge._native import (
Bundled,
@ -26,6 +27,16 @@ Source = Remote | File | Bundled
CATALOG: Final = Catalog.bound_to(sys.modules["litellm"], "model_cost")
_REPORTED_INVALID_ROWS: Final[set[str]] = set() # mutable-ok: logs each bad live row once per process
def report_invalid(invalid: Invalid) -> None:
"""Logs a row that fails validation, once per key per process (decision 20)"""
if invalid.key in _REPORTED_INVALID_ROWS:
return
_REPORTED_INVALID_ROWS.add(invalid.key)
verbose_logger.warning("litellm.model_cost[%r] fails validation: %s", invalid.key, "; ".join(invalid.errors))
def default_source() -> Source:
"""Main's reload rule, the bundled map under offline mode or ``LITELLM_LOCAL_MODEL_COST_MAP``"""
@ -54,4 +65,5 @@ __all__ = [
"Unavailable",
"default_source",
"overlay",
"report_invalid",
]

56
litellm/_cost/__init__.py Normal file
View file

@ -0,0 +1,56 @@
"""Internal pricing API backed by the Rust ``cost`` crate. Not public, its shape may change in any release"""
from collections.abc import Generator, Mapping
from contextlib import contextmanager
from contextvars import ContextVar
from typing import Final, Literal
from litellm._catalog import Invalid, report_invalid
from litellm.rust_bridge._native import CallContext, Line, Priced, Prices, Unpriced, Usage
Channel = Literal["online", "batch", "decisions"]
_CALL_CUSTOM_PRICING: Final[ContextVar[Mapping[str, object] | None]] = ContextVar(
"litellm_call_custom_pricing", default=None
)
@contextmanager
def custom_pricing_for_call(custom_pricing: Mapping[str, object] | None) -> Generator[None]:
"""Prices every token meter of the call inside the block with ``custom_pricing`` over its row (decision 12).
The provider cost calculators keep their signatures, so the overlay reaches them through the call's context"""
token: Final = _CALL_CUSTOM_PRICING.set(custom_pricing)
try:
yield
finally:
_CALL_CUSTOM_PRICING.reset(token)
def call_custom_pricing() -> Mapping[str, object] | None:
return _CALL_CUSTOM_PRICING.get()
def as_main_cost(outcome: Priced | Unpriced | Invalid) -> tuple[float, float]:
"""Main's ``(prompt, completion)`` pair, ``(0.0, 0.0)`` for anything unpriced (decisions 14 and 20)"""
match outcome:
case Priced():
return outcome.prompt, outcome.completion
case Unpriced():
return 0.0, 0.0
case Invalid():
report_invalid(outcome)
return 0.0, 0.0
__all__ = [
"CallContext",
"Channel",
"Line",
"Priced",
"Prices",
"Unpriced",
"Usage",
"as_main_cost",
"call_custom_pricing",
"custom_pricing_for_call",
]

File diff suppressed because it is too large Load diff

View file

@ -12,6 +12,9 @@ from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
from typing_extensions import ReadOnly
import litellm
from litellm._catalog import Invalid
from litellm._cost import CallContext, Prices, as_main_cost, call_custom_pricing
from litellm._cost import Usage as TokenUsage
from litellm._internal_context import current_billing_time
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.llm_cost_calc.tiered_pricing import (
@ -892,7 +895,7 @@ def _get_token_base_cost(
)
def calculate_cost_component(model_info: ModelInfo, cost_key: str, usage_value: float | None) -> float:
def calculate_cost_component(model_info: Mapping[str, object], cost_key: str, usage_value: float | None) -> float:
"""
Generic cost calculator for any usage component
@ -910,7 +913,7 @@ def calculate_cost_component(model_info: ModelInfo, cost_key: str, usage_value:
return 0.0
def get_cost_per_unit(model_info: ModelInfo, cost_key: str, default_value: float | None = 0.0) -> float | None:
def get_cost_per_unit(model_info: Mapping[str, object], cost_key: str, default_value: float | None = 0.0) -> float | None:
# Sometimes the cost per unit is a string (e.g.: If a value like "3e-7" was read from the config.yaml)
cost_per_unit: Final = model_info.get(cost_key)
if isinstance(cost_per_unit, float):
@ -1185,144 +1188,7 @@ def parse_completion_tokens_details(usage: Usage) -> CompletionTokensDetailsResu
)
def _modality_token_cost(
model_info: ModelInfo,
modality_cost_key: str,
tokens: int,
prompt_base_cost: float,
service_tier: str | None,
) -> float:
"""
Audio, image and video prompt tokens bill at their modality rate when the model has one.
Without one they are ordinary prompt tokens, so they bill at `prompt_base_cost`, which
`_get_token_base_cost` has already resolved for the service tier and long-context tier.
"""
tiered_key: Final = get_service_tier_cost_key(modality_cost_key, service_tier)
cost_key: Final = tiered_key if model_info.get(tiered_key) is not None else modality_cost_key
if model_info.get(cost_key) is None:
return float(tokens) * prompt_base_cost
return calculate_cost_component(model_info, cost_key, tokens)
def _calculate_input_cost(
prompt_tokens_details: PromptTokensDetailsResult,
model_info: ModelInfo,
prompt_base_cost: float,
cache_read_cost: float,
cache_creation_cost: float,
cache_creation_cost_by_ttl: Mapping[str, float],
service_tier: str | None = None,
) -> float:
"""
Calculates the input cost for a given model, prompt tokens, and completion tokens.
A model priced per character (`input_cost_per_character`) bills the reported character count
in place of its text tokens, so the same prompt is never charged under both units.
"""
bills_characters: Final = (
prompt_tokens_details["character_count"] > 0 and model_info.get("input_cost_per_character") is not None
)
prompt_cost = 0.0 if bills_characters else float(prompt_tokens_details["text_tokens"]) * prompt_base_cost
### CACHE READ COST - Now uses tiered pricing
cache_hit_audio_tokens: Final = prompt_tokens_details["cache_hit_audio_tokens"]
audio_cache_read_rate: Final = get_cost_per_unit(
model_info,
get_service_tier_cost_key("cache_read_input_audio_token_cost", service_tier),
None,
)
prompt_cost += float(prompt_tokens_details["cache_hit_tokens"] - cache_hit_audio_tokens) * cache_read_cost
prompt_cost += float(cache_hit_audio_tokens) * (
audio_cache_read_rate if audio_cache_read_rate is not None else cache_read_cost
)
### AUDIO COST
if prompt_tokens_details["audio_tokens"] and not (
prompt_tokens_details["audio_length_seconds"] and model_info.get("input_cost_per_audio_per_second") is not None
):
prompt_cost += _modality_token_cost(
model_info,
"input_cost_per_audio_token",
prompt_tokens_details["audio_tokens"],
prompt_base_cost,
service_tier,
)
### IMAGE TOKEN COST
if prompt_tokens_details["image_tokens"] and not (
prompt_tokens_details["image_count"] and model_info.get("input_cost_per_image") is not None
):
prompt_cost += _modality_token_cost(
model_info,
"input_cost_per_image_token",
prompt_tokens_details["image_tokens"],
prompt_base_cost,
service_tier,
)
### VIDEO TOKEN COST
if prompt_tokens_details["video_tokens"] and not (
prompt_tokens_details["video_length_seconds"] and model_info.get("input_cost_per_video_per_second") is not None
):
prompt_cost += _modality_token_cost(
model_info,
"input_cost_per_video_token",
prompt_tokens_details["video_tokens"],
prompt_base_cost,
service_tier,
)
### CACHE WRITING COST - Now uses tiered pricing
if (
prompt_tokens_details["cache_creation_tokens"]
or prompt_tokens_details["cache_creation_token_details"] is not None
):
prompt_cost += calculate_cache_writing_cost(
cache_creation_tokens=prompt_tokens_details["cache_creation_tokens"],
cache_creation_token_details=prompt_tokens_details["cache_creation_token_details"],
cache_creation_cost=cache_creation_cost,
cache_creation_cost_by_ttl=cache_creation_cost_by_ttl,
)
### CHARACTER COST
if prompt_tokens_details["character_count"]:
prompt_cost += calculate_cost_component(
model_info,
"input_cost_per_character",
prompt_tokens_details["character_count"],
)
### IMAGE COUNT COST
if prompt_tokens_details["image_count"]:
prompt_cost += calculate_cost_component(
model_info, "input_cost_per_image", prompt_tokens_details["image_count"]
)
### VIDEO LENGTH COST
if prompt_tokens_details["video_length_seconds"]:
prompt_cost += calculate_cost_component(
model_info,
"input_cost_per_video_per_second",
prompt_tokens_details["video_length_seconds"],
)
### AUDIO LENGTH COST
if prompt_tokens_details["audio_length_seconds"]:
prompt_cost += calculate_cost_component(
model_info,
"input_cost_per_audio_per_second",
prompt_tokens_details["audio_length_seconds"],
)
if prompt_tokens_details["query_count"]:
prompt_cost += calculate_cost_component(
model_info, "input_cost_per_query", prompt_tokens_details["query_count"]
)
return prompt_cost
def get_regional_uplift_multiplier(model_info: ModelInfo, data_residency: str | None) -> float:
def get_regional_uplift_multiplier(model_info: Mapping[str, object], data_residency: str | None) -> float:
"""
Resolve the per-model regional-processing uplift multiplier for a given
data-residency region.
@ -1341,7 +1207,7 @@ def get_regional_uplift_multiplier(model_info: ModelInfo, data_residency: str |
if residency not in _VALID_DATA_RESIDENCIES:
return 1.0
multiplier: Final = model_info.get(f"regional_processing_uplift_multiplier_{residency}")
if multiplier is None:
if not isinstance(multiplier, (int, float, str)):
return 1.0
try:
return float(multiplier)
@ -1356,7 +1222,7 @@ def get_regional_uplift_multiplier(model_info: ModelInfo, data_residency: str |
_get_regional_uplift_multiplier = get_regional_uplift_multiplier
def get_vertex_regional_endpoint_uplift(model_info: ModelInfo, vertex_location: str | None) -> float:
def get_vertex_regional_endpoint_uplift(model_info: Mapping[str, object], vertex_location: str | None) -> float:
"""
Resolve the per-model uplift multiplier for Vertex AI non-global (regional and
multi-region) endpoints.
@ -1476,183 +1342,177 @@ def generic_cost_per_token(
Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd
"""
## GET MODEL INFO
# A caller that already resolved the deployment's effective rates passes them in
# rather than handing back a name for this to re-resolve. A name cannot express a
# per-deployment override: those are registered under the deployment id and kept off
# the shared model-name key, so resolving from the name here reads the public rate.
resolved_model_info: Final = apply_provider_cache_read_default(
get_model_info(model=model, custom_llm_provider=custom_llm_provider) if model_info is None else model_info,
custom_llm_provider,
custom_pricing: Final = call_custom_pricing()
row: Final = _pricing_row(model, custom_llm_provider, model_info, has_overlay=custom_pricing is not None)
prices: Final = Prices.from_row(
dict(row) if usage.prompt_tokens > 0 else _without_tiers(row), key=str(row.get("key") or model)
)
## CALCULATE INPUT COST
### Cost of processing (non-cache hit + cache hit) + Cost of cache-writing (cache writing)
prompt_cost = 0.0
### PROCESSING COST
prompt_tokens_details = PromptTokensDetailsResult(
cache_hit_tokens=0,
cache_hit_audio_tokens=0,
cache_creation_tokens=0,
cache_creation_token_details=None,
text_tokens=usage.prompt_tokens,
audio_tokens=0,
image_tokens=0,
video_tokens=0,
character_count=0,
image_count=0,
video_length_seconds=0.0,
audio_length_seconds=0.0,
query_count=0,
if isinstance(prices, Invalid):
return as_main_cost(prices)
details: Final = _billable_prompt_details(usage)
call: Final = CallContext(
service_tier=service_tier,
channel="online",
data_residency=data_residency,
vertex_location=vertex_location,
at=current_time if current_time is not None else current_billing_time(),
custom_pricing=None if custom_pricing is None else dict(custom_pricing),
)
if usage.prompt_tokens_details:
prompt_tokens_details = parse_prompt_tokens_details(usage)
prompt_cost, completion_cost = as_main_cost(prices.price(_token_usage(usage, details, row), call))
uplift: Final = get_regional_uplift_multiplier(row, data_residency) * get_vertex_regional_endpoint_uplift(
row, vertex_location
)
return prompt_cost + _unmetered_prompt_cost(details, row) * uplift, completion_cost
## EDGE CASE - text tokens not set or includes cached tokens (double-counting)
## Some providers (like xAI) report text_tokens = prompt_tokens (including cached)
## We detect this when: text_tokens + cached_tokens + other > prompt_tokens
## Ref: https://github.com/BerriAI/litellm/issues/19680, #14874, #14875
cache_hit: Final = prompt_tokens_details["cache_hit_tokens"]
text_tokens = prompt_tokens_details["text_tokens"]
audio_tokens = prompt_tokens_details["audio_tokens"]
cache_creation: Final = prompt_tokens_details["cache_creation_tokens"]
image_tokens = prompt_tokens_details["image_tokens"]
video_tokens = prompt_tokens_details["video_tokens"]
def _pricing_row(
model: str, custom_llm_provider: str, model_info: ModelInfo | None, *, has_overlay: bool
) -> Mapping[str, object]:
"""The row main priced, as written rather than `get_model_info`'s copy, which writes `0` for a missing token
price. A call with custom pricing prices an unmapped model from its overlay alone"""
if model_info is not None:
return model_info
try:
resolved: Final = get_model_info(model=model, custom_llm_provider=custom_llm_provider)
except Exception: # noqa: BLE001 # get_model_info raises a bare Exception for an unmapped model
if has_overlay:
return MappingProxyType({})
raise
key: Final = resolved.get("key")
row: Final = litellm.model_cost.get(key)
return {**row, "key": key} if isinstance(row, Mapping) else resolved
# Check for double-counting: sum of details > prompt_tokens means overlap
total_details: Final = text_tokens + cache_hit + audio_tokens + cache_creation + image_tokens + video_tokens
has_double_counting: Final = (cache_hit > 0 or cache_creation > 0) and total_details > usage.prompt_tokens
if has_double_counting:
# cached and per-modality counts are both subsets of prompt_tokens and may overlap, so a
# modality can only bill what the cache did not already cover or the overlap is billed twice
def _without_tiers(row: Mapping[str, object]) -> dict[str, object]:
"""Main picks no tier for an empty prompt and bills the row's flat prices"""
return {name: value for name, value in row.items() if name != "tiered_pricing"}
def _billable_prompt_details(usage: Usage) -> PromptTokensDetailsResult:
"""Prompt token details with modality counts capped by what the cache did not cover.
Some providers (like xAI) report text_tokens = prompt_tokens, cached tokens included, and cached
and per-modality counts can overlap (#19680, #14874, #14875)
"""
details: Final = (
parse_prompt_tokens_details(usage)
if usage.prompt_tokens_details
else PromptTokensDetailsResult(
cache_hit_tokens=0,
cache_hit_audio_tokens=0,
cache_creation_tokens=0,
cache_creation_token_details=None,
text_tokens=usage.prompt_tokens,
audio_tokens=0,
image_tokens=0,
video_tokens=0,
character_count=0,
image_count=0,
video_length_seconds=0.0,
audio_length_seconds=0.0,
query_count=0,
)
)
cache_hit: Final = details["cache_hit_tokens"]
cache_creation: Final = details["cache_creation_tokens"]
modality_tokens: Final = details["audio_tokens"] + details["image_tokens"] + details["video_tokens"]
total_details: Final = details["text_tokens"] + cache_hit + cache_creation + modality_tokens
if (cache_hit > 0 or cache_creation > 0) and total_details > usage.prompt_tokens:
uncached_budget: Final = max(usage.prompt_tokens - cache_hit - cache_creation, 0)
billable_audio: Final = min(audio_tokens, uncached_budget)
billable_image: Final = min(image_tokens, uncached_budget - billable_audio)
billable_video: Final = min(video_tokens, uncached_budget - billable_audio - billable_image)
prompt_tokens_details["audio_tokens"] = billable_audio
prompt_tokens_details["image_tokens"] = billable_image
prompt_tokens_details["video_tokens"] = billable_video
prompt_tokens_details["text_tokens"] = uncached_budget - billable_audio - billable_image - billable_video
elif text_tokens == 0 and prompt_tokens_details["image_count"] == 0:
# Clamp to zero: inconsistent streaming usage
prompt_tokens_details["text_tokens"] = max(
usage.prompt_tokens - cache_hit - audio_tokens - cache_creation - image_tokens - video_tokens, 0
)
audio: Final = min(details["audio_tokens"], uncached_budget)
image: Final = min(details["image_tokens"], uncached_budget - audio)
video: Final = min(details["video_tokens"], uncached_budget - audio - image)
return {
**details,
"audio_tokens": audio,
"image_tokens": image,
"video_tokens": video,
"text_tokens": uncached_budget - audio - image - video,
}
if details["text_tokens"] == 0 and details["image_count"] == 0:
return {
**details,
"text_tokens": max(usage.prompt_tokens - cache_hit - cache_creation - modality_tokens, 0),
}
return details
billing_time: Final = current_time if current_time is not None else current_billing_time()
base_costs: Final = _get_token_base_cost(
model_info=resolved_model_info,
usage=usage,
service_tier=service_tier,
current_time=billing_time,
threshold_is_inclusive=_uses_inclusive_token_thresholds(custom_llm_provider),
)
completion_base_cost: Final = base_costs.output_rate
resolved_cache_read_cost: Final = resolve_explicit_cache_read_cost_rate(
model_info=resolved_model_info,
usage=usage,
current_time=billing_time,
service_tier=service_tier,
)
effective_cache_read_cost: Final = (
base_costs.cache_read_rate if resolved_cache_read_cost is None else resolved_cache_read_cost
def _cache_write_tokens(details: PromptTokensDetailsResult) -> tuple[int, int, int]:
"""(default, 1 hour, 12 hour) cache write tokens. Writes no lifetime count covers bill at the default rate"""
written: Final = details["cache_creation_tokens"]
by_ttl: Final = details["cache_creation_token_details"]
if by_ttl is None:
return written, 0, 0
short: Final = (by_ttl.ephemeral_5m_input_tokens or 0) + (by_ttl.ephemeral_30m_input_tokens or 0)
one_hour: Final = by_ttl.ephemeral_1h_input_tokens or 0
twelve_hours: Final = by_ttl.ephemeral_12h_input_tokens or 0
return short + max(written - short - one_hour - twelve_hours, 0), one_hour, twelve_hours
def _token_usage(usage: Usage, details: PromptTokensDetailsResult, row: Mapping[str, object]) -> TokenUsage:
"""The token meters main billed. A prompt billed per character, image or second of media does not bill the
same tokens again"""
cache_type: Final = getattr(usage.prompt_tokens_details, "cache_type", None)
cached_text: Final = details["cache_hit_tokens"] - details["cache_hit_audio_tokens"]
explicit_cache: Final = cache_type == "ephemeral"
cache_write_5m, cache_write_1h, cache_write_12h = _cache_write_tokens(details)
completion: Final = _completion_token_counts(usage)
return TokenUsage(
prompt_tokens=usage.prompt_tokens,
text=0
if details["character_count"] > 0 and row.get("input_cost_per_character") is not None
else details["text_tokens"],
audio=0
if details["audio_length_seconds"] and row.get("input_cost_per_audio_per_second") is not None
else details["audio_tokens"],
image=0 if details["image_count"] and row.get("input_cost_per_image") is not None else details["image_tokens"],
video=0
if details["video_length_seconds"] and row.get("input_cost_per_video_per_second") is not None
else details["video_tokens"],
cache_read=0 if explicit_cache else cached_text,
cache_read_explicit=cached_text if explicit_cache else 0,
cache_read_audio=details["cache_hit_audio_tokens"],
cache_write_5m=cache_write_5m,
cache_write_1h=cache_write_1h,
cache_write_12h=cache_write_12h,
output_text=completion["text_tokens"],
output_audio=completion["audio_tokens"],
output_image=completion["image_tokens"],
output_video=completion["video_tokens"],
reasoning=completion["reasoning_tokens"],
)
prompt_cost = _calculate_input_cost(
prompt_tokens_details=prompt_tokens_details,
model_info=resolved_model_info,
prompt_base_cost=base_costs.input_rate,
cache_read_cost=effective_cache_read_cost,
cache_creation_cost=base_costs.cache_creation_rate,
cache_creation_cost_by_ttl=base_costs.cache_creation_rates_by_ttl,
service_tier=service_tier,
def _completion_token_counts(usage: Usage) -> CompletionTokensDetailsResult:
"""Completion tokens by meter. Text is the remainder of a breakdown, and with no breakdown every token is text"""
details: Final = (
parse_completion_tokens_details(usage)
if usage.completion_tokens_details is not None
else CompletionTokensDetailsResult(
audio_tokens=0, text_tokens=0, reasoning_tokens=0, image_tokens=0, video_tokens=0
)
)
breakdown: Final = (
details["audio_tokens"] + details["reasoning_tokens"] + details["image_tokens"] + details["video_tokens"]
)
if details["text_tokens"] > 0:
return details
if breakdown > 0:
return {**details, "text_tokens": max(0, usage.completion_tokens - breakdown)}
return CompletionTokensDetailsResult(
audio_tokens=0, text_tokens=usage.completion_tokens, reasoning_tokens=0, image_tokens=0, video_tokens=0
)
## CALCULATE OUTPUT COST
text_tokens = 0
audio_tokens = 0
reasoning_tokens = 0
image_tokens = 0
video_tokens = 0
is_text_tokens_total = False
if usage.completion_tokens_details is not None:
completion_tokens_details: Final = parse_completion_tokens_details(usage)
audio_tokens = completion_tokens_details["audio_tokens"]
text_tokens = completion_tokens_details["text_tokens"]
reasoning_tokens = completion_tokens_details["reasoning_tokens"]
image_tokens = completion_tokens_details["image_tokens"]
video_tokens = completion_tokens_details["video_tokens"]
# Handle text_tokens calculation:
# 1. If text_tokens is explicitly provided and > 0, use it
# 2. If there's a breakdown (reasoning/audio/image/video tokens), calculate text_tokens as the remainder
# 3. If no breakdown at all, assume all completion_tokens are text_tokens
has_token_breakdown: Final = image_tokens > 0 or audio_tokens > 0 or reasoning_tokens > 0 or video_tokens > 0
if text_tokens == 0:
if has_token_breakdown:
# Calculate text tokens as remainder when we have a breakdown
# This handles cases like OpenAI's reasoning models where text_tokens isn't provided
text_tokens = max(
0,
usage.completion_tokens - reasoning_tokens - audio_tokens - image_tokens - video_tokens,
)
else:
# No breakdown at all, all tokens are text tokens
text_tokens = usage.completion_tokens
is_text_tokens_total = True
## TEXT COST
completion_cost = float(text_tokens) * completion_base_cost
## AUDIO COST
if not is_text_tokens_total and audio_tokens is not None and audio_tokens > 0:
_output_cost_per_audio_token = get_cost_per_unit(resolved_model_info, "output_cost_per_audio_token", None)
_output_cost_per_audio_token = (
_output_cost_per_audio_token if _output_cost_per_audio_token is not None else completion_base_cost
)
completion_cost += float(audio_tokens) * _output_cost_per_audio_token
## REASONING COST
if not is_text_tokens_total and reasoning_tokens and reasoning_tokens > 0:
completion_cost += float(reasoning_tokens) * _resolve_billed_reasoning_rate(
model_info=resolved_model_info,
usage=usage,
service_tier=service_tier,
completion_base_cost=completion_base_cost,
current_time=billing_time,
)
## IMAGE COST
if not is_text_tokens_total and image_tokens and image_tokens > 0:
_output_cost_per_image_token = get_cost_per_unit(resolved_model_info, "output_cost_per_image_token", None)
_output_cost_per_image_token = (
_output_cost_per_image_token if _output_cost_per_image_token is not None else completion_base_cost
)
completion_cost += float(image_tokens) * _output_cost_per_image_token
## VIDEO COST
if not is_text_tokens_total and video_tokens and video_tokens > 0:
_output_cost_per_video_token = get_cost_per_unit(resolved_model_info, "output_cost_per_video_token", None)
_output_cost_per_video_token = (
_output_cost_per_video_token if _output_cost_per_video_token is not None else completion_base_cost
)
completion_cost += float(video_tokens) * _output_cost_per_video_token
## REGIONAL DATA-RESIDENCY UPLIFT
# Applied as a flat multiplier across all token costs for the request
# when the upstream is a regionalized OpenAI host (eu./us.api.openai.com).
uplift: Final = get_regional_uplift_multiplier(resolved_model_info, data_residency)
if uplift != 1.0:
prompt_cost *= uplift
completion_cost *= uplift
vertex_uplift: Final = get_vertex_regional_endpoint_uplift(resolved_model_info, vertex_location)
if vertex_uplift != 1.0:
prompt_cost *= vertex_uplift
completion_cost *= vertex_uplift
return prompt_cost, completion_cost
def _unmetered_prompt_cost(details: PromptTokensDetailsResult, row: Mapping[str, object]) -> float:
"""Prompt meters the Rust cost core does not price yet, characters, image counts, media seconds and queries"""
return (
calculate_cost_component(row, "input_cost_per_character", details["character_count"])
+ calculate_cost_component(row, "input_cost_per_image", details["image_count"])
+ calculate_cost_component(row, "input_cost_per_video_per_second", details["video_length_seconds"])
+ calculate_cost_component(row, "input_cost_per_audio_per_second", details["audio_length_seconds"])
+ calculate_cost_component(row, "input_cost_per_query", details["query_count"])
)
def _coerce_token_count(value: object) -> int:

View file

@ -155,7 +155,6 @@ from litellm.types.llms.openai import AllMessageValues
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import (
BaseUrlRoutingHint,
CustomPricingLiteLLMParams,
ModelResponseStream,
ProviderRoutingHints,
RawRequestTypedDict,
@ -1119,7 +1118,6 @@ from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the o
)
from litellm.responses.bridge_check import (
bridge_reasoning_effort,
keep_registered_mode,
str_keyed_mapping,
)
from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the old definition site
@ -1236,93 +1234,6 @@ def _locals_snapshot(values: Mapping[str, object]) -> Mapping[str, object]:
return values
def _build_custom_pricing_entry(
custom_llm_provider: str,
kwargs: dict,
model_info: dict | None = None,
) -> dict:
"""Build a complete model cost entry from kwargs and model_info.
Collects all CustomPricingLiteLLMParams fields present in kwargs and
merges metadata from model_info (mode, supports_prompt_caching, max_tokens)
so that register_model() receives the full pricing configuration.
"""
entry: Final[dict] = {"litellm_provider": custom_llm_provider}
for field_name in CustomPricingLiteLLMParams.model_fields:
value = kwargs.get(field_name)
if value is not None:
entry[field_name] = value
if model_info and isinstance(model_info, dict):
for key in ("mode", "supports_prompt_caching", "max_tokens"):
if key in model_info and model_info[key] is not None:
entry.setdefault(key, model_info[key])
return entry
def _get_router_deployment_id(kwargs: dict) -> str | None:
for metadata_key in ("litellm_metadata", "metadata"):
metadata = kwargs.get(metadata_key) or {}
if not isinstance(metadata, dict):
continue
deployment_model_info = metadata.get("model_info") or {}
if not isinstance(deployment_model_info, dict):
continue
deployment_id = deployment_model_info.get("id")
if deployment_id is not None:
return str(deployment_id)
return None
def _register_custom_pricing_for_request(
model: str,
custom_llm_provider: str,
kwargs: dict,
model_info: dict | None,
) -> None:
"""Register per-request custom pricing in litellm.model_cost.
Router-originated requests (identified by the deployment id the router puts
in metadata) get their full pricing registered under that unique id only;
the shared ``{provider}/{model}`` key receives the entry with pricing fields
stripped, mirroring Router._create_deployment. This keeps one deployment's
pricing overrides (e.g. a zero-cost wildcard) from clobbering built-in
pricing used by sibling deployments of the same backend model. Direct SDK
calls keep the legacy behavior of registering the shared key with pricing.
Either way the shared key's ``mode`` follows ``keep_registered_mode``, as on
deployment registration, since a call routes by its own ``model_info``.
"""
entry: Final = _build_custom_pricing_entry(
custom_llm_provider=custom_llm_provider,
kwargs=kwargs,
model_info=model_info,
)
shared_key: Final = f"{custom_llm_provider}/{model}"
deployment_id: Final = _get_router_deployment_id(kwargs)
if deployment_id is None:
litellm.register_model({shared_key: keep_registered_mode(shared_key, entry)}, persist_across_reloads=False)
return
shared_entry: Final = keep_registered_mode(
shared_key, CustomPricingLiteLLMParams.strip_custom_pricing_fields(entry)
)
litellm.register_model(
{deployment_id: entry},
persist_across_reloads=False,
warning_display_name=shared_key,
custom_llm_provider=custom_llm_provider,
add_to_provider_model_sets=False,
)
litellm.register_model(
{shared_key: shared_entry},
persist_across_reloads=False,
warning_display_name=shared_key,
custom_llm_provider=custom_llm_provider,
add_to_provider_model_sets=False,
)
def _dispatch_metadata(ctx: CompletionDispatchContext) -> Mapping[str, object] | None:
return ctx.metadata
@ -5757,19 +5668,6 @@ def completion(
"aws_region_name", None
) # support region-based pricing for bedrock
### REGISTER CUSTOM MODEL PRICING -- IF GIVEN ###
if (
(input_cost_per_token is not None and output_cost_per_token is not None)
or input_cost_per_second is not None
or output_cost_per_second is not None
or cost_per_second is not None
):
_register_custom_pricing_for_request(
model=model,
custom_llm_provider=custom_llm_provider,
kwargs=kwargs,
model_info=model_info,
)
### BUILD CUSTOM PROMPT TEMPLATE -- IF GIVEN ###
custom_prompt_dict = {}
if initial_prompt_value or roles or final_prompt_value or bos_token or eos_token:
@ -6713,12 +6611,6 @@ def embedding(
headers.update(proxy_headers)
except Exception as e:
verbose_logger.warning("Failed to get proxy auth headers: %s", e)
### CUSTOM MODEL COST ###
input_cost_per_token: Final = kwargs.get("input_cost_per_token", None)
output_cost_per_token: Final = kwargs.get("output_cost_per_token", None)
cost_per_second: Final = kwargs.get("cost_per_second", None)
input_cost_per_second: Final = kwargs.get("input_cost_per_second", None)
output_cost_per_second: Final = kwargs.get("output_cost_per_second", None)
openai_params: Final = [
"user",
"dimensions",
@ -6765,20 +6657,6 @@ def embedding(
**non_default_params,
)
### REGISTER CUSTOM MODEL PRICING -- IF GIVEN ###
if (
(input_cost_per_token is not None and output_cost_per_token is not None)
or input_cost_per_second is not None
or output_cost_per_second is not None
or cost_per_second is not None
):
_register_custom_pricing_for_request(
model=model,
custom_llm_provider=custom_llm_provider,
kwargs=kwargs,
model_info=kwargs.get("model_info"),
)
logging: Final[LiteLLMLoggingObj] = litellm_logging_obj
logging.update_environment_variables(
model=model,

View file

@ -103,6 +103,7 @@ class Provenance:
@final
class Resolved:
key: str
prices: Prices
def model_info(self, provider: str | None = None) -> dict[str, JsonValue]: ...
def supports(self, capability: str) -> bool: ...
def row(self) -> dict[str, JsonValue]: ...
@ -118,6 +119,69 @@ class Invalid:
def model_info(self, provider: str | None = None) -> dict[str, JsonValue]: ...
def row(self) -> dict[str, JsonValue]: ...
@final
class Prices:
@staticmethod
def from_row(row: Mapping[str, object], key: str = "model_info") -> Prices | Invalid: ...
def price(self, usage: Usage, call: CallContext) -> Priced | Unpriced | Invalid: ...
@final
class Usage:
def __init__(
self,
*,
prompt_tokens: int,
text: int,
output_text: int,
audio: int = 0,
image: int = 0,
video: int = 0,
cache_read: int = 0,
cache_read_audio: int = 0,
cache_read_explicit: int = 0,
cache_write_5m: int = 0,
cache_write_1h: int = 0,
cache_write_12h: int = 0,
output_audio: int = 0,
output_image: int = 0,
output_video: int = 0,
reasoning: int = 0,
) -> None: ...
@final
class CallContext:
def __init__(
self,
*,
service_tier: str | None,
channel: Literal["online", "batch", "decisions"],
data_residency: str | None,
vertex_location: str | None,
at: datetime,
custom_pricing: Mapping[str, object] | None,
) -> None: ...
@final
class Line:
meter: str
quantity: int
unit_price: float
amount: float
fallback_from: str | None
@final
class Priced:
total: float
prompt: float
completion: float
lines: tuple[Line, ...]
missing: tuple[str, ...]
@final
class Unpriced:
missing: tuple[str, ...]
@final
class Registered:
written: list[str]

View file

@ -6081,9 +6081,6 @@ def get_provider_info(model: str, custom_llm_provider: str | None) -> ProviderSp
return json_provider.provider_info if json_provider is not None and json_provider.provider_info else None
_REPORTED_INVALID_ROWS: Final[set[str]] = set() # mutable-ok: logs each bad live row once per process
def _model_not_mapped_message(model: str, custom_llm_provider: str | None) -> str:
return (
f"This model isn't mapped yet. model={model}, custom_llm_provider={custom_llm_provider}. "
@ -6192,15 +6189,12 @@ def get_model_info_helper(
and strip it. Tried last so no model that already resolves through 1-6 can change.
"""
from litellm._catalog import CATALOG, Invalid, Missing
from litellm._catalog import CATALOG, Invalid, Missing, report_invalid
resolution: Final = CATALOG.resolve(model, requested_provider)
key: str | None = None if isinstance(resolution, Missing) else resolution.key
if isinstance(resolution, Invalid) and resolution.key not in _REPORTED_INVALID_ROWS:
_REPORTED_INVALID_ROWS.add(resolution.key)
verbose_logger.warning(
"litellm.model_cost[%r] fails validation: %s", resolution.key, "; ".join(resolution.errors)
)
if isinstance(resolution, Invalid):
report_invalid(resolution)
_model_info: dict[str, Any] | None = None if isinstance(resolution, Missing) else resolution.row()
if _model_info is not None and key is not None and (_model_info.get("mode") or "chat") in _BACKFILL_MODES:

View file

@ -10,9 +10,7 @@ from litellm._internal_context import pinned_billing_time
from litellm.litellm_core_utils.llm_cost_calc.utils import (
BilledTokenRates,
CostCalculatorUtils,
PromptTokensDetailsResult,
TokenRates,
_calculate_input_cost,
_get_token_base_cost,
_is_off_peak,
_is_within_off_peak_window,
@ -1919,7 +1917,7 @@ def test_calculate_cost_component_with_string_values():
def test_string_cost_values_edge_cases():
"""Test edge cases for string cost value handling."""
"""A numeric string is a price, a word is not, and a row with one bad price is unpriced (decision 20)"""
from unittest.mock import patch
# Test with mixed string and float cost values
@ -1954,14 +1952,7 @@ def test_string_cost_values_edge_cases():
model="test-model", usage=usage, custom_llm_provider="test-provider"
)
# Expected costs:
# Prompt: 1000 * 1e-6 + 100 * 0 (invalid string becomes 0)
# Completion: 500 * 2e-6 + 50 * 2e-6 (audio tokens fall back to base cost when output_cost_per_audio_token is None)
expected_prompt_cost = 1000 * 1e-6
expected_completion_cost = 500 * 2e-6 + 50 * 2e-6
assert round(prompt_cost, 12) == round(expected_prompt_cost, 12)
assert round(completion_cost, 12) == round(expected_completion_cost, 12)
assert (prompt_cost, completion_cost) == (0.0, 0.0)
def test_string_cost_values_with_threshold():
@ -2225,46 +2216,38 @@ def test_cache_writing_cost_with_zero_creation_tokens_and_ephemeral_details():
"""
Regression test: when cache_creation_tokens is 0 but cache_creation_token_details
has non-zero ephemeral tokens, the cost must still be calculated.
This ensures the guard in _calculate_input_cost doesn't skip
calculate_cache_writing_cost when only ephemeral token details are present.
"""
cache_creation_cost = 3.75e-06
cache_creation_cost_above_1hr = 6e-06
prompt_tokens_details: PromptTokensDetailsResult = {
"cache_hit_tokens": 0,
"cache_hit_audio_tokens": 0,
"cache_creation_tokens": 0,
"cache_creation_token_details": CacheCreationTokenDetails(
ephemeral_5m_input_tokens=100,
ephemeral_1h_input_tokens=200,
usage = Usage(
prompt_tokens=300,
completion_tokens=0,
total_tokens=300,
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=0,
cache_creation_tokens=0,
cache_creation_token_details=CacheCreationTokenDetails(
ephemeral_5m_input_tokens=100,
ephemeral_1h_input_tokens=200,
),
),
"text_tokens": 0,
"audio_tokens": 0,
"image_tokens": 0,
"video_tokens": 0,
"character_count": 0,
"image_count": 0,
"video_length_seconds": 0.0,
"audio_length_seconds": 0.0,
"query_count": 0,
}
model_info: ModelInfo = {}
result = _calculate_input_cost(
prompt_tokens_details=prompt_tokens_details,
model_info=model_info,
prompt_base_cost=0.0,
cache_read_cost=0.0,
cache_creation_cost=cache_creation_cost,
cache_creation_cost_by_ttl={"1hr": cache_creation_cost_above_1hr, "12hr": cache_creation_cost_above_1hr},
)
# Expected: (100 * 3.75e-06) + (200 * 6e-06) = 0.000375 + 0.0012 = 0.001575
expected = (100 * cache_creation_cost) + (200 * cache_creation_cost_above_1hr)
assert result > 0, "Cost should not be zero when ephemeral token details are present"
assert round(result, 6) == round(expected, 6)
prompt_cost, _ = generic_cost_per_token(
model="litellm-test-ephemeral-cache-writes",
usage=usage,
custom_llm_provider="anthropic",
model_info=cast(
ModelInfo,
{
"input_cost_per_token": 0.0,
"cache_creation_input_token_cost": cache_creation_cost,
"cache_creation_input_token_cost_above_1hr": cache_creation_cost_above_1hr,
},
),
)
assert prompt_cost == pytest.approx(100 * cache_creation_cost + 200 * cache_creation_cost_above_1hr)
def test_service_tier_ultrafast_pricing():

View file

@ -528,7 +528,8 @@ async def test_completion_sagemaker(
assert route.called
assert response.choices[0].message.content == "SageMaker reply"
assert litellm.completion_cost(completion_response=response) == pytest.approx(rate)
assert litellm.completion_cost(completion_response=response, custom_cost_per_second=rate) == pytest.approx(rate)
assert "sagemaker/jumpstart-dft-hf-textgeneration1-mp-20240815-185614" not in litellm.model_cost
@pytest.mark.asyncio

View file

@ -10590,3 +10590,46 @@ def test_bedrock_registered_cache_prices_preserve_dynamic_threshold_rates(
assert entry is not None
assert entry[1]["input_cost_per_token_above_300k_tokens"] == 0.03
assert entry[1][tier_field] == (tier_override if tier_override is not None else 0.007)
@pytest.mark.parametrize(
"bad_price",
[pytest.param("abc", id="word"), pytest.param(-1.0, id="negative"), pytest.param(float("nan"), id="nan")],
)
def test_a_directly_written_bad_price_is_unpriced_and_logged_once(
bad_price: object, caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Decision 20. On main the same 1000 prompt tokens gave a TypeError, -0.001 or nan"""
import logging
from litellm._logging import verbose_logger
key: Final = f"litellm-test-bad-price-{type(bad_price).__name__}-{bad_price!r}"
monkeypatch.setitem(
litellm.model_cost,
key,
{"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": bad_price, "output_cost_per_token": 1e-6},
)
response: Final = litellm.ModelResponse(
model=key, usage=litellm.Usage(prompt_tokens=1000, completion_tokens=0, total_tokens=1000)
)
with caplog.at_level(logging.WARNING, logger=verbose_logger.name):
per_token: Final = cost_per_token(model=key, prompt_tokens=1000, completion_tokens=0, custom_llm_provider="openai")
total: Final = completion_cost(completion_response=response, model=key, custom_llm_provider="openai")
assert per_token == (0.0, 0.0)
assert total == 0.0
assert [record for record in caplog.records if key in record.getMessage() and "fails validation" in record.getMessage()] != []
assert len([record for record in caplog.records if key in record.getMessage()]) == 1
def test_a_directly_written_null_price_is_unpriced_like_a_missing_one(monkeypatch: pytest.MonkeyPatch) -> None:
key: Final = "litellm-test-null-price"
monkeypatch.setitem(
litellm.model_cost,
key,
{"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": None, "output_cost_per_token": 1e-6},
)
assert cost_per_token(model=key, prompt_tokens=1000, completion_tokens=0, custom_llm_provider="openai") == (0.0, 0.0)

View file

@ -1,12 +1,6 @@
"""
Test that register_model() in completion() and embedding() passes all
custom pricing fields from kwargs and model_info, not just the base
input/output costs.
Previously, only input_cost_per_token, output_cost_per_token, and
litellm_provider were forwarded. Fields like cache_read_input_token_cost,
mode, and supports_prompt_caching were dropped, causing incorrect cost
calculations for DB-sourced models with prompt caching pricing.
Custom pricing through register_model() and per call. A router deployment's prices are registered under its id, and a
direct SDK call's prices price that call only, never the shared row (decision 12)
"""
import copy
@ -21,7 +15,7 @@ import pytest
import litellm
from litellm._catalog import Fetched, File
from litellm.litellm_core_utils.get_model_cost_map import reload_model_cost_map
from litellm.main import _build_custom_pricing_entry
from litellm.types.utils import EmbeddingResponse, ModelResponse
from litellm.utils import _invalidate_model_cost_lowercase_map
@ -44,131 +38,6 @@ def _restore_model_cost_entries(original_entries):
_invalidate_model_cost_lowercase_map()
def test_build_custom_pricing_entry_includes_all_kwargs_fields():
"""All CustomPricingLiteLLMParams fields present in kwargs should be
included in the resulting entry dict."""
kwargs = {
"input_cost_per_token": 0.001,
"output_cost_per_token": 0.002,
"cache_read_input_token_cost": 0.00025,
"cache_creation_input_token_cost": 0.005,
"output_cost_per_reasoning_token": 0.01,
"input_cost_per_audio_token": 0.003,
"unrelated_kwarg": "should_be_ignored",
}
entry = _build_custom_pricing_entry(
custom_llm_provider="openai",
kwargs=kwargs,
)
assert entry["litellm_provider"] == "openai"
assert entry["input_cost_per_token"] == 0.001
assert entry["output_cost_per_token"] == 0.002
assert entry["cache_read_input_token_cost"] == 0.00025
assert entry["cache_creation_input_token_cost"] == 0.005
assert entry["output_cost_per_reasoning_token"] == 0.01
assert entry["input_cost_per_audio_token"] == 0.003
assert "unrelated_kwarg" not in entry
def test_build_custom_pricing_entry_merges_model_info_metadata():
"""Fields from model_info (mode, supports_prompt_caching, max_tokens)
should be merged into the entry when present."""
kwargs = {
"input_cost_per_token": 0.001,
"output_cost_per_token": 0.002,
}
model_info = {
"id": "deployment-123",
"mode": "chat",
"supports_prompt_caching": True,
"max_tokens": 128000,
}
entry = _build_custom_pricing_entry(
custom_llm_provider="openai",
kwargs=kwargs,
model_info=model_info,
)
assert entry["mode"] == "chat"
assert entry["supports_prompt_caching"] is True
assert entry["max_tokens"] == 128000
def test_build_custom_pricing_entry_setdefault_does_not_override_existing():
"""model_info uses setdefault, so it should not override a key that is
already present in the entry dict. Currently CustomPricingLiteLLMParams
and the model_info keys (mode, supports_prompt_caching, max_tokens) do
not overlap, but if they ever do, setdefault ensures the kwargs-sourced
value wins."""
kwargs = {
"input_cost_per_token": 0.001,
"output_cost_per_token": 0.002,
}
model_info = {
"mode": "chat",
"supports_prompt_caching": True,
"max_tokens": 128000,
}
entry = _build_custom_pricing_entry(
custom_llm_provider="openai",
kwargs=kwargs,
model_info=model_info,
)
assert entry["mode"] == "chat"
assert entry["supports_prompt_caching"] is True
assert entry["max_tokens"] == 128000
# Verify setdefault behavior: if a model_info key already exists in
# the entry (e.g. from a future CustomPricingLiteLLMParams addition),
# setdefault must not overwrite it.
entry["mode"] = "embedding" # simulate pre-existing value
# Re-apply setdefault the same way _build_custom_pricing_entry does
entry.setdefault("mode", model_info["mode"])
assert entry["mode"] == "embedding" # must NOT revert to "chat"
def test_build_custom_pricing_entry_skips_none_values():
"""Fields with None values in kwargs should not be included."""
kwargs = {
"input_cost_per_token": 0.001,
"output_cost_per_token": None, # explicitly None
"cache_read_input_token_cost": None,
}
entry = _build_custom_pricing_entry(
custom_llm_provider="openai",
kwargs=kwargs,
)
assert entry["input_cost_per_token"] == 0.001
assert "output_cost_per_token" not in entry
assert "cache_read_input_token_cost" not in entry
def test_build_custom_pricing_entry_handles_no_model_info():
"""Should work correctly when model_info is None."""
kwargs = {
"input_cost_per_token": 0.001,
"output_cost_per_token": 0.002,
}
entry = _build_custom_pricing_entry(
custom_llm_provider="openai",
kwargs=kwargs,
model_info=None,
)
assert entry["litellm_provider"] == "openai"
assert entry["input_cost_per_token"] == 0.001
assert entry["output_cost_per_token"] == 0.002
assert "mode" not in entry
def test_register_model_receives_cache_pricing_fields():
"""End-to-end: when register_model is called with a full pricing entry,
the cache pricing fields should be present in litellm.model_cost."""
@ -199,23 +68,6 @@ def test_register_model_receives_cache_pricing_fields():
litellm.model_cost.pop(model_key, None)
def test_build_custom_pricing_entry_time_based():
"""Time-based pricing fields should be included correctly."""
kwargs = {
"input_cost_per_second": 0.01,
"output_cost_per_second": 0.02,
}
entry = _build_custom_pricing_entry(
custom_llm_provider="openai",
kwargs=kwargs,
)
assert entry["litellm_provider"] == "openai"
assert entry["input_cost_per_second"] == 0.01
assert entry["output_cost_per_second"] == 0.02
def test_register_model_strips_none_litellm_provider():
"""``get_model_info`` returns ``litellm_provider: None`` for deployments
registered without a provider (e.g. ``Router.add_deployment`` flows).
@ -603,8 +455,7 @@ def test_embedding_router_zero_pricing_does_not_clobber_builtin_pricing():
litellm.get_model_info(model=shared_key)["input_cost_per_token"]
== builtin_input_cost
), "wildcard deployment's zero pricing leaked into the shared model_cost key"
assert litellm.model_cost[deployment_id]["input_cost_per_token"] == 0.0
assert litellm.model_cost[deployment_id]["output_cost_per_token"] == 0.0
assert deployment_id not in litellm.model_cost
sibling_response = litellm.embedding(
model=shared_key,
@ -620,20 +471,15 @@ def test_embedding_router_zero_pricing_does_not_clobber_builtin_pricing():
_restore_model_cost_entries(snapshot)
def test_embedding_router_custom_pricing_costs_request_via_deployment_id():
"""The request that carries custom pricing must still be costed with that
pricing (via its deployment id entry), while the shared backend key keeps
the built-in rate for siblings.
def test_embedding_router_custom_pricing_costs_request_with_its_own_prices():
"""The request that carries custom pricing is costed with that pricing, while the shared backend key keeps the
built-in rate for siblings
"""
shared_key = "openai/text-embedding-3-small"
deployment_id = "lit3991-wildcard-embed-custom"
override_input_cost = 5e-05
snapshot = _snapshot_model_cost_entries(
[shared_key, "text-embedding-3-small", deployment_id]
)
builtin_input_cost = litellm.get_model_info(model=shared_key)[
"input_cost_per_token"
]
snapshot = _snapshot_model_cost_entries([shared_key, "text-embedding-3-small", deployment_id])
builtin_input_cost = litellm.get_model_info(model=shared_key)["input_cost_per_token"]
assert builtin_input_cost != override_input_cost
try:
@ -648,19 +494,8 @@ def test_embedding_router_custom_pricing_costs_request_via_deployment_id():
mock_response=[0.1, 0.2],
)
request_cost = litellm.completion_cost(
completion_response=response,
model=shared_key,
custom_llm_provider="openai",
call_type="embedding",
custom_pricing=True,
router_model_id=deployment_id,
)
assert request_cost == pytest.approx(10 * override_input_cost)
assert (
litellm.get_model_info(model=shared_key)["input_cost_per_token"]
== builtin_input_cost
)
assert _response_cost(response) == pytest.approx(10 * override_input_cost)
assert litellm.get_model_info(model=shared_key)["input_cost_per_token"] == builtin_input_cost
finally:
_restore_model_cost_entries(snapshot)
@ -693,43 +528,80 @@ def test_completion_router_zero_pricing_does_not_clobber_builtin_pricing():
litellm.get_model_info(model=shared_key)["input_cost_per_token"]
== builtin_input_cost
), "wildcard deployment's zero pricing leaked into the shared model_cost key"
assert litellm.model_cost[deployment_id]["input_cost_per_token"] == 0.0
assert deployment_id not in litellm.model_cost
finally:
_restore_model_cost_entries(snapshot)
def test_embedding_direct_sdk_custom_pricing_still_registers_shared_key():
"""Direct SDK calls (no router deployment id in metadata) keep the legacy
behavior: custom pricing is registered under ``{provider}/{model}`` and the
request is costed with it.
"""
model_key = "openai/lit3991-direct-sdk-embed-model"
override_input_cost = 3e-05
try:
response = litellm.embedding(
model=model_key,
input=["hello"],
api_key="fake-key",
input_cost_per_token=override_input_cost,
output_cost_per_token=override_input_cost * 2,
mock_response=[0.1, 0.2],
)
def test_embedding_direct_sdk_custom_pricing_prices_the_call_without_registering_the_shared_key():
model_key: Final = "openai/lit3991-direct-sdk-embed-model"
override_input_cost: Final = 3e-05
response: Final = litellm.embedding(
model=model_key,
input=["hello"],
api_key="fake-key",
input_cost_per_token=override_input_cost,
output_cost_per_token=override_input_cost * 2,
mock_response=[0.1, 0.2],
)
assert (
litellm.model_cost[model_key]["input_cost_per_token"]
== override_input_cost
)
cost = litellm.completion_cost(
completion_response=response,
model=model_key,
custom_llm_provider="openai",
call_type="embedding",
custom_pricing=True,
)
assert cost == pytest.approx(10 * override_input_cost)
finally:
litellm.model_cost.pop(model_key, None)
_invalidate_model_cost_lowercase_map()
assert model_key not in litellm.model_cost
assert isinstance(response, EmbeddingResponse)
assert _response_cost(response) == pytest.approx(response.usage.prompt_tokens * override_input_cost)
def _response_cost(response: object) -> float:
assert isinstance(response, (ModelResponse, EmbeddingResponse))
cost: Final = response._hidden_params["response_cost"] # pyright: ignore[reportPrivateUsage] # the SDK reports a call's cost only here
assert isinstance(cost, float)
return cost
def _tenant_call_cost(**custom_pricing: float) -> tuple[float, litellm.Usage]:
response: Final = litellm.completion(
model="openai/gpt-4o",
messages=[{"role": "user", "content": "price this call"}],
api_key="fake-key",
mock_response="priced",
**custom_pricing,
)
assert isinstance(response, ModelResponse)
usage: Final = response.usage
assert isinstance(usage, litellm.Usage)
return _response_cost(response), usage
def test_one_tenants_custom_pricing_never_reprices_another_tenants_calls():
"""Two tenants in one SDK process. Tenant A's negotiated price prices only A's call"""
catalog_row_before: Final = copy.deepcopy(litellm.model_cost["gpt-4o"])
shared_key_before: Final = copy.deepcopy(litellm.model_cost.get("openai/gpt-4o"))
tenant_b_before, usage = _tenant_call_cost()
tenant_a, tenant_a_usage = _tenant_call_cost(input_cost_per_token=1e-06, output_cost_per_token=4e-06)
tenant_b_after, _ = _tenant_call_cost()
input_rate, output_rate = catalog_row_before["input_cost_per_token"], catalog_row_before["output_cost_per_token"]
assert isinstance(input_rate, float) and isinstance(output_rate, float)
catalog_cost: Final = usage.prompt_tokens * input_rate + usage.completion_tokens * output_rate
assert tenant_b_before == pytest.approx(catalog_cost)
assert tenant_a == pytest.approx(tenant_a_usage.prompt_tokens * 1e-06 + tenant_a_usage.completion_tokens * 4e-06)
assert tenant_b_after == pytest.approx(catalog_cost)
assert litellm.model_cost["gpt-4o"] == catalog_row_before
assert litellm.model_cost.get("openai/gpt-4o") == shared_key_before
def test_custom_pricing_never_writes_the_calls_model_info_onto_the_shared_row():
model_key: Final = "openai/litellm-test-custom-priced-responses-mode"
litellm.completion(
model=model_key,
messages=[{"role": "user", "content": "hello"}],
api_key="fake-key",
input_cost_per_token=1e-06,
output_cost_per_token=2e-06,
model_info={"mode": "responses"},
mock_response="hello back",
)
assert model_key not in litellm.model_cost
@pytest.mark.parametrize("route", ["", "converse/", "invoke/"])
@ -1468,22 +1340,18 @@ def test_completion_cost_applies_off_peak_only_deployment_pricing():
del router
def test_completion_registers_cost_per_second_pricing():
def test_completion_prices_per_second_custom_pricing_without_registering_it():
model_key: Final = "openai/test-cost-per-second-registration"
original_entries: Final = _snapshot_model_cost_entries([model_key])
response: Final = litellm.completion(
model=model_key,
messages=[{"role": "user", "content": "hello"}],
api_key="fake-key",
cost_per_second=0.02,
mock_response="hello back",
)
try:
litellm.completion(
model=model_key,
messages=[{"role": "user", "content": "hello"}],
api_key="fake-key",
cost_per_second=0.02,
mock_response="hello back",
)
assert litellm.model_cost[model_key]["cost_per_second"] == 0.02
finally:
_restore_model_cost_entries(original_entries)
assert model_key not in litellm.model_cost
assert _response_cost(response) >= 0
def test_update_model_cost():
@ -1533,33 +1401,3 @@ def test_register_model_scopes_builtin_match_to_the_given_provider():
assert litellm.model_cost[builtin_key] == builtin_row_before
finally:
_restore_model_cost_entries(model_cost_entries)
def test_per_request_custom_pricing_scopes_a_colliding_deployment_id_to_its_provider():
from litellm.main import _register_custom_pricing_for_request
deployment_id: Final = "baseten/zai-org/glm-5.2"
builtin_key: Final = "baseten/zai-org/GLM-5.2"
shared_key: Final = "openai/zai-org/GLM-5.2"
model_cost_entries: Final = _snapshot_model_cost_entries((deployment_id, builtin_key, shared_key))
builtin_row_before: Final = copy.deepcopy(litellm.model_cost[builtin_key])
openai_models_before: Final = frozenset(litellm.open_ai_chat_completion_models)
try:
_register_custom_pricing_for_request(
model="zai-org/GLM-5.2",
custom_llm_provider="openai",
kwargs={
"input_cost_per_token": 0.00000096,
"output_cost_per_token": 0.00000302,
"metadata": {"model_info": {"id": deployment_id}},
},
model_info={"mode": "chat"},
)
entry: Final = litellm.model_cost[deployment_id]
assert entry["input_cost_per_token"] == 0.00000096
assert entry["output_cost_per_token"] == 0.00000302
assert litellm.model_cost[builtin_key] == builtin_row_before
finally:
_restore_model_cost_entries(model_cost_entries)
litellm.open_ai_chat_completion_models.intersection_update(openai_models_before)

View file

@ -2243,7 +2243,7 @@ def test_register_model_price_is_used_for_completion_cost() -> None:
@respx.mock
@pytest.mark.usefixtures("isolated_openai_model_sets", "local_model_cost_map")
def test_completion_registers_test_owned_model_prices() -> None:
def test_completion_prices_per_call_prices_without_registering_them() -> None:
model: Final = "migration-owned-pricing-model"
messages: Final = [{"role": "user", "content": "price control"}]
litellm.register_model(
@ -2286,8 +2286,7 @@ def test_completion_registers_test_owned_model_prices() -> None:
)
assert json.loads(route.calls[0].request.content) == {"model": model, "messages": messages}
assert litellm.model_cost[f"openai/{model}"]["input_cost_per_token"] == 0.001
assert litellm.model_cost[f"openai/{model}"]["output_cost_per_token"] == 0.002
assert f"openai/{model}" not in litellm.model_cost
assert litellm.model_cost[model]["input_cost_per_token"] == 0.0001
assert response._hidden_params["response_cost"] == pytest.approx(0.018)