From c57274fa12f52fca33425b09a895de9d761659be Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Wed, 28 Oct 2026 01:55:35 -0700 Subject: [PATCH] refactor(cost): price token usage with the Rust cost core [08/12] cost::calculate prices token meters over the typed Prices every row lowers into, with one line per meter and the borrowed rate named in fallback_from. generic_cost_per_token prices through Prices.price and as_main_cost, per-call custom pricing is an overlay that never writes the shared row, and a directly written bad price is unpriced with one log line Co-Authored-By: Claude Opus 5.5 --- litellm-rust/Cargo.lock | 2 + litellm-rust/crates/cost/Cargo.toml | 2 +- litellm-rust/crates/cost/README.md | 2 +- litellm-rust/crates/cost/src/calculate.rs | 796 ++++++++++ litellm-rust/crates/cost/src/lib.rs | 2 + litellm-rust/crates/cost/src/price.rs | 2 +- litellm-rust/crates/cost/src/schedule.rs | 96 ++ .../crates/model-catalog/src/error.rs | 2 +- .../crates/model-catalog/src/lower.rs | 15 +- litellm-rust/crates/python-bridge/Cargo.toml | 2 + litellm-rust/crates/python-bridge/src/cost.rs | 457 ++++++ litellm-rust/crates/python-bridge/src/lib.rs | 3 + .../src/model_registry/resolve.rs | 17 + litellm/_catalog/__init__.py | 12 + litellm/_cost/__init__.py | 56 + litellm/cost_calculator.py | 1410 ++++++++--------- .../litellm_core_utils/llm_cost_calc/utils.py | 472 ++---- litellm/main.py | 122 -- litellm/rust_bridge/_native.pyi | 64 + litellm/utils.py | 12 +- .../llm_cost_calc/test_llm_cost_calc_utils.py | 73 +- .../test_sagemaker_completion_handler.py | 3 +- tests/unit/test_cost_calculator.py | 43 + .../test_register_model_custom_pricing.py | 342 ++-- tests/unit/test_utils.py | 5 +- 25 files changed, 2562 insertions(+), 1450 deletions(-) create mode 100644 litellm-rust/crates/cost/src/calculate.rs create mode 100644 litellm-rust/crates/python-bridge/src/cost.rs create mode 100644 litellm/_cost/__init__.py diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 6f8e3b52432..e82511c9e1f 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -4284,6 +4284,7 @@ dependencies = [ "fancy-regex 0.19.2", "futures-util", "indexmap 2.14.0", + "jiff", "litellm-auth", "litellm-auth-aws", "litellm-cache", @@ -4292,6 +4293,7 @@ dependencies = [ "litellm-cache-response", "litellm-callbacks-legacy-python", "litellm-core-utils", + "litellm-cost", "litellm-host", "litellm-host-python", "litellm-http", diff --git a/litellm-rust/crates/cost/Cargo.toml b/litellm-rust/crates/cost/Cargo.toml index bb2fa6be63e..bb9bebe9929 100644 --- a/litellm-rust/crates/cost/Cargo.toml +++ b/litellm-rust/crates/cost/Cargo.toml @@ -10,7 +10,7 @@ serde = ["dep:serde"] schema = ["serde", "dep:schemars"] [dependencies] -jiff = { version = "0.2.37", default-features = false, features = ["std"] } +jiff = { version = "0.2.37", default-features = false, features = ["std", "tzdb-bundle-always"] } schemars = { workspace = true, optional = true } serde = { workspace = true, optional = true } thiserror.workspace = true diff --git a/litellm-rust/crates/cost/README.md b/litellm-rust/crates/cost/README.md index f8fc9403917..77d906a528f 100644 --- a/litellm-rust/crates/cost/README.md +++ b/litellm-rust/crates/cost/README.md @@ -1,3 +1,3 @@ # litellm-cost -This crate holds the price data model that `litellm-model-catalog` lowers each catalog entry into, plus the usage, call context and priced outcome types that `calculate` will compute over. There is no `calculate` until the token pricing PR lands. `UnitPrice` is USD per unit and `Usd` is an amount, and only the operations that make sense between them exist, so a price is never added to an amount. `Prices::band` and `Prices::rate` resolve a token rate the way Python's `_get_token_base_cost` does +This crate holds the price data model that `litellm-model-catalog` lowers each catalog entry into, plus the usage, call context and priced outcome types. `calculate` prices a call's token usage over a row's `Prices` and `Adjustments`, one `Line` per meter with usage, and names the meter whose rate a line borrowed in `fallback_from`. Character, second and per-unit meters land in later PRs and are reported as `missing` until then. `UnitPrice` is USD per unit and `Usd` is an amount, and only the operations that make sense between them exist, so a price is never added to an amount. `Prices::band` picks the prompt band the way Python's `_get_token_base_cost` does diff --git a/litellm-rust/crates/cost/src/calculate.rs b/litellm-rust/crates/cost/src/calculate.rs new file mode 100644 index 00000000000..c9d08c27ff0 --- /dev/null +++ b/litellm-rust/crates/cost/src/calculate.rs @@ -0,0 +1,796 @@ +use crate::call::CallContext; +use crate::money::{Multiplier, UnitPrice}; +use crate::price::{ + Adjustments, Band, CacheReadFallback, CacheReadMode, CacheTtl, Channel, Feature, InputUnit, + Meter, OffPeakRates, OutputUnit, Price, Prices, ServiceTier, TierScope, TokenKind, Uplift, + When, +}; +use crate::priced::{Line, Outcome, Priced, Quantity, Side}; +use crate::usage::{DurationMeter, Usage}; + +const TEXT_INPUT: Meter = Meter::Input(InputUnit::Token(TokenKind::Text)); +const TEXT_OUTPUT: Meter = Meter::Output(OutputUnit::Token(TokenKind::Text)); +const IMAGE_OUTPUT: Meter = Meter::Output(OutputUnit::Token(TokenKind::Image)); +const REASONING: Meter = Meter::Output(OutputUnit::ReasoningToken); +const CACHE_READ: Meter = Meter::CacheRead { + kind: TokenKind::Text, + mode: CacheReadMode::Implicit, +}; +const CACHE_WRITE: Meter = Meter::CacheWrite { + kind: TokenKind::Text, + ttl: CacheTtl::Default, +}; + +/// Prices one call's token usage (decisions 21 and 30). Every meter with usage is a `Line`, one that borrowed another +/// meter's rate names it in `fallback_from`, and one with no price and no fallback is listed in `missing` +pub fn calculate( + prices: &Prices, + adjustments: &Adjustments, + usage: &Usage, + call: &CallContext, +) -> Outcome { + let rates = Rates::new(prices, usage, call); + let uplifts = uplifts(adjustments, call); + let tokens = &usage.tokens; + let billed: Vec<(Meter, Option, u64, Side)> = tokens + .input + .iter() + .map(|(kind, count)| { + ( + Meter::Input(InputUnit::Token(*kind)), + rates.input(*kind), + *count, + Side::Prompt, + ) + }) + .chain(tokens.cache_read.iter().map(|((kind, mode), count)| { + let meter = Meter::CacheRead { + kind: *kind, + mode: *mode, + }; + (meter, rates.cache_read(*kind, *mode), *count, Side::Prompt) + })) + .chain(tokens.cache_write.iter().map(|((kind, ttl), count)| { + let meter = Meter::CacheWrite { + kind: *kind, + ttl: *ttl, + }; + (meter, rates.cache_write(*ttl), *count, Side::Prompt) + })) + .chain(tokens.output.iter().map(|(kind, count)| { + let meter = Meter::Output(OutputUnit::Token(*kind)); + (meter, rates.output(*kind), *count, Side::Completion) + })) + .chain([( + REASONING, + rates.reasoning(), + tokens.reasoning, + Side::Completion, + )]) + .filter(|(.., count, _)| *count > 0) + .collect(); + let lines: Vec = billed + .iter() + .filter_map(|(meter, rate, count, side)| { + Some(rate.clone()?.line(meter.clone(), *count, *side, &uplifts)) + }) + .collect(); + let missing: Vec = billed + .into_iter() + .filter_map(|(meter, rate, ..)| rate.is_none().then_some(meter)) + .chain(without_an_arm(usage)) + .collect(); + if lines.is_empty() && !missing.is_empty() { + return Outcome::Unpriced { missing }; + } + let total = lines.iter().map(|line| line.amount).sum(); + Outcome::Priced(Priced { + lines, + missing, + total, + }) +} + +/// One resolved rate, and the meter it came from when that is not the billed meter's own +#[derive(Clone, Debug)] +struct Rate { + price: UnitPrice, + borrowed: Option, +} + +impl Rate { + fn own(price: UnitPrice) -> Self { + Self { + price, + borrowed: None, + } + } + + fn borrowed_from(self, meter: &Meter) -> Self { + Self { + borrowed: Some(self.borrowed.unwrap_or_else(|| meter.clone())), + ..self + } + } + + fn halved(self) -> Option { + Some(Self { + price: UnitPrice::new(self.price.get() * 0.5)?, + ..self + }) + } + + fn line(self, meter: Meter, count: u64, side: Side, uplifts: &[Multiplier]) -> Line { + let quantity = Quantity::Count(count); + Line { + quantity, + unit_price: self.price, + amount: uplifts + .iter() + .fold(self.price * quantity, |amount, factor| amount * *factor), + fallback_from: self.borrowed, + meter, + side, + } + } +} + +/// The rates one call resolves, main's `_get_token_base_cost` and its modality, cache and reasoning rules +struct Rates<'a> { + prices: &'a Prices, + overlay: Option<&'a Prices>, + channel: Channel, + tier: ServiceTier, + band: Band, + whole_call: bool, + off_peak: Option, + text_input: Option, + text_output: Option, +} + +impl<'a> Rates<'a> { + /// A tier table bills the whole call at the tier the prompt picks, unless that tier sets no input price. Then + /// the row is priced as if it had no tiers, as main's `_select_priced_tier` does + fn new(prices: &'a Prices, usage: &Usage, call: &'a CallContext) -> Self { + let band = prices.band(call.tier, usage.prompt_tokens); + let tier_input = Price::online( + TEXT_INPUT, + When { + tier: ServiceTier::Standard, + band, + }, + ); + let whole_call = prices.billing().tier_scope == TierScope::WholeCall + && prices.get(&tier_input).is_some(); + let lookup = Self { + prices, + overlay: call.overlay.as_ref(), + channel: call.channel, + tier: call.tier, + band: match (prices.billing().tier_scope, whole_call) { + (TierScope::WholeCall, false) => Band::Base, + _ => band, + }, + whole_call, + off_peak: prices + .off_peak() + .filter(|off_peak| off_peak.schedule().is_open(call.at)) + .map(|off_peak| *off_peak.rates()), + text_input: None, + text_output: None, + }; + let text_input = lookup + .off_peak(|rates| rates.input) + .or_else(|| lookup.banded(&TEXT_INPUT)); + let text_output = lookup + .off_peak(|rates| rates.output) + .or_else(|| lookup.output_or_image_rate()); + Self { + text_input, + text_output, + ..lookup + } + } + + fn input(&self, kind: TokenKind) -> Option { + match kind { + TokenKind::Text => self.text_input.clone(), + _ => self + .banded(&Meter::Input(InputUnit::Token(kind))) + .or_else(|| self.borrowed_input()), + } + } + + fn output(&self, kind: TokenKind) -> Option { + match kind { + TokenKind::Text => self.text_output.clone(), + _ => self + .banded(&Meter::Output(OutputUnit::Token(kind))) + .or_else(|| self.borrowed_output()), + } + } + + fn cache_read(&self, kind: TokenKind, mode: CacheReadMode) -> Option { + let borrowed_implicit = || { + self.implicit_cache_read() + .map(|rate| rate.borrowed_from(&CACHE_READ)) + }; + match (kind, mode) { + (TokenKind::Text, CacheReadMode::Implicit) => self.implicit_cache_read(), + (TokenKind::Text, CacheReadMode::Explicit) => { + let own = self.banded(&Meter::CacheRead { kind, mode }); + match (self.off_peak(|rates| rates.explicit_cache_read), own) { + (Some(off_peak), _) => Some(off_peak), + (None, Some(own)) => { + Some(self.off_peak(|rates| rates.cache_read).unwrap_or(own)) + } + (None, None) => borrowed_implicit(), + } + } + _ => self + .banded(&Meter::CacheRead { kind, mode }) + .or_else(borrowed_implicit), + } + } + + fn cache_write(&self, ttl: CacheTtl) -> Option { + match ttl { + CacheTtl::Default => self.default_cache_write(), + CacheTtl::OneHour | CacheTtl::TwelveHours => self + .banded(&Meter::CacheWrite { + kind: TokenKind::Text, + ttl, + }) + .or_else(|| { + self.default_cache_write() + .map(|rate| rate.borrowed_from(&CACHE_WRITE)) + }), + } + } + + /// Off-peak first, then the tier's reasoning price. A tier that prices output and not reasoning bills reasoning at + /// its output rate, otherwise the standard reasoning price, then the output rate. Reasoning prices are never banded + /// outside a whole-call tier table + fn reasoning(&self) -> Option { + let tier_base = When { + tier: self.tier, + band: Band::Base, + }; + let tier_prices_output_only = + || self.tier != ServiceTier::Standard && self.at(&TEXT_OUTPUT, &[tier_base]).is_some(); + self.off_peak(|rates| rates.reasoning) + .or_else(|| match self.whole_call() { + true => self.banded(&REASONING), + false => self.at(&REASONING, &[tier_base]).or_else(|| { + (!tier_prices_output_only()) + .then(|| self.at(&REASONING, &[When::default()])) + .flatten() + }), + }) + .or_else(|| self.borrowed_output()) + } + + fn borrowed_input(&self) -> Option { + self.text_input + .clone() + .map(|rate| rate.borrowed_from(&TEXT_INPUT)) + } + + fn borrowed_output(&self) -> Option { + self.text_output + .clone() + .map(|rate| rate.borrowed_from(&TEXT_OUTPUT)) + } + + fn implicit_cache_read(&self) -> Option { + self.off_peak(|rates| rates.cache_read) + .or_else(|| self.banded(&CACHE_READ)) + .or_else(|| match self.prices.billing().cache_read_fallback { + CacheReadFallback::InputRate => self.borrowed_input(), + CacheReadFallback::HalfInputRate => self.borrowed_input()?.halved(), + }) + } + + fn default_cache_write(&self) -> Option { + self.off_peak(|rates| rates.cache_write) + .or_else(|| self.banded(&CACHE_WRITE)) + .or_else(|| self.borrowed_input()) + } + + /// main reads a missing or zero output price as unknown and bills image models at their image token rate. A + /// whole-call tier with no output price bills the row's own output price + fn output_or_image_rate(&self) -> Option { + let own = self.banded(&TEXT_OUTPUT).or_else(|| { + self.whole_call() + .then(|| self.at(&TEXT_OUTPUT, &[When::default()]))? + }); + match own { + Some(rate) if rate.price.get() != 0.0 => Some(rate), + own => self + .at(&IMAGE_OUTPUT, &[When::default()]) + .map(|rate| rate.borrowed_from(&IMAGE_OUTPUT)) + .or(own), + } + } + + fn off_peak(&self, rate: fn(&OffPeakRates) -> Option) -> Option { + self.off_peak.as_ref().and_then(rate).map(Rate::own) + } + + fn whole_call(&self) -> bool { + self.whole_call + } + + /// The meter's own rate in this call's band, tried at `(tier, band)`, `(standard, band)`, `(tier, base)` and + /// `(standard, base)`. A whole-call tier table prices every meter at the band's own rate only + fn banded(&self, meter: &Meter) -> Option { + let standard = ServiceTier::Standard; + let whens: &[When] = match self.whole_call() { + true => &[When { + tier: standard, + band: self.band, + }], + false => &[ + When { + tier: self.tier, + band: self.band, + }, + When { + tier: standard, + band: self.band, + }, + When { + tier: self.tier, + band: Band::Base, + }, + When::default(), + ], + }; + self.at(meter, whens) + } + + /// The first price for `meter` at `whens` on this call's channel. A per-call overlay that prices the meter at all + /// replaces the row's prices for it (decision 12). Decisions bills the online prices it does not set, and a batch + /// call with no batch price bills half the online price, main's `_batch_or_half` + fn at(&self, meter: &Meter, whens: &[When]) -> Option { + let source = match self.overlay { + Some(overlay) if overlay.iter().any(|(price, _)| &price.meter == meter) => overlay, + _ => self.prices, + }; + let on = |channel: Channel| { + whens.iter().find_map(|when| { + source.get(&Price { + meter: meter.clone(), + channel, + when: *when, + }) + }) + }; + match self.channel { + Channel::Online => on(Channel::Online).map(Rate::own), + Channel::Decisions => on(Channel::Decisions) + .or_else(|| on(Channel::Online)) + .map(Rate::own), + Channel::Batch => on(Channel::Batch).map(Rate::own).or_else(|| { + Rate::own(on(Channel::Online)?) + .borrowed_from(meter) + .halved() + }), + } + } +} + +/// Meters with usage that no arm prices yet. Characters and seconds land in 09, the per-unit meters in 10 +fn without_an_arm(usage: &Usage) -> impl Iterator + '_ { + let counted = [ + (usage.characters.prompt, Meter::Input(InputUnit::Character)), + ( + usage.characters.completion, + Meter::Output(OutputUnit::Character), + ), + (usage.images.input, Meter::Input(InputUnit::Image)), + (usage.images.input_pixels, Meter::Input(InputUnit::Pixel)), + ( + usage.images.output.len() as u64, + Meter::Output(OutputUnit::Image(None)), + ), + ( + usage.search.queries, + Meter::Input(InputUnit::Query { over_results: None }), + ), + ( + usage.search.maps_grounding, + Meter::Feature(Feature::GoogleMapsGroundingQuery), + ), + ( + usage.search.browser_opens, + Meter::Feature(Feature::BrowserOpen), + ), + ( + usage.search.additional_results, + Meter::Feature(Feature::AdditionalResult), + ), + (usage.text_messages, Meter::Input(InputUnit::TextMessage)), + ( + usage.tokens.citation, + Meter::Feature(Feature::CitationToken), + ), + ]; + let web_search = usage + .search + .web_search + .filter(|search| search.requests > 0) + .map(|search| Meter::WebSearch(search.context)); + let durations = usage + .durations + .iter() + .filter(|(_, duration)| !duration.is_zero()) + .map(|(meter, _)| match meter { + DurationMeter::WallClock => Meter::WallClockSecond, + DurationMeter::InputAudio => Meter::Input(InputUnit::AudioSecond), + DurationMeter::LiveVoice => Meter::Input(InputUnit::Second), + DurationMeter::OutputVideo { resolution } => { + Meter::Output(OutputUnit::Second(resolution.clone())) + } + }); + counted + .into_iter() + .filter_map(|(count, meter)| (count > 0).then_some(meter)) + .chain(web_search) + .chain(durations) +} + +fn uplifts(adjustments: &Adjustments, call: &CallContext) -> Vec { + let residency = call + .residency + .and_then(|residency| adjustments.get(&Uplift::Residency(residency))); + let endpoint = call + .non_global_endpoint + .then(|| adjustments.get(&Uplift::NonGlobalEndpoint)) + .flatten(); + residency.into_iter().chain(endpoint).collect() +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + use std::num::NonZeroU64; + + use rstest::rstest; + + use super::*; + use crate::price::{Billing, Residency}; + + fn price( + meter: &Meter, + channel: Channel, + tier: ServiceTier, + band: Band, + rate: f64, + ) -> (Price, UnitPrice) { + ( + Price { + meter: meter.clone(), + channel, + when: When { tier, band }, + }, + UnitPrice::new(rate).unwrap(), + ) + } + + fn standard(meter: &Meter, rate: f64) -> (Price, UnitPrice) { + price( + meter, + Channel::Online, + ServiceTier::Standard, + Band::Base, + rate, + ) + } + + fn row(rates: impl IntoIterator) -> Prices { + Prices::new(rates.into_iter().collect(), None, Billing::default()) + } + + fn call() -> CallContext { + CallContext { + tier: ServiceTier::Standard, + channel: Channel::Online, + residency: None, + non_global_endpoint: false, + speed: None, + inference_geo: None, + at: jiff::Timestamp::UNIX_EPOCH, + overlay: None, + } + } + + fn usage(input: u64, cache_read: u64, output: u64) -> Usage { + let mut usage = Usage { + prompt_tokens: input + cache_read, + ..Usage::default() + }; + usage.tokens.input = BTreeMap::from([(TokenKind::Text, input)]); + usage.tokens.cache_read = + BTreeMap::from([((TokenKind::Text, CacheReadMode::Implicit), cache_read)]); + usage.tokens.output = BTreeMap::from([(TokenKind::Text, output)]); + usage + } + + fn priced(outcome: Outcome) -> Priced { + match outcome { + Outcome::Priced(priced) => priced, + Outcome::Unpriced { missing } => panic!("unpriced, missing {missing:?}"), + } + } + + fn total(prices: &Prices, usage: &Usage, call: &CallContext) -> f64 { + priced(calculate(prices, &Adjustments::default(), usage, call)) + .total + .get() + } + + fn close(actual: f64, expected: f64) { + assert!( + (actual - expected).abs() <= expected.abs() * 1e-12, + "{actual} != {expected}" + ); + } + + #[rstest] + fn a_cache_read_with_no_price_borrows_the_input_rate_as_a_visible_line() { + let prices = row([standard(&TEXT_INPUT, 1e-6), standard(&TEXT_OUTPUT, 2e-6)]); + let outcome = priced(calculate( + &prices, + &Adjustments::default(), + &usage(200, 800, 10), + &call(), + )); + let cache_line = outcome + .lines + .iter() + .find(|line| line.meter == CACHE_READ) + .unwrap(); + assert_eq!(cache_line.fallback_from, Some(TEXT_INPUT)); + close(outcome.total.get(), 1000.0 * 1e-6 + 10.0 * 2e-6); + } + + #[rstest] + fn a_meter_with_usage_and_no_price_is_missing_and_the_rest_is_still_priced() { + let prices = row([standard(&TEXT_OUTPUT, 2e-6)]); + let outcome = priced(calculate( + &prices, + &Adjustments::default(), + &usage(100, 0, 10), + &call(), + )); + assert_eq!(outcome.missing, [TEXT_INPUT]); + close(outcome.total.get(), 10.0 * 2e-6); + } + + #[rstest] + fn a_meter_no_arm_prices_yet_is_missing() { + let prices = row([standard(&TEXT_INPUT, 1e-6)]); + let usage = Usage { + characters: crate::usage::Characters { + prompt: 40, + completion: 0, + }, + ..usage(100, 0, 0) + }; + let outcome = priced(calculate(&prices, &Adjustments::default(), &usage, &call())); + assert_eq!(outcome.missing, [Meter::Input(InputUnit::Character)]); + } + + #[rstest] + fn a_call_with_no_priced_meter_is_unpriced() { + let outcome = calculate( + &row([]), + &Adjustments::default(), + &usage(100, 0, 10), + &call(), + ); + assert_eq!( + outcome, + Outcome::Unpriced { + missing: vec![TEXT_INPUT, TEXT_OUTPUT] + } + ); + } + + #[rstest] + fn the_overlay_replaces_the_meters_it_prices_and_keeps_the_rest() { + let prices = row([standard(&TEXT_INPUT, 1e-6), standard(&TEXT_OUTPUT, 2e-6)]); + let call = CallContext { + overlay: Some(row([standard(&TEXT_INPUT, 5e-6)])), + ..call() + }; + close( + total(&prices, &usage(100, 0, 10), &call), + 100.0 * 5e-6 + 10.0 * 2e-6, + ); + } + + #[rstest] + #[case::requested_region(Residency::Eu, true)] + #[case::other_region(Residency::Us, false)] + fn a_residency_uplift_multiplies_every_line( + #[case] priced_region: Residency, + #[case] applies: bool, + ) { + let prices = row([standard(&TEXT_INPUT, 1e-6), standard(&TEXT_OUTPUT, 2e-6)]); + let adjustments = Adjustments::new([( + Uplift::Residency(priced_region), + Multiplier::new(1.1).unwrap(), + )]); + let call = CallContext { + residency: Some(Residency::Eu), + ..call() + }; + let outcome = priced(calculate(&prices, &adjustments, &usage(100, 0, 10), &call)); + let base = 100.0 * 1e-6 + 10.0 * 2e-6; + close(outcome.total.get(), if applies { base * 1.1 } else { base }); + } + + #[rstest] + #[case::missing(None)] + #[case::zero(Some(0.0))] + fn an_image_model_with_no_output_price_bills_output_at_its_image_token_rate( + #[case] output: Option, + ) { + let prices = row(std::iter::once(standard(&IMAGE_OUTPUT, 4e-5)) + .chain(output.map(|rate| standard(&TEXT_OUTPUT, rate)))); + let outcome = priced(calculate( + &prices, + &Adjustments::default(), + &usage(0, 0, 10), + &call(), + )); + assert_eq!(outcome.lines[0].fallback_from, Some(IMAGE_OUTPUT)); + close(outcome.total.get(), 10.0 * 4e-5); + } + + #[rstest] + #[case::below_the_band(1_000, 1e-6)] + #[case::above_the_band(300_000, 2e-6)] + fn the_prompt_band_prices_every_token_meter(#[case] prompt: u64, #[case] input_rate: f64) { + let over = Band::over_prompt_tokens(200_000); + let prices = row([ + standard(&TEXT_INPUT, 1e-6), + price( + &TEXT_INPUT, + Channel::Online, + ServiceTier::Standard, + over, + 2e-6, + ), + ]); + let usage = Usage { + prompt_tokens: prompt, + ..usage(100, 0, 0) + }; + close(total(&prices, &usage, &call()), 100.0 * input_rate); + } + + #[rstest] + #[case::batch_price(Some(3e-7), 3e-7, None)] + #[case::half_the_online_price(None, 5e-7, Some(TEXT_INPUT))] + fn a_batch_call_bills_its_batch_price_or_half_the_online_one( + #[case] batch: Option, + #[case] rate: f64, + #[case] fallback: Option, + ) { + let batch_price = batch.map(|rate| { + price( + &TEXT_INPUT, + Channel::Batch, + ServiceTier::Standard, + Band::Base, + rate, + ) + }); + let prices = row(std::iter::once(standard(&TEXT_INPUT, 1e-6)).chain(batch_price)); + let call = CallContext { + channel: Channel::Batch, + ..call() + }; + let outcome = priced(calculate( + &prices, + &Adjustments::default(), + &usage(100, 0, 0), + &call, + )); + assert_eq!(outcome.lines[0].fallback_from, fallback); + close(outcome.total.get(), 100.0 * rate); + } + + #[rstest] + #[case::tier_reasoning_price(Some(9e-6), true, 9e-6)] + #[case::tier_output_without_reasoning(None, true, 7e-6)] + #[case::standard_reasoning_price(None, false, 3e-6)] + fn reasoning_follows_mains_tier_rules( + #[case] tier_reasoning: Option, + #[case] tier_output: bool, + #[case] expected: f64, + ) { + let priority = |meter: &Meter, rate| { + price( + meter, + Channel::Online, + ServiceTier::Priority, + Band::Base, + rate, + ) + }; + let prices = row([standard(&TEXT_OUTPUT, 2e-6), standard(&REASONING, 3e-6)] + .into_iter() + .chain(tier_reasoning.map(|rate| priority(&REASONING, rate))) + .chain(tier_output.then(|| priority(&TEXT_OUTPUT, 7e-6)))); + let mut usage = usage(0, 0, 0); + usage.tokens.reasoning = 10; + let call = CallContext { + tier: ServiceTier::Priority, + ..call() + }; + close(total(&prices, &usage, &call), 10.0 * expected); + } + + #[rstest] + fn a_long_cache_write_with_no_price_borrows_the_default_write_rate() { + let prices = row([standard(&TEXT_INPUT, 1e-6), standard(&CACHE_WRITE, 1.25e-6)]); + let mut usage = usage(0, 0, 0); + usage.tokens.cache_write = BTreeMap::from([((TokenKind::Text, CacheTtl::OneHour), 100)]); + let outcome = priced(calculate(&prices, &Adjustments::default(), &usage, &call())); + assert_eq!(outcome.lines[0].fallback_from, Some(CACHE_WRITE)); + close(outcome.total.get(), 100.0 * 1.25e-6); + } + + #[rstest] + fn a_half_input_cache_read_row_bills_cache_reads_at_half_the_input_rate() { + let prices = Prices::new( + [standard(&TEXT_INPUT, 1e-6)].into_iter().collect(), + None, + Billing { + cache_read_fallback: CacheReadFallback::HalfInputRate, + ..Billing::default() + }, + ); + close(total(&prices, &usage(0, 100, 0), &call()), 100.0 * 0.5e-6); + } + + #[rstest] + #[case::low_tier(1_000, 1e-6)] + #[case::high_tier(40_000, 4e-6)] + fn a_whole_call_tier_bills_cache_reads_at_its_own_input_rate( + #[case] prompt: u64, + #[case] rate: f64, + ) { + let over = Band::OverPromptTokens(NonZeroU64::new(32_000).unwrap()); + let prices = Prices::new( + [ + standard(&TEXT_INPUT, 1e-6), + standard(&CACHE_READ, 0.1e-6), + price( + &TEXT_INPUT, + Channel::Online, + ServiceTier::Standard, + over, + 4e-6, + ), + ] + .into_iter() + .collect(), + None, + Billing { + tier_scope: TierScope::WholeCall, + ..Billing::default() + }, + ); + let usage = Usage { + prompt_tokens: prompt, + ..usage(0, 100, 0) + }; + let expected = if prompt > 32_000 { rate } else { 0.1e-6 }; + close(total(&prices, &usage, &call()), 100.0 * expected); + } +} diff --git a/litellm-rust/crates/cost/src/lib.rs b/litellm-rust/crates/cost/src/lib.rs index 192dfa361d6..89e8acb5827 100644 --- a/litellm-rust/crates/cost/src/lib.rs +++ b/litellm-rust/crates/cost/src/lib.rs @@ -1,3 +1,4 @@ +pub mod calculate; pub mod call; mod error; pub mod money; @@ -6,4 +7,5 @@ pub mod priced; pub mod schedule; pub mod usage; +pub use calculate::calculate; pub use error::*; diff --git a/litellm-rust/crates/cost/src/price.rs b/litellm-rust/crates/cost/src/price.rs index 34934263b7d..50a485127ae 100644 --- a/litellm-rust/crates/cost/src/price.rs +++ b/litellm-rust/crates/cost/src/price.rs @@ -68,7 +68,7 @@ pub struct Billing { /// From `web_search_billing_unit`, whether web search and Maps grounding bill per query or once per prompt pub web_search: WebSearchUnit, /// Whether `tiered_pricing` bills each meter in its own tier or the whole call at the tier the input picks. - /// Dashscope rows bill the whole call (inferred, 10 checks no other row has that shape) + /// A prompt-token tier table bills the whole call, main's `_get_tiered_base_costs` pub tier_scope: TierScope, } diff --git a/litellm-rust/crates/cost/src/schedule.rs b/litellm-rust/crates/cost/src/schedule.rs index dd33301288f..73ca26b54d9 100644 --- a/litellm-rust/crates/cost/src/schedule.rs +++ b/litellm-rust/crates/cost/src/schedule.rs @@ -31,6 +31,54 @@ pub enum Weekday { Sunday, } +impl Schedule { + /// Whether `at` is off-peak. A window applies on its weekdays, read on the calendar `weekday_timezone` names (UTC + /// when it names none or an unknown zone), or every day. Its start is inclusive and its end exclusive, a window may + /// wrap past midnight, and one whose start equals its end covers the whole day + pub fn is_open(&self, at: jiff::Timestamp) -> bool { + let utc = at.to_zoned(jiff::tz::TimeZone::UTC); + let second_of_day = u32::from(utc.hour().unsigned_abs()) * 3600 + + u32::from(utc.minute().unsigned_abs()) * 60 + + u32::from(utc.second().unsigned_abs()); + let calendar = self + .weekday_timezone + .as_deref() + .map(str::trim) + .filter(|zone| !zone.is_empty()) + .and_then(|zone| jiff::tz::TimeZone::get(zone).ok()) + .unwrap_or(jiff::tz::TimeZone::UTC); + let weekday = Weekday::from_iso( + at.to_zoned(calendar) + .weekday() + .to_monday_one_offset() + .unsigned_abs(), + ); + self.windows.iter().any(|window| { + let on_day = window + .weekdays + .as_ref() + .is_none_or(|days| weekday.is_some_and(|weekday| days.contains(&weekday))); + on_day + && window + .hours + .iter() + .any(|hours| hours.contains(second_of_day)) + }) + } +} + +impl UtcWindow { + fn contains(self, second_of_day: u32) -> bool { + let start = u32::from(self.start_minute) * 60; + let end = u32::from(self.end_minute) * 60; + if start < end { + (start..end).contains(&second_of_day) + } else { + second_of_day >= start || second_of_day < end + } + } +} + impl FromStr for UtcWindow { type Err = InvalidUtcWindow; @@ -215,6 +263,54 @@ mod tests { assert_eq!(name.parse().ok(), expected); } + fn schedule( + hours: &str, + weekdays: Option>, + weekday_timezone: Option<&str>, + ) -> Schedule { + Schedule { + windows: vec![Window { + hours: vec![hours.parse().unwrap()], + weekdays, + }], + weekday_timezone: weekday_timezone.map(str::to_owned), + } + } + + #[rstest] + #[case::inside("2026-10-28T10:00:00Z", "10:00-18:30", true)] + #[case::end_is_exclusive("2026-10-28T18:30:00Z", "10:00-18:30", false)] + #[case::wrapped_after_midnight("2026-10-28T01:59:59Z", "22:00-02:00", true)] + #[case::wrapped_outside("2026-10-28T12:00:00Z", "22:00-02:00", false)] + #[case::whole_day("2026-10-28T12:00:00Z", "00:00-00:00", true)] + fn a_window_opens_at_its_start_and_closes_at_its_end( + #[case] at: &str, + #[case] hours: &str, + #[case] open: bool, + ) { + assert_eq!( + schedule(hours, None, None).is_open(at.parse().unwrap()), + open + ); + } + + #[rstest] + #[case::utc_wednesday(None, Weekday::Wednesday, true)] + #[case::shanghai_thursday(Some("Asia/Shanghai"), Weekday::Thursday, true)] + #[case::shanghai_not_wednesday(Some("Asia/Shanghai"), Weekday::Wednesday, false)] + #[case::unknown_zone_reads_utc(Some("Mars/Base"), Weekday::Wednesday, true)] + fn weekdays_are_read_on_the_named_calendar( + #[case] weekday_timezone: Option<&str>, + #[case] weekday: Weekday, + #[case] open: bool, + ) { + let at = "2026-10-28T17:00:00Z".parse().unwrap(); + assert_eq!( + schedule("16:30-00:30", Some(vec![weekday]), weekday_timezone).is_open(at), + open + ); + } + #[rstest] #[case::monday(1, Some(Weekday::Monday))] #[case::sunday(7, Some(Weekday::Sunday))] diff --git a/litellm-rust/crates/model-catalog/src/error.rs b/litellm-rust/crates/model-catalog/src/error.rs index e6c068242a0..dcca980e71a 100644 --- a/litellm-rust/crates/model-catalog/src/error.rs +++ b/litellm-rust/crates/model-catalog/src/error.rs @@ -86,7 +86,7 @@ impl Located { Self::new(path_to_pointer(path), error) } - pub(crate) fn prepend(&mut self, key: &str) { + pub fn prepend(&mut self, key: &str) { let prefix = escape_pointer_segment(key); self.path = if self.path.is_empty() { format!("/{prefix}") diff --git a/litellm-rust/crates/model-catalog/src/lower.rs b/litellm-rust/crates/model-catalog/src/lower.rs index 0454adcee7a..379e5978c15 100644 --- a/litellm-rust/crates/model-catalog/src/lower.rs +++ b/litellm-rust/crates/model-catalog/src/lower.rs @@ -141,7 +141,12 @@ fn billing(wire: &EntryWire, rates: &BTreeMap) -> Billing { Some(WebSearchBillingUnit::PerQuery) => WebSearchUnit::PerQuery, Some(WebSearchBillingUnit::PerPrompt) | None => WebSearchUnit::PerPrompt, }, - tier_scope: if provider == Some("dashscope") && wire.tier_rows.is_some() { + tier_scope: if wire + .tier_rows + .iter() + .flatten() + .any(|row| row.range.is_some()) + { TierScope::WholeCall } else { TierScope::PerMeter @@ -958,10 +963,14 @@ mod tests { json!({"litellm_provider": "gemini", "web_search_billing_unit": "per_query"}), Billing { web_search: WebSearchUnit::PerQuery, ..Billing::default() } )] - #[case::dashscope_tiers_bill_the_whole_call( - json!({"litellm_provider": "dashscope", "tiered_pricing": [{"range": [0, 32000], "input_cost_per_token": 1e-6}]}), + #[case::token_tiers_bill_the_whole_call( + json!({"litellm_provider": "openai", "tiered_pricing": [{"range": [0, 32000], "input_cost_per_token": 1e-6}]}), Billing { tier_scope: TierScope::WholeCall, ..Billing::default() } )] + #[case::result_tiers_bill_per_meter( + json!({"litellm_provider": "exa_ai", "tiered_pricing": [{"max_results_range": [1, 25], "input_cost_per_query": 5e-3}]}), + Billing::default() + )] fn billing_is_lowered_from_the_row_and_its_provider( #[case] entry: Value, #[case] expected: Billing, diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index 832dfeeac58..98c2d91274b 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -48,7 +48,9 @@ litellm-secrets-types.workspace = true litellm-llms-types.workspace = true litellm-router-types.workspace = true litellm-host-python.workspace = true +litellm-cost.workspace = true litellm-model-catalog.workspace = true +jiff = { version = "0.2.37", default-features = false, features = ["std"] } litellm-model-catalog-refresh.workspace = true indexmap = "2.14.0" litellm-token-counter = { path = "../token-counter", default-features = false } diff --git a/litellm-rust/crates/python-bridge/src/cost.rs b/litellm-rust/crates/python-bridge/src/cost.rs new file mode 100644 index 00000000000..ce01f7a3f7c --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/cost.rs @@ -0,0 +1,457 @@ +use std::borrow::Cow; +use std::collections::BTreeMap; +use std::sync::Arc; + +use litellm_cost::call::CallContext as CostCallContext; +use litellm_cost::price::{ + Adjustments, CacheReadMode, CacheTtl, Channel, InputUnit, Meter, OutputUnit, + Prices as CostPrices, Residency, ServiceTier, TokenKind, +}; +use litellm_cost::priced::{Outcome, Quantity, Side}; +use litellm_cost::usage::Usage as CostUsage; +use litellm_host_python::from_py; +use litellm_model_catalog::rejects_row; +use litellm_model_catalog::wire::{EntryWire, Fields, FlatKey}; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::*; +use pyo3::types::{PyDict, PyTuple}; + +use crate::model_registry::Invalid; + +/// A row's typed prices and uplifts, an immutable snapshot taken when the row was resolved +#[derive(Default)] +#[pyclass(frozen, name = "Prices", module = "litellm.rust_bridge._native")] +pub(crate) struct PricesHandle { + prices: Arc, + adjustments: Arc, +} + +#[pyclass(frozen, module = "litellm.rust_bridge._native")] +pub(crate) struct Usage(CostUsage); + +/// The call facts pricing reads. A custom pricing overlay that fails validation is kept as its errors +#[pyclass(frozen, module = "litellm.rust_bridge._native")] +pub(crate) struct CallContext { + call: CostCallContext, + invalid_overlay: Option<(Vec, Arc)>, +} + +#[pyclass(frozen, module = "litellm.rust_bridge._native")] +pub(crate) struct Priced { + #[pyo3(get)] + total: f64, + #[pyo3(get)] + prompt: f64, + #[pyo3(get)] + completion: f64, + lines: Vec>, + missing: Vec, +} + +#[pyclass(frozen, module = "litellm.rust_bridge._native")] +pub(crate) struct Line { + #[pyo3(get)] + meter: String, + #[pyo3(get)] + quantity: u64, + #[pyo3(get)] + unit_price: f64, + #[pyo3(get)] + amount: f64, + #[pyo3(get)] + fallback_from: Option, +} + +#[pyclass(frozen, module = "litellm.rust_bridge._native")] +pub(crate) struct Unpriced { + missing: Vec, +} + +impl PricesHandle { + /// Lowers a row the way the catalog lowers its entries, `Err` with every error that rejects the row + pub(crate) fn lower(key: &str, fields: &Fields) -> Result> { + let wire = EntryWire::read(&as_main_reads_tiers(fields)); + let adjustments = Adjustments::from(&wire); + let (prices, price_errors) = match CostPrices::try_from(&wire) { + Ok(prices) => (Some(prices), Vec::new()), + Err(errors) => (None, errors), + }; + let errors: Vec = wire + .problems + .into_iter() + .chain(price_errors) + .filter(|error| rejects_row(&error.error)) + .map(|mut error| { + error.prepend(key); + error.to_string() + }) + .collect(); + match prices { + Some(prices) if errors.is_empty() => Ok(Self { + prices: Arc::new(prices), + adjustments: Arc::new(adjustments), + }), + _ => Err(errors), + } + } +} + +#[pymethods] +impl PricesHandle { + /// Prices for a row that did not come from `resolve`, such as a deployment's `model_info` + #[staticmethod] + #[pyo3(signature = (row, key = "model_info"))] + fn from_row(py: Python<'_>, row: &Bound<'_, PyDict>, key: &str) -> PyResult> { + let (fields, converted) = json_fields(row); + let lowered = match converted { + true => Self::lower(key, &fields), + false => Err(vec![format!("{key} must contain JSON-compatible prices")]), + }; + match lowered { + Ok(prices) => Ok(prices.into_pyobject(py)?.into_any().unbind()), + Err(errors) => Ok(Invalid::new(key.to_owned(), errors, Arc::new(fields)) + .into_pyobject(py)? + .into_any() + .unbind()), + } + } + + /// Pure arithmetic over owned values, it calls no Python and never reads `litellm.model_cost` + fn price(&self, py: Python<'_>, usage: &Usage, call: &CallContext) -> PyResult> { + if let Some((errors, fields)) = &call.invalid_overlay { + return Ok( + Invalid::new("custom_pricing".to_owned(), errors.clone(), fields.clone()) + .into_pyobject(py)? + .into_any() + .unbind(), + ); + } + match litellm_cost::calculate(&self.prices, &self.adjustments, &usage.0, &call.call) { + Outcome::Unpriced { missing } => Ok(Unpriced { + missing: missing.iter().map(meter_name).collect(), + } + .into_pyobject(py)? + .into_any() + .unbind()), + Outcome::Priced(priced) => { + let side_total = |side: Side| { + priced + .lines + .iter() + .filter(|line| line.side == side) + .map(|line| line.amount.get()) + .sum() + }; + let lines = priced + .lines + .iter() + .map(|line| { + Py::new( + py, + Line { + meter: meter_name(&line.meter), + quantity: match line.quantity { + Quantity::Count(count) => count, + Quantity::Duration(duration) => duration.as_secs(), + }, + unit_price: line.unit_price.get(), + amount: line.amount.get(), + fallback_from: line.fallback_from.as_ref().map(meter_name), + }, + ) + }) + .collect::>()?; + Ok(Priced { + total: priced.total.get(), + prompt: side_total(Side::Prompt), + completion: side_total(Side::Completion), + lines, + missing: priced.missing.iter().map(meter_name).collect(), + } + .into_pyobject(py)? + .into_any() + .unbind()) + } + } + } +} + +#[pymethods] +impl Usage { + #[new] + #[pyo3(signature = ( + *, prompt_tokens, text, output_text, audio = 0, image = 0, video = 0, cache_read = 0, cache_read_audio = 0, + cache_read_explicit = 0, cache_write_5m = 0, cache_write_1h = 0, cache_write_12h = 0, output_audio = 0, + output_image = 0, output_video = 0, reasoning = 0, + ))] + #[expect( + clippy::too_many_arguments, + reason = "one keyword argument per token meter, the Python signature" + )] + fn new( + prompt_tokens: u64, + text: u64, + output_text: u64, + audio: u64, + image: u64, + video: u64, + cache_read: u64, + cache_read_audio: u64, + cache_read_explicit: u64, + cache_write_5m: u64, + cache_write_1h: u64, + cache_write_12h: u64, + output_audio: u64, + output_image: u64, + output_video: u64, + reasoning: u64, + ) -> Self { + let mut usage = CostUsage { + prompt_tokens, + ..CostUsage::default() + }; + usage.tokens.input = BTreeMap::from([ + (TokenKind::Text, text), + (TokenKind::Audio, audio), + (TokenKind::Image, image), + (TokenKind::Video, video), + ]); + usage.tokens.cache_read = BTreeMap::from([ + ((TokenKind::Text, CacheReadMode::Implicit), cache_read), + ( + (TokenKind::Text, CacheReadMode::Explicit), + cache_read_explicit, + ), + ( + (TokenKind::Audio, CacheReadMode::Implicit), + cache_read_audio, + ), + ]); + usage.tokens.cache_write = BTreeMap::from([ + ((TokenKind::Text, CacheTtl::Default), cache_write_5m), + ((TokenKind::Text, CacheTtl::OneHour), cache_write_1h), + ((TokenKind::Text, CacheTtl::TwelveHours), cache_write_12h), + ]); + usage.tokens.output = BTreeMap::from([ + (TokenKind::Text, output_text), + (TokenKind::Audio, output_audio), + (TokenKind::Image, output_image), + (TokenKind::Video, output_video), + ]); + usage.tokens.reasoning = reasoning; + Self(usage) + } +} + +#[pymethods] +impl CallContext { + #[new] + #[pyo3(signature = (*, service_tier, channel, data_residency, vertex_location, at, custom_pricing))] + fn new( + service_tier: Option<&str>, + channel: &str, + data_residency: Option<&str>, + vertex_location: Option<&str>, + at: &Bound<'_, PyAny>, + custom_pricing: Option<&Bound<'_, PyDict>>, + ) -> PyResult { + let channel = match channel { + "online" => Channel::Online, + "batch" => Channel::Batch, + "decisions" => Channel::Decisions, + other => return Err(PyValueError::new_err(format!("unknown channel {other:?}"))), + }; + let overlay = custom_pricing.map(|pricing| { + let (fields, converted) = json_fields(pricing); + let lowered = match converted { + true => PricesHandle::lower("custom_pricing", &fields), + false => Err(vec![ + "custom_pricing must contain JSON-compatible prices".to_owned(), + ]), + }; + lowered + .map(|handle| Arc::unwrap_or_clone(handle.prices)) + .map_err(|errors| (errors, Arc::new(fields))) + }); + let (overlay, invalid_overlay) = match overlay { + None => (None, None), + Some(Ok(prices)) => (Some(prices), None), + Some(Err(invalid)) => (None, Some(invalid)), + }; + Ok(Self { + call: CostCallContext { + tier: service_tier.map_or(ServiceTier::Standard, tier), + channel, + residency: data_residency.and_then(residency), + non_global_endpoint: vertex_location + .is_some_and(|location| !location.eq_ignore_ascii_case("global")), + speed: None, + inference_geo: None, + at: timestamp(at)?, + overlay, + }, + invalid_overlay, + }) + } +} + +#[pymethods] +impl Priced { + #[getter] + fn lines<'py>(&self, py: Python<'py>) -> PyResult> { + PyTuple::new(py, self.lines.iter().map(|line| line.clone_ref(py))) + } + + #[getter] + fn missing<'py>(&self, py: Python<'py>) -> PyResult> { + PyTuple::new(py, &self.missing) + } +} + +#[pymethods] +impl Unpriced { + #[getter] + fn missing<'py>(&self, py: Python<'py>) -> PyResult> { + PyTuple::new(py, &self.missing) + } +} + +/// The row's fields that convert to JSON, and whether every price did. A price only Python can hold, such as `nan`, +/// makes the row invalid (decision 20). Any other such value, a router deployment's `datetime` say, is left out +fn json_fields(row: &Bound<'_, PyDict>) -> (Fields, bool) { + let converted: Vec<(String, Option)> = row + .iter() + .filter_map(|(name, value)| { + let non_finite = value + .extract::() + .is_ok_and(|number| !number.is_finite()); + Some(( + name.extract::().ok()?, + from_py(&value).ok().filter(|_| !non_finite), + )) + }) + .collect(); + let prices_convert = converted + .iter() + .all(|(name, value)| value.is_some() || FlatKey::parse(name).is_none()); + let fields = converted + .into_iter() + .filter_map(|(name, value)| Some((name, value?))) + .collect(); + (fields, prices_convert) +} + +/// Flat prices a priced tier table replaces, main's `_get_tiered_base_costs`. The row's own output price stays as the +/// fallback for a tier that sets none +const TIER_REPLACED_PRICES: [&str; 5] = [ + "input_cost_per_token", + "cache_read_input_token_cost", + "cache_creation_input_token_cost", + "cache_creation_input_token_cost_above_1hr", + "cache_creation_input_token_cost_above_12hr", +]; + +/// main bills a call with a prompt-token tier table at its tier's rates and never at the row's flat ones, and ignores +/// a table whose first tier sets no input price. The catalog lowers the first tier and the flat prices into one band, +/// so the flat prices the table replaces are dropped before lowering +fn as_main_reads_tiers(fields: &Fields) -> Cow<'_, Fields> { + let first_tier = fields + .get("tiered_pricing") + .and_then(serde_json::Value::as_array) + .and_then(|tiers| { + tiers + .iter() + .filter_map(serde_json::Value::as_object) + .find(|tier| { + tier.get("range") + .and_then(|range| range.get(0)) + .and_then(serde_json::Value::as_u64) + == Some(0) + }) + }); + let Some(first_tier) = first_tier else { + return Cow::Borrowed(fields); + }; + let priced = first_tier.contains_key("input_cost_per_token"); + let replaced = |name: &str| match priced { + true => TIER_REPLACED_PRICES.contains(&name) || first_tier.contains_key(name), + false => name == "tiered_pricing", + }; + Cow::Owned( + fields + .iter() + .filter(|(name, _)| !replaced(name)) + .map(|(name, value)| (name.clone(), value.clone())) + .collect(), + ) +} + +/// main's `_SERVICE_TIER_TO_COST_KEY_SUFFIX`, `"fast"` bills the priority tier +fn tier(service_tier: &str) -> ServiceTier { + match service_tier.to_lowercase().as_str() { + "priority" | "fast" => ServiceTier::Priority, + "flex" => ServiceTier::Flex, + "balanced" => ServiceTier::Balanced, + "ultrafast" => ServiceTier::Ultrafast, + _ => ServiceTier::Standard, + } +} + +fn residency(data_residency: &str) -> Option { + match data_residency.to_lowercase().as_str() { + "eu" => Some(Residency::Eu), + "us" => Some(Residency::Us), + _ => None, + } +} + +/// An aware datetime is read as its instant, a naive one as UTC, as main's off-peak check does +fn timestamp(at: &Bound<'_, PyAny>) -> PyResult { + let aware = if at.getattr("tzinfo")?.is_none() { + let utc = at + .py() + .import("datetime")? + .getattr("timezone")? + .getattr("utc")?; + let replace_kwargs = PyDict::new(at.py()); + replace_kwargs.set_item("tzinfo", utc)?; + at.call_method("replace", (), Some(&replace_kwargs))? + } else { + at.clone() + }; + let seconds: f64 = aware.call_method0("timestamp")?.extract()?; + jiff::Timestamp::from_microsecond((seconds * 1e6).round() as i64) + .map_err(|error| PyValueError::new_err(format!("billing time out of range: {error}"))) +} + +/// main's flat key name for the meter, the name a caller already knows from the JSON +fn meter_name(meter: &Meter) -> String { + let token = |kind: &TokenKind| match kind { + TokenKind::Text => "", + TokenKind::Audio => "audio_", + TokenKind::Image => "image_", + TokenKind::Video => "video_", + }; + match meter { + Meter::Input(InputUnit::Token(kind)) => format!("input_cost_per_{}token", token(kind)), + Meter::Output(OutputUnit::Token(kind)) => format!("output_cost_per_{}token", token(kind)), + Meter::Output(OutputUnit::ReasoningToken) => "output_cost_per_reasoning_token".to_owned(), + Meter::CacheRead { + kind, + mode: CacheReadMode::Implicit, + } => format!("cache_read_input_{}token_cost", token(kind)), + Meter::CacheRead { + kind, + mode: CacheReadMode::Explicit, + } => format!("explicit_cache_read_input_{}token_cost", token(kind)), + Meter::CacheWrite { kind, ttl } => { + let ttl = match ttl { + CacheTtl::Default => "", + CacheTtl::OneHour => "_above_1hr", + CacheTtl::TwelveHours => "_above_12hr", + }; + format!("cache_creation_input_{}token_cost{ttl}", token(kind)) + } + other => format!("{other:?}"), + } +} diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index 0c8a4885263..1c7f69ad25d 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -1,6 +1,7 @@ mod cache; mod callable; mod coercion; +mod cost; mod credentials; mod diagnostics; mod errors; @@ -24,6 +25,8 @@ mod _native { types::{PyModule, PyTuple}, }; + #[pymodule_export] + use crate::cost::{CallContext, Line, Priced, PricesHandle, Unpriced, Usage}; #[cfg(feature = "panic-test")] #[pymodule_export] use crate::diagnostics::_panic_for_test; diff --git a/litellm-rust/crates/python-bridge/src/model_registry/resolve.rs b/litellm-rust/crates/python-bridge/src/model_registry/resolve.rs index feb11d0e432..e2313679150 100644 --- a/litellm-rust/crates/python-bridge/src/model_registry/resolve.rs +++ b/litellm-rust/crates/python-bridge/src/model_registry/resolve.rs @@ -11,6 +11,7 @@ use pyo3::types::{PyDict, PyTuple}; use serde_json::Value; use super::name_index::NameIndex; +use crate::cost::PricesHandle; const CANDIDATES_MODULE: &str = "litellm.rust_bridge.host.model_names"; @@ -52,6 +53,22 @@ impl Resolved { fn row(&self, py: Python<'_>) -> PyResult> { to_py(py, self.fields.as_ref()) } + + /// The row's typed prices and uplifts. `resolve` already reported a row whose prices do not lower as `Invalid` + #[getter] + fn prices(&self) -> PricesHandle { + PricesHandle::lower(&self.key, &self.fields).unwrap_or_default() + } +} + +impl Invalid { + pub(crate) fn new(key: String, errors: Vec, fields: Arc) -> Self { + Self { + key, + errors, + fields, + } + } } #[pymethods] diff --git a/litellm/_catalog/__init__.py b/litellm/_catalog/__init__.py index 441fb5a0f8f..e372aecf569 100644 --- a/litellm/_catalog/__init__.py +++ b/litellm/_catalog/__init__.py @@ -4,6 +4,7 @@ import os import sys from typing import Final +from litellm._logging import verbose_logger from litellm.litellm_core_utils.offline_mode import is_offline_mode from litellm.rust_bridge._native import ( Bundled, @@ -26,6 +27,16 @@ Source = Remote | File | Bundled CATALOG: Final = Catalog.bound_to(sys.modules["litellm"], "model_cost") +_REPORTED_INVALID_ROWS: Final[set[str]] = set() # mutable-ok: logs each bad live row once per process + + +def report_invalid(invalid: Invalid) -> None: + """Logs a row that fails validation, once per key per process (decision 20)""" + if invalid.key in _REPORTED_INVALID_ROWS: + return + _REPORTED_INVALID_ROWS.add(invalid.key) + verbose_logger.warning("litellm.model_cost[%r] fails validation: %s", invalid.key, "; ".join(invalid.errors)) + def default_source() -> Source: """Main's reload rule, the bundled map under offline mode or ``LITELLM_LOCAL_MODEL_COST_MAP``""" @@ -54,4 +65,5 @@ __all__ = [ "Unavailable", "default_source", "overlay", + "report_invalid", ] diff --git a/litellm/_cost/__init__.py b/litellm/_cost/__init__.py new file mode 100644 index 00000000000..25e5dccbd7a --- /dev/null +++ b/litellm/_cost/__init__.py @@ -0,0 +1,56 @@ +"""Internal pricing API backed by the Rust ``cost`` crate. Not public, its shape may change in any release""" + +from collections.abc import Generator, Mapping +from contextlib import contextmanager +from contextvars import ContextVar +from typing import Final, Literal + +from litellm._catalog import Invalid, report_invalid +from litellm.rust_bridge._native import CallContext, Line, Priced, Prices, Unpriced, Usage + +Channel = Literal["online", "batch", "decisions"] + +_CALL_CUSTOM_PRICING: Final[ContextVar[Mapping[str, object] | None]] = ContextVar( + "litellm_call_custom_pricing", default=None +) + + +@contextmanager +def custom_pricing_for_call(custom_pricing: Mapping[str, object] | None) -> Generator[None]: + """Prices every token meter of the call inside the block with ``custom_pricing`` over its row (decision 12). + The provider cost calculators keep their signatures, so the overlay reaches them through the call's context""" + token: Final = _CALL_CUSTOM_PRICING.set(custom_pricing) + try: + yield + finally: + _CALL_CUSTOM_PRICING.reset(token) + + +def call_custom_pricing() -> Mapping[str, object] | None: + return _CALL_CUSTOM_PRICING.get() + + +def as_main_cost(outcome: Priced | Unpriced | Invalid) -> tuple[float, float]: + """Main's ``(prompt, completion)`` pair, ``(0.0, 0.0)`` for anything unpriced (decisions 14 and 20)""" + match outcome: + case Priced(): + return outcome.prompt, outcome.completion + case Unpriced(): + return 0.0, 0.0 + case Invalid(): + report_invalid(outcome) + return 0.0, 0.0 + + +__all__ = [ + "CallContext", + "Channel", + "Line", + "Priced", + "Prices", + "Unpriced", + "Usage", + "as_main_cost", + "call_custom_pricing", + "custom_pricing_for_call", +] diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index bc1bdf64825..793475890bc 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -16,6 +16,7 @@ from typing_extensions import ReadOnly, TypedDict, assert_never import litellm import litellm._logging from litellm import verbose_logger +from litellm._cost import call_custom_pricing, custom_pricing_for_call from litellm.constants import ( DEFAULT_MAX_LRU_CACHE_SIZE, DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND, @@ -43,6 +44,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import ( generic_cost_per_token, get_batch_cost_rates, get_billable_input_tokens, + get_cost_per_unit, get_regional_uplift_multiplier, get_service_tier_cost_key, get_token_type_cost_breakdown, @@ -128,6 +130,7 @@ from litellm.types.llms.xai import RealtimeMeteredUsage from litellm.types.rerank import RerankBilledUnits, RerankResponse from litellm.types.utils import ( CallTypesLiteral, + CustomPricingLiteLLMParams, LiteLLMRealtimeStreamLoggingObject, LlmProviders, LlmProvidersSet, @@ -219,57 +222,6 @@ _AREALTIME_CALL_TYPE: Final = CallTypes.arealtime.value _MCP_CALL_TYPE: Final = CallTypes.call_mcp_tool.value -def _cost_per_token_custom_pricing_helper( - prompt_tokens: float = 0, - completion_tokens: float = 0, - response_time_ms: float | None = 0.0, - cached_tokens: float = 0, - cache_creation_tokens: float = 0, - ### CUSTOM PRICING ### - custom_cost_per_token: CostPerToken | None = None, - custom_cost_per_second: float | None = None, -) -> tuple[float, float] | None: - """Internal helper function for calculating cost, if custom pricing given. - - prompt_tokens is assumed to include both cached_tokens and cache_creation_tokens - (OpenAI-compatible convention). Anthropic-style usage where prompt_tokens excludes - cache tokens is handled at the caller (cost_per_token) before invoking this helper. - """ - if custom_cost_per_token is None and custom_cost_per_second is None: - return None - - if custom_cost_per_token is not None: - input_cost_per_token: Final = custom_cost_per_token["input_cost_per_token"] - output_cost_per_token: Final = custom_cost_per_token["output_cost_per_token"] - - cache_read_input_token_cost: Final = custom_cost_per_token.get( - "cache_read_input_token_cost", - input_cost_per_token, - ) - cache_creation_input_token_cost: Final = custom_cost_per_token.get( - "cache_creation_input_token_cost", - input_cost_per_token, - ) - - regular_prompt_tokens: Final = max( - prompt_tokens - cached_tokens - cache_creation_tokens, - 0, - ) - - input_cost: Final = ( - regular_prompt_tokens * input_cost_per_token - + cached_tokens * cache_read_input_token_cost - + cache_creation_tokens * cache_creation_input_token_cost - ) - output_cost = completion_tokens * output_cost_per_token - return input_cost, output_cost - elif custom_cost_per_second is not None: - output_cost = custom_cost_per_second * (response_time_ms or 0.0) / 1000 - return 0, output_cost - - return None - - def _get_additional_costs( model: str, custom_llm_provider: str | None, @@ -358,8 +310,8 @@ _WALL_CLOCK_PRICED_MODES: Final = frozenset({"audio_transcription", "chat", "com def _has_token_or_tiered_pricing(model_info: ModelInfoBase) -> bool: return ( - (model_info.get("input_cost_per_token") or 0.0) > 0 - or (model_info.get("output_cost_per_token") or 0.0) > 0 + (get_cost_per_unit(model_info, "input_cost_per_token") or 0.0) > 0 + or (get_cost_per_unit(model_info, "output_cost_per_token") or 0.0) > 0 or model_info.get("tiered_pricing") is not None ) @@ -369,15 +321,65 @@ def _bills_wall_clock_seconds(model_info: ModelInfoBase) -> bool: return mode is None or mode in _WALL_CLOCK_PRICED_MODES +def _flat_custom_pricing_usage( + usage_object: Usage | None, + prompt_tokens: int, + completion_tokens: int, + cache_read_input_tokens: int | None, + cache_creation_input_tokens: int | None, +) -> Usage: + """The usage flat custom rates bill: prompt tokens including cache tokens, cache reads, cache writes and + completion tokens, nothing else. + + OpenAI-compatible usage reports cache tokens in ``prompt_tokens_details`` and counts them in ``prompt_tokens``. + Anthropic reports them at the top level and leaves them out of ``prompt_tokens`` + """ + details: Final = getattr(usage_object, "prompt_tokens_details", None) + detail_read: Final = float(getattr(details, "cached_tokens", 0) or 0) + detail_creation: Final = float( + getattr(details, "cache_write_tokens", 0) or getattr(details, "cache_creation_tokens", 0) or 0 + ) + anthropic_read: Final[int | None] = getattr(usage_object, "cache_read_input_tokens", None) + anthropic_creation: Final[int | None] = getattr(usage_object, "cache_creation_input_tokens", None) + reported_read: Final = detail_read if anthropic_read is None else float(anthropic_read) + reported_creation: Final = detail_creation if anthropic_creation is None else float(anthropic_creation) + cache_read: Final = reported_read or float(cache_read_input_tokens or 0) + cache_creation: Final = reported_creation or float(cache_creation_input_tokens or 0) + anthropic_style: Final = ( + anthropic_read is not None + or anthropic_creation is not None + or (not reported_read and bool(cache_read_input_tokens)) + or (not reported_creation and bool(cache_creation_input_tokens)) + ) + billed_prompt: Final = prompt_tokens + (cache_read + cache_creation if anthropic_style else 0) + return Usage( + prompt_tokens=int(billed_prompt), + completion_tokens=completion_tokens, + total_tokens=int(billed_prompt) + completion_tokens, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=int(cache_read), cache_creation_tokens=int(cache_creation) + ), + ) + + +def _row_with_call_pricing(model: str, custom_llm_provider: str | None) -> ModelInfo | None: + """The row with the call's custom prices over it. A call with custom pricing prices an unmapped model from them""" + custom_pricing: Final = call_custom_pricing() or {} + try: + row: Final = cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) + except Exception: # noqa: BLE001 # the lookup raises plain Exception for an unmapped model + return cast(ModelInfo, custom_pricing) if custom_pricing else None # cast-ok: the call's prices are its row + return cast(ModelInfo, {**row, **custom_pricing}) # cast-ok: the call's prices over its row + + def _per_second_pricing_cost( model: str, custom_llm_provider: str | None, response_time_ms: float | None, audio_seconds: float = 0.0, ) -> tuple[float, float] | None: - try: - model_info: Final = cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) - except Exception: # noqa: BLE001 # the lookup raises plain Exception for an unmapped model + model_info: Final = _row_with_call_pricing(model, custom_llm_provider) + if model_info is None: return None if _has_token_or_tiered_pricing(model_info) or not _bills_wall_clock_seconds(model_info): return None @@ -479,61 +481,18 @@ def cost_per_token( cache_read_input_tokens=cache_read_input_tokens, ) - ## CUSTOM PRICING ## - # Normalize cache token counts across providers: - # - OpenAI-compatible: usage.prompt_tokens_details.cached_tokens - # (prompt_tokens already INCLUDES cached_tokens) - # - Anthropic: usage.cache_read_input_tokens / cache_creation_input_tokens - # (prompt_tokens does NOT include these — adjust before calling helper) - _cache_read_tokens: float = 0 - _cache_creation_tokens: float = 0 - _is_anthropic_style = False - - if usage_object is not None: - _pt_details: Final[PromptTokensDetailsWrapper | None] = getattr(usage_object, "prompt_tokens_details", None) - if _pt_details is not None: - _cache_read_tokens = float(getattr(_pt_details, "cached_tokens", 0) or 0) - # OpenAI-compatible providers report cache-write tokens under - # either `cache_write_tokens` (kimi-k2) or `cache_creation_tokens`. - # Mirror db_spend_update_writer to stay symmetric. - _cache_creation_tokens = float( - getattr(_pt_details, "cache_write_tokens", 0) or getattr(_pt_details, "cache_creation_tokens", 0) or 0 + if custom_cost_per_token is not None: + with custom_pricing_for_call(dict(custom_cost_per_token)): + return generic_cost_per_token( + model=model, + usage=_flat_custom_pricing_usage( + usage_object, prompt_tokens, completion_tokens, cache_read_input_tokens, cache_creation_input_tokens + ), + custom_llm_provider=custom_llm_provider or "", + model_info=cast(ModelInfo, {}), # cast-ok: flat custom rates price the call without the row ) - - _anthropic_read: Final[int | None] = getattr(usage_object, "cache_read_input_tokens", None) - _anthropic_create: Final[int | None] = getattr(usage_object, "cache_creation_input_tokens", None) - if _anthropic_read is not None or _anthropic_create is not None: - _is_anthropic_style = True - if _anthropic_read is not None: - _cache_read_tokens = float(_anthropic_read) - if _anthropic_create is not None: - _cache_creation_tokens = float(_anthropic_create) - - if not _cache_read_tokens and cache_read_input_tokens: - _cache_read_tokens = float(cache_read_input_tokens) - _is_anthropic_style = True - if not _cache_creation_tokens and cache_creation_input_tokens: - _cache_creation_tokens = float(cache_creation_input_tokens) - _is_anthropic_style = True - - # Anthropic reports prompt_tokens as input_tokens (excluding cache tokens). - # Adjust so the helper's "prompt_tokens includes cache tokens" invariant holds. - _normalized_prompt_tokens = float(prompt_tokens) - if _is_anthropic_style: - _normalized_prompt_tokens += _cache_read_tokens + _cache_creation_tokens - - response_cost: Final = _cost_per_token_custom_pricing_helper( - prompt_tokens=_normalized_prompt_tokens, - completion_tokens=completion_tokens, - response_time_ms=response_time_ms, - cached_tokens=_cache_read_tokens, - cache_creation_tokens=_cache_creation_tokens, - custom_cost_per_second=custom_cost_per_second, - custom_cost_per_token=custom_cost_per_token, - ) - - if response_cost is not None: - return response_cost[0], response_cost[1] + if custom_cost_per_second is not None: + return 0.0, custom_cost_per_second * (response_time_ms or 0.0) / 1000 model_cost_ref: Final = litellm.model_cost # Only callers that explicitly pass `custom_llm_provider` get the @@ -719,6 +678,16 @@ def cost_per_token( ) ) is not None: return per_second_cost + elif call_custom_pricing() is not None and _cost_map_model_info(model, custom_llm_provider) is None: + return generic_cost_per_token( + model=model, + usage=usage_block, + custom_llm_provider=custom_llm_provider, + service_tier=service_tier, + data_residency=data_residency, + vertex_location=vertex_location, + model_info=cast(ModelInfo, {}), # cast-ok: an unmapped model is priced from the call's prices alone + ) elif custom_llm_provider == "vertex_ai": cost_router: Final = google_cost_router( model=model_without_prefix, @@ -1505,6 +1474,26 @@ def _response_time_ms_for_cost( return 0.0 +def _call_custom_pricing( + custom_pricing: bool | None, + router_model_id: str | None, + litellm_logging_obj: LitellmLoggingObject | None, +) -> dict[str, object] | None: + """A direct SDK call's own prices, applied over the row for this call only and never written to it (decision 12). + A router deployment's prices are registered under its id when the deployment is added""" + if not custom_pricing or litellm_logging_obj is None: + return None + if router_model_id is not None and _raw_cost_map_entry(router_model_id) is not None: + return None + litellm_params: Final = litellm_logging_obj.litellm_params or {} + pricing: Final = { + field: value + for field in CustomPricingLiteLLMParams.model_fields + if (value := litellm_params.get(field)) is not None + } + return pricing or None + + def completion_cost( completion_response: object | None = None, model: str | None = None, @@ -1549,641 +1538,652 @@ def completion_cost( Raises if no candidate model name resolves to pricing. Register the model via `litellm.register_model` or add it to model_prices_and_context_window.json. """ - try: - call_type = infer_call_type(call_type, completion_response) or "completion" - if isinstance(completion_response, DecisionsResponse) and completion_response.usage is None: - raise ValueError("Cannot calculate Decisions cost: provider did not report usage") - if pricing_provider is not None: - custom_llm_provider = pricing_provider # rebind-ok: model_info.pricing_provider overrides the routing provider for cost-map lookups + with custom_pricing_for_call(_call_custom_pricing(custom_pricing, router_model_id, litellm_logging_obj)): + try: + call_type = infer_call_type(call_type, completion_response) or "completion" + if isinstance(completion_response, DecisionsResponse) and completion_response.usage is None: + raise ValueError("Cannot calculate Decisions cost: provider did not report usage") + if pricing_provider is not None: + custom_llm_provider = pricing_provider # rebind-ok: model_info.pricing_provider overrides the routing provider for cost-map lookups - if call_type == CallTypes.aresponses_websocket.value and isinstance( - completion_response, LiteLLMRealtimeStreamLoggingObject - ): - ws_tier_parts: Final = _split_responses_ws_logging_object_by_service_tier(completion_response) - if ws_tier_parts is not None: - return sum( - completion_cost( - completion_response=part, - model=model, - prompt=prompt, - messages=messages, - completion=completion, - total_time=total_time, - call_type=call_type, - custom_llm_provider=custom_llm_provider, - region_name=region_name, - size=size, - quality=quality, - n=n, - custom_cost_per_token=custom_cost_per_token, - custom_cost_per_second=custom_cost_per_second, - optional_params=optional_params, - custom_pricing=custom_pricing, - base_model=base_model, - pricing_provider=pricing_provider, - standard_built_in_tools_params=standard_built_in_tools_params, - litellm_model_name=litellm_model_name, - router_model_id=router_model_id, - litellm_logging_obj=litellm_logging_obj, - service_tier=service_tier, - data_residency=data_residency, - vertex_location=vertex_location, + if call_type == CallTypes.aresponses_websocket.value and isinstance( + completion_response, LiteLLMRealtimeStreamLoggingObject + ): + ws_tier_parts: Final = _split_responses_ws_logging_object_by_service_tier(completion_response) + if ws_tier_parts is not None: + return sum( + completion_cost( + completion_response=part, + model=model, + prompt=prompt, + messages=messages, + completion=completion, + total_time=total_time, + call_type=call_type, + custom_llm_provider=custom_llm_provider, + region_name=region_name, + size=size, + quality=quality, + n=n, + custom_cost_per_token=custom_cost_per_token, + custom_cost_per_second=custom_cost_per_second, + optional_params=optional_params, + custom_pricing=custom_pricing, + base_model=base_model, + pricing_provider=pricing_provider, + standard_built_in_tools_params=standard_built_in_tools_params, + litellm_model_name=litellm_model_name, + router_model_id=router_model_id, + litellm_logging_obj=litellm_logging_obj, + service_tier=service_tier, + data_residency=data_residency, + vertex_location=vertex_location, + ) + for part in ws_tier_parts ) - for part in ws_tier_parts + + if ( + (call_type == "aimage_generation" or call_type == "image_generation") + and model is not None + and isinstance(model, str) + and len(model) == 0 + and custom_llm_provider == "azure" + ): + model = "dall-e-2" # for dall-e-2, azure expects an empty model name + # Handle Inputs to completion_cost + prompt_tokens = 0 + prompt_characters: int | None = None + completion_tokens = 0 + completion_characters: int | None = None + cache_creation_input_tokens: int | None = None + cache_read_input_tokens: int | None = None + audio_transcription_file_duration: float = 0.0 + provider_usage_object: Final = get_usage_object(completion_response=completion_response) + cost_per_token_usage_object: Final[Usage | None] = ( + _without_provider_stated_cost(provider_usage_object) if custom_pricing else provider_usage_object + ) + rerank_billed_units: RerankBilledUnits | None = None + + explicit_tier: Final = _normalize_service_tier(service_tier) + if explicit_tier is not None: + service_tier = explicit_tier + else: + service_tier = _resolve_billable_service_tier( # rebind-ok: resolved from request then response + requested=optional_params.get("service_tier") if optional_params is not None else None, + served=_served_service_tier(completion_response, cost_per_token_usage_object), ) - if ( - (call_type == "aimage_generation" or call_type == "image_generation") - and model is not None - and isinstance(model, str) - and len(model) == 0 - and custom_llm_provider == "azure" - ): - model = "dall-e-2" # for dall-e-2, azure expects an empty model name - # Handle Inputs to completion_cost - prompt_tokens = 0 - prompt_characters: int | None = None - completion_tokens = 0 - completion_characters: int | None = None - cache_creation_input_tokens: int | None = None - cache_read_input_tokens: int | None = None - audio_transcription_file_duration: float = 0.0 - provider_usage_object: Final = get_usage_object(completion_response=completion_response) - cost_per_token_usage_object: Final[Usage | None] = ( - _without_provider_stated_cost(provider_usage_object) if custom_pricing else provider_usage_object - ) - rerank_billed_units: RerankBilledUnits | None = None - - explicit_tier: Final = _normalize_service_tier(service_tier) - if explicit_tier is not None: - service_tier = explicit_tier - else: - service_tier = _resolve_billable_service_tier( # rebind-ok: resolved from request then response - requested=optional_params.get("service_tier") if optional_params is not None else None, - served=_served_service_tier(completion_response, cost_per_token_usage_object), + explicit_pricing: Final = custom_pricing is True or base_model is not None + custom_priced_request_model: Final = model if custom_pricing is True else None + configured_cost_per_query: Final = ( + _configured_search_cost_per_query(litellm_logging_obj) if custom_pricing else None + ) + base_model_provider: Final = ( + _base_model_pricing_provider( + base_model, + _get_provider_for_cost_calc( + model=model, + custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None, + ), + ) + if base_model is not None and custom_pricing is not True + else None + ) + invoked_model_id: Final = _invoked_model_id_for_cost( + _LOGGED_LITELLM_PARAMS.validate_python(optional_params) if optional_params is not None else None, + litellm_logging_obj, + ) + selected_model: Final = select_model_name_for_cost_calc( + model=model, + completion_response=completion_response, + custom_llm_provider=base_model_provider or custom_llm_provider, + custom_pricing=custom_pricing, + base_model=base_model, + router_model_id=router_model_id, + region_name=region_name, + invoked_model_id=invoked_model_id, ) - explicit_pricing: Final = custom_pricing is True or base_model is not None - custom_priced_request_model: Final = model if custom_pricing is True else None - configured_cost_per_query: Final = ( - _configured_search_cost_per_query(litellm_logging_obj) if custom_pricing else None - ) - base_model_provider: Final = ( - _base_model_pricing_provider( - base_model, - _get_provider_for_cost_calc( - model=model, - custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None, + bedrock_model_info: Final = _bedrock_pricing_info( + model or _get_response_model(completion_response), + invoked_model_id, + _get_provider_for_cost_calc(model, custom_llm_provider), + explicit_pricing, + ) + requested_model: Final = model + request_pricing_provider: Final = custom_llm_provider if isinstance(custom_llm_provider, str) else None + raw_candidates: Final = ( + ( + selected_model, + base_model_provider if base_model_provider is not None else request_pricing_provider, ), + (_get_response_model(completion_response), request_pricing_provider), + *(((model, request_pricing_provider),) if model is not None else ()), + ) + potential_model_names: Final = ( + raw_candidates if custom_pricing is True else _priced_candidates_before_unpriced_rules(raw_candidates) ) - if base_model is not None and custom_pricing is not True - else None - ) - invoked_model_id: Final = _invoked_model_id_for_cost( - _LOGGED_LITELLM_PARAMS.validate_python(optional_params) if optional_params is not None else None, - litellm_logging_obj, - ) - selected_model: Final = select_model_name_for_cost_calc( - model=model, - completion_response=completion_response, - custom_llm_provider=base_model_provider or custom_llm_provider, - custom_pricing=custom_pricing, - base_model=base_model, - router_model_id=router_model_id, - region_name=region_name, - invoked_model_id=invoked_model_id, - ) - bedrock_model_info: Final = _bedrock_pricing_info( - model or _get_response_model(completion_response), - invoked_model_id, - _get_provider_for_cost_calc(model, custom_llm_provider), - explicit_pricing, - ) - requested_model: Final = model - request_pricing_provider: Final = custom_llm_provider if isinstance(custom_llm_provider, str) else None - raw_candidates: Final = ( - ( - selected_model, - base_model_provider if base_model_provider is not None else request_pricing_provider, - ), - (_get_response_model(completion_response), request_pricing_provider), - *(((model, request_pricing_provider),) if model is not None else ()), - ) - potential_model_names: Final = ( - raw_candidates if custom_pricing is True else _priced_candidates_before_unpriced_rules(raw_candidates) - ) + for idx, (model, candidate_pricing_provider) in enumerate(potential_model_names): + try: + if verbose_logger.isEnabledFor(logging.DEBUG): + verbose_logger.debug("selected model name for cost calculation: %s", model) - for idx, (model, candidate_pricing_provider) in enumerate(potential_model_names): - try: - if verbose_logger.isEnabledFor(logging.DEBUG): - verbose_logger.debug("selected model name for cost calculation: %s", model) - - if completion_response is not None and ( - isinstance(completion_response, (BaseModel, dict)) - or isinstance(completion_response, HttpxBinaryResponseContent) - and completion_response.usage is not None - ): - if isinstance(completion_response, dict): - usage_obj: dict | Usage | None = completion_response.get("usage", {}) - else: - usage_obj = getattr(completion_response, "usage", {}) - if isinstance(usage_obj, BaseModel) and not _is_known_usage_objects(usage_obj=usage_obj): - _usage_for_dump = cast(BaseModel, usage_obj) - setattr( - completion_response, - "usage", - _usage_from_model(_usage_for_dump), - ) - if isinstance(usage_obj, (DecisionsUsage, OpenAIDecisionUsage)): - _usage = _decisions_usage(usage_obj).model_dump() - elif isinstance(usage_obj, ExtractionUsage): - _usage = _extraction_usage(usage_obj).model_dump() - elif usage_obj is None: - _usage = {} - elif isinstance(usage_obj, BaseModel): - _usage = cast(BaseModel, usage_obj).model_dump() - else: - _usage = usage_obj - - if litellm.AnthropicConfig.is_anthropic_usage_object(_usage): - _usage = ( - litellm.AnthropicConfig() - .calculate_usage(usage_object=_usage, reasoning_content=None) - .model_dump() - ) - elif ResponseAPILoggingUtils.is_response_api_usage(_usage): - _usage = ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(_usage).model_dump() - elif TranscriptionUsageObjectTransformation.is_transcription_usage_object(_usage): - tr_usage = TranscriptionUsageObjectTransformation.transform_transcription_usage_object( - cast( - TranscriptionUsageDurationObject | TranscriptionUsageTokensObject, - _usage, - ) - ) - if tr_usage is not None: - _usage = tr_usage.model_dump() - elif InteractionsUsageObjectTransformation.is_interactions_usage_object(_usage): - _usage = InteractionsUsageObjectTransformation.transform_interactions_usage_object( - _usage - ).model_dump() - else: - _usage = _usage - - # get input/output tokens from completion_response - prompt_tokens = _usage.get("prompt_tokens", 0) - completion_tokens = _usage.get("completion_tokens", 0) - cache_creation_input_tokens = _usage.get("cache_creation_input_tokens", 0) - cache_read_input_tokens = _usage.get("cache_read_input_tokens", 0) - if ( - "prompt_tokens_details" in _usage - and _usage["prompt_tokens_details"] != {} - and _usage["prompt_tokens_details"] + if completion_response is not None and ( + isinstance(completion_response, (BaseModel, dict)) + or isinstance(completion_response, HttpxBinaryResponseContent) + and completion_response.usage is not None ): - prompt_tokens_details = _usage.get("prompt_tokens_details") or {} - cache_read_input_tokens = prompt_tokens_details.get("cached_tokens", 0) + if isinstance(completion_response, dict): + usage_obj: dict | Usage | None = completion_response.get("usage", {}) + else: + usage_obj = getattr(completion_response, "usage", {}) + if isinstance(usage_obj, BaseModel) and not _is_known_usage_objects(usage_obj=usage_obj): + _usage_for_dump = cast(BaseModel, usage_obj) + setattr( + completion_response, + "usage", + _usage_from_model(_usage_for_dump), + ) + if isinstance(usage_obj, (DecisionsUsage, OpenAIDecisionUsage)): + _usage = _decisions_usage(usage_obj).model_dump() + elif isinstance(usage_obj, ExtractionUsage): + _usage = _extraction_usage(usage_obj).model_dump() + elif usage_obj is None: + _usage = {} + elif isinstance(usage_obj, BaseModel): + _usage = cast(BaseModel, usage_obj).model_dump() + else: + _usage = usage_obj - hidden_params = getattr(completion_response, "_hidden_params", None) - if hidden_params is not None: - custom_llm_provider = pricing_provider or hidden_params.get( - "custom_llm_provider", custom_llm_provider or None - ) - region_name = hidden_params.get("region_name", region_name) + if litellm.AnthropicConfig.is_anthropic_usage_object(_usage): + _usage = ( + litellm.AnthropicConfig() + .calculate_usage(usage_object=_usage, reasoning_content=None) + .model_dump() + ) + elif ResponseAPILoggingUtils.is_response_api_usage(_usage): + _usage = ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage( + _usage + ).model_dump() + elif TranscriptionUsageObjectTransformation.is_transcription_usage_object(_usage): + tr_usage = TranscriptionUsageObjectTransformation.transform_transcription_usage_object( + cast( + TranscriptionUsageDurationObject | TranscriptionUsageTokensObject, + _usage, + ) + ) + if tr_usage is not None: + _usage = tr_usage.model_dump() + elif InteractionsUsageObjectTransformation.is_interactions_usage_object(_usage): + _usage = InteractionsUsageObjectTransformation.transform_interactions_usage_object( + _usage + ).model_dump() + else: + _usage = _usage + + # get input/output tokens from completion_response + prompt_tokens = _usage.get("prompt_tokens", 0) + completion_tokens = _usage.get("completion_tokens", 0) + cache_creation_input_tokens = _usage.get("cache_creation_input_tokens", 0) + cache_read_input_tokens = _usage.get("cache_read_input_tokens", 0) + if ( + "prompt_tokens_details" in _usage + and _usage["prompt_tokens_details"] != {} + and _usage["prompt_tokens_details"] + ): + prompt_tokens_details = _usage.get("prompt_tokens_details") or {} + cache_read_input_tokens = prompt_tokens_details.get("cached_tokens", 0) + + hidden_params = getattr(completion_response, "_hidden_params", None) + if hidden_params is not None: + custom_llm_provider = pricing_provider or hidden_params.get( + "custom_llm_provider", custom_llm_provider or None + ) + region_name = hidden_params.get("region_name", region_name) + + else: + if model is None: + raise ValueError( + f"Model is None and does not exist in passed completion_response. Passed completion_response={completion_response}, model={model}" + ) + if len(messages) > 0: + prompt_tokens = token_counter(model=model, messages=messages) + elif len(prompt) > 0: + prompt_tokens = token_counter(model=model, text=prompt) + completion_tokens = token_counter(model=model, text=completion) + + # Handle A2A calls before model check - A2A doesn't require a model + if call_type in _A2A_CALL_TYPES: + from litellm.a2a_protocol.cost_calculator import A2ACostCalculator + + return A2ACostCalculator.calculate_a2a_cost(litellm_logging_obj=litellm_logging_obj) - else: if model is None: raise ValueError( f"Model is None and does not exist in passed completion_response. Passed completion_response={completion_response}, model={model}" ) - if len(messages) > 0: - prompt_tokens = token_counter(model=model, messages=messages) - elif len(prompt) > 0: - prompt_tokens = token_counter(model=model, text=prompt) - completion_tokens = token_counter(model=model, text=completion) - - # Handle A2A calls before model check - A2A doesn't require a model - if call_type in _A2A_CALL_TYPES: - from litellm.a2a_protocol.cost_calculator import A2ACostCalculator - - return A2ACostCalculator.calculate_a2a_cost(litellm_logging_obj=litellm_logging_obj) - - if model is None: - raise ValueError( - f"Model is None and does not exist in passed completion_response. Passed completion_response={completion_response}, model={model}" - ) - if custom_llm_provider is None: - try: - model, custom_llm_provider, _, _ = litellm.get_llm_provider( - model=model - ) # strip the llm provider from the model name -> for image gen cost calculation - except Exception as e: - verbose_logger.debug( - "litellm.cost_calculator.py::completion_cost() - Error inferring custom_llm_provider - %s", - e, - ) - if CostCalculatorUtils.call_type_has_image_response(call_type) and isinstance( - completion_response, ImageResponse - ): - ### IMAGE GENERATION COST CALCULATION ### - return CostCalculatorUtils.route_image_generation_cost_calculator( - model=model, - custom_llm_provider=custom_llm_provider, - completion_response=completion_response, - quality=quality, - n=n, - size=size, - optional_params=optional_params, - call_type=call_type, - model_info=_deployment_model_info(litellm_logging_obj, custom_pricing, router_model_id), - vertex_location=vertex_location, - ) - elif call_type in _VIDEO_CALL_TYPES: - ### VIDEO GENERATION COST CALCULATION ### - _video_model_info: ModelInfo | None = _deployment_model_info( - litellm_logging_obj, custom_pricing, router_model_id - ) - - usage_obj = getattr(completion_response, "usage", None) - duration_seconds: float | None = None - video_resolution: str | None = None - provider_reported_cost: float | None = None - if completion_response is not None and usage_obj: - # Handle both dict and Pydantic Usage object - if isinstance(usage_obj, dict): - duration_seconds = usage_obj.get("duration_seconds", None) - _vr = usage_obj.get("video_resolution", None) - provider_reported_cost = usage_obj.get("provider_reported_cost_usd", None) - _vc = usage_obj.get("video_count", None) - else: - duration_seconds = getattr(usage_obj, "duration_seconds", None) - _vr = getattr(usage_obj, "video_resolution", None) - provider_reported_cost = getattr(usage_obj, "provider_reported_cost_usd", None) - _vc = getattr(usage_obj, "video_count", None) - if _vr is not None: - video_resolution = str(_vr).strip().lower() - video_count = _vc if isinstance(_vc, int) and not isinstance(_vc, bool) and _vc > 1 else 1 - - if _video_model_info is None and provider_reported_cost is not None: - return float(provider_reported_cost) - - if duration_seconds is not None: - # Calculate cost based on video duration using video-specific cost calculation - from litellm.llms.openai.cost_calculation import ( - video_generation_cost, + if custom_llm_provider is None: + try: + model, custom_llm_provider, _, _ = litellm.get_llm_provider( + model=model + ) # strip the llm provider from the model name -> for image gen cost calculation + except Exception as e: + verbose_logger.debug( + "litellm.cost_calculator.py::completion_cost() - Error inferring custom_llm_provider - %s", + e, ) - - return video_generation_cost( - model=model, - duration_seconds=duration_seconds, - custom_llm_provider=custom_llm_provider, - model_info=_video_model_info, - video_resolution=video_resolution, - ) * video_count + _video_input_image_cost( - requested_model or model, - custom_llm_provider, - _video_model_info, - getattr(completion_response, "usage", None), - ) - # Fallback to default video cost calculation if no duration available - return default_video_cost_calculator( - model=model, - duration_seconds=0.0, # Default to 0 if no duration available - custom_llm_provider=custom_llm_provider, - model_info=_video_model_info, - video_resolution=video_resolution, - ) - elif call_type in _SPEECH_CALL_TYPES: - prompt_characters = litellm.utils.count_characters(text=prompt) - elif call_type in _TRANSCRIPTION_CALL_TYPES: - # Check _hidden_params first (duration stored there to - # avoid polluting the response body), then fall back to - # the response attribute (for verbose_json responses that - # naturally include duration from the provider). - _hidden = getattr(completion_response, "_hidden_params", {}) or {} - audio_transcription_file_duration = _hidden.get( - "audio_transcription_duration", - getattr(completion_response, "duration", 0.0), - ) or ( - completion_response.usage.seconds - if isinstance(completion_response, TranscriptionResponse) - and isinstance(completion_response.usage, TranscriptionUsageDurationObject) - and _bills_provider_transcription_seconds(requested_model, custom_llm_provider) - else 0.0 - ) - elif call_type in _RERANK_CALL_TYPES: - if completion_response is not None and isinstance(completion_response, RerankResponse): - meta_obj = completion_response.meta - if meta_obj is not None: - billed_units: RerankBilledUnits = meta_obj.get("billed_units") or {} - else: - billed_units = {} - - rerank_billed_units = RerankBilledUnits( - search_units=billed_units.get("search_units"), - total_tokens=billed_units.get("total_tokens"), - ) - - search_units = billed_units.get("search_units") or 1 # cohere charges per request by default. - completion_tokens = search_units - elif call_type in _SEARCH_CALL_TYPES: - from litellm.search import search_provider_cost_per_query - - # Extract number_of_queries from optional_params or default to 1 - number_of_queries = 1 - if optional_params is not None: - # Check if query is a list (multiple queries) - query = optional_params.get("query") - if isinstance(query, list): - number_of_queries = len(query) - elif query is not None: - number_of_queries = 1 - - search_model = model or "" - if custom_llm_provider and "/" not in search_model: - # If model is like "tavily-search", construct "tavily/search" for cost lookup - search_model = f"{custom_llm_provider}/search" - - ( - prompt_cost, - completion_cost_result, - ) = ( - (number_of_queries * configured_cost_per_query, 0.0) - if configured_cost_per_query is not None - else search_provider_cost_per_query( - model=search_model, + if CostCalculatorUtils.call_type_has_image_response(call_type) and isinstance( + completion_response, ImageResponse + ): + ### IMAGE GENERATION COST CALCULATION ### + return CostCalculatorUtils.route_image_generation_cost_calculator( + model=model, custom_llm_provider=custom_llm_provider, - number_of_queries=number_of_queries, + completion_response=completion_response, + quality=quality, + n=n, + size=size, optional_params=optional_params, + call_type=call_type, + model_info=_deployment_model_info(litellm_logging_obj, custom_pricing, router_model_id), + vertex_location=vertex_location, ) - ) - - # Return the total cost (prompt_cost + completion_cost, but for search it's just prompt_cost) - _final_cost = prompt_cost + completion_cost_result - - # Apply discount - original_cost = _final_cost - ( - _final_cost, - discount_percent, - discount_amount, - ) = _apply_cost_discount( - base_cost=_final_cost, - custom_llm_provider=custom_llm_provider, - model=model, - region_name=region_name if isinstance(region_name, str) else None, - ) - - # Apply margin from module-level config if configured - ( - _final_cost, - margin_percent, - margin_fixed_amount, - margin_total_amount, - ) = _apply_cost_margin( - base_cost=_final_cost, - custom_llm_provider=custom_llm_provider, - ) - - # Store cost breakdown in logging object if available - _store_cost_breakdown_in_logging_obj( - litellm_logging_obj=litellm_logging_obj, - prompt_tokens_cost_usd_dollar=prompt_cost, - completion_tokens_cost_usd_dollar=completion_cost_result, - cost_for_built_in_tools_cost_usd_dollar=0.0, - total_cost_usd_dollar=_final_cost, - original_cost=original_cost, - discount_percent=discount_percent, - discount_amount=discount_amount, - margin_percent=margin_percent, - margin_fixed_amount=margin_fixed_amount, - margin_total_amount=margin_total_amount, - service_tier=service_tier, - data_residency=data_residency, - ) - - return _final_cost - elif call_type == _AREALTIME_CALL_TYPE and isinstance( - completion_response, LiteLLMRealtimeStreamLoggingObject - ): - if cost_per_token_usage_object is None or custom_llm_provider is None: - raise ValueError( - f"usage object and custom_llm_provider must be provided for realtime stream cost calculation. Got cost_per_token_usage_object={cost_per_token_usage_object}, custom_llm_provider={custom_llm_provider}" - ) - return handle_realtime_stream_cost_calculation( - results=completion_response.results, - combined_usage_object=cost_per_token_usage_object, - custom_llm_provider=custom_llm_provider, - litellm_model_name=model, - data_residency=data_residency, - litellm_logging_obj=litellm_logging_obj, - custom_pricing_model=selected_model if custom_pricing else None, - base_pricing_model=(selected_model if base_model is not None and not custom_pricing else None), - ) - elif call_type == _MCP_CALL_TYPE: - from litellm.proxy.mcp_server.cost_calculator import ( - MCPCostCalculator, - ) - - return MCPCostCalculator.calculate_mcp_tool_call_cost(litellm_logging_obj=litellm_logging_obj) - response_time_ms = _response_time_ms_for_cost( - completion_response=completion_response, - litellm_logging_obj=litellm_logging_obj, - total_time=total_time, - ) - model_for_discount = custom_priced_request_model or model - # Calculate cost based on prompt_tokens, completion_tokens - if ( - "togethercomputer" in model or "together_ai" in model or custom_llm_provider == "together_ai" - ) and not has_together_registry_pricing(model, litellm.model_cost): - model = get_model_params_and_category(model, call_type=CallTypes(call_type)) - - # replicate llms are calculate based on time for request running - # see https://replicate.com/pricing - elif (model in litellm.replicate_models or "replicate" in model) and model not in litellm.model_cost: - # for unmapped replicate model, default to replicate's time tracking logic - return get_replicate_completion_pricing(completion_response, response_time_ms) - - if model is None: - raise ValueError( - f"Model is None and does not exist in passed completion_response. Passed completion_response={completion_response}, model={model}" - ) - - if custom_llm_provider is not None and custom_llm_provider == "vertex_ai": - # Calculate the prompt characters + response characters - if len(messages) > 0: - prompt_string = litellm.utils.get_formatted_prompt( - data={"messages": messages}, call_type="completion" + elif call_type in _VIDEO_CALL_TYPES: + ### VIDEO GENERATION COST CALCULATION ### + _video_model_info: ModelInfo | None = _deployment_model_info( + litellm_logging_obj, custom_pricing, router_model_id ) - prompt_characters = litellm.utils.count_characters(text=prompt_string) - if completion_response is not None and isinstance(completion_response, ModelResponse): - completion_string = litellm.utils.get_response_string(response_obj=completion_response) - completion_characters = litellm.utils.count_characters(text=completion_string) + usage_obj = getattr(completion_response, "usage", None) + duration_seconds: float | None = None + video_resolution: str | None = None + provider_reported_cost: float | None = None + if completion_response is not None and usage_obj: + # Handle both dict and Pydantic Usage object + if isinstance(usage_obj, dict): + duration_seconds = usage_obj.get("duration_seconds", None) + _vr = usage_obj.get("video_resolution", None) + provider_reported_cost = usage_obj.get("provider_reported_cost_usd", None) + _vc = usage_obj.get("video_count", None) + else: + duration_seconds = getattr(usage_obj, "duration_seconds", None) + _vr = getattr(usage_obj, "video_resolution", None) + provider_reported_cost = getattr(usage_obj, "provider_reported_cost_usd", None) + _vc = getattr(usage_obj, "video_count", None) + if _vr is not None: + video_resolution = str(_vr).strip().lower() + video_count = _vc if isinstance(_vc, int) and not isinstance(_vc, bool) and _vc > 1 else 1 - # Get the original request model for router detection - request_model_for_cost = None - if litellm_logging_obj is not None: - request_model_for_cost = litellm_logging_obj.model + if _video_model_info is None and provider_reported_cost is not None: + return float(provider_reported_cost) - ( - prompt_tokens_cost_usd_dollar, - completion_tokens_cost_usd_dollar, - ) = cost_per_token( - model=model, - prompt_tokens=prompt_tokens or 0, - completion_tokens=completion_tokens or 0, - custom_llm_provider=_pricing_provider_for_cost_candidate( - candidate_provider=candidate_pricing_provider, - request_provider=request_pricing_provider, - current_provider=custom_llm_provider, - ), - response_time_ms=response_time_ms, - region_name=None if explicit_pricing else region_name, - custom_cost_per_second=custom_cost_per_second, - custom_cost_per_token=custom_cost_per_token, - prompt_characters=prompt_characters, - completion_characters=completion_characters, - cache_creation_input_tokens=cache_creation_input_tokens, - cache_read_input_tokens=cache_read_input_tokens, - usage_object=cost_per_token_usage_object, - call_type=call_type, - audio_transcription_file_duration=audio_transcription_file_duration, - rerank_billed_units=rerank_billed_units, - service_tier=service_tier, - data_residency=data_residency, - vertex_location=vertex_location, - response=completion_response, - custom_model_info=_ocr_model_info(litellm_logging_obj, custom_pricing, router_model_id), - bedrock_model_info=bedrock_model_info, - ) + if duration_seconds is not None: + # Calculate cost based on video duration using video-specific cost calculation + from litellm.llms.openai.cost_calculation import ( + video_generation_cost, + ) - # Get additional costs from provider (e.g., routing fees, infrastructure costs) - if custom_llm_provider in ("azure_ai", "azure") and not azure_ai_is_model_router_name(model): - model_for_additional_costs = request_model_for_cost - if completion_response is not None: - hidden_params = getattr(completion_response, "_hidden_params", None) or {} - hidden_model = hidden_params.get("model") or hidden_params.get("litellm_model_name") - if ( - hidden_model - and ( - "model_router" in (hidden_model or "").lower() - or "model-router" in (hidden_model or "").lower() + return video_generation_cost( + model=model, + duration_seconds=duration_seconds, + custom_llm_provider=custom_llm_provider, + model_info=_video_model_info, + video_resolution=video_resolution, + ) * video_count + _video_input_image_cost( + requested_model or model, + custom_llm_provider, + _video_model_info, + getattr(completion_response, "usage", None), + ) + # Fallback to default video cost calculation if no duration available + return default_video_cost_calculator( + model=model, + duration_seconds=0.0, # Default to 0 if no duration available + custom_llm_provider=custom_llm_provider, + model_info=_video_model_info, + video_resolution=video_resolution, + ) + elif call_type in _SPEECH_CALL_TYPES: + prompt_characters = litellm.utils.count_characters(text=prompt) + elif call_type in _TRANSCRIPTION_CALL_TYPES: + # Check _hidden_params first (duration stored there to + # avoid polluting the response body), then fall back to + # the response attribute (for verbose_json responses that + # naturally include duration from the provider). + _hidden = getattr(completion_response, "_hidden_params", {}) or {} + audio_transcription_file_duration = _hidden.get( + "audio_transcription_duration", + getattr(completion_response, "duration", 0.0), + ) or ( + completion_response.usage.seconds + if isinstance(completion_response, TranscriptionResponse) + and isinstance(completion_response.usage, TranscriptionUsageDurationObject) + and _bills_provider_transcription_seconds(requested_model, custom_llm_provider) + else 0.0 + ) + elif call_type in _RERANK_CALL_TYPES: + if completion_response is not None and isinstance(completion_response, RerankResponse): + meta_obj = completion_response.meta + if meta_obj is not None: + billed_units: RerankBilledUnits = meta_obj.get("billed_units") or {} + else: + billed_units = {} + + rerank_billed_units = RerankBilledUnits( + search_units=billed_units.get("search_units"), + total_tokens=billed_units.get("total_tokens"), ) - or model_for_additional_costs is None - ): - model_for_additional_costs = hidden_model - if model_for_additional_costs is None: - model_for_additional_costs = model - additional_costs = _get_additional_costs( - model=model_for_additional_costs, - custom_llm_provider=custom_llm_provider, + + search_units = ( + billed_units.get("search_units") or 1 + ) # cohere charges per request by default. + completion_tokens = search_units + elif call_type in _SEARCH_CALL_TYPES: + from litellm.search import search_provider_cost_per_query + + # Extract number_of_queries from optional_params or default to 1 + number_of_queries = 1 + if optional_params is not None: + # Check if query is a list (multiple queries) + query = optional_params.get("query") + if isinstance(query, list): + number_of_queries = len(query) + elif query is not None: + number_of_queries = 1 + + search_model = model or "" + if custom_llm_provider and "/" not in search_model: + # If model is like "tavily-search", construct "tavily/search" for cost lookup + search_model = f"{custom_llm_provider}/search" + + ( + prompt_cost, + completion_cost_result, + ) = ( + (number_of_queries * configured_cost_per_query, 0.0) + if configured_cost_per_query is not None + else search_provider_cost_per_query( + model=search_model, + custom_llm_provider=custom_llm_provider, + number_of_queries=number_of_queries, + optional_params=optional_params, + ) + ) + + # Return the total cost (prompt_cost + completion_cost, but for search it's just prompt_cost) + _final_cost = prompt_cost + completion_cost_result + + # Apply discount + original_cost = _final_cost + ( + _final_cost, + discount_percent, + discount_amount, + ) = _apply_cost_discount( + base_cost=_final_cost, + custom_llm_provider=custom_llm_provider, + model=model, + region_name=region_name if isinstance(region_name, str) else None, + ) + + # Apply margin from module-level config if configured + ( + _final_cost, + margin_percent, + margin_fixed_amount, + margin_total_amount, + ) = _apply_cost_margin( + base_cost=_final_cost, + custom_llm_provider=custom_llm_provider, + ) + + # Store cost breakdown in logging object if available + _store_cost_breakdown_in_logging_obj( + litellm_logging_obj=litellm_logging_obj, + prompt_tokens_cost_usd_dollar=prompt_cost, + completion_tokens_cost_usd_dollar=completion_cost_result, + cost_for_built_in_tools_cost_usd_dollar=0.0, + total_cost_usd_dollar=_final_cost, + original_cost=original_cost, + discount_percent=discount_percent, + discount_amount=discount_amount, + margin_percent=margin_percent, + margin_fixed_amount=margin_fixed_amount, + margin_total_amount=margin_total_amount, + service_tier=service_tier, + data_residency=data_residency, + ) + + return _final_cost + elif call_type == _AREALTIME_CALL_TYPE and isinstance( + completion_response, LiteLLMRealtimeStreamLoggingObject + ): + if cost_per_token_usage_object is None or custom_llm_provider is None: + raise ValueError( + f"usage object and custom_llm_provider must be provided for realtime stream cost calculation. Got cost_per_token_usage_object={cost_per_token_usage_object}, custom_llm_provider={custom_llm_provider}" + ) + return handle_realtime_stream_cost_calculation( + results=completion_response.results, + combined_usage_object=cost_per_token_usage_object, + custom_llm_provider=custom_llm_provider, + litellm_model_name=model, + data_residency=data_residency, + litellm_logging_obj=litellm_logging_obj, + custom_pricing_model=selected_model if custom_pricing else None, + base_pricing_model=( + selected_model if base_model is not None and not custom_pricing else None + ), + ) + elif call_type == _MCP_CALL_TYPE: + from litellm.proxy.mcp_server.cost_calculator import ( + MCPCostCalculator, + ) + + return MCPCostCalculator.calculate_mcp_tool_call_cost(litellm_logging_obj=litellm_logging_obj) + response_time_ms = _response_time_ms_for_cost( + completion_response=completion_response, + litellm_logging_obj=litellm_logging_obj, + total_time=total_time, + ) + model_for_discount = custom_priced_request_model or model + # Calculate cost based on prompt_tokens, completion_tokens + if ( + "togethercomputer" in model or "together_ai" in model or custom_llm_provider == "together_ai" + ) and not has_together_registry_pricing(model, litellm.model_cost): + model = get_model_params_and_category(model, call_type=CallTypes(call_type)) + + # replicate llms are calculate based on time for request running + # see https://replicate.com/pricing + elif ( + model in litellm.replicate_models or "replicate" in model + ) and model not in litellm.model_cost: + # for unmapped replicate model, default to replicate's time tracking logic + return get_replicate_completion_pricing(completion_response, response_time_ms) + + if model is None: + raise ValueError( + f"Model is None and does not exist in passed completion_response. Passed completion_response={completion_response}, model={model}" + ) + + if custom_llm_provider is not None and custom_llm_provider == "vertex_ai": + # Calculate the prompt characters + response characters + if len(messages) > 0: + prompt_string = litellm.utils.get_formatted_prompt( + data={"messages": messages}, call_type="completion" + ) + + prompt_characters = litellm.utils.count_characters(text=prompt_string) + if completion_response is not None and isinstance(completion_response, ModelResponse): + completion_string = litellm.utils.get_response_string(response_obj=completion_response) + completion_characters = litellm.utils.count_characters(text=completion_string) + + # Get the original request model for router detection + request_model_for_cost = None + if litellm_logging_obj is not None: + request_model_for_cost = litellm_logging_obj.model + + ( + prompt_tokens_cost_usd_dollar, + completion_tokens_cost_usd_dollar, + ) = cost_per_token( + model=model, prompt_tokens=prompt_tokens or 0, completion_tokens=completion_tokens or 0, + custom_llm_provider=_pricing_provider_for_cost_candidate( + candidate_provider=candidate_pricing_provider, + request_provider=request_pricing_provider, + current_provider=custom_llm_provider, + ), + response_time_ms=response_time_ms, + region_name=None if explicit_pricing else region_name, + custom_cost_per_second=custom_cost_per_second, + custom_cost_per_token=custom_cost_per_token, + prompt_characters=prompt_characters, + completion_characters=completion_characters, + cache_creation_input_tokens=cache_creation_input_tokens, + cache_read_input_tokens=cache_read_input_tokens, + usage_object=cost_per_token_usage_object, + call_type=call_type, + audio_transcription_file_duration=audio_transcription_file_duration, + rerank_billed_units=rerank_billed_units, + service_tier=service_tier, + data_residency=data_residency, + vertex_location=vertex_location, + response=completion_response, + custom_model_info=_ocr_model_info(litellm_logging_obj, custom_pricing, router_model_id), + bedrock_model_info=bedrock_model_info, ) - else: - additional_costs = None - _final_cost = prompt_tokens_cost_usd_dollar + completion_tokens_cost_usd_dollar - cost_for_built_in_tools = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( - model=model, - response_object=completion_response, - usage=cost_per_token_usage_object, - standard_built_in_tools_params=standard_built_in_tools_params, - custom_llm_provider=custom_llm_provider, - ) - _final_cost += cost_for_built_in_tools - if additional_costs: - _final_cost += sum(additional_costs.values()) + # Get additional costs from provider (e.g., routing fees, infrastructure costs) + if custom_llm_provider in ("azure_ai", "azure") and not azure_ai_is_model_router_name(model): + model_for_additional_costs = request_model_for_cost + if completion_response is not None: + hidden_params = getattr(completion_response, "_hidden_params", None) or {} + hidden_model = hidden_params.get("model") or hidden_params.get("litellm_model_name") + if ( + hidden_model + and ( + "model_router" in (hidden_model or "").lower() + or "model-router" in (hidden_model or "").lower() + ) + or model_for_additional_costs is None + ): + model_for_additional_costs = hidden_model + if model_for_additional_costs is None: + model_for_additional_costs = model + additional_costs = _get_additional_costs( + model=model_for_additional_costs, + custom_llm_provider=custom_llm_provider, + prompt_tokens=prompt_tokens or 0, + completion_tokens=completion_tokens or 0, + ) + else: + additional_costs = None - original_cost = _final_cost - if litellm.cost_discount_config: - ( - _final_cost, - discount_percent, - discount_amount, - ) = _apply_cost_discount( - base_cost=_final_cost, - custom_llm_provider=custom_llm_provider, - model=model_for_discount, - region_name=region_name if isinstance(region_name, str) else None, - ) - else: - discount_percent = 0.0 - discount_amount = 0.0 - - # Apply margin from module-level config if configured - if litellm.cost_margin_config: - ( - _final_cost, - margin_percent, - margin_fixed_amount, - margin_total_amount, - ) = _apply_cost_margin( - base_cost=_final_cost, + _final_cost = prompt_tokens_cost_usd_dollar + completion_tokens_cost_usd_dollar + cost_for_built_in_tools = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( + model=model, + response_object=completion_response, + usage=cost_per_token_usage_object, + standard_built_in_tools_params=standard_built_in_tools_params, custom_llm_provider=custom_llm_provider, ) - else: - margin_percent = 0.0 - margin_fixed_amount = 0.0 - margin_total_amount = 0.0 + _final_cost += cost_for_built_in_tools + if additional_costs: + _final_cost += sum(additional_costs.values()) - # Store cost breakdown in logging object if available - if litellm_logging_obj is not None: - _reasoning_cost: float | None = None - _cache_read_cost: float | None = None - _cache_creation_cost: float | None = None - _billed_token_rates: BilledTokenRates | None = None - if cost_per_token_usage_object is not None and model: - _token_type_breakdown = get_token_type_cost_breakdown( - model=model, - custom_llm_provider=_pricing_provider_for_cost_candidate( - candidate_provider=candidate_pricing_provider, - request_provider=request_pricing_provider, - current_provider=custom_llm_provider, - ), - usage=cost_per_token_usage_object, + original_cost = _final_cost + if litellm.cost_discount_config: + ( + _final_cost, + discount_percent, + discount_amount, + ) = _apply_cost_discount( + base_cost=_final_cost, + custom_llm_provider=custom_llm_provider, + model=model_for_discount, + region_name=region_name if isinstance(region_name, str) else None, + ) + else: + discount_percent = 0.0 + discount_amount = 0.0 + + # Apply margin from module-level config if configured + if litellm.cost_margin_config: + ( + _final_cost, + margin_percent, + margin_fixed_amount, + margin_total_amount, + ) = _apply_cost_margin( + base_cost=_final_cost, + custom_llm_provider=custom_llm_provider, + ) + else: + margin_percent = 0.0 + margin_fixed_amount = 0.0 + margin_total_amount = 0.0 + + # Store cost breakdown in logging object if available + if litellm_logging_obj is not None: + _reasoning_cost: float | None = None + _cache_read_cost: float | None = None + _cache_creation_cost: float | None = None + _billed_token_rates: BilledTokenRates | None = None + if cost_per_token_usage_object is not None and model: + _token_type_breakdown = get_token_type_cost_breakdown( + model=model, + custom_llm_provider=_pricing_provider_for_cost_candidate( + candidate_provider=candidate_pricing_provider, + request_provider=request_pricing_provider, + current_provider=custom_llm_provider, + ), + usage=cost_per_token_usage_object, + service_tier=service_tier, + data_residency=data_residency, + vertex_location=vertex_location, + custom_cost_per_token=custom_cost_per_token, + model_info=bedrock_model_info, + ) + _reasoning_cost = _token_type_breakdown.reasoning_cost + _cache_read_cost = _token_type_breakdown.cache_read_cost + _cache_creation_cost = _token_type_breakdown.cache_creation_cost + _billed_token_rates = _token_type_breakdown.rates + _store_cost_breakdown_in_logging_obj( + litellm_logging_obj=litellm_logging_obj, + prompt_tokens_cost_usd_dollar=prompt_tokens_cost_usd_dollar, + completion_tokens_cost_usd_dollar=completion_tokens_cost_usd_dollar, + cost_for_built_in_tools_cost_usd_dollar=cost_for_built_in_tools, + total_cost_usd_dollar=_final_cost, + original_cost=original_cost, + additional_costs=additional_costs, + discount_percent=discount_percent, + discount_amount=discount_amount, + margin_percent=margin_percent, + margin_fixed_amount=margin_fixed_amount, + margin_total_amount=margin_total_amount, + cache_read_cost=_cache_read_cost, + cache_creation_cost=_cache_creation_cost, + reasoning_cost=_reasoning_cost, service_tier=service_tier, data_residency=data_residency, vertex_location=vertex_location, - custom_cost_per_token=custom_cost_per_token, - model_info=bedrock_model_info, + billed_token_rates=_billed_token_rates, ) - _reasoning_cost = _token_type_breakdown.reasoning_cost - _cache_read_cost = _token_type_breakdown.cache_read_cost - _cache_creation_cost = _token_type_breakdown.cache_creation_cost - _billed_token_rates = _token_type_breakdown.rates - _store_cost_breakdown_in_logging_obj( - litellm_logging_obj=litellm_logging_obj, - prompt_tokens_cost_usd_dollar=prompt_tokens_cost_usd_dollar, - completion_tokens_cost_usd_dollar=completion_tokens_cost_usd_dollar, - cost_for_built_in_tools_cost_usd_dollar=cost_for_built_in_tools, - total_cost_usd_dollar=_final_cost, - original_cost=original_cost, - additional_costs=additional_costs, - discount_percent=discount_percent, - discount_amount=discount_amount, - margin_percent=margin_percent, - margin_fixed_amount=margin_fixed_amount, - margin_total_amount=margin_total_amount, - cache_read_cost=_cache_read_cost, - cache_creation_cost=_cache_creation_cost, - reasoning_cost=_reasoning_cost, - service_tier=service_tier, - data_residency=data_residency, - vertex_location=vertex_location, - billed_token_rates=_billed_token_rates, - ) - return _final_cost - except Exception as e: - verbose_logger.debug( - "litellm.cost_calculator.py::completion_cost() - Error calculating cost for model=%s - %s", model, e - ) - if idx == len(potential_model_names) - 1: - raise e - raise Exception(f"Unable to calculate cost for received potential model names - {potential_model_names}") - except Exception as e: - raise e + return _final_cost + except Exception as e: + verbose_logger.debug( + "litellm.cost_calculator.py::completion_cost() - Error calculating cost for model=%s - %s", + model, + e, + ) + if idx == len(potential_model_names) - 1: + raise e + raise Exception(f"Unable to calculate cost for received potential model names - {potential_model_names}") + except Exception as e: + raise e def get_response_cost_from_hidden_params( diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 3b49f239afe..10f443b0234 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -12,6 +12,9 @@ from zoneinfo import ZoneInfo, ZoneInfoNotFoundError from typing_extensions import ReadOnly import litellm +from litellm._catalog import Invalid +from litellm._cost import CallContext, Prices, as_main_cost, call_custom_pricing +from litellm._cost import Usage as TokenUsage from litellm._internal_context import current_billing_time from litellm._logging import verbose_logger from litellm.litellm_core_utils.llm_cost_calc.tiered_pricing import ( @@ -892,7 +895,7 @@ def _get_token_base_cost( ) -def calculate_cost_component(model_info: ModelInfo, cost_key: str, usage_value: float | None) -> float: +def calculate_cost_component(model_info: Mapping[str, object], cost_key: str, usage_value: float | None) -> float: """ Generic cost calculator for any usage component @@ -910,7 +913,7 @@ def calculate_cost_component(model_info: ModelInfo, cost_key: str, usage_value: return 0.0 -def get_cost_per_unit(model_info: ModelInfo, cost_key: str, default_value: float | None = 0.0) -> float | None: +def get_cost_per_unit(model_info: Mapping[str, object], cost_key: str, default_value: float | None = 0.0) -> float | None: # Sometimes the cost per unit is a string (e.g.: If a value like "3e-7" was read from the config.yaml) cost_per_unit: Final = model_info.get(cost_key) if isinstance(cost_per_unit, float): @@ -1185,144 +1188,7 @@ def parse_completion_tokens_details(usage: Usage) -> CompletionTokensDetailsResu ) -def _modality_token_cost( - model_info: ModelInfo, - modality_cost_key: str, - tokens: int, - prompt_base_cost: float, - service_tier: str | None, -) -> float: - """ - Audio, image and video prompt tokens bill at their modality rate when the model has one. - Without one they are ordinary prompt tokens, so they bill at `prompt_base_cost`, which - `_get_token_base_cost` has already resolved for the service tier and long-context tier. - """ - tiered_key: Final = get_service_tier_cost_key(modality_cost_key, service_tier) - cost_key: Final = tiered_key if model_info.get(tiered_key) is not None else modality_cost_key - if model_info.get(cost_key) is None: - return float(tokens) * prompt_base_cost - return calculate_cost_component(model_info, cost_key, tokens) - - -def _calculate_input_cost( - prompt_tokens_details: PromptTokensDetailsResult, - model_info: ModelInfo, - prompt_base_cost: float, - cache_read_cost: float, - cache_creation_cost: float, - cache_creation_cost_by_ttl: Mapping[str, float], - service_tier: str | None = None, -) -> float: - """ - Calculates the input cost for a given model, prompt tokens, and completion tokens. - - A model priced per character (`input_cost_per_character`) bills the reported character count - in place of its text tokens, so the same prompt is never charged under both units. - """ - bills_characters: Final = ( - prompt_tokens_details["character_count"] > 0 and model_info.get("input_cost_per_character") is not None - ) - prompt_cost = 0.0 if bills_characters else float(prompt_tokens_details["text_tokens"]) * prompt_base_cost - - ### CACHE READ COST - Now uses tiered pricing - cache_hit_audio_tokens: Final = prompt_tokens_details["cache_hit_audio_tokens"] - audio_cache_read_rate: Final = get_cost_per_unit( - model_info, - get_service_tier_cost_key("cache_read_input_audio_token_cost", service_tier), - None, - ) - prompt_cost += float(prompt_tokens_details["cache_hit_tokens"] - cache_hit_audio_tokens) * cache_read_cost - prompt_cost += float(cache_hit_audio_tokens) * ( - audio_cache_read_rate if audio_cache_read_rate is not None else cache_read_cost - ) - - ### AUDIO COST - if prompt_tokens_details["audio_tokens"] and not ( - prompt_tokens_details["audio_length_seconds"] and model_info.get("input_cost_per_audio_per_second") is not None - ): - prompt_cost += _modality_token_cost( - model_info, - "input_cost_per_audio_token", - prompt_tokens_details["audio_tokens"], - prompt_base_cost, - service_tier, - ) - - ### IMAGE TOKEN COST - if prompt_tokens_details["image_tokens"] and not ( - prompt_tokens_details["image_count"] and model_info.get("input_cost_per_image") is not None - ): - prompt_cost += _modality_token_cost( - model_info, - "input_cost_per_image_token", - prompt_tokens_details["image_tokens"], - prompt_base_cost, - service_tier, - ) - - ### VIDEO TOKEN COST - if prompt_tokens_details["video_tokens"] and not ( - prompt_tokens_details["video_length_seconds"] and model_info.get("input_cost_per_video_per_second") is not None - ): - prompt_cost += _modality_token_cost( - model_info, - "input_cost_per_video_token", - prompt_tokens_details["video_tokens"], - prompt_base_cost, - service_tier, - ) - - ### CACHE WRITING COST - Now uses tiered pricing - if ( - prompt_tokens_details["cache_creation_tokens"] - or prompt_tokens_details["cache_creation_token_details"] is not None - ): - prompt_cost += calculate_cache_writing_cost( - cache_creation_tokens=prompt_tokens_details["cache_creation_tokens"], - cache_creation_token_details=prompt_tokens_details["cache_creation_token_details"], - cache_creation_cost=cache_creation_cost, - cache_creation_cost_by_ttl=cache_creation_cost_by_ttl, - ) - - ### CHARACTER COST - if prompt_tokens_details["character_count"]: - prompt_cost += calculate_cost_component( - model_info, - "input_cost_per_character", - prompt_tokens_details["character_count"], - ) - - ### IMAGE COUNT COST - if prompt_tokens_details["image_count"]: - prompt_cost += calculate_cost_component( - model_info, "input_cost_per_image", prompt_tokens_details["image_count"] - ) - - ### VIDEO LENGTH COST - if prompt_tokens_details["video_length_seconds"]: - prompt_cost += calculate_cost_component( - model_info, - "input_cost_per_video_per_second", - prompt_tokens_details["video_length_seconds"], - ) - - ### AUDIO LENGTH COST - if prompt_tokens_details["audio_length_seconds"]: - prompt_cost += calculate_cost_component( - model_info, - "input_cost_per_audio_per_second", - prompt_tokens_details["audio_length_seconds"], - ) - - if prompt_tokens_details["query_count"]: - prompt_cost += calculate_cost_component( - model_info, "input_cost_per_query", prompt_tokens_details["query_count"] - ) - - return prompt_cost - - -def get_regional_uplift_multiplier(model_info: ModelInfo, data_residency: str | None) -> float: +def get_regional_uplift_multiplier(model_info: Mapping[str, object], data_residency: str | None) -> float: """ Resolve the per-model regional-processing uplift multiplier for a given data-residency region. @@ -1341,7 +1207,7 @@ def get_regional_uplift_multiplier(model_info: ModelInfo, data_residency: str | if residency not in _VALID_DATA_RESIDENCIES: return 1.0 multiplier: Final = model_info.get(f"regional_processing_uplift_multiplier_{residency}") - if multiplier is None: + if not isinstance(multiplier, (int, float, str)): return 1.0 try: return float(multiplier) @@ -1356,7 +1222,7 @@ def get_regional_uplift_multiplier(model_info: ModelInfo, data_residency: str | _get_regional_uplift_multiplier = get_regional_uplift_multiplier -def get_vertex_regional_endpoint_uplift(model_info: ModelInfo, vertex_location: str | None) -> float: +def get_vertex_regional_endpoint_uplift(model_info: Mapping[str, object], vertex_location: str | None) -> float: """ Resolve the per-model uplift multiplier for Vertex AI non-global (regional and multi-region) endpoints. @@ -1476,183 +1342,177 @@ def generic_cost_per_token( Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd """ - ## GET MODEL INFO - # A caller that already resolved the deployment's effective rates passes them in - # rather than handing back a name for this to re-resolve. A name cannot express a - # per-deployment override: those are registered under the deployment id and kept off - # the shared model-name key, so resolving from the name here reads the public rate. - resolved_model_info: Final = apply_provider_cache_read_default( - get_model_info(model=model, custom_llm_provider=custom_llm_provider) if model_info is None else model_info, - custom_llm_provider, + custom_pricing: Final = call_custom_pricing() + row: Final = _pricing_row(model, custom_llm_provider, model_info, has_overlay=custom_pricing is not None) + prices: Final = Prices.from_row( + dict(row) if usage.prompt_tokens > 0 else _without_tiers(row), key=str(row.get("key") or model) ) - - ## CALCULATE INPUT COST - ### Cost of processing (non-cache hit + cache hit) + Cost of cache-writing (cache writing) - prompt_cost = 0.0 - ### PROCESSING COST - prompt_tokens_details = PromptTokensDetailsResult( - cache_hit_tokens=0, - cache_hit_audio_tokens=0, - cache_creation_tokens=0, - cache_creation_token_details=None, - text_tokens=usage.prompt_tokens, - audio_tokens=0, - image_tokens=0, - video_tokens=0, - character_count=0, - image_count=0, - video_length_seconds=0.0, - audio_length_seconds=0.0, - query_count=0, + if isinstance(prices, Invalid): + return as_main_cost(prices) + details: Final = _billable_prompt_details(usage) + call: Final = CallContext( + service_tier=service_tier, + channel="online", + data_residency=data_residency, + vertex_location=vertex_location, + at=current_time if current_time is not None else current_billing_time(), + custom_pricing=None if custom_pricing is None else dict(custom_pricing), ) - if usage.prompt_tokens_details: - prompt_tokens_details = parse_prompt_tokens_details(usage) + prompt_cost, completion_cost = as_main_cost(prices.price(_token_usage(usage, details, row), call)) + uplift: Final = get_regional_uplift_multiplier(row, data_residency) * get_vertex_regional_endpoint_uplift( + row, vertex_location + ) + return prompt_cost + _unmetered_prompt_cost(details, row) * uplift, completion_cost - ## EDGE CASE - text tokens not set or includes cached tokens (double-counting) - ## Some providers (like xAI) report text_tokens = prompt_tokens (including cached) - ## We detect this when: text_tokens + cached_tokens + other > prompt_tokens - ## Ref: https://github.com/BerriAI/litellm/issues/19680, #14874, #14875 - cache_hit: Final = prompt_tokens_details["cache_hit_tokens"] - text_tokens = prompt_tokens_details["text_tokens"] - audio_tokens = prompt_tokens_details["audio_tokens"] - cache_creation: Final = prompt_tokens_details["cache_creation_tokens"] - image_tokens = prompt_tokens_details["image_tokens"] - video_tokens = prompt_tokens_details["video_tokens"] +def _pricing_row( + model: str, custom_llm_provider: str, model_info: ModelInfo | None, *, has_overlay: bool +) -> Mapping[str, object]: + """The row main priced, as written rather than `get_model_info`'s copy, which writes `0` for a missing token + price. A call with custom pricing prices an unmapped model from its overlay alone""" + if model_info is not None: + return model_info + try: + resolved: Final = get_model_info(model=model, custom_llm_provider=custom_llm_provider) + except Exception: # noqa: BLE001 # get_model_info raises a bare Exception for an unmapped model + if has_overlay: + return MappingProxyType({}) + raise + key: Final = resolved.get("key") + row: Final = litellm.model_cost.get(key) + return {**row, "key": key} if isinstance(row, Mapping) else resolved - # Check for double-counting: sum of details > prompt_tokens means overlap - total_details: Final = text_tokens + cache_hit + audio_tokens + cache_creation + image_tokens + video_tokens - has_double_counting: Final = (cache_hit > 0 or cache_creation > 0) and total_details > usage.prompt_tokens - if has_double_counting: - # cached and per-modality counts are both subsets of prompt_tokens and may overlap, so a - # modality can only bill what the cache did not already cover or the overlap is billed twice +def _without_tiers(row: Mapping[str, object]) -> dict[str, object]: + """Main picks no tier for an empty prompt and bills the row's flat prices""" + return {name: value for name, value in row.items() if name != "tiered_pricing"} + + +def _billable_prompt_details(usage: Usage) -> PromptTokensDetailsResult: + """Prompt token details with modality counts capped by what the cache did not cover. + + Some providers (like xAI) report text_tokens = prompt_tokens, cached tokens included, and cached + and per-modality counts can overlap (#19680, #14874, #14875) + """ + details: Final = ( + parse_prompt_tokens_details(usage) + if usage.prompt_tokens_details + else PromptTokensDetailsResult( + cache_hit_tokens=0, + cache_hit_audio_tokens=0, + cache_creation_tokens=0, + cache_creation_token_details=None, + text_tokens=usage.prompt_tokens, + audio_tokens=0, + image_tokens=0, + video_tokens=0, + character_count=0, + image_count=0, + video_length_seconds=0.0, + audio_length_seconds=0.0, + query_count=0, + ) + ) + cache_hit: Final = details["cache_hit_tokens"] + cache_creation: Final = details["cache_creation_tokens"] + modality_tokens: Final = details["audio_tokens"] + details["image_tokens"] + details["video_tokens"] + total_details: Final = details["text_tokens"] + cache_hit + cache_creation + modality_tokens + if (cache_hit > 0 or cache_creation > 0) and total_details > usage.prompt_tokens: uncached_budget: Final = max(usage.prompt_tokens - cache_hit - cache_creation, 0) - billable_audio: Final = min(audio_tokens, uncached_budget) - billable_image: Final = min(image_tokens, uncached_budget - billable_audio) - billable_video: Final = min(video_tokens, uncached_budget - billable_audio - billable_image) - prompt_tokens_details["audio_tokens"] = billable_audio - prompt_tokens_details["image_tokens"] = billable_image - prompt_tokens_details["video_tokens"] = billable_video - prompt_tokens_details["text_tokens"] = uncached_budget - billable_audio - billable_image - billable_video - elif text_tokens == 0 and prompt_tokens_details["image_count"] == 0: - # Clamp to zero: inconsistent streaming usage - prompt_tokens_details["text_tokens"] = max( - usage.prompt_tokens - cache_hit - audio_tokens - cache_creation - image_tokens - video_tokens, 0 - ) + audio: Final = min(details["audio_tokens"], uncached_budget) + image: Final = min(details["image_tokens"], uncached_budget - audio) + video: Final = min(details["video_tokens"], uncached_budget - audio - image) + return { + **details, + "audio_tokens": audio, + "image_tokens": image, + "video_tokens": video, + "text_tokens": uncached_budget - audio - image - video, + } + if details["text_tokens"] == 0 and details["image_count"] == 0: + return { + **details, + "text_tokens": max(usage.prompt_tokens - cache_hit - cache_creation - modality_tokens, 0), + } + return details - billing_time: Final = current_time if current_time is not None else current_billing_time() - base_costs: Final = _get_token_base_cost( - model_info=resolved_model_info, - usage=usage, - service_tier=service_tier, - current_time=billing_time, - threshold_is_inclusive=_uses_inclusive_token_thresholds(custom_llm_provider), - ) - completion_base_cost: Final = base_costs.output_rate - resolved_cache_read_cost: Final = resolve_explicit_cache_read_cost_rate( - model_info=resolved_model_info, - usage=usage, - current_time=billing_time, - service_tier=service_tier, - ) - effective_cache_read_cost: Final = ( - base_costs.cache_read_rate if resolved_cache_read_cost is None else resolved_cache_read_cost + +def _cache_write_tokens(details: PromptTokensDetailsResult) -> tuple[int, int, int]: + """(default, 1 hour, 12 hour) cache write tokens. Writes no lifetime count covers bill at the default rate""" + written: Final = details["cache_creation_tokens"] + by_ttl: Final = details["cache_creation_token_details"] + if by_ttl is None: + return written, 0, 0 + short: Final = (by_ttl.ephemeral_5m_input_tokens or 0) + (by_ttl.ephemeral_30m_input_tokens or 0) + one_hour: Final = by_ttl.ephemeral_1h_input_tokens or 0 + twelve_hours: Final = by_ttl.ephemeral_12h_input_tokens or 0 + return short + max(written - short - one_hour - twelve_hours, 0), one_hour, twelve_hours + + +def _token_usage(usage: Usage, details: PromptTokensDetailsResult, row: Mapping[str, object]) -> TokenUsage: + """The token meters main billed. A prompt billed per character, image or second of media does not bill the + same tokens again""" + cache_type: Final = getattr(usage.prompt_tokens_details, "cache_type", None) + cached_text: Final = details["cache_hit_tokens"] - details["cache_hit_audio_tokens"] + explicit_cache: Final = cache_type == "ephemeral" + cache_write_5m, cache_write_1h, cache_write_12h = _cache_write_tokens(details) + completion: Final = _completion_token_counts(usage) + return TokenUsage( + prompt_tokens=usage.prompt_tokens, + text=0 + if details["character_count"] > 0 and row.get("input_cost_per_character") is not None + else details["text_tokens"], + audio=0 + if details["audio_length_seconds"] and row.get("input_cost_per_audio_per_second") is not None + else details["audio_tokens"], + image=0 if details["image_count"] and row.get("input_cost_per_image") is not None else details["image_tokens"], + video=0 + if details["video_length_seconds"] and row.get("input_cost_per_video_per_second") is not None + else details["video_tokens"], + cache_read=0 if explicit_cache else cached_text, + cache_read_explicit=cached_text if explicit_cache else 0, + cache_read_audio=details["cache_hit_audio_tokens"], + cache_write_5m=cache_write_5m, + cache_write_1h=cache_write_1h, + cache_write_12h=cache_write_12h, + output_text=completion["text_tokens"], + output_audio=completion["audio_tokens"], + output_image=completion["image_tokens"], + output_video=completion["video_tokens"], + reasoning=completion["reasoning_tokens"], ) - prompt_cost = _calculate_input_cost( - prompt_tokens_details=prompt_tokens_details, - model_info=resolved_model_info, - prompt_base_cost=base_costs.input_rate, - cache_read_cost=effective_cache_read_cost, - cache_creation_cost=base_costs.cache_creation_rate, - cache_creation_cost_by_ttl=base_costs.cache_creation_rates_by_ttl, - service_tier=service_tier, + +def _completion_token_counts(usage: Usage) -> CompletionTokensDetailsResult: + """Completion tokens by meter. Text is the remainder of a breakdown, and with no breakdown every token is text""" + details: Final = ( + parse_completion_tokens_details(usage) + if usage.completion_tokens_details is not None + else CompletionTokensDetailsResult( + audio_tokens=0, text_tokens=0, reasoning_tokens=0, image_tokens=0, video_tokens=0 + ) + ) + breakdown: Final = ( + details["audio_tokens"] + details["reasoning_tokens"] + details["image_tokens"] + details["video_tokens"] + ) + if details["text_tokens"] > 0: + return details + if breakdown > 0: + return {**details, "text_tokens": max(0, usage.completion_tokens - breakdown)} + return CompletionTokensDetailsResult( + audio_tokens=0, text_tokens=usage.completion_tokens, reasoning_tokens=0, image_tokens=0, video_tokens=0 ) - ## CALCULATE OUTPUT COST - text_tokens = 0 - audio_tokens = 0 - reasoning_tokens = 0 - image_tokens = 0 - video_tokens = 0 - is_text_tokens_total = False - if usage.completion_tokens_details is not None: - completion_tokens_details: Final = parse_completion_tokens_details(usage) - audio_tokens = completion_tokens_details["audio_tokens"] - text_tokens = completion_tokens_details["text_tokens"] - reasoning_tokens = completion_tokens_details["reasoning_tokens"] - image_tokens = completion_tokens_details["image_tokens"] - video_tokens = completion_tokens_details["video_tokens"] - # Handle text_tokens calculation: - # 1. If text_tokens is explicitly provided and > 0, use it - # 2. If there's a breakdown (reasoning/audio/image/video tokens), calculate text_tokens as the remainder - # 3. If no breakdown at all, assume all completion_tokens are text_tokens - has_token_breakdown: Final = image_tokens > 0 or audio_tokens > 0 or reasoning_tokens > 0 or video_tokens > 0 - if text_tokens == 0: - if has_token_breakdown: - # Calculate text tokens as remainder when we have a breakdown - # This handles cases like OpenAI's reasoning models where text_tokens isn't provided - text_tokens = max( - 0, - usage.completion_tokens - reasoning_tokens - audio_tokens - image_tokens - video_tokens, - ) - else: - # No breakdown at all, all tokens are text tokens - text_tokens = usage.completion_tokens - is_text_tokens_total = True - ## TEXT COST - completion_cost = float(text_tokens) * completion_base_cost - - ## AUDIO COST - if not is_text_tokens_total and audio_tokens is not None and audio_tokens > 0: - _output_cost_per_audio_token = get_cost_per_unit(resolved_model_info, "output_cost_per_audio_token", None) - _output_cost_per_audio_token = ( - _output_cost_per_audio_token if _output_cost_per_audio_token is not None else completion_base_cost - ) - completion_cost += float(audio_tokens) * _output_cost_per_audio_token - - ## REASONING COST - if not is_text_tokens_total and reasoning_tokens and reasoning_tokens > 0: - completion_cost += float(reasoning_tokens) * _resolve_billed_reasoning_rate( - model_info=resolved_model_info, - usage=usage, - service_tier=service_tier, - completion_base_cost=completion_base_cost, - current_time=billing_time, - ) - - ## IMAGE COST - if not is_text_tokens_total and image_tokens and image_tokens > 0: - _output_cost_per_image_token = get_cost_per_unit(resolved_model_info, "output_cost_per_image_token", None) - _output_cost_per_image_token = ( - _output_cost_per_image_token if _output_cost_per_image_token is not None else completion_base_cost - ) - completion_cost += float(image_tokens) * _output_cost_per_image_token - - ## VIDEO COST - if not is_text_tokens_total and video_tokens and video_tokens > 0: - _output_cost_per_video_token = get_cost_per_unit(resolved_model_info, "output_cost_per_video_token", None) - _output_cost_per_video_token = ( - _output_cost_per_video_token if _output_cost_per_video_token is not None else completion_base_cost - ) - completion_cost += float(video_tokens) * _output_cost_per_video_token - - ## REGIONAL DATA-RESIDENCY UPLIFT - # Applied as a flat multiplier across all token costs for the request - # when the upstream is a regionalized OpenAI host (eu./us.api.openai.com). - uplift: Final = get_regional_uplift_multiplier(resolved_model_info, data_residency) - if uplift != 1.0: - prompt_cost *= uplift - completion_cost *= uplift - - vertex_uplift: Final = get_vertex_regional_endpoint_uplift(resolved_model_info, vertex_location) - if vertex_uplift != 1.0: - prompt_cost *= vertex_uplift - completion_cost *= vertex_uplift - - return prompt_cost, completion_cost +def _unmetered_prompt_cost(details: PromptTokensDetailsResult, row: Mapping[str, object]) -> float: + """Prompt meters the Rust cost core does not price yet, characters, image counts, media seconds and queries""" + return ( + calculate_cost_component(row, "input_cost_per_character", details["character_count"]) + + calculate_cost_component(row, "input_cost_per_image", details["image_count"]) + + calculate_cost_component(row, "input_cost_per_video_per_second", details["video_length_seconds"]) + + calculate_cost_component(row, "input_cost_per_audio_per_second", details["audio_length_seconds"]) + + calculate_cost_component(row, "input_cost_per_query", details["query_count"]) + ) def _coerce_token_count(value: object) -> int: diff --git a/litellm/main.py b/litellm/main.py index 9f412e1f24b..7a09aede666 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -155,7 +155,6 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ( BaseUrlRoutingHint, - CustomPricingLiteLLMParams, ModelResponseStream, ProviderRoutingHints, RawRequestTypedDict, @@ -1119,7 +1118,6 @@ from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the o ) from litellm.responses.bridge_check import ( bridge_reasoning_effort, - keep_registered_mode, str_keyed_mapping, ) from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the old definition site @@ -1236,93 +1234,6 @@ def _locals_snapshot(values: Mapping[str, object]) -> Mapping[str, object]: return values -def _build_custom_pricing_entry( - custom_llm_provider: str, - kwargs: dict, - model_info: dict | None = None, -) -> dict: - """Build a complete model cost entry from kwargs and model_info. - - Collects all CustomPricingLiteLLMParams fields present in kwargs and - merges metadata from model_info (mode, supports_prompt_caching, max_tokens) - so that register_model() receives the full pricing configuration. - """ - entry: Final[dict] = {"litellm_provider": custom_llm_provider} - - for field_name in CustomPricingLiteLLMParams.model_fields: - value = kwargs.get(field_name) - if value is not None: - entry[field_name] = value - - if model_info and isinstance(model_info, dict): - for key in ("mode", "supports_prompt_caching", "max_tokens"): - if key in model_info and model_info[key] is not None: - entry.setdefault(key, model_info[key]) - - return entry - - -def _get_router_deployment_id(kwargs: dict) -> str | None: - for metadata_key in ("litellm_metadata", "metadata"): - metadata = kwargs.get(metadata_key) or {} - if not isinstance(metadata, dict): - continue - deployment_model_info = metadata.get("model_info") or {} - if not isinstance(deployment_model_info, dict): - continue - deployment_id = deployment_model_info.get("id") - if deployment_id is not None: - return str(deployment_id) - return None - - -def _register_custom_pricing_for_request( - model: str, - custom_llm_provider: str, - kwargs: dict, - model_info: dict | None, -) -> None: - """Register per-request custom pricing in litellm.model_cost. - - Router-originated requests (identified by the deployment id the router puts - in metadata) get their full pricing registered under that unique id only; - the shared ``{provider}/{model}`` key receives the entry with pricing fields - stripped, mirroring Router._create_deployment. This keeps one deployment's - pricing overrides (e.g. a zero-cost wildcard) from clobbering built-in - pricing used by sibling deployments of the same backend model. Direct SDK - calls keep the legacy behavior of registering the shared key with pricing. - Either way the shared key's ``mode`` follows ``keep_registered_mode``, as on - deployment registration, since a call routes by its own ``model_info``. - """ - entry: Final = _build_custom_pricing_entry( - custom_llm_provider=custom_llm_provider, - kwargs=kwargs, - model_info=model_info, - ) - shared_key: Final = f"{custom_llm_provider}/{model}" - deployment_id: Final = _get_router_deployment_id(kwargs) - if deployment_id is None: - litellm.register_model({shared_key: keep_registered_mode(shared_key, entry)}, persist_across_reloads=False) - return - shared_entry: Final = keep_registered_mode( - shared_key, CustomPricingLiteLLMParams.strip_custom_pricing_fields(entry) - ) - litellm.register_model( - {deployment_id: entry}, - persist_across_reloads=False, - warning_display_name=shared_key, - custom_llm_provider=custom_llm_provider, - add_to_provider_model_sets=False, - ) - litellm.register_model( - {shared_key: shared_entry}, - persist_across_reloads=False, - warning_display_name=shared_key, - custom_llm_provider=custom_llm_provider, - add_to_provider_model_sets=False, - ) - - def _dispatch_metadata(ctx: CompletionDispatchContext) -> Mapping[str, object] | None: return ctx.metadata @@ -5757,19 +5668,6 @@ def completion( "aws_region_name", None ) # support region-based pricing for bedrock - ### REGISTER CUSTOM MODEL PRICING -- IF GIVEN ### - if ( - (input_cost_per_token is not None and output_cost_per_token is not None) - or input_cost_per_second is not None - or output_cost_per_second is not None - or cost_per_second is not None - ): - _register_custom_pricing_for_request( - model=model, - custom_llm_provider=custom_llm_provider, - kwargs=kwargs, - model_info=model_info, - ) ### BUILD CUSTOM PROMPT TEMPLATE -- IF GIVEN ### custom_prompt_dict = {} if initial_prompt_value or roles or final_prompt_value or bos_token or eos_token: @@ -6713,12 +6611,6 @@ def embedding( headers.update(proxy_headers) except Exception as e: verbose_logger.warning("Failed to get proxy auth headers: %s", e) - ### CUSTOM MODEL COST ### - input_cost_per_token: Final = kwargs.get("input_cost_per_token", None) - output_cost_per_token: Final = kwargs.get("output_cost_per_token", None) - cost_per_second: Final = kwargs.get("cost_per_second", None) - input_cost_per_second: Final = kwargs.get("input_cost_per_second", None) - output_cost_per_second: Final = kwargs.get("output_cost_per_second", None) openai_params: Final = [ "user", "dimensions", @@ -6765,20 +6657,6 @@ def embedding( **non_default_params, ) - ### REGISTER CUSTOM MODEL PRICING -- IF GIVEN ### - if ( - (input_cost_per_token is not None and output_cost_per_token is not None) - or input_cost_per_second is not None - or output_cost_per_second is not None - or cost_per_second is not None - ): - _register_custom_pricing_for_request( - model=model, - custom_llm_provider=custom_llm_provider, - kwargs=kwargs, - model_info=kwargs.get("model_info"), - ) - logging: Final[LiteLLMLoggingObj] = litellm_logging_obj logging.update_environment_variables( model=model, diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi index 46b04113e74..3502d125821 100644 --- a/litellm/rust_bridge/_native.pyi +++ b/litellm/rust_bridge/_native.pyi @@ -103,6 +103,7 @@ class Provenance: @final class Resolved: key: str + prices: Prices def model_info(self, provider: str | None = None) -> dict[str, JsonValue]: ... def supports(self, capability: str) -> bool: ... def row(self) -> dict[str, JsonValue]: ... @@ -118,6 +119,69 @@ class Invalid: def model_info(self, provider: str | None = None) -> dict[str, JsonValue]: ... def row(self) -> dict[str, JsonValue]: ... +@final +class Prices: + @staticmethod + def from_row(row: Mapping[str, object], key: str = "model_info") -> Prices | Invalid: ... + def price(self, usage: Usage, call: CallContext) -> Priced | Unpriced | Invalid: ... + +@final +class Usage: + def __init__( + self, + *, + prompt_tokens: int, + text: int, + output_text: int, + audio: int = 0, + image: int = 0, + video: int = 0, + cache_read: int = 0, + cache_read_audio: int = 0, + cache_read_explicit: int = 0, + cache_write_5m: int = 0, + cache_write_1h: int = 0, + cache_write_12h: int = 0, + output_audio: int = 0, + output_image: int = 0, + output_video: int = 0, + reasoning: int = 0, + ) -> None: ... + + +@final +class CallContext: + def __init__( + self, + *, + service_tier: str | None, + channel: Literal["online", "batch", "decisions"], + data_residency: str | None, + vertex_location: str | None, + at: datetime, + custom_pricing: Mapping[str, object] | None, + ) -> None: ... + +@final +class Line: + meter: str + quantity: int + unit_price: float + amount: float + fallback_from: str | None + +@final +class Priced: + total: float + prompt: float + completion: float + lines: tuple[Line, ...] + missing: tuple[str, ...] + +@final +class Unpriced: + missing: tuple[str, ...] + @final class Registered: written: list[str] diff --git a/litellm/utils.py b/litellm/utils.py index f7545e83466..c8f36564cf4 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6081,9 +6081,6 @@ def get_provider_info(model: str, custom_llm_provider: str | None) -> ProviderSp return json_provider.provider_info if json_provider is not None and json_provider.provider_info else None -_REPORTED_INVALID_ROWS: Final[set[str]] = set() # mutable-ok: logs each bad live row once per process - - def _model_not_mapped_message(model: str, custom_llm_provider: str | None) -> str: return ( f"This model isn't mapped yet. model={model}, custom_llm_provider={custom_llm_provider}. " @@ -6192,15 +6189,12 @@ def get_model_info_helper( and strip it. Tried last so no model that already resolves through 1-6 can change. """ - from litellm._catalog import CATALOG, Invalid, Missing + from litellm._catalog import CATALOG, Invalid, Missing, report_invalid resolution: Final = CATALOG.resolve(model, requested_provider) key: str | None = None if isinstance(resolution, Missing) else resolution.key - if isinstance(resolution, Invalid) and resolution.key not in _REPORTED_INVALID_ROWS: - _REPORTED_INVALID_ROWS.add(resolution.key) - verbose_logger.warning( - "litellm.model_cost[%r] fails validation: %s", resolution.key, "; ".join(resolution.errors) - ) + if isinstance(resolution, Invalid): + report_invalid(resolution) _model_info: dict[str, Any] | None = None if isinstance(resolution, Missing) else resolution.row() if _model_info is not None and key is not None and (_model_info.get("mode") or "chat") in _BACKFILL_MODES: diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 21b527250c4..0b3933a0598 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -10,9 +10,7 @@ from litellm._internal_context import pinned_billing_time from litellm.litellm_core_utils.llm_cost_calc.utils import ( BilledTokenRates, CostCalculatorUtils, - PromptTokensDetailsResult, TokenRates, - _calculate_input_cost, _get_token_base_cost, _is_off_peak, _is_within_off_peak_window, @@ -1919,7 +1917,7 @@ def test_calculate_cost_component_with_string_values(): def test_string_cost_values_edge_cases(): - """Test edge cases for string cost value handling.""" + """A numeric string is a price, a word is not, and a row with one bad price is unpriced (decision 20)""" from unittest.mock import patch # Test with mixed string and float cost values @@ -1954,14 +1952,7 @@ def test_string_cost_values_edge_cases(): model="test-model", usage=usage, custom_llm_provider="test-provider" ) - # Expected costs: - # Prompt: 1000 * 1e-6 + 100 * 0 (invalid string becomes 0) - # Completion: 500 * 2e-6 + 50 * 2e-6 (audio tokens fall back to base cost when output_cost_per_audio_token is None) - expected_prompt_cost = 1000 * 1e-6 - expected_completion_cost = 500 * 2e-6 + 50 * 2e-6 - - assert round(prompt_cost, 12) == round(expected_prompt_cost, 12) - assert round(completion_cost, 12) == round(expected_completion_cost, 12) + assert (prompt_cost, completion_cost) == (0.0, 0.0) def test_string_cost_values_with_threshold(): @@ -2225,46 +2216,38 @@ def test_cache_writing_cost_with_zero_creation_tokens_and_ephemeral_details(): """ Regression test: when cache_creation_tokens is 0 but cache_creation_token_details has non-zero ephemeral tokens, the cost must still be calculated. - This ensures the guard in _calculate_input_cost doesn't skip - calculate_cache_writing_cost when only ephemeral token details are present. """ cache_creation_cost = 3.75e-06 cache_creation_cost_above_1hr = 6e-06 - - prompt_tokens_details: PromptTokensDetailsResult = { - "cache_hit_tokens": 0, - "cache_hit_audio_tokens": 0, - "cache_creation_tokens": 0, - "cache_creation_token_details": CacheCreationTokenDetails( - ephemeral_5m_input_tokens=100, - ephemeral_1h_input_tokens=200, + usage = Usage( + prompt_tokens=300, + completion_tokens=0, + total_tokens=300, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=0, + cache_creation_tokens=0, + cache_creation_token_details=CacheCreationTokenDetails( + ephemeral_5m_input_tokens=100, + ephemeral_1h_input_tokens=200, + ), ), - "text_tokens": 0, - "audio_tokens": 0, - "image_tokens": 0, - "video_tokens": 0, - "character_count": 0, - "image_count": 0, - "video_length_seconds": 0.0, - "audio_length_seconds": 0.0, - "query_count": 0, - } - - model_info: ModelInfo = {} - - result = _calculate_input_cost( - prompt_tokens_details=prompt_tokens_details, - model_info=model_info, - prompt_base_cost=0.0, - cache_read_cost=0.0, - cache_creation_cost=cache_creation_cost, - cache_creation_cost_by_ttl={"1hr": cache_creation_cost_above_1hr, "12hr": cache_creation_cost_above_1hr}, ) - # Expected: (100 * 3.75e-06) + (200 * 6e-06) = 0.000375 + 0.0012 = 0.001575 - expected = (100 * cache_creation_cost) + (200 * cache_creation_cost_above_1hr) - assert result > 0, "Cost should not be zero when ephemeral token details are present" - assert round(result, 6) == round(expected, 6) + prompt_cost, _ = generic_cost_per_token( + model="litellm-test-ephemeral-cache-writes", + usage=usage, + custom_llm_provider="anthropic", + model_info=cast( + ModelInfo, + { + "input_cost_per_token": 0.0, + "cache_creation_input_token_cost": cache_creation_cost, + "cache_creation_input_token_cost_above_1hr": cache_creation_cost_above_1hr, + }, + ), + ) + + assert prompt_cost == pytest.approx(100 * cache_creation_cost + 200 * cache_creation_cost_above_1hr) def test_service_tier_ultrafast_pricing(): diff --git a/tests/unit/llms/sagemaker/test_sagemaker_completion_handler.py b/tests/unit/llms/sagemaker/test_sagemaker_completion_handler.py index 0980c9f5c66..4ce51fe46bd 100644 --- a/tests/unit/llms/sagemaker/test_sagemaker_completion_handler.py +++ b/tests/unit/llms/sagemaker/test_sagemaker_completion_handler.py @@ -528,7 +528,8 @@ async def test_completion_sagemaker( assert route.called assert response.choices[0].message.content == "SageMaker reply" - assert litellm.completion_cost(completion_response=response) == pytest.approx(rate) + assert litellm.completion_cost(completion_response=response, custom_cost_per_second=rate) == pytest.approx(rate) + assert "sagemaker/jumpstart-dft-hf-textgeneration1-mp-20240815-185614" not in litellm.model_cost @pytest.mark.asyncio diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index dc5c8bd6692..d47227b1b98 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -10590,3 +10590,46 @@ def test_bedrock_registered_cache_prices_preserve_dynamic_threshold_rates( assert entry is not None assert entry[1]["input_cost_per_token_above_300k_tokens"] == 0.03 assert entry[1][tier_field] == (tier_override if tier_override is not None else 0.007) + + +@pytest.mark.parametrize( + "bad_price", + [pytest.param("abc", id="word"), pytest.param(-1.0, id="negative"), pytest.param(float("nan"), id="nan")], +) +def test_a_directly_written_bad_price_is_unpriced_and_logged_once( + bad_price: object, caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch +) -> None: + """Decision 20. On main the same 1000 prompt tokens gave a TypeError, -0.001 or nan""" + import logging + + from litellm._logging import verbose_logger + + key: Final = f"litellm-test-bad-price-{type(bad_price).__name__}-{bad_price!r}" + monkeypatch.setitem( + litellm.model_cost, + key, + {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": bad_price, "output_cost_per_token": 1e-6}, + ) + response: Final = litellm.ModelResponse( + model=key, usage=litellm.Usage(prompt_tokens=1000, completion_tokens=0, total_tokens=1000) + ) + + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + per_token: Final = cost_per_token(model=key, prompt_tokens=1000, completion_tokens=0, custom_llm_provider="openai") + total: Final = completion_cost(completion_response=response, model=key, custom_llm_provider="openai") + + assert per_token == (0.0, 0.0) + assert total == 0.0 + assert [record for record in caplog.records if key in record.getMessage() and "fails validation" in record.getMessage()] != [] + assert len([record for record in caplog.records if key in record.getMessage()]) == 1 + + +def test_a_directly_written_null_price_is_unpriced_like_a_missing_one(monkeypatch: pytest.MonkeyPatch) -> None: + key: Final = "litellm-test-null-price" + monkeypatch.setitem( + litellm.model_cost, + key, + {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": None, "output_cost_per_token": 1e-6}, + ) + + assert cost_per_token(model=key, prompt_tokens=1000, completion_tokens=0, custom_llm_provider="openai") == (0.0, 0.0) diff --git a/tests/unit/test_register_model_custom_pricing.py b/tests/unit/test_register_model_custom_pricing.py index 9a08f0c6352..f125ec14bed 100644 --- a/tests/unit/test_register_model_custom_pricing.py +++ b/tests/unit/test_register_model_custom_pricing.py @@ -1,12 +1,6 @@ """ -Test that register_model() in completion() and embedding() passes all -custom pricing fields from kwargs and model_info, not just the base -input/output costs. - -Previously, only input_cost_per_token, output_cost_per_token, and -litellm_provider were forwarded. Fields like cache_read_input_token_cost, -mode, and supports_prompt_caching were dropped, causing incorrect cost -calculations for DB-sourced models with prompt caching pricing. +Custom pricing through register_model() and per call. A router deployment's prices are registered under its id, and a +direct SDK call's prices price that call only, never the shared row (decision 12) """ import copy @@ -21,7 +15,7 @@ import pytest import litellm from litellm._catalog import Fetched, File from litellm.litellm_core_utils.get_model_cost_map import reload_model_cost_map -from litellm.main import _build_custom_pricing_entry +from litellm.types.utils import EmbeddingResponse, ModelResponse from litellm.utils import _invalidate_model_cost_lowercase_map @@ -44,131 +38,6 @@ def _restore_model_cost_entries(original_entries): _invalidate_model_cost_lowercase_map() -def test_build_custom_pricing_entry_includes_all_kwargs_fields(): - """All CustomPricingLiteLLMParams fields present in kwargs should be - included in the resulting entry dict.""" - kwargs = { - "input_cost_per_token": 0.001, - "output_cost_per_token": 0.002, - "cache_read_input_token_cost": 0.00025, - "cache_creation_input_token_cost": 0.005, - "output_cost_per_reasoning_token": 0.01, - "input_cost_per_audio_token": 0.003, - "unrelated_kwarg": "should_be_ignored", - } - - entry = _build_custom_pricing_entry( - custom_llm_provider="openai", - kwargs=kwargs, - ) - - assert entry["litellm_provider"] == "openai" - assert entry["input_cost_per_token"] == 0.001 - assert entry["output_cost_per_token"] == 0.002 - assert entry["cache_read_input_token_cost"] == 0.00025 - assert entry["cache_creation_input_token_cost"] == 0.005 - assert entry["output_cost_per_reasoning_token"] == 0.01 - assert entry["input_cost_per_audio_token"] == 0.003 - assert "unrelated_kwarg" not in entry - - -def test_build_custom_pricing_entry_merges_model_info_metadata(): - """Fields from model_info (mode, supports_prompt_caching, max_tokens) - should be merged into the entry when present.""" - kwargs = { - "input_cost_per_token": 0.001, - "output_cost_per_token": 0.002, - } - model_info = { - "id": "deployment-123", - "mode": "chat", - "supports_prompt_caching": True, - "max_tokens": 128000, - } - - entry = _build_custom_pricing_entry( - custom_llm_provider="openai", - kwargs=kwargs, - model_info=model_info, - ) - - assert entry["mode"] == "chat" - assert entry["supports_prompt_caching"] is True - assert entry["max_tokens"] == 128000 - - -def test_build_custom_pricing_entry_setdefault_does_not_override_existing(): - """model_info uses setdefault, so it should not override a key that is - already present in the entry dict. Currently CustomPricingLiteLLMParams - and the model_info keys (mode, supports_prompt_caching, max_tokens) do - not overlap, but if they ever do, setdefault ensures the kwargs-sourced - value wins.""" - kwargs = { - "input_cost_per_token": 0.001, - "output_cost_per_token": 0.002, - } - model_info = { - "mode": "chat", - "supports_prompt_caching": True, - "max_tokens": 128000, - } - - entry = _build_custom_pricing_entry( - custom_llm_provider="openai", - kwargs=kwargs, - model_info=model_info, - ) - - assert entry["mode"] == "chat" - assert entry["supports_prompt_caching"] is True - assert entry["max_tokens"] == 128000 - - # Verify setdefault behavior: if a model_info key already exists in - # the entry (e.g. from a future CustomPricingLiteLLMParams addition), - # setdefault must not overwrite it. - entry["mode"] = "embedding" # simulate pre-existing value - # Re-apply setdefault the same way _build_custom_pricing_entry does - entry.setdefault("mode", model_info["mode"]) - assert entry["mode"] == "embedding" # must NOT revert to "chat" - - -def test_build_custom_pricing_entry_skips_none_values(): - """Fields with None values in kwargs should not be included.""" - kwargs = { - "input_cost_per_token": 0.001, - "output_cost_per_token": None, # explicitly None - "cache_read_input_token_cost": None, - } - - entry = _build_custom_pricing_entry( - custom_llm_provider="openai", - kwargs=kwargs, - ) - - assert entry["input_cost_per_token"] == 0.001 - assert "output_cost_per_token" not in entry - assert "cache_read_input_token_cost" not in entry - - -def test_build_custom_pricing_entry_handles_no_model_info(): - """Should work correctly when model_info is None.""" - kwargs = { - "input_cost_per_token": 0.001, - "output_cost_per_token": 0.002, - } - - entry = _build_custom_pricing_entry( - custom_llm_provider="openai", - kwargs=kwargs, - model_info=None, - ) - - assert entry["litellm_provider"] == "openai" - assert entry["input_cost_per_token"] == 0.001 - assert entry["output_cost_per_token"] == 0.002 - assert "mode" not in entry - - def test_register_model_receives_cache_pricing_fields(): """End-to-end: when register_model is called with a full pricing entry, the cache pricing fields should be present in litellm.model_cost.""" @@ -199,23 +68,6 @@ def test_register_model_receives_cache_pricing_fields(): litellm.model_cost.pop(model_key, None) -def test_build_custom_pricing_entry_time_based(): - """Time-based pricing fields should be included correctly.""" - kwargs = { - "input_cost_per_second": 0.01, - "output_cost_per_second": 0.02, - } - - entry = _build_custom_pricing_entry( - custom_llm_provider="openai", - kwargs=kwargs, - ) - - assert entry["litellm_provider"] == "openai" - assert entry["input_cost_per_second"] == 0.01 - assert entry["output_cost_per_second"] == 0.02 - - def test_register_model_strips_none_litellm_provider(): """``get_model_info`` returns ``litellm_provider: None`` for deployments registered without a provider (e.g. ``Router.add_deployment`` flows). @@ -603,8 +455,7 @@ def test_embedding_router_zero_pricing_does_not_clobber_builtin_pricing(): litellm.get_model_info(model=shared_key)["input_cost_per_token"] == builtin_input_cost ), "wildcard deployment's zero pricing leaked into the shared model_cost key" - assert litellm.model_cost[deployment_id]["input_cost_per_token"] == 0.0 - assert litellm.model_cost[deployment_id]["output_cost_per_token"] == 0.0 + assert deployment_id not in litellm.model_cost sibling_response = litellm.embedding( model=shared_key, @@ -620,20 +471,15 @@ def test_embedding_router_zero_pricing_does_not_clobber_builtin_pricing(): _restore_model_cost_entries(snapshot) -def test_embedding_router_custom_pricing_costs_request_via_deployment_id(): - """The request that carries custom pricing must still be costed with that - pricing (via its deployment id entry), while the shared backend key keeps - the built-in rate for siblings. +def test_embedding_router_custom_pricing_costs_request_with_its_own_prices(): + """The request that carries custom pricing is costed with that pricing, while the shared backend key keeps the + built-in rate for siblings """ shared_key = "openai/text-embedding-3-small" deployment_id = "lit3991-wildcard-embed-custom" override_input_cost = 5e-05 - snapshot = _snapshot_model_cost_entries( - [shared_key, "text-embedding-3-small", deployment_id] - ) - builtin_input_cost = litellm.get_model_info(model=shared_key)[ - "input_cost_per_token" - ] + snapshot = _snapshot_model_cost_entries([shared_key, "text-embedding-3-small", deployment_id]) + builtin_input_cost = litellm.get_model_info(model=shared_key)["input_cost_per_token"] assert builtin_input_cost != override_input_cost try: @@ -648,19 +494,8 @@ def test_embedding_router_custom_pricing_costs_request_via_deployment_id(): mock_response=[0.1, 0.2], ) - request_cost = litellm.completion_cost( - completion_response=response, - model=shared_key, - custom_llm_provider="openai", - call_type="embedding", - custom_pricing=True, - router_model_id=deployment_id, - ) - assert request_cost == pytest.approx(10 * override_input_cost) - assert ( - litellm.get_model_info(model=shared_key)["input_cost_per_token"] - == builtin_input_cost - ) + assert _response_cost(response) == pytest.approx(10 * override_input_cost) + assert litellm.get_model_info(model=shared_key)["input_cost_per_token"] == builtin_input_cost finally: _restore_model_cost_entries(snapshot) @@ -693,43 +528,80 @@ def test_completion_router_zero_pricing_does_not_clobber_builtin_pricing(): litellm.get_model_info(model=shared_key)["input_cost_per_token"] == builtin_input_cost ), "wildcard deployment's zero pricing leaked into the shared model_cost key" - assert litellm.model_cost[deployment_id]["input_cost_per_token"] == 0.0 + assert deployment_id not in litellm.model_cost finally: _restore_model_cost_entries(snapshot) -def test_embedding_direct_sdk_custom_pricing_still_registers_shared_key(): - """Direct SDK calls (no router deployment id in metadata) keep the legacy - behavior: custom pricing is registered under ``{provider}/{model}`` and the - request is costed with it. - """ - model_key = "openai/lit3991-direct-sdk-embed-model" - override_input_cost = 3e-05 - try: - response = litellm.embedding( - model=model_key, - input=["hello"], - api_key="fake-key", - input_cost_per_token=override_input_cost, - output_cost_per_token=override_input_cost * 2, - mock_response=[0.1, 0.2], - ) +def test_embedding_direct_sdk_custom_pricing_prices_the_call_without_registering_the_shared_key(): + model_key: Final = "openai/lit3991-direct-sdk-embed-model" + override_input_cost: Final = 3e-05 + response: Final = litellm.embedding( + model=model_key, + input=["hello"], + api_key="fake-key", + input_cost_per_token=override_input_cost, + output_cost_per_token=override_input_cost * 2, + mock_response=[0.1, 0.2], + ) - assert ( - litellm.model_cost[model_key]["input_cost_per_token"] - == override_input_cost - ) - cost = litellm.completion_cost( - completion_response=response, - model=model_key, - custom_llm_provider="openai", - call_type="embedding", - custom_pricing=True, - ) - assert cost == pytest.approx(10 * override_input_cost) - finally: - litellm.model_cost.pop(model_key, None) - _invalidate_model_cost_lowercase_map() + assert model_key not in litellm.model_cost + assert isinstance(response, EmbeddingResponse) + assert _response_cost(response) == pytest.approx(response.usage.prompt_tokens * override_input_cost) + + +def _response_cost(response: object) -> float: + assert isinstance(response, (ModelResponse, EmbeddingResponse)) + cost: Final = response._hidden_params["response_cost"] # pyright: ignore[reportPrivateUsage] # the SDK reports a call's cost only here + assert isinstance(cost, float) + return cost + + +def _tenant_call_cost(**custom_pricing: float) -> tuple[float, litellm.Usage]: + response: Final = litellm.completion( + model="openai/gpt-4o", + messages=[{"role": "user", "content": "price this call"}], + api_key="fake-key", + mock_response="priced", + **custom_pricing, + ) + assert isinstance(response, ModelResponse) + usage: Final = response.usage + assert isinstance(usage, litellm.Usage) + return _response_cost(response), usage + + +def test_one_tenants_custom_pricing_never_reprices_another_tenants_calls(): + """Two tenants in one SDK process. Tenant A's negotiated price prices only A's call""" + catalog_row_before: Final = copy.deepcopy(litellm.model_cost["gpt-4o"]) + shared_key_before: Final = copy.deepcopy(litellm.model_cost.get("openai/gpt-4o")) + tenant_b_before, usage = _tenant_call_cost() + tenant_a, tenant_a_usage = _tenant_call_cost(input_cost_per_token=1e-06, output_cost_per_token=4e-06) + tenant_b_after, _ = _tenant_call_cost() + + input_rate, output_rate = catalog_row_before["input_cost_per_token"], catalog_row_before["output_cost_per_token"] + assert isinstance(input_rate, float) and isinstance(output_rate, float) + catalog_cost: Final = usage.prompt_tokens * input_rate + usage.completion_tokens * output_rate + assert tenant_b_before == pytest.approx(catalog_cost) + assert tenant_a == pytest.approx(tenant_a_usage.prompt_tokens * 1e-06 + tenant_a_usage.completion_tokens * 4e-06) + assert tenant_b_after == pytest.approx(catalog_cost) + assert litellm.model_cost["gpt-4o"] == catalog_row_before + assert litellm.model_cost.get("openai/gpt-4o") == shared_key_before + + +def test_custom_pricing_never_writes_the_calls_model_info_onto_the_shared_row(): + model_key: Final = "openai/litellm-test-custom-priced-responses-mode" + litellm.completion( + model=model_key, + messages=[{"role": "user", "content": "hello"}], + api_key="fake-key", + input_cost_per_token=1e-06, + output_cost_per_token=2e-06, + model_info={"mode": "responses"}, + mock_response="hello back", + ) + + assert model_key not in litellm.model_cost @pytest.mark.parametrize("route", ["", "converse/", "invoke/"]) @@ -1468,22 +1340,18 @@ def test_completion_cost_applies_off_peak_only_deployment_pricing(): del router -def test_completion_registers_cost_per_second_pricing(): +def test_completion_prices_per_second_custom_pricing_without_registering_it(): model_key: Final = "openai/test-cost-per-second-registration" - original_entries: Final = _snapshot_model_cost_entries([model_key]) + response: Final = litellm.completion( + model=model_key, + messages=[{"role": "user", "content": "hello"}], + api_key="fake-key", + cost_per_second=0.02, + mock_response="hello back", + ) - try: - litellm.completion( - model=model_key, - messages=[{"role": "user", "content": "hello"}], - api_key="fake-key", - cost_per_second=0.02, - mock_response="hello back", - ) - - assert litellm.model_cost[model_key]["cost_per_second"] == 0.02 - finally: - _restore_model_cost_entries(original_entries) + assert model_key not in litellm.model_cost + assert _response_cost(response) >= 0 def test_update_model_cost(): @@ -1533,33 +1401,3 @@ def test_register_model_scopes_builtin_match_to_the_given_provider(): assert litellm.model_cost[builtin_key] == builtin_row_before finally: _restore_model_cost_entries(model_cost_entries) - - -def test_per_request_custom_pricing_scopes_a_colliding_deployment_id_to_its_provider(): - from litellm.main import _register_custom_pricing_for_request - - deployment_id: Final = "baseten/zai-org/glm-5.2" - builtin_key: Final = "baseten/zai-org/GLM-5.2" - shared_key: Final = "openai/zai-org/GLM-5.2" - model_cost_entries: Final = _snapshot_model_cost_entries((deployment_id, builtin_key, shared_key)) - builtin_row_before: Final = copy.deepcopy(litellm.model_cost[builtin_key]) - openai_models_before: Final = frozenset(litellm.open_ai_chat_completion_models) - try: - _register_custom_pricing_for_request( - model="zai-org/GLM-5.2", - custom_llm_provider="openai", - kwargs={ - "input_cost_per_token": 0.00000096, - "output_cost_per_token": 0.00000302, - "metadata": {"model_info": {"id": deployment_id}}, - }, - model_info={"mode": "chat"}, - ) - - entry: Final = litellm.model_cost[deployment_id] - assert entry["input_cost_per_token"] == 0.00000096 - assert entry["output_cost_per_token"] == 0.00000302 - assert litellm.model_cost[builtin_key] == builtin_row_before - finally: - _restore_model_cost_entries(model_cost_entries) - litellm.open_ai_chat_completion_models.intersection_update(openai_models_before) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 2a2d1f8584f..914589be55d 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -2243,7 +2243,7 @@ def test_register_model_price_is_used_for_completion_cost() -> None: @respx.mock @pytest.mark.usefixtures("isolated_openai_model_sets", "local_model_cost_map") -def test_completion_registers_test_owned_model_prices() -> None: +def test_completion_prices_per_call_prices_without_registering_them() -> None: model: Final = "migration-owned-pricing-model" messages: Final = [{"role": "user", "content": "price control"}] litellm.register_model( @@ -2286,8 +2286,7 @@ def test_completion_registers_test_owned_model_prices() -> None: ) assert json.loads(route.calls[0].request.content) == {"model": model, "messages": messages} - assert litellm.model_cost[f"openai/{model}"]["input_cost_per_token"] == 0.001 - assert litellm.model_cost[f"openai/{model}"]["output_cost_per_token"] == 0.002 + assert f"openai/{model}" not in litellm.model_cost assert litellm.model_cost[model]["input_cost_per_token"] == 0.0001 assert response._hidden_params["response_cost"] == pytest.approx(0.018)