diff --git a/litellm-rust/crates/cost/src/calculate.rs b/litellm-rust/crates/cost/src/calculate.rs index abcd7e87e76..3d7a3429078 100644 --- a/litellm-rust/crates/cost/src/calculate.rs +++ b/litellm-rust/crates/cost/src/calculate.rs @@ -5,9 +5,11 @@ use crate::price::{ Meter, OffPeakRates, OutputUnit, Price, Prices, ServiceTier, TierScope, TokenKind, Uplift, When, }; -use crate::price::{Resolution, TextUnit, WallClock}; +use crate::price::{Resolution, SearchDefaults, TextUnit, WallClock, WebSearchUnit}; use crate::priced::{Line, Outcome, Priced, Quantity, Side}; -use crate::usage::{DurationMeter, Usage}; +use std::num::NonZeroU64; + +use crate::usage::{DurationMeter, ImageOut, Usage}; const TEXT_INPUT: Meter = Meter::Input(InputUnit::Token(TokenKind::Text)); const TEXT_OUTPUT: Meter = Meter::Output(OutputUnit::Token(TokenKind::Text)); @@ -35,6 +37,8 @@ pub fn calculate( let billed: Vec = token_meters(&rates, usage) .chain(character_meters(&rates, usage)) .chain(second_meters(&rates, usage)) + .chain(image_meters(&rates, usage)) + .chain(search_meters(&rates, usage)) .chain([( Meter::Input(InputUnit::TextMessage), rates.banded(&Meter::Input(InputUnit::TextMessage)), @@ -55,7 +59,6 @@ pub fn calculate( let missing: Vec = billed .into_iter() .filter_map(|(meter, rate, ..)| rate.is_none().then_some(meter)) - .chain(without_an_arm(usage)) .collect(); if lines.is_empty() && !missing.is_empty() { return Outcome::Unpriced { missing }; @@ -437,6 +440,38 @@ impl<'a> Rates<'a> { ) } + /// `search_provider_cost_per_query`'s tier walk. The tier whose range holds `max_results`, past the last range the + /// highest tier, without tiers the flat query price + fn query_rate(&self, max_results: u64) -> (Meter, Option) { + let tiers: Vec<(Option, UnitPrice)> = self + .prices + .iter() + .filter_map(|(price, rate)| match price.meter { + Meter::Input(InputUnit::Query { over_results }) + if price.channel == Channel::Online => + { + Some((over_results, rate)) + } + _ => None, + }) + .collect(); + let floor = |over: Option| over.map_or(0, NonZeroU64::get); + let held = tiers + .iter() + .filter(|(over, _)| max_results > floor(*over)) + .max_by_key(|(over, _)| floor(*over)) + .or_else(|| tiers.iter().max_by_key(|(over, _)| floor(*over))); + match held { + Some((over, rate)) => ( + Meter::Input(InputUnit::Query { + over_results: *over, + }), + Some(Rate::own(*rate)), + ), + None => (Meter::Input(InputUnit::Query { over_results: None }), None), + } + } + /// The first of `meters` with a price, borrowed when it is not the first fn first_of(&self, meters: &[Meter]) -> Option { let (own, rest) = meters.split_first()?; @@ -556,47 +591,142 @@ impl<'a> Rates<'a> { } } -/// Meters with usage that no arm prices yet, the per-unit meters 10 adds -fn without_an_arm(usage: &Usage) -> impl Iterator + '_ { - let counted = [ - (usage.images.input, Meter::Input(InputUnit::Image)), - (usage.images.input_pixels, Meter::Input(InputUnit::Pixel)), - ( - usage.images.output.len() as u64, - Meter::Output(OutputUnit::Image(None)), - ), - ( - usage.search.queries, - Meter::Input(InputUnit::Query { over_results: None }), - ), - ( - usage.search.maps_grounding, - Meter::Feature(Feature::GoogleMapsGroundingQuery), - ), - ( - usage.search.browser_opens, - Meter::Feature(Feature::BrowserOpen), - ), - ( - usage.search.additional_results, - Meter::Feature(Feature::AdditionalResult), - ), - ( - usage.tokens.citation, - Meter::Feature(Feature::CitationToken), - ), - ]; - let web_search = usage - .search - .web_search - .filter(|search| search.requests > 0) - .map(|search| Meter::WebSearch(search.context)); - counted - .into_iter() - .filter_map(|(count, meter)| (count > 0).then_some(meter)) - .chain(web_search) +const MEGAPIXEL: u64 = 1_048_576; + +/// main's `default_image_cost_calculator` and fal's image rules. A requested size with its own price bills that +/// price, then the first of the input per-image price, the output per-image price and the input per-pixel price +/// times the image's pixels, then fal's output per-pixel price per started megapixel +fn image_meters(rates: &Rates<'_>, usage: &Usage) -> Vec { + let per_image = |image: &ImageOut| -> Billed { + let pixels = u64::from(image.width.unwrap_or(0)) * u64::from(image.height.unwrap_or(0)); + let sized = image.size.clone().and_then(|size| { + let meter = Meter::Output(OutputUnit::Image(Some(size))); + Some((meter.clone(), rates.banded(&meter)?)) + }); + let candidates = [ + (Meter::Input(InputUnit::Image), 1), + (Meter::Output(OutputUnit::Image(None)), 1), + (Meter::Input(InputUnit::Pixel), pixels), + ( + Meter::Output(OutputUnit::Pixel), + MEGAPIXEL * pixels.div_ceil(MEGAPIXEL), + ), + ]; + let flat = || { + candidates + .into_iter() + .filter(|(_, quantity)| *quantity > 0) + .find_map(|(meter, quantity)| { + Some((meter.clone(), rates.banded(&meter)?, quantity)) + }) + }; + match sized.map(|(meter, rate)| (meter, rate, 1)).or_else(flat) { + Some((meter, rate, quantity)) => ( + meter, + Some(rate), + Quantity::Count(quantity), + Side::Completion, + ), + None => ( + Meter::Output(OutputUnit::Image(None)), + None, + Quantity::Count(1), + Side::Completion, + ), + } + }; + let input_image = Meter::Input(InputUnit::Image); + let input_pixel = Meter::Input(InputUnit::Pixel); + usage + .images + .output + .iter() + .map(per_image) + .chain([ + ( + input_image.clone(), + rates.banded(&input_image), + Quantity::Count(usage.images.input), + Side::Prompt, + ), + ( + input_pixel.clone(), + rates.banded(&input_pixel), + Quantity::Count(usage.images.input_pixels), + Side::Prompt, + ), + ]) + .collect() } +/// Search query tiers, web search, Maps grounding, citation tokens, Groq browser opens and Parallel AI's extra +/// results. Web search and Maps bill once per prompt unless the row bills them per query +fn search_meters(rates: &Rates<'_>, usage: &Usage) -> Vec { + let search = &usage.search; + let per_prompt = rates.prices.billing().web_search == WebSearchUnit::PerPrompt; + let billed_requests = |requests: u64| match per_prompt { + true => requests.min(1), + false => requests, + }; + let google = rates.prices.billing().search_defaults == SearchDefaults::Google; + let default = |price: f64| google.then(|| UnitPrice::new(price)).flatten(); + let web_search = search.web_search.map(|web_search| { + let meter = Meter::WebSearch(web_search.context); + let rate = rates.banded(&meter).or_else(|| { + default(GOOGLE_WEB_SEARCH_DEFAULT).map(|price| Rate::own(price).borrowed_from(&meter)) + }); + ( + meter, + rate, + Quantity::Count(billed_requests(web_search.requests)), + Side::Prompt, + ) + }); + let maps = Meter::Feature(Feature::GoogleMapsGroundingQuery); + let maps_default = match per_prompt { + true => GOOGLE_MAPS_DEFAULT_PER_PROMPT, + false => GOOGLE_MAPS_DEFAULT_PER_QUERY, + }; + let maps_line = ( + maps.clone(), + rates + .banded(&maps) + .or_else(|| default(maps_default).map(|price| Rate::own(price).borrowed_from(&maps))), + Quantity::Count(billed_requests(search.maps_grounding)), + Side::Prompt, + ); + let feature = |feature: Feature, count: u64| { + let meter = Meter::Feature(feature); + ( + meter.clone(), + rates.banded(&meter), + Quantity::Count(count), + Side::Prompt, + ) + }; + let query = (search.queries > 0).then(|| { + let (meter, rate) = rates.query_rate(search.max_results.unwrap_or(DEFAULT_MAX_RESULTS)); + (meter, rate, Quantity::Count(search.queries), Side::Prompt) + }); + query + .into_iter() + .chain(web_search) + .chain([ + maps_line, + feature(Feature::CitationToken, usage.tokens.citation), + feature(Feature::BrowserOpen, search.browser_opens), + feature(Feature::AdditionalResult, search.additional_results), + ]) + .collect() +} + +/// Gemini's defaults for a Google row with no web search or Maps grounding price +const GOOGLE_WEB_SEARCH_DEFAULT: f64 = 0.035; +const GOOGLE_MAPS_DEFAULT_PER_QUERY: f64 = 0.014; +const GOOGLE_MAPS_DEFAULT_PER_PROMPT: f64 = 0.025; +/// The results a search returns when the request names no `max_results`, main's default +const DEFAULT_MAX_RESULTS: u64 = 10; + fn uplifts(adjustments: &Adjustments, call: &CallContext) -> Vec { let residency = call .residency @@ -605,7 +735,19 @@ fn uplifts(adjustments: &Adjustments, call: &CallContext) -> Vec { .non_global_endpoint .then(|| adjustments.get(&Uplift::NonGlobalEndpoint)) .flatten(); - residency.into_iter().chain(endpoint).collect() + let speed = call + .speed + .and_then(|speed| adjustments.get(&Uplift::Speed(speed))); + let geo = call + .inference_geo + .clone() + .and_then(|geo| adjustments.get(&Uplift::InferenceGeo(geo))); + residency + .into_iter() + .chain(endpoint) + .chain(speed) + .chain(geo) + .collect() } #[cfg(test)] @@ -616,7 +758,7 @@ mod tests { use rstest::rstest; use super::*; - use crate::price::{Billing, Residency}; + use crate::price::{Billing, Residency, SearchContextSize}; fn price( meter: &Meter, @@ -1152,4 +1294,194 @@ mod tests { let expected = if prompt > 32_000 { rate } else { 0.1e-6 }; close(total(&prices, &usage, &call()), 100.0 * expected); } + + fn images(images: Vec) -> Usage { + Usage { + images: crate::usage::Images { + output: images, + ..crate::usage::Images::default() + }, + ..Usage::default() + } + } + + fn image(size: Option<&str>, width: u32, height: u32) -> ImageOut { + ImageOut { + size: size.and_then(crate::price::ImageSize::new), + width: Some(width), + height: Some(height), + } + } + + #[rstest] + #[case::input_price_wins(Some(0.04), Some(0.08), None, 0.04)] + #[case::output_price(None, Some(0.08), None, 0.08)] + #[case::input_pixels(None, None, Some(1e-8), 1e-8 * 1024.0 * 1024.0)] + fn an_image_bills_the_first_per_image_price_in_mains_order( + #[case] input: Option, + #[case] output: Option, + #[case] per_pixel: Option, + #[case] expected: f64, + ) { + let prices = row(input + .map(|rate| standard(&Meter::Input(InputUnit::Image), rate)) + .into_iter() + .chain(output.map(|rate| standard(&Meter::Output(OutputUnit::Image(None)), rate))) + .chain(per_pixel.map(|rate| standard(&Meter::Input(InputUnit::Pixel), rate)))); + close( + total( + &prices, + &images(vec![image(None, 1024, 1024), image(None, 1024, 1024)]), + &call(), + ), + 2.0 * expected, + ); + } + + #[rstest] + fn a_requested_size_with_its_own_price_bills_that_price() { + let size = crate::price::ImageSize::new("1K").unwrap(); + let prices = row([ + standard(&Meter::Output(OutputUnit::Image(None)), 0.08), + standard(&Meter::Output(OutputUnit::Image(Some(size))), 0.03), + ]); + close( + total( + &prices, + &images(vec![image(Some("1K"), 1024, 1024)]), + &call(), + ), + 0.03, + ); + } + + #[rstest] + fn output_pixels_bill_per_started_megapixel() { + let prices = row([standard(&Meter::Output(OutputUnit::Pixel), 1e-8)]); + let outcome = priced(calculate( + &prices, + &Adjustments::default(), + &images(vec![image(None, 1500, 1000)]), + &call(), + )); + assert_eq!(outcome.lines[0].quantity, Quantity::Count(2 * MEGAPIXEL)); + } + + fn queries(count: u64, max_results: Option) -> Usage { + Usage { + search: crate::usage::Search { + queries: count, + max_results, + ..crate::usage::Search::default() + }, + ..Usage::default() + } + } + + #[rstest] + #[case::first_tier(Some(10), 0.005)] + #[case::second_tier(Some(50), 0.025)] + #[case::past_the_last_range(Some(500), 0.025)] + #[case::default_max_results(None, 0.005)] + fn a_query_bills_the_tier_its_max_results_falls_in( + #[case] max_results: Option, + #[case] rate: f64, + ) { + let query = |over: u64| { + Meter::Input(InputUnit::Query { + over_results: NonZeroU64::new(over), + }) + }; + let prices = row([standard(&query(0), 0.005), standard(&query(25), 0.025)]); + close( + total(&prices, &queries(2, max_results), &call()), + 2.0 * rate, + ); + } + + fn web_searches(requests: u64) -> Usage { + Usage { + search: crate::usage::Search { + web_search: Some(crate::usage::WebSearch { + requests, + context: SearchContextSize::Medium, + }), + ..crate::usage::Search::default() + }, + ..Usage::default() + } + } + + fn billed(rates: impl IntoIterator, billing: Billing) -> Prices { + Prices::new(rates.into_iter().collect(), None, billing) + } + + #[rstest] + #[case::per_prompt(WebSearchUnit::PerPrompt, 0.01)] + #[case::per_query(WebSearchUnit::PerQuery, 0.03)] + fn web_search_bills_once_per_prompt_unless_the_row_bills_per_query( + #[case] web_search: WebSearchUnit, + #[case] expected: f64, + ) { + let prices = billed( + [standard(&Meter::WebSearch(SearchContextSize::Medium), 0.01)], + Billing { + web_search, + ..Billing::default() + }, + ); + close(total(&prices, &web_searches(3), &call()), expected); + } + + #[rstest] + #[case::google_row(SearchDefaults::Google, Some(0.035))] + #[case::other_row(SearchDefaults::None, None)] + fn a_google_row_without_a_web_search_price_bills_geminis_default( + #[case] search_defaults: SearchDefaults, + #[case] expected: Option, + ) { + let prices = billed( + [standard(&TEXT_INPUT, 1e-6)], + Billing { + search_defaults, + web_search: WebSearchUnit::PerPrompt, + ..Billing::default() + }, + ); + let outcome = calculate(&prices, &Adjustments::default(), &web_searches(3), &call()); + let web_search_total = match outcome { + Outcome::Priced(priced) => Some(priced.total.get()), + Outcome::Unpriced { .. } => None, + }; + assert_eq!(web_search_total, expected); + } + + #[rstest] + #[case::fast(Some(crate::call::Speed::Fast), None, 2.0)] + #[case::geo(None, Some("US"), 1.1)] + #[case::neither(None, None, 1.0)] + fn speed_and_geo_uplifts_multiply_every_line( + #[case] speed: Option, + #[case] geo: Option<&str>, + #[case] factor: f64, + ) { + let prices = row([standard(&TEXT_INPUT, 1e-6), standard(&TEXT_OUTPUT, 2e-6)]); + let adjustments = Adjustments::new([ + ( + Uplift::Speed(crate::call::Speed::Fast), + Multiplier::new(2.0).unwrap(), + ), + ( + Uplift::InferenceGeo(crate::call::InferenceGeo::new("us")), + Multiplier::new(1.1).unwrap(), + ), + ]); + let call = CallContext { + speed, + inference_geo: geo.map(crate::call::InferenceGeo::new), + ..call() + }; + let outcome = priced(calculate(&prices, &adjustments, &usage(100, 0, 10), &call)); + close(outcome.total.get(), (100.0 * 1e-6 + 10.0 * 2e-6) * factor); + } } diff --git a/litellm-rust/crates/cost/src/price.rs b/litellm-rust/crates/cost/src/price.rs index 50a485127ae..afe6e0c82c9 100644 --- a/litellm-rust/crates/cost/src/price.rs +++ b/litellm-rust/crates/cost/src/price.rs @@ -65,8 +65,11 @@ pub struct Billing { pub cache_read_fallback: CacheReadFallback, /// Whether a row with no token price is priced by wall clock, from its `mode` pub wall_clock: WallClock, - /// From `web_search_billing_unit`, whether web search and Maps grounding bill per query or once per prompt + /// From `web_search_billing_unit`, whether web search and Maps grounding bill per query or once per prompt. A + /// Google row without one bills per prompt, Gemini's default, and every other row per query pub web_search: WebSearchUnit, + /// Prices a Google row bills web search and Maps grounding at when it sets none, Gemini's defaults on main + pub search_defaults: SearchDefaults, /// Whether `tiered_pricing` bills each meter in its own tier or the whole call at the tier the input picks. /// A prompt-token tier table bills the whole call, main's `_get_tiered_base_costs` pub tier_scope: TierScope, @@ -105,11 +108,18 @@ pub enum WallClock { #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub enum WebSearchUnit { - PerQuery, #[default] + PerQuery, PerPrompt, } +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum SearchDefaults { + #[default] + None, + Google, +} + #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub enum TierScope { #[default] diff --git a/litellm-rust/crates/model-catalog/src/lower.rs b/litellm-rust/crates/model-catalog/src/lower.rs index 379e5978c15..4cd0772b6b7 100644 --- a/litellm-rust/crates/model-catalog/src/lower.rs +++ b/litellm-rust/crates/model-catalog/src/lower.rs @@ -2,11 +2,13 @@ use std::collections::BTreeMap; use std::collections::btree_map::Entry; use std::num::NonZeroU64; -use litellm_cost::money::UnitPrice; +use litellm_cost::call::{InferenceGeo, Speed}; +use litellm_cost::money::{Multiplier, UnitPrice}; use litellm_cost::price::{ Adjustments, Band, BandEdge, Billing, CacheReadFallback, CacheReadMode, CacheTtl, GuardrailUnit, InputUnit, Meter, OffPeak, OffPeakRates, OutputUnit, Price, Prices, Residency, - SearchContextSize, TextUnit, TierScope, TokenKind, Uplift, WallClock, WebSearchUnit, When, + SearchContextSize, SearchDefaults, TextUnit, TierScope, TokenKind, Uplift, WallClock, + WebSearchUnit, When, }; use litellm_cost::schedule::{Schedule, Window}; @@ -92,6 +94,10 @@ impl TryFrom<&EntryWire> for Prices { fn billing(wire: &EntryWire, rates: &BTreeMap) -> Billing { let provider = wire.attributes.litellm_provider.as_deref(); + let google = provider.is_some_and(|provider| { + provider == "gemini" + || (provider.starts_with("vertex_ai") && !provider.contains("anthropic")) + }); let prices_meter = |priced: fn(&Meter) -> bool| rates.keys().any(|price| priced(&price.meter)); let prices_tokens = prices_meter(|meter| { matches!( @@ -137,9 +143,14 @@ fn billing(wire: &EntryWire, rates: &BTreeMap) -> Billing { Some(Mode::AudioTranscription) => WallClock::AudioFile, Some(_) => WallClock::Never, }, - web_search: match wire.attributes.web_search_billing_unit { - Some(WebSearchBillingUnit::PerQuery) => WebSearchUnit::PerQuery, - Some(WebSearchBillingUnit::PerPrompt) | None => WebSearchUnit::PerPrompt, + web_search: match (wire.attributes.web_search_billing_unit, google) { + (Some(WebSearchBillingUnit::PerQuery), _) | (None, false) => WebSearchUnit::PerQuery, + (Some(WebSearchBillingUnit::PerPrompt), _) | (None, true) => WebSearchUnit::PerPrompt, + }, + search_defaults: if google { + SearchDefaults::Google + } else { + SearchDefaults::None }, tier_scope: if wire .tier_rows @@ -155,7 +166,21 @@ fn billing(wire: &EntryWire, rates: &BTreeMap) -> Billing { } impl From<&EntryWire> for Adjustments { + /// `provider_specific_entry.fast` is Anthropic's speed uplift and every other number in it a geo uplift, as + /// main's `get_provider_specific_speed_multiplier` and `get_provider_specific_geo_multiplier` read it fn from(wire: &EntryWire) -> Self { + let provider_specific = wire + .attributes + .provider_specific_entry + .iter() + .flatten() + .filter_map(|(key, value)| { + let uplift = match key.as_str() { + "fast" => Uplift::Speed(Speed::Fast), + geo => Uplift::InferenceGeo(InferenceGeo::new(geo)), + }; + Some((uplift, Multiplier::new(value.as_f64()?)?)) + }); Adjustments::new( [ (Uplift::NonGlobalEndpoint, wire.regional_endpoint_uplift), @@ -163,7 +188,8 @@ impl From<&EntryWire> for Adjustments { (Uplift::Residency(Residency::Us), wire.residency_uplift_us), ] .into_iter() - .filter_map(|(uplift, factor)| Some((uplift, factor?))), + .filter_map(|(uplift, factor)| Some((uplift, factor?))) + .chain(provider_specific), ) } } @@ -929,11 +955,16 @@ mod tests { #[rstest] #[case::tokens_win_over_characters( json!({"litellm_provider": "vertex_ai-language-models", "input_cost_per_character": 1e-6, "input_cost_per_token": 1e-6}), - Billing::default() + Billing { web_search: WebSearchUnit::PerPrompt, search_defaults: SearchDefaults::Google, ..Billing::default() } )] #[case::characters_only( json!({"litellm_provider": "vertex_ai-language-models", "input_cost_per_character": 1e-6}), - Billing { text_unit: TextUnit::Character, ..Billing::default() } + Billing { + text_unit: TextUnit::Character, + web_search: WebSearchUnit::PerPrompt, + search_defaults: SearchDefaults::Google, + ..Billing::default() + } )] #[case::xai_band_is_inclusive( json!({"litellm_provider": "xai", "input_cost_per_token": 1e-6}), @@ -961,7 +992,15 @@ mod tests { )] #[case::gemini_per_query_search( json!({"litellm_provider": "gemini", "web_search_billing_unit": "per_query"}), - Billing { web_search: WebSearchUnit::PerQuery, ..Billing::default() } + Billing { web_search: WebSearchUnit::PerQuery, search_defaults: SearchDefaults::Google, ..Billing::default() } + )] + #[case::vertex_rows_take_googles_search_defaults( + json!({"litellm_provider": "vertex_ai-language-models"}), + Billing { web_search: WebSearchUnit::PerPrompt, search_defaults: SearchDefaults::Google, ..Billing::default() } + )] + #[case::vertex_claude_rows_bill_search_like_anthropic( + json!({"litellm_provider": "vertex_ai-anthropic_models"}), + Billing::default() )] #[case::token_tiers_bill_the_whole_call( json!({"litellm_provider": "openai", "tiered_pricing": [{"range": [0, 32000], "input_cost_per_token": 1e-6}]}), @@ -977,4 +1016,26 @@ mod tests { ) { assert_eq!(prices(entry).unwrap().billing(), expected); } + + #[rstest] + fn provider_specific_entry_lowers_to_speed_and_geo_uplifts() { + let wire = EntryWire::read( + json!({"litellm_provider": "anthropic", "provider_specific_entry": {"fast": 6.0, "us": 1.1}}) + .as_object() + .unwrap(), + ); + let adjustments = Adjustments::from(&wire); + assert_eq!( + adjustments + .get(&Uplift::Speed(Speed::Fast)) + .map(Multiplier::get), + Some(6.0) + ); + assert_eq!( + adjustments + .get(&Uplift::InferenceGeo(InferenceGeo::new("US"))) + .map(Multiplier::get), + Some(1.1) + ); + } } diff --git a/litellm-rust/crates/model-catalog/src/resolve/image.rs b/litellm-rust/crates/model-catalog/src/resolve/image.rs index d24e170b32c..d2686a6897b 100644 --- a/litellm-rust/crates/model-catalog/src/resolve/image.rs +++ b/litellm-rust/crates/model-catalog/src/resolve/image.rs @@ -16,3 +16,161 @@ pub struct ImageKey { pub height: u32, pub model: String, } + +impl ImageCandidates { + /// main's `default_image_cost_calculator` key spellings, in its order + pub fn spellings(&self) -> Vec { + let (first, second) = self.size.unwrap_or((1024, 1024)); + let size = format!("{first}-x-{second}"); + let provider_prefix = self + .provider + .as_deref() + .map(|provider| format!("{provider}/")); + let without_provider = provider_prefix + .as_deref() + .and_then(|prefix| self.model.strip_prefix(prefix)) + .map(|rest| rest.replace(provider_prefix.as_deref().unwrap_or_default(), "")); + let base = match (&self.provider, &without_provider) { + (Some(provider), Some(rest)) => format!("{provider}/{size}/{rest}"), + _ => format!("{size}/{}", self.model), + }; + let quality = self + .quality + .as_deref() + .filter(|quality| !quality.is_empty()); + let with_quality = |name: &str| { + quality.map_or_else(|| name.to_owned(), |quality| format!("{quality}/{name}")) + }; + let provider_first = quality + .zip(self.provider.as_deref()) + .map(|(quality, provider)| { + format!( + "{provider}/{quality}/{size}/{}", + without_provider.as_deref().unwrap_or(&self.model) + ) + }); + let last_segment = self.model.rsplit('/').next().unwrap_or(&self.model); + let model_without_provider = format!("{size}/{last_segment}"); + [ + Some(with_quality(&base)), + provider_first, + Some(base.clone()), + Some(format!("high/{base}")), + Some(with_quality(&model_without_provider)), + Some(model_without_provider.clone()), + Some(self.model.clone()), + without_provider, + ] + .into_iter() + .flatten() + .collect() + } +} + +impl ImageKey { + /// A `{provider}/{quality}/{W}-x-{H}/{model}` key, `None` for any other shape + pub fn parse(key: &str, provider: &str) -> Option { + let rest = key.strip_prefix(provider)?.strip_prefix('/')?; + let (quality, rest) = rest.split_once('/')?; + let (size, model) = rest.split_once('/')?; + let (width, height) = size.split_once("-x-")?; + let width: u32 = width.parse().ok().filter(|width| *width > 0)?; + let height: u32 = height.parse().ok().filter(|height| *height > 0)?; + Some(Self { + quality: Some(quality.to_owned()), + width, + height, + model: model.to_owned(), + }) + } + + fn pixels(&self) -> u64 { + u64::from(self.width) * u64::from(self.height) + } +} + +/// fal's nearest size match. Of the `{provider}/{quality}/{W}-x-{H}/{model}` keys the row filter keeps, the one whose +/// pixel count is closest to the target's, the smaller one on a tie +pub fn nearest_image_key<'a>( + keys: impl IntoIterator, + provider: &str, + model: &str, + quality: &str, + target: (u32, u32), +) -> Option<&'a str> { + let target_pixels = u64::from(target.0) * u64::from(target.1); + keys.into_iter() + .filter_map(|key| Some((key, ImageKey::parse(key, provider)?))) + .filter(|(_, parsed)| parsed.model == model && parsed.quality.as_deref() == Some(quality)) + .min_by_key(|(_, parsed)| (parsed.pixels().abs_diff(target_pixels), parsed.pixels())) + .map(|(key, _)| key) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[rstest] + fn the_spellings_follow_mains_order() { + let candidates = ImageCandidates { + model: "azure/dall-e-3".into(), + provider: Some("azure".into()), + size: Some((1024, 1792)), + quality: Some("hd".into()), + }; + assert_eq!( + candidates.spellings(), + [ + "hd/azure/1024-x-1792/dall-e-3", + "azure/hd/1024-x-1792/dall-e-3", + "azure/1024-x-1792/dall-e-3", + "high/azure/1024-x-1792/dall-e-3", + "hd/1024-x-1792/dall-e-3", + "1024-x-1792/dall-e-3", + "azure/dall-e-3", + "dall-e-3", + ] + ); + } + + #[rstest] + fn without_a_provider_prefix_or_quality_the_spellings_are_the_bare_ones() { + let candidates = ImageCandidates { + model: "dall-e-2".into(), + provider: Some("openai".into()), + size: None, + quality: None, + }; + assert_eq!( + candidates.spellings(), + [ + "1024-x-1024/dall-e-2", + "1024-x-1024/dall-e-2", + "high/1024-x-1024/dall-e-2", + "1024-x-1024/dall-e-2", + "1024-x-1024/dall-e-2", + "dall-e-2", + ] + ); + } + + #[rstest] + #[case::exact((1024, 1024), "fal_ai/high/1024-x-1024/flux")] + #[case::nearest((1100, 1100), "fal_ai/high/1024-x-1024/flux")] + #[case::tie_takes_the_smaller((768, 768), "fal_ai/high/512-x-512/flux")] + fn the_nearest_size_wins(#[case] target: (u32, u32), #[case] expected: &str) { + let keys = [ + "fal_ai/high/512-x-512/flux", + "fal_ai/high/1024-x-1024/flux", + "fal_ai/low/1024-x-1024/flux", + "fal_ai/high/1024-x-1024/other", + "fal_ai/flux", + ]; + assert_eq!( + nearest_image_key(keys, "fal_ai", "flux", "high", target), + Some(expected) + ); + } +} diff --git a/litellm-rust/crates/python-bridge/src/cost.rs b/litellm-rust/crates/python-bridge/src/cost.rs index 93343c0210d..554cb56a8bb 100644 --- a/litellm-rust/crates/python-bridge/src/cost.rs +++ b/litellm-rust/crates/python-bridge/src/cost.rs @@ -4,12 +4,17 @@ use std::sync::Arc; use std::time::Duration; use litellm_cost::call::CallContext as CostCallContext; +use litellm_cost::call::{InferenceGeo, Speed}; use litellm_cost::price::{ - Adjustments, CacheReadMode, CacheTtl, Channel, InputUnit, Meter, OutputUnit, - Prices as CostPrices, Residency, Resolution, ServiceTier, TextUnit, TokenKind, + Adjustments, CacheReadMode, CacheTtl, Channel, Feature, ImageSize, InputUnit, Meter, + OutputUnit, Prices as CostPrices, Residency, Resolution, SearchContextSize, ServiceTier, + TextUnit, TokenKind, }; use litellm_cost::priced::{Outcome, Quantity, Side}; -use litellm_cost::usage::{Characters, DurationMeter, Usage as CostUsage}; +use litellm_cost::usage::{ + Characters, DurationMeter, ImageOut as CostImageOut, Images, Search, Usage as CostUsage, + WebSearch, +}; use litellm_host_python::from_py; use litellm_model_catalog::rejects_row; use litellm_model_catalog::wire::{EntryWire, Fields, FlatKey}; @@ -190,7 +195,9 @@ impl Usage { cache_read_explicit = 0, cache_write_5m = 0, cache_write_1h = 0, cache_write_12h = 0, output_audio = 0, output_image = 0, output_video = 0, reasoning = 0, prompt_characters = 0, completion_characters = 0, wall_clock = None, input_audio = None, live_voice = None, output_video_duration = None, - video_resolution = None, text_messages = 0, + video_resolution = None, text_messages = 0, images_in = 0, images_out = Vec::new(), pixels_in = 0, + search_queries = 0, max_results = None, web_search_requests = 0, web_search_context = "medium", + maps_grounding_requests = 0, citation_tokens = 0, browser_opens = 0, additional_results = 0, ))] #[expect( clippy::too_many_arguments, @@ -221,7 +228,18 @@ impl Usage { output_video_duration: Option, video_resolution: Option<&str>, text_messages: u64, - ) -> Self { + images_in: u64, + images_out: Vec>, + pixels_in: u64, + search_queries: u64, + max_results: Option, + web_search_requests: u64, + web_search_context: &str, + maps_grounding_requests: u64, + citation_tokens: u64, + browser_opens: u64, + additional_results: u64, + ) -> PyResult { let mut usage = CostUsage { prompt_tokens, ..CostUsage::default() @@ -276,7 +294,34 @@ impl Usage { .filter_map(|(meter, duration)| Some((meter, duration?))) .collect(); usage.text_messages = text_messages; - Self(usage) + usage.tokens.citation = citation_tokens; + usage.images = Images { + input: images_in, + input_pixels: pixels_in, + output: images_out.iter().map(|image| image.0.clone()).collect(), + }; + let context = match web_search_context { + "low" => SearchContextSize::Low, + "medium" => SearchContextSize::Medium, + "high" => SearchContextSize::High, + other => { + return Err(PyValueError::new_err(format!( + "unknown search context size {other:?}" + ))); + } + }; + usage.search = Search { + queries: search_queries, + max_results, + web_search: (web_search_requests > 0).then_some(WebSearch { + requests: web_search_requests, + context, + }), + maps_grounding: maps_grounding_requests, + browser_opens, + additional_results, + }; + Ok(Self(usage)) } /// The provider-reported prompt size, which picks the band @@ -286,10 +331,34 @@ impl Usage { } } +/// One returned image. `size` picks a size-keyed price, `width` and `height` the per-pixel ones +#[pyclass(frozen, module = "litellm.rust_bridge._native")] +pub(crate) struct ImageOut(CostImageOut); + +#[pymethods] +impl ImageOut { + #[new] + #[pyo3(signature = (*, size = None, width = None, height = None))] + fn new(size: Option<&str>, width: Option, height: Option) -> Self { + Self(CostImageOut { + size: size.and_then(ImageSize::new), + width, + height, + }) + } +} + #[pymethods] impl CallContext { #[new] - #[pyo3(signature = (*, service_tier, channel, data_residency, vertex_location, at, custom_pricing))] + #[pyo3(signature = ( + *, service_tier, channel, data_residency, vertex_location, at, custom_pricing, speed = None, + inference_geo = None, + ))] + #[expect( + clippy::too_many_arguments, + reason = "one keyword argument per call fact, the Python signature" + )] fn new( service_tier: Option<&str>, channel: &str, @@ -297,6 +366,8 @@ impl CallContext { vertex_location: Option<&str>, at: &Bound<'_, PyAny>, custom_pricing: Option<&Bound<'_, PyDict>>, + speed: Option<&str>, + inference_geo: Option<&str>, ) -> PyResult { let channel = match channel { "online" => Channel::Online, @@ -328,8 +399,12 @@ impl CallContext { residency: data_residency.and_then(residency), non_global_endpoint: vertex_location .is_some_and(|location| !location.eq_ignore_ascii_case("global")), - speed: None, - inference_geo: None, + speed: (speed == Some("fast")).then_some(Speed::Fast), + inference_geo: inference_geo + .filter(|geo| { + !matches!(geo.to_lowercase().as_str(), "global" | "not_available") + }) + .map(InferenceGeo::new), at: timestamp(at)?, overlay, }, @@ -518,6 +593,28 @@ fn meter_name(meter: &Meter) -> String { Meter::Input(InputUnit::AudioSecond) => "input_cost_per_audio_per_second".to_owned(), Meter::Output(OutputUnit::VideoSecond) => "output_cost_per_video_per_second".to_owned(), Meter::Input(InputUnit::TextMessage) => "input_cost_per_text_message".to_owned(), + Meter::Input(InputUnit::Image) => "input_cost_per_image".to_owned(), + Meter::Output(OutputUnit::Image(None)) => "output_cost_per_image".to_owned(), + Meter::Output(OutputUnit::Image(Some(size))) => { + format!("output_cost_per_image_{}", size.as_str()) + } + Meter::Input(InputUnit::Pixel) => "input_cost_per_pixel".to_owned(), + Meter::Output(OutputUnit::Pixel) => "output_cost_per_pixel".to_owned(), + Meter::Input(InputUnit::Query { .. }) => "input_cost_per_query".to_owned(), + Meter::WebSearch(context) => format!( + "search_context_cost_per_query.search_context_size_{}", + match context { + SearchContextSize::Low => "low", + SearchContextSize::Medium => "medium", + SearchContextSize::High => "high", + } + ), + Meter::Feature(Feature::GoogleMapsGroundingQuery) => { + "google_maps_grounding_cost_per_query".to_owned() + } + Meter::Feature(Feature::CitationToken) => "citation_cost_per_token".to_owned(), + Meter::Feature(Feature::BrowserOpen) => "browser_open_cost_per_request".to_owned(), + Meter::Feature(Feature::AdditionalResult) => "cost_per_additional_result".to_owned(), other => format!("{other:?}"), } } diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index 1c7f69ad25d..69b5415c6e5 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -26,7 +26,7 @@ mod _native { }; #[pymodule_export] - use crate::cost::{CallContext, Line, Priced, PricesHandle, Unpriced, Usage}; + use crate::cost::{CallContext, ImageOut, Line, Priced, PricesHandle, Unpriced, Usage}; #[cfg(feature = "panic-test")] #[pymodule_export] use crate::diagnostics::_panic_for_test; diff --git a/litellm-rust/crates/python-bridge/src/model_registry.rs b/litellm-rust/crates/python-bridge/src/model_registry.rs index 6a7ec87a7ae..778afd17fe4 100644 --- a/litellm-rust/crates/python-bridge/src/model_registry.rs +++ b/litellm-rust/crates/python-bridge/src/model_registry.rs @@ -245,6 +245,69 @@ impl CatalogHandle { }) } + /// The row for an image call, through main's size and quality key spellings + #[pyo3(signature = (model, provider = None, *, size = None, quality = None))] + fn resolve_image( + &self, + py: Python<'_>, + model: &str, + provider: Option<&str>, + size: Option<(u32, u32)>, + quality: Option<&str>, + ) -> PyResult> { + let dict = self.bound_dict(py)?; + let keys = resolve::LiveKeys { + dict: &dict, + index: &self.index, + }; + let candidates = litellm_model_catalog::ImageCandidates { + model: model.to_owned(), + provider: provider.map(str::to_owned), + size, + quality: quality.map(str::to_owned), + }; + resolve::resolve_image(py, &keys, &candidates) + } + + /// fal's nearest size row, `{provider}/{quality}/{W}-x-{H}/{model}` with an `output_cost_per_image` price, from the + /// provider's keys in the index + #[pyo3(signature = (model, provider, *, quality, size))] + fn nearest_image_key( + &self, + py: Python<'_>, + model: &str, + provider: &str, + quality: &str, + size: (u32, u32), + ) -> PyResult> { + let dict = self.bound_dict(py)?; + let keys = resolve::LiveKeys { + dict: &dict, + index: &self.index, + }; + let priced: Vec = keys + .provider_keys(provider) + .into_iter() + .filter(|key| { + keys.fields_of(key) + .and_then(|fields| { + fields + .get("output_cost_per_image") + .map(serde_json::Value::is_number) + }) + .unwrap_or(false) + }) + .collect(); + Ok(litellm_model_catalog::nearest_image_key( + priced.iter().map(String::as_str), + provider, + model, + quality, + size, + ) + .map(str::to_owned)) + } + /// The keys whose row names this `litellm_provider`, from the index fn provider_keys(&self, py: Python<'_>, provider: &str) -> PyResult> { let dict = self.bound_dict(py)?; diff --git a/litellm-rust/crates/python-bridge/src/model_registry/resolve.rs b/litellm-rust/crates/python-bridge/src/model_registry/resolve.rs index e2313679150..e86b87c3877 100644 --- a/litellm-rust/crates/python-bridge/src/model_registry/resolve.rs +++ b/litellm-rust/crates/python-bridge/src/model_registry/resolve.rs @@ -4,7 +4,7 @@ use std::sync::{Arc, Mutex, PoisonError}; use litellm_host_python::{from_py, to_py}; use litellm_model_catalog::wire::Fields; use litellm_model_catalog::{ - Candidates, Catalog, Keys, Resolution, RowRef, model_info, rejects_row, + Candidates, Catalog, ImageCandidates, Keys, Resolution, RowRef, model_info, rejects_row, }; use pyo3::prelude::*; use pyo3::types::{PyDict, PyTuple}; @@ -167,34 +167,50 @@ pub(crate) fn resolve( Resolution::Missing { tried } => { Ok(Missing { tried }.into_pyobject(py)?.into_any().unbind()) } - Resolution::Found { key, .. } => { - let (fields, converted) = keys.fields(&key).unwrap_or_default(); - let errors: Vec = if converted { - Catalog::row_errors(&key, &fields) - .iter() - .filter(|error| rejects_row(&error.error)) - .map(ToString::to_string) - .collect() - } else { - vec![format!("{key} must contain JSON-compatible values")] - }; - let fields = Arc::new(fields); - if errors.is_empty() { - Ok(Resolved { key, fields } - .into_pyobject(py)? - .into_any() - .unbind()) - } else { - Ok(Invalid { - key, - errors, - fields, - } - .into_pyobject(py)? - .into_any() - .unbind()) - } + Resolution::Found { key, .. } => found(py, keys, key), + } +} + +/// The row at `key` as `Resolved`, or `Invalid` when it fails validation +pub(crate) fn found(py: Python<'_>, keys: &LiveKeys<'_, '_>, key: String) -> PyResult> { + let (fields, converted) = keys.fields(&key).unwrap_or_default(); + let errors: Vec = if converted { + Catalog::row_errors(&key, &fields) + .iter() + .filter(|error| rejects_row(&error.error)) + .map(ToString::to_string) + .collect() + } else { + vec![format!("{key} must contain JSON-compatible values")] + }; + let fields = Arc::new(fields); + if errors.is_empty() { + Ok(Resolved { key, fields } + .into_pyobject(py)? + .into_any() + .unbind()) + } else { + Ok(Invalid { + key, + errors, + fields, } + .into_pyobject(py)? + .into_any() + .unbind()) + } +} + +/// main's `default_image_cost_calculator` lookup, the first of its eight key spellings in the live dict +pub(crate) fn resolve_image( + py: Python<'_>, + keys: &LiveKeys<'_, '_>, + candidates: &ImageCandidates, +) -> PyResult> { + let tried = candidates.spellings(); + match tried.iter().find(|key| keys.contains(key)) { + Some(key) => found(py, keys, key.clone()), + None => Ok(Missing { tried }.into_pyobject(py)?.into_any().unbind()), } } diff --git a/litellm/_cost/__init__.py b/litellm/_cost/__init__.py index 25e5dccbd7a..7615392ba22 100644 --- a/litellm/_cost/__init__.py +++ b/litellm/_cost/__init__.py @@ -6,7 +6,7 @@ from contextvars import ContextVar from typing import Final, Literal from litellm._catalog import Invalid, report_invalid -from litellm.rust_bridge._native import CallContext, Line, Priced, Prices, Unpriced, Usage +from litellm.rust_bridge._native import CallContext, ImageOut, Line, Priced, Prices, Unpriced, Usage Channel = Literal["online", "batch", "decisions"] @@ -45,6 +45,7 @@ def as_main_cost(outcome: Priced | Unpriced | Invalid) -> tuple[float, float]: __all__ = [ "CallContext", "Channel", + "ImageOut", "Line", "Priced", "Prices", diff --git a/litellm/constants.py b/litellm/constants.py index e8bba6dd762..aba2c63a134 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -693,7 +693,6 @@ FIREWORKS_AI_16_B: Final = get_env_int("FIREWORKS_AI_16_B", 16) FIREWORKS_AI_80_B: Final = get_env_int("FIREWORKS_AI_80_B", 80) # https://docs.fireworks.ai/guides/prompt-caching (accessed 2026-09-19): serverless cached prompt tokens # default to a 50% discount off the input rate -FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO: Final = 0.5 #### Logging callback constants #### REDACTED_BY_LITELM_STRING: Final = "REDACTED_BY_LITELM" MAX_LANGFUSE_INITIALIZED_CLIENTS: Final = get_env_int("MAX_LANGFUSE_INITIALIZED_CLIENTS", 50) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 5f1b7cf8c4b..e10d58bcdb5 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -16,6 +16,7 @@ from typing_extensions import ReadOnly, TypedDict, assert_never import litellm import litellm._logging from litellm import verbose_logger +from litellm._catalog import CATALOG, Missing from litellm._cost import Priced, as_main_cost, call_custom_pricing, custom_pricing_for_call from litellm._cost import Usage as TokenUsage from litellm.constants import ( @@ -43,13 +44,13 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import ( calculate_cache_writing_cost, calculate_cost_component, cost_per_text, - generic_cost_per_token, get_batch_cost_rates, get_billable_input_tokens, get_regional_uplift_multiplier, get_service_tier_cost_key, get_token_type_cost_breakdown, get_vertex_regional_endpoint_uplift, + image_cost, parse_prompt_tokens_details, price_usage, pricing_row, @@ -111,7 +112,6 @@ from litellm.types.extraction import ExtractionResponse, ExtractionUsage from litellm.types.llms.base import CachedTokensDetails, LiteLLMBaseModel from litellm.types.llms.openai import ( HttpxBinaryResponseContent, - ImageGenerationRequestQuality, OpenAIModerationResponse, OpenAIRealtimeStreamList, OpenAIRealtimeStreamResponseBaseObject, @@ -436,7 +436,7 @@ def cost_per_token( if custom_cost_per_token is not None: with custom_pricing_for_call(dict(custom_cost_per_token)): - return generic_cost_per_token( + return cost_per_text( model=model, usage=_flat_custom_pricing_usage( usage_object, prompt_tokens, completion_tokens, cache_read_input_tokens, cache_creation_input_tokens @@ -514,7 +514,7 @@ def cost_per_token( litellm.model_cost.get(decisions_model_info["key"], {}) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # model cost map is dynamically loaded ) if decisions_overrides: - return generic_cost_per_token( + return cost_per_text( model=model, usage=usage_block, custom_llm_provider=custom_llm_provider, @@ -580,7 +580,7 @@ def cost_per_token( ) elif call_type == "atranscription" or call_type == "transcription": if _transcription_usage_has_token_details(usage_block): - return generic_cost_per_token( + return cost_per_text( model=model_without_prefix, usage=usage_block, custom_llm_provider=custom_llm_provider, @@ -613,7 +613,7 @@ def cost_per_token( ) is not None: return per_second_cost elif call_custom_pricing() is not None and _cost_map_model_info(model, custom_llm_provider) is None: - return generic_cost_per_token( + return cost_per_text( model=model, usage=usage_block, custom_llm_provider=custom_llm_provider, @@ -646,7 +646,7 @@ def cost_per_token( elif custom_llm_provider == "bedrock": resolved_bedrock_info: Final = bedrock_model_info or _bedrock_pricing_info(model, None, "bedrock", False) if resolved_bedrock_info is not None: - return generic_cost_per_token( + return cost_per_text( model=model, usage=usage_block, custom_llm_provider="bedrock", @@ -699,7 +699,7 @@ def cost_per_token( service_tier=service_tier, ) else: - return generic_cost_per_token( + return cost_per_text( model=model, usage=usage_block, custom_llm_provider=custom_llm_provider, @@ -2259,15 +2259,15 @@ def _video_input_image_cost( ) -> float: match usage: case {"billable_input_image_count": int(billable_input_images)} if billable_input_images > 0: - deployment_rate: Final = model_info.get("input_cost_per_image") if model_info is not None else None - image_rate: Final = ( - deployment_rate - if deployment_rate is not None - else litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider).get( - "input_cost_per_image" - ) + row: Final = ( + model_info + if model_info is not None and model_info.get("input_cost_per_image") is not None + else pricing_row(model, custom_llm_provider, None) ) - return billable_input_images * image_rate if isinstance(image_rate, (int, float)) else 0.0 + prompt_cost, completion_cost = as_main_cost( + price_usage(row, TokenUsage(prompt_tokens=0, text=0, output_text=0, images_in=billable_input_images)) + ) + return prompt_cost + completion_cost case _: return 0.0 @@ -2682,57 +2682,19 @@ def default_image_cost_calculator( # Parse dimensions height, width = map(int, size_str.split("-x-")) - # Build model names for cost lookup - base_model_name = f"{size_str}/{model}" - model_name_without_custom_llm_provider: str | None = None - if custom_llm_provider and model.startswith(f"{custom_llm_provider}/"): - model_name_without_custom_llm_provider = model.replace(f"{custom_llm_provider}/", "") - base_model_name = f"{custom_llm_provider}/{size_str}/{model_name_without_custom_llm_provider}" - model_name_with_quality: Final = f"{quality}/{base_model_name}" if quality else base_model_name - provider_first_model_name_with_quality: Final = ( - f"{custom_llm_provider}/{quality}/{size_str}/{model_name_without_custom_llm_provider or model}" - if quality and custom_llm_provider - else None - ) - - # gpt-image-1 models use low, medium, high quality. If user did not specify quality, use medium fot gpt-image-1 model family - model_name_with_v2_quality: Final = f"{ImageGenerationRequestQuality.HIGH.value}/{base_model_name}" - - verbose_logger.debug("Looking up cost for models: %s, %s", model_name_with_quality, base_model_name) - - model_without_provider: Final = f"{size_str}/{model.split('/')[-1]}" - model_with_quality_without_provider = f"{quality}/{model_without_provider}" if quality else model_without_provider - - models_to_check: Final = ( - model_name_with_quality, - provider_first_model_name_with_quality, - base_model_name, - model_name_with_v2_quality, - model_with_quality_without_provider, - model_without_provider, - model, - model_name_without_custom_llm_provider, - ) - matched_model: Final = next( - (_model for _model in models_to_check if _model is not None and _model in litellm.model_cost), None - ) - if matched_model is None and model_info is None: + resolution: Final = CATALOG.resolve_image(model, custom_llm_provider, size=(height, width), quality=quality or None) + models_to_check: Final = tuple(resolution.tried) if isinstance(resolution, Missing) else (resolution.key,) + if isinstance(resolution, Missing) and model_info is None: raise Exception(f"Model not found in cost map. Tried checking {models_to_check}") - shared_cost_info: Final = litellm.model_cost[matched_model] if matched_model is not None else None + shared_cost_info: Final = None if isinstance(resolution, Missing) else litellm.model_cost[resolution.key] price_tables: Final = tuple(table for table in (model_info, shared_cost_info) if table is not None) image_count: Final = n if n is not None else 1 - unit_counts: Final = ( - ("input_cost_per_image", image_count), - ("output_cost_per_image", image_count), - ("input_cost_per_pixel", height * width * image_count), - ) cost: Final = next( ( - price * units + table_cost for price_table in price_tables - for cost_key, units in unit_counts - if isinstance(price := price_table.get(cost_key), (int, float)) + if (table_cost := image_cost(price_table, image_count, width=int(width), height=int(height))) is not None ), None, ) @@ -3382,7 +3344,7 @@ def _candidate_realtime_token_costs( data_residency: str | None, ) -> tuple[float, float] | None: try: - return generic_cost_per_token( + return cost_per_text( model=model_name, usage=combined_usage_object, custom_llm_provider=custom_llm_provider, diff --git a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py index 46d0ea1c6bf..817f3c9fce2 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py +++ b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py @@ -281,21 +281,33 @@ class StandardBuiltInToolCostTracking: tool_usage: Final = _responses_tool_usage(response_object) if tool_usage is None or tool_usage.image_gen is None or tool_usage.image_gen.total_tokens <= 0: return None + from litellm._cost import Usage as TokenUsage + from litellm._cost import as_main_cost + from litellm.litellm_core_utils.llm_cost_calc.utils import price_usage, pricing_row + try: - model_info: Final = litellm.get_model_info(model=tool_model, custom_llm_provider=custom_llm_provider) + row: Final = pricing_row(tool_model, custom_llm_provider, None) except Exception as e: # noqa: BLE001 # get_model_info raises bare Exception for unmapped models; fall back verbose_logger.debug("Could not resolve pricing for image tool model %s: %s", tool_model, e) return None image_gen: Final = tool_usage.image_gen input_details: Final = image_gen.input_tokens_details output_details: Final = image_gen.output_tokens_details - token_cost: Final = ( - (input_details.text_tokens if input_details else 0) * (model_info.get("input_cost_per_token") or 0) - + (input_details.image_tokens if input_details else 0) * (model_info.get("input_cost_per_image_token") or 0) - + (output_details.image_tokens if output_details else 0) - * (model_info.get("output_cost_per_image_token") or 0) - + (output_details.text_tokens if output_details else 0) * (model_info.get("output_cost_per_token") or 0) + text_in: Final = input_details.text_tokens if input_details else 0 + image_in: Final = input_details.image_tokens if input_details else 0 + prompt_cost, completion_cost = as_main_cost( + price_usage( + row, + TokenUsage( + prompt_tokens=text_in + image_in, + text=text_in, + image=image_in, + output_text=output_details.text_tokens if output_details else 0, + output_image=output_details.image_tokens if output_details else 0, + ), + ) ) + token_cost: Final = prompt_cost + completion_cost return token_cost if token_cost > 0 else None @staticmethod diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index be53ddae460..c1e61a8aef6 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -13,7 +13,16 @@ from typing_extensions import ReadOnly import litellm from litellm._catalog import Invalid -from litellm._cost import CallContext, Channel, Priced, Prices, Unpriced, as_main_cost, call_custom_pricing +from litellm._cost import ( + CallContext, + Channel, + ImageOut, + Priced, + Prices, + Unpriced, + as_main_cost, + call_custom_pricing, +) from litellm._cost import Usage as TokenUsage from litellm._internal_context import current_billing_time from litellm._logging import verbose_logger @@ -21,7 +30,6 @@ from litellm.litellm_core_utils.llm_cost_calc.tiered_pricing import ( select_tier_for_input, tier_rate, ) -from litellm.llms.fireworks_ai.cache_pricing import with_default_cache_read_rate from litellm.types.utils import ( CacheCreationTokenDetails, CallTypes, @@ -109,19 +117,10 @@ _BATCH_RATE_PREFIXES: Final = ( ) _BATCH_TIER_PREFIXES: Final = (*_BATCH_RATE_PREFIXES, *_TTL_PRICED_CACHE_WRITE_RATE_KEYS.values()) _BATCH_TIER_KEY: Final = re.compile(rf"^({'|'.join(_BATCH_TIER_PREFIXES)})_above_(\d+k?)_tokens{_BATCH_KEY_SUFFIX}$") -_NON_STANDARD_THRESHOLD_SUFFIXES: Final = (*_SERVICE_TIER_SUFFIXES, _BATCH_KEY_SUFFIX) - - def _uses_inclusive_token_thresholds(custom_llm_provider: str | None) -> bool: return custom_llm_provider in _INCLUSIVE_THRESHOLD_PROVIDERS -def apply_provider_cache_read_default(model_info: ModelInfo, custom_llm_provider: str | None) -> ModelInfo: - if custom_llm_provider == "fireworks_ai": - return with_default_cache_read_rate(model_info) - return model_info - - def _get_token_detail_value(details: object, key: str) -> int | None: if isinstance(details, dict): value = details.get(key) @@ -173,15 +172,6 @@ def get_web_search_requests_from_usage(usage: Usage) -> int | None: return get_web_search_requests(getattr(usage, "server_tool_use", None)) -def is_above_128k(tokens: float) -> bool: - if tokens > 128000: - return True - return False - - -_is_above_128k = is_above_128k - - def get_billable_input_tokens(usage: Usage) -> int: """ Returns the number of billable input tokens. @@ -219,10 +209,6 @@ def _parse_token_threshold(threshold: str) -> float: return float(threshold.replace("k", "")) * (1000 if "k" in threshold else 1) -def _parse_above_token_threshold(key: str) -> float: - return _parse_token_threshold(key.rsplit("_above_", 1)[1].split("_tokens")[0]) - - def _prompt_exceeds_threshold(prompt_tokens: int, threshold: float, inclusive: bool) -> bool: return prompt_tokens > threshold or (inclusive and prompt_tokens == threshold) @@ -324,46 +310,6 @@ def _select_priced_tier(model_info: ModelInfo, usage: Usage) -> Mapping[str, obj return tier -def _get_tiered_reasoning_rate(model_info: ModelInfo, usage: Usage) -> float | None: - tier: Final = _select_priced_tier(model_info=model_info, usage=usage) - if tier is None: - return None - if "output_cost_per_reasoning_token" not in tier and "output_cost_per_token" not in tier: - return None - return tier_rate(tier, "output_cost_per_reasoning_token", "output_cost_per_token") - - -def _get_tiered_base_costs(model_info: ModelInfo, usage: Usage) -> tuple[TokenRates, Mapping[str, object]] | None: - """ - Resolve the base rates from a model's ``tiered_pricing`` table, if it has one. - - Tiered pricing is all-or-nothing: one tier is picked from the request's input tokens - and every token of the request is billed at that tier's rate. Rates the tier does not - declare fall back to the tier's input rate, so a request never mixes tiers. - - An output rate is the exception: a tier table that spells out only input rates would - otherwise serve every completion for free, so the model's own output rate stands in. - """ - tier: Final = _select_priced_tier(model_info=model_info, usage=usage) - if tier is None: - return None - - completion_cost: Final = ( - tier_rate(tier, "output_cost_per_token") - if "output_cost_per_token" in tier - else get_cost_per_unit(model_info, "output_cost_per_token") or 0.0 - ) - cache_creation_rate: Final = tier_rate(tier, "cache_creation_input_token_cost", "input_cost_per_token") - rates: Final = TokenRates( - input_rate=tier_rate(tier, "input_cost_per_token"), - output_rate=completion_cost, - cache_read_rate=tier_rate(tier, "cache_read_input_token_cost", "input_cost_per_token"), - cache_creation_rate=cache_creation_rate, - reasoning_rate=None, - ) - return rates, tier - - def _is_within_off_peak_window(off_peak_hours_utc: str | Sequence[str], current_time: datetime | None = None) -> bool: """Return True if current_time (UTC, defaulting to now) falls inside any off-peak window. @@ -601,215 +547,6 @@ def apply_off_peak_pricing(model_info: ModelInfo, current_time: datetime | None, ) -def _apply_off_peak_to_base_costs( - model_info: ModelInfo, - current_time: datetime | None, - base_rates: TokenRates, - tier: Mapping[str, object], -) -> TokenRates: - """Apply off-peak rates to the base costs resolved from a pricing tier. off_peak_pricing has no - field for the long-TTL cache-creation rates, so a tier that declares one keeps it, and a tier - that does not bills that TTL at the off-peak-adjusted write rate. Reasoning is left to - _resolve_billed_reasoning_rate. - """ - rates: Final = apply_off_peak_pricing(model_info, current_time, base_rates) - return replace( - rates, - cache_creation_rates_by_ttl=MappingProxyType( - { - ttl: rates.cache_creation_rate if tier.get(key) is None else tier_rate(tier, key) - for ttl, key in _TTL_PRICED_CACHE_WRITE_RATE_KEYS.items() - } - ), - ) - - -def _first_present_key(model_info: ModelInfo, preferred_key: str, fallback_key: str) -> str: - return preferred_key if model_info.get(preferred_key) is not None else fallback_key - - -def _get_token_base_cost( - model_info: ModelInfo, - usage: Usage, - service_tier: str | None = None, - current_time: datetime | None = None, - *, - threshold_is_inclusive: bool = False, -) -> TokenRates: - """ - Return prompt cost, completion cost, and cache costs for a given model and usage. - - If input_tokens > threshold and `input_cost_per_token_above_[x]k_tokens` or `input_cost_per_token_above_[x]_tokens` is set, - then we use the corresponding threshold cost for all token types. - - `threshold_is_inclusive` switches that comparison to >=, for providers such as xAI - that bill the higher tier once the prompt reaches the threshold. - - An absent cache-creation or cache-read rate always resolves to the resolved input - rate, the way the tiered table and custom deployment pricing already do, since a - provider that publishes no cache price bills cached tokens as ordinary input. An - absent long-TTL write rate resolves to the cache-creation rate, off-peak included. An - explicit 0.0 stays a real price for all of them. - """ - tiered_base_costs: Final = _get_tiered_base_costs(model_info=model_info, usage=usage) - if tiered_base_costs is not None: - return _apply_off_peak_to_base_costs(model_info, current_time, *tiered_base_costs) - - # Get service tier aware cost keys - input_cost_key: Final = get_service_tier_cost_key("input_cost_per_token", service_tier) - output_cost_key: Final = get_service_tier_cost_key("output_cost_per_token", service_tier) - cache_creation_cost_key: Final = get_service_tier_cost_key("cache_creation_input_token_cost", service_tier) - cache_read_cost_key: Final = get_service_tier_cost_key("cache_read_input_token_cost", service_tier) - - prompt_base_cost = cast( # cast-ok: model pricing data is external - float, get_cost_per_unit(model_info, input_cost_key) - ) - completion_base_cost = cast( # cast-ok: model pricing data is external - float, get_cost_per_unit(model_info, output_cost_key) - ) - - # For image generation models that don't have output_cost_per_token, - # use output_cost_per_image_token as the base cost (all output tokens are image tokens) - if completion_base_cost == 0.0: - output_image_cost: Final = get_cost_per_unit(model_info, "output_cost_per_image_token", None) - if output_image_cost is not None: - completion_base_cost = output_image_cost - cache_creation_cost = get_cost_per_unit(model_info, cache_creation_cost_key, default_value=None) - cache_creation_keys_by_ttl: Mapping[str, str] = MappingProxyType( - {ttl: get_service_tier_cost_key(key, service_tier) for ttl, key in _TTL_PRICED_CACHE_WRITE_RATE_KEYS.items()} - ) - cache_read_cost = get_cost_per_unit(model_info, cache_read_cost_key, default_value=None) - - ## CHECK IF ABOVE THRESHOLD - # Optimization: collect threshold keys first to avoid sorting all model_info keys. - # Standard thresholds and thresholds suffixed for this request's service tier both count. - tier_key_suffix: Final = get_service_tier_cost_key("", service_tier) - threshold_prefixes: Final = tuple(f"{prefix}_above_" for prefix in _BATCH_RATE_PREFIXES) - threshold_keys: Final = [ - k - for k in model_info - if k.startswith(threshold_prefixes) - and "_tokens" in k - and ( - not k.endswith(_NON_STANDARD_THRESHOLD_SUFFIXES) or (tier_key_suffix != "" and k.endswith(tier_key_suffix)) - ) - ] - - # Only sort the threshold keys (typically 1-2 keys instead of 66+) - threshold: float | None = None - for key in sorted(threshold_keys, key=_parse_above_token_threshold): - value = model_info.get(key) - if value is not None: - try: - # Handle both formats: _above_128k_tokens and _above_128_tokens - threshold_str = key.rsplit("_above_", 1)[1].split("_tokens")[0] - threshold = _parse_above_token_threshold(key) - if _prompt_exceeds_threshold(usage.prompt_tokens, threshold, threshold_is_inclusive): - # Prefer a service_tier-specific above-threshold key when available, - # e.g. input_cost_per_token_priority_above_200k_tokens for Gemini - # ON_DEMAND_PRIORITY. Falls back to the standard key automatically - # via _get_cost_per_unit's service_tier fallback logic. - tiered_input_key = ( - get_service_tier_cost_key( - f"input_cost_per_token_above_{threshold_str}_tokens", - service_tier, - ) - if service_tier - else f"input_cost_per_token_above_{threshold_str}_tokens" - ) - prompt_base_cost = cast( - float, - get_cost_per_unit(model_info, tiered_input_key, prompt_base_cost), - ) - tiered_output_key = ( - get_service_tier_cost_key( - f"output_cost_per_token_above_{threshold_str}_tokens", - service_tier, - ) - if service_tier - else f"output_cost_per_token_above_{threshold_str}_tokens" - ) - completion_base_cost = cast( - float, - get_cost_per_unit( - model_info, - tiered_output_key, - completion_base_cost, - ), - ) - - # Apply tiered pricing to cache costs - cache_creation_tiered_key = ( - get_service_tier_cost_key( - f"cache_creation_input_token_cost_above_{threshold_str}_tokens", - service_tier, - ) - if service_tier - else f"cache_creation_input_token_cost_above_{threshold_str}_tokens" - ) - cache_read_tiered_key = ( - get_service_tier_cost_key( - f"cache_read_input_token_cost_above_{threshold_str}_tokens", - service_tier, - ) - if service_tier - else f"cache_read_input_token_cost_above_{threshold_str}_tokens" - ) - - cache_creation_cost = get_cost_per_unit(model_info, cache_creation_tiered_key, cache_creation_cost) - - cache_creation_keys_by_ttl = MappingProxyType( - { - ttl: _first_present_key( - model_info, - get_service_tier_cost_key( - f"{_TTL_PRICED_CACHE_WRITE_RATE_KEYS[ttl]}_above_{threshold_str}_tokens", - service_tier, - ), - key, - ) - for ttl, key in cache_creation_keys_by_ttl.items() - } - ) - cache_read_cost = get_cost_per_unit(model_info, cache_read_tiered_key, cache_read_cost) - except (IndexError, ValueError): - continue - except Exception: - continue - - input_rate_for_missing_cache_rates: Final = _off_peak_rate( - _open_off_peak_block(model_info, current_time) or MappingProxyType({}), - "input_cost_per_token", - prompt_base_cost, - ) - resolved_cache_read_cost: Final = input_rate_for_missing_cache_rates if cache_read_cost is None else cache_read_cost - resolved_cache_creation_cost: Final = ( - input_rate_for_missing_cache_rates if cache_creation_cost is None else cache_creation_cost - ) - base_rates: Final = apply_off_peak_pricing( - model_info, - current_time, - TokenRates( - input_rate=prompt_base_cost, - output_rate=completion_base_cost, - cache_read_rate=resolved_cache_read_cost, - cache_creation_rate=resolved_cache_creation_cost, - reasoning_rate=None, - ), - ) - return replace( - base_rates, - cache_creation_rates_by_ttl=MappingProxyType( - { - ttl: base_rates.cache_creation_rate - if (rate := get_cost_per_unit(model_info, key, default_value=None)) is None - else rate - for ttl, key in cache_creation_keys_by_ttl.items() - } - ), - ) - - def calculate_cost_component(model_info: Mapping[str, object], cost_key: str, usage_value: float | None) -> float: """ Generic cost calculator for any usage component @@ -1164,111 +901,6 @@ def get_vertex_regional_endpoint_uplift(model_info: Mapping[str, object], vertex return 1.0 -def get_provider_specific_geo_multiplier(model_info: ModelInfo, usage: Usage) -> float: - """ - Resolve the provider-specific regional pricing multiplier for the geo the - request was served from (``usage.inference_geo``), e.g. Anthropic's ``us: 1.1`` - stored under ``provider_specific_entry``. The regional surcharge applies to - every token type, so per-type cost breakdowns must scale by it too. - - Returns 1.0 when the request was served globally or the model carries no - multiplier for the geo. - """ - inference_geo: Final = getattr(usage, "inference_geo", None) - if not isinstance(inference_geo, str) or inference_geo.lower() in ("global", "not_available"): - return 1.0 - provider_specific_entry: Final[dict[str, float]] = model_info.get("provider_specific_entry") or {} - return float(provider_specific_entry.get(inference_geo.lower(), 1.0)) - - -def get_provider_specific_speed_multiplier(model_info: ModelInfo, usage: Usage) -> float: - provider_specific_entry: Final = model_info.get("provider_specific_entry") - if getattr(usage, "speed", None) != "fast" or not provider_specific_entry: - return 1.0 - return float(provider_specific_entry.get("fast", 1.0)) - - -def _resolve_reasoning_token_cost( - model_info: ModelInfo, - service_tier: str | None, - completion_base_cost: float, -) -> float: - tier_reasoning_key: Final = get_service_tier_cost_key("output_cost_per_reasoning_token", service_tier) - if model_info.get(tier_reasoning_key) is not None: - tier_reasoning_cost: Final = get_cost_per_unit(model_info, tier_reasoning_key, None) - if tier_reasoning_cost is not None: - return tier_reasoning_cost - tier_output_key: Final = get_service_tier_cost_key("output_cost_per_token", service_tier) - if tier_output_key != "output_cost_per_token" and model_info.get(tier_output_key) is not None: - return completion_base_cost - standard_reasoning_cost: Final = get_cost_per_unit(model_info, "output_cost_per_reasoning_token", None) - return standard_reasoning_cost if standard_reasoning_cost is not None else completion_base_cost - - -def _resolve_billed_reasoning_rate( - model_info: ModelInfo, - usage: Usage, - service_tier: str | None, - completion_base_cost: float, - current_time: datetime | None, -) -> float: - off_peak: Final = _open_off_peak_block(model_info, current_time) - off_peak_reasoning_rate: Final = ( - None if off_peak is None else _parse_off_peak_rate(off_peak.get("output_cost_per_reasoning_token")) - ) - if off_peak_reasoning_rate is not None: - return off_peak_reasoning_rate - tiered_reasoning_rate: Final = _get_tiered_reasoning_rate(model_info=model_info, usage=usage) - if tiered_reasoning_rate is not None: - return tiered_reasoning_rate - return _resolve_reasoning_token_cost( - model_info=model_info, - service_tier=service_tier, - completion_base_cost=completion_base_cost, - ) - - -def generic_cost_per_token( - model: str, - usage: Usage, - custom_llm_provider: str, - service_tier: str | None = None, - data_residency: str | None = None, - model_info: ModelInfo | None = None, - vertex_location: str | None = None, - current_time: datetime | None = None, -) -> tuple[float, float]: - """ - Calculates the cost per token for a given model, prompt tokens, and completion tokens. - - Handles context caching as well. - - Input: - - model: str, the model name without provider prefix - - usage: LiteLLM Usage block, containing anthropic caching information - - data_residency: optional OpenAI data-residency region (e.g. "eu", "us"), - used to apply the per-model regional-processing uplift multiplier. - - vertex_location: optional Vertex AI location the request was served from - (e.g. "us-east5", "global"), used to apply the per-model - regional-endpoint uplift multiplier when non-global. - - current_time: the moment the request is billed at, for off_peak_pricing; defaults to now, UTC - - Returns: - Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd - """ - - return cost_per_text( - model, - usage, - custom_llm_provider, - service_tier=service_tier, - data_residency=data_residency, - model_info=model_info, - vertex_location=vertex_location, - current_time=current_time, - ) - - def cost_per_text( model: str, usage: Usage, @@ -1293,6 +925,8 @@ def cost_per_text( data_residency=data_residency, vertex_location=vertex_location, current_time=current_time, + speed=_usage_str(usage, "speed"), + inference_geo=_usage_str(usage, "inference_geo"), key=model, ) ) @@ -1311,6 +945,8 @@ def price_usage( vertex_location: str | None = None, current_time: datetime | None = None, channel: Channel = "online", + speed: str | None = None, + inference_geo: str | None = None, key: str = "model_info", ) -> Priced | Unpriced | Invalid: """Prices ``usage`` over ``row`` with the call's custom pricing over it, the one place a cost is computed""" @@ -1329,10 +965,40 @@ def price_usage( vertex_location=vertex_location, at=current_time if current_time is not None else current_billing_time(), custom_pricing=None if custom_pricing is None else dict(custom_pricing), + speed=speed, + inference_geo=inference_geo, ), ) +def image_cost( + row: Mapping[str, object], images: int, width: int | None = None, height: int | None = None +) -> float | None: + """``images`` images priced per image or per pixel over ``row``, None when it prices neither""" + priced: Final = price_usage( + row, + TokenUsage( + prompt_tokens=0, + text=0, + output_text=0, + images_out=[ImageOut(width=width, height=height)] * max(images, 0), + ), + ) + return priced.total if isinstance(priced, Priced) and priced.lines else None + + +def search_cost(model_info: Mapping[str, object], usage: TokenUsage) -> float: + """The search, grounding and tool meters of ``usage`` priced over ``model_info``""" + prompt_cost, completion_cost = as_main_cost(price_usage(model_info, usage)) + return prompt_cost + completion_cost + + +def _usage_str(usage: Usage, name: str) -> str | None: + """Anthropic's ``speed`` and ``inference_geo``, which pick the row's speed and geo uplifts""" + value: Final = getattr(usage, name, None) + return value if isinstance(value, str) else None + + def pricing_row(model: str, custom_llm_provider: str | None, model_info: ModelInfo | None) -> Mapping[str, object]: """The row main priced, as written rather than `get_model_info`'s copy, which writes `0` for a missing token price. A call with custom pricing prices an unmapped model from its overlay alone""" @@ -1356,8 +1022,13 @@ def bills_characters(row: Mapping[str, object]) -> bool: def _without_tiers(row: Mapping[str, object]) -> dict[str, object]: - """Main picks no tier for an empty prompt and bills the row's flat prices""" - return {name: value for name, value in row.items() if name != "tiered_pricing"} + """Main picks no prompt-token tier for an empty prompt and bills the row's flat prices. A search row's result + tiers stay""" + tiers: Final = row.get("tiered_pricing") + token_tiers: Final = isinstance(tiers, list) and any( + isinstance(tier, Mapping) and "range" in tier for tier in tiers + ) + return {name: value for name, value in row.items() if not (token_tiers and name == "tiered_pricing")} def _billable_prompt_details(usage: Usage) -> PromptTokensDetailsResult: @@ -1426,9 +1097,8 @@ def _token_usage( ) -> TokenUsage: """The meters main billed. A prompt billed per image or second of media does not bill the same tokens again, and a row that bills characters bills no text tokens (decision 32)""" - cache_type: Final = getattr(usage.prompt_tokens_details, "cache_type", None) cached_text: Final = details["cache_hit_tokens"] - details["cache_hit_audio_tokens"] - explicit_cache: Final = cache_type == "ephemeral" + explicit_cache: Final = _reads_explicit_cache(usage) cache_write_5m, cache_write_1h, cache_write_12h = _cache_write_tokens(details) completion: Final = _completion_token_counts(usage) return TokenUsage( @@ -1571,6 +1241,10 @@ def _custom_pricing_rates(custom_cost_per_token: CostPerToken) -> BilledTokenRat ) +def _reads_explicit_cache(usage: Usage) -> bool: + return getattr(usage.prompt_tokens_details, "cache_type", None) == "ephemeral" + + def _cost_map_billed_rates( model_info: ModelInfo, usage: Usage, @@ -1579,54 +1253,55 @@ def _cost_map_billed_rates( data_residency: str | None, vertex_location: str | None, current_time: datetime | None, + *, + provider_multipliers: bool = True, ) -> BilledTokenRates: - billing_time: Final = current_time if current_time is not None else current_billing_time() - base_costs: Final = _get_token_base_cost( - model_info=model_info, - usage=usage, - service_tier=service_tier, - current_time=billing_time, - threshold_is_inclusive=_uses_inclusive_token_thresholds(custom_llm_provider), - ) - provider_cache_read_cost_rate: Final = resolve_explicit_cache_read_cost_rate( - model_info=model_info, - usage=usage, - current_time=billing_time, - service_tier=service_tier, - ) - effective_cache_read_cost_rate: Final = ( - base_costs.cache_read_rate if provider_cache_read_cost_rate is None else provider_cache_read_cost_rate - ) - - reasoning_rate: Final = _resolve_billed_reasoning_rate( - model_info=model_info, - usage=usage, - service_tier=service_tier, - completion_base_cost=base_costs.output_rate, - current_time=billing_time, - ) - audio_cache_read_rate: Final = get_cost_per_unit( + """The rate of every token meter this call would bill, read off one priced probe: one unit of each meter at the + call's prompt size, so the band, tier, off-peak window and uplifts are the ones the call itself billed at""" + explicit_cache: Final = _reads_explicit_cache(usage) + priced: Final = price_usage( model_info, - get_service_tier_cost_key("cache_read_input_audio_token_cost", service_tier), - None, - ) - multiplier: Final = ( - get_regional_uplift_multiplier(model_info, data_residency) - * get_vertex_regional_endpoint_uplift(model_info, vertex_location) - * get_provider_specific_geo_multiplier(model_info=model_info, usage=usage) - * get_provider_specific_speed_multiplier(model_info=model_info, usage=usage) - ) - return BilledTokenRates( - input_cost_per_token=base_costs.input_rate, - output_cost_per_token=base_costs.output_rate, - cache_read_input_token_cost=effective_cache_read_cost_rate, - cache_read_input_audio_token_cost=( - audio_cache_read_rate if audio_cache_read_rate is not None else effective_cache_read_cost_rate + TokenUsage( + prompt_tokens=usage.prompt_tokens, + text=1, + output_text=1, + cache_read=0 if explicit_cache else 1, + cache_read_explicit=1 if explicit_cache else 0, + cache_read_audio=1, + cache_write_5m=1, + cache_write_1h=1, + cache_write_12h=1, + reasoning=1, ), - cache_creation_input_token_cost=base_costs.cache_creation_rate, - cache_creation_input_token_cost_by_ttl=base_costs.cache_creation_rates_by_ttl, - output_cost_per_reasoning_token=reasoning_rate, - ).scaled(multiplier) + service_tier=service_tier, + data_residency=data_residency, + vertex_location=vertex_location, + current_time=current_time, + speed=_usage_str(usage, "speed") if provider_multipliers else None, + inference_geo=_usage_str(usage, "inference_geo") if provider_multipliers else None, + key=str(model_info.get("key") or custom_llm_provider or "model_info"), + ) + rates: Final = MappingProxyType( + {line.meter: line.amount for line in priced.lines} if isinstance(priced, Priced) else {} + ) + cache_read: Final = rates.get( + "explicit_cache_read_input_token_cost" if explicit_cache else "cache_read_input_token_cost", 0.0 + ) + cache_write: Final = rates.get("cache_creation_input_token_cost", 0.0) + return BilledTokenRates( + input_cost_per_token=rates.get("input_cost_per_token", 0.0), + output_cost_per_token=rates.get("output_cost_per_token", 0.0), + cache_read_input_token_cost=cache_read, + cache_read_input_audio_token_cost=rates.get("cache_read_input_audio_token_cost", cache_read), + cache_creation_input_token_cost=cache_write, + cache_creation_input_token_cost_by_ttl=MappingProxyType( + { + "1hr": rates.get("cache_creation_input_token_cost_above_1hr", cache_write), + "12hr": rates.get("cache_creation_input_token_cost_above_12hr", cache_write), + } + ), + output_cost_per_reasoning_token=rates.get("output_cost_per_reasoning_token", 0.0), + ) def get_billed_token_rates( @@ -1645,16 +1320,11 @@ def get_billed_token_rates( if custom_cost_per_token is not None: return _custom_pricing_rates(custom_cost_per_token) try: - resolved_model_info: Final = apply_provider_cache_read_default( - model_info - if model_info is not None - else get_model_info(model=model, custom_llm_provider=custom_llm_provider), - custom_llm_provider, - ) + resolved_model_info: Final = pricing_row(model, custom_llm_provider, model_info) except Exception: # noqa: BLE001 # get_model_info raises a bare Exception for an unmapped model: no rates return None return _cost_map_billed_rates( - model_info=resolved_model_info, + model_info=cast(ModelInfo, resolved_model_info), # cast-ok: the row as written, priced by its own meters usage=usage, custom_llm_provider=custom_llm_provider, service_tier=service_tier, @@ -1680,7 +1350,7 @@ def get_token_type_cost_breakdown( object and model pricing alone. This works for every provider, including Perplexity/Cerebras/Dashscope whose - cost calculators bypass ``generic_cost_per_token``, because cache tokens always + cost calculators bypass ``cost_per_text``, because cache tokens always land on ``prompt_tokens_details`` (via the Usage constructor and provider transformations) and reasoning tokens on ``completion_tokens_details``. It reuses the same rate resolution as the total-cost path (``get_billed_token_rates``) so the @@ -1743,25 +1413,18 @@ def calculate_prompt_caching_savings( ``billed_at`` is the request's completion time, so off-peak windows resolve as the biller saw them rather than at the later spend write. """ - model_info_with_cache_read_default: Final = apply_provider_cache_read_default(model_info, custom_llm_provider) - base_costs: Final = _get_token_base_cost( - model_info=model_info_with_cache_read_default, - usage=usage, - service_tier=service_tier, - current_time=billed_at, - threshold_is_inclusive=_uses_inclusive_token_thresholds(custom_llm_provider), - ) - provider_cache_read_cost: Final = resolve_explicit_cache_read_cost_rate( + rates: Final = _cost_map_billed_rates( model_info=model_info, usage=usage, - current_time=billed_at, + custom_llm_provider=custom_llm_provider, service_tier=service_tier, + data_residency=data_residency, + vertex_location=vertex_location, + current_time=billed_at, + provider_multipliers=False, ) - prompt_base_cost: Final = base_costs.input_rate - effective_cache_read_cost: Final = ( - base_costs.cache_read_rate if provider_cache_read_cost is None else provider_cache_read_cost - ) - write_rate: Final = base_costs.cache_creation_rate or prompt_base_cost + input_rate: Final = rates.input_cost_per_token + write_rate: Final = rates.cache_creation_input_token_cost or input_rate prompt_tokens_details: Final = parse_prompt_tokens_details(usage) cache_read_tokens: Final = max(prompt_tokens_details["cache_hit_tokens"], 0) cache_creation_tokens: Final = max(prompt_tokens_details["cache_creation_tokens"], 0) @@ -1776,19 +1439,16 @@ def calculate_prompt_caching_savings( if details is not None else None ) - read_discount: Final = cache_read_tokens * max(prompt_base_cost - effective_cache_read_cost, 0.0) + read_discount: Final = cache_read_tokens * max(input_rate - rates.cache_read_input_token_cost, 0.0) write_premium: Final = calculate_cache_writing_cost( cache_creation_tokens=cache_creation_tokens, cache_creation_token_details=cache_creation_details, - cache_creation_cost=write_rate - prompt_base_cost, + cache_creation_cost=write_rate - input_rate, cache_creation_cost_by_ttl=MappingProxyType( - {ttl: rate - prompt_base_cost for ttl, rate in base_costs.cache_creation_rates_by_ttl.items()} + {ttl: rate - input_rate for ttl, rate in rates.cache_creation_input_token_cost_by_ttl.items()} ), ) - uplift: Final = get_regional_uplift_multiplier(model_info, data_residency) * get_vertex_regional_endpoint_uplift( - model_info, vertex_location - ) - return (read_discount - write_premium) * uplift + return read_discount - write_premium def calculate_image_response_cost_from_usage( @@ -1875,7 +1535,7 @@ def calculate_image_response_cost_from_usage( completion_tokens_details=completion_tokens_details, ) - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model, usage=normalized_usage, custom_llm_provider=custom_llm_provider, diff --git a/litellm/llms/aiml/image_generation/cost_calculator.py b/litellm/llms/aiml/image_generation/cost_calculator.py index ff49052dc25..e74cc162541 100644 --- a/litellm/llms/aiml/image_generation/cost_calculator.py +++ b/litellm/llms/aiml/image_generation/cost_calculator.py @@ -1,7 +1,7 @@ from typing import Final import litellm -from litellm.litellm_core_utils.llm_cost_calc.utils import resolve_image_model_info +from litellm.litellm_core_utils.llm_cost_calc.utils import image_cost, resolve_image_model_info from litellm.types.utils import ImageResponse, ModelInfo @@ -18,11 +18,10 @@ def cost_calculator( custom_llm_provider=litellm.LlmProviders.AIML.value, model_info=model_info, ) - output_cost_per_image: Final[float] = _model_info.get("output_cost_per_image") or 0.0 num_images: int = 0 if isinstance(image_response, ImageResponse): if image_response.data: num_images = len(image_response.data) - return output_cost_per_image * num_images + return image_cost(_model_info, num_images) or 0.0 else: raise ValueError(f"image_response must be of type ImageResponse got type={type(image_response)}") diff --git a/litellm/llms/amazon_nova/cost_calculation.py b/litellm/llms/amazon_nova/cost_calculation.py index 6e691cf279e..6aca46b601b 100644 --- a/litellm/llms/amazon_nova/cost_calculation.py +++ b/litellm/llms/amazon_nova/cost_calculation.py @@ -5,7 +5,7 @@ Helper util for handling amazon nova cost calculation from typing import TYPE_CHECKING -from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text if TYPE_CHECKING: from litellm.types.utils import Usage @@ -16,4 +16,4 @@ def cost_per_token(model: str, usage: "Usage") -> tuple[float, float]: Calculates the cost per token for a given model, prompt tokens, and completion tokens. Follows the same logic as Anthropic's cost per token calculation. """ - return generic_cost_per_token(model=model, usage=usage, custom_llm_provider="amazon_nova") + return cost_per_text(model=model, usage=usage, custom_llm_provider="amazon_nova") diff --git a/litellm/llms/anthropic/cost_calculation.py b/litellm/llms/anthropic/cost_calculation.py index 94c8149eff3..2d8bdd4ad34 100644 --- a/litellm/llms/anthropic/cost_calculation.py +++ b/litellm/llms/anthropic/cost_calculation.py @@ -8,16 +8,13 @@ from typing import TYPE_CHECKING, Final, Optional from pydantic import ValidationError from litellm.litellm_core_utils.llm_cost_calc.utils import ( - generic_cost_per_token, - get_provider_specific_geo_multiplier, - get_provider_specific_speed_multiplier, + cost_per_text, get_web_search_requests_from_usage, ) from litellm.types.llms.base import LiteLLMBaseModel if TYPE_CHECKING: from litellm.types.utils import ModelInfo, Usage -import litellm def cost_per_token( @@ -36,7 +33,7 @@ def cost_per_token( Returns: Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd """ - prompt_cost, completion_cost = generic_cost_per_token( + return cost_per_text( model=model, usage=usage, custom_llm_provider="anthropic", @@ -44,28 +41,6 @@ def cost_per_token( model_info=model_info, ) - # Apply provider_specific_entry multipliers for geo/speed routing - try: - effective_info: Final = ( - model_info - if model_info is not None - else litellm.get_model_info(model=model, custom_llm_provider="anthropic") - ) - geo_multiplier: Final = get_provider_specific_geo_multiplier(model_info=effective_info, usage=usage) - speed_multiplier: Final = get_provider_specific_speed_multiplier(model_info=effective_info, usage=usage) - - if speed_multiplier != 1.0: - prompt_cost *= speed_multiplier - completion_cost *= speed_multiplier - - if geo_multiplier != 1.0: - prompt_cost *= geo_multiplier - completion_cost *= geo_multiplier - except Exception: - pass - - return prompt_cost, completion_cost - class _AnthropicServerToolUseProbe(LiteLLMBaseModel): web_search_requests: int | None = None @@ -99,29 +74,15 @@ def get_cost_for_anthropic_web_search( model_info: Optional["ModelInfo"] = None, usage: Optional["Usage"] = None, ) -> float: - """ - Get the cost of using a web search tool for Anthropic. - """ - from litellm.types.utils import SearchContextCostPerQuery + """``server_tool_use.web_search_requests`` at the row's medium search context price""" + from litellm._cost import Usage as TokenUsage + from litellm.litellm_core_utils.llm_cost_calc.utils import search_cost - ## Check if web search requests are in the usage object - if model_info is None: - return 0.0 - - if usage is None: + if model_info is None or usage is None: return 0.0 web_search_requests: Final = get_web_search_requests_from_usage(usage) if web_search_requests is None: return 0.0 - - ## Get the cost per web search request - search_context_pricing: Final[SearchContextCostPerQuery] = ( - model_info.get("search_context_cost_per_query") or SearchContextCostPerQuery() + return search_cost( + model_info, TokenUsage(prompt_tokens=0, text=0, output_text=0, web_search_requests=web_search_requests) ) - cost_per_web_search_request: Final = search_context_pricing.get("search_context_size_medium", 0.0) - if cost_per_web_search_request is None or cost_per_web_search_request == 0.0: - return 0.0 - - ## Calculate the total cost - total_cost: Final = cost_per_web_search_request * web_search_requests - return total_cost diff --git a/litellm/llms/azure/cost_calculation.py b/litellm/llms/azure/cost_calculation.py index 057e9dbb9d9..ba5fbc3e39b 100644 --- a/litellm/llms/azure/cost_calculation.py +++ b/litellm/llms/azure/cost_calculation.py @@ -3,7 +3,7 @@ Helper util for handling azure openai-specific cost calculation - e.g.: prompt caching, audio tokens """ -from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text from litellm.types.utils import Usage @@ -25,7 +25,7 @@ def cost_per_token( """ ## Use generic cost calculator for all other cases ## This properly handles: text tokens, audio tokens, cached tokens, reasoning tokens, etc. - return generic_cost_per_token( + return cost_per_text( model=model, usage=usage, custom_llm_provider="azure", diff --git a/litellm/llms/azure_ai/cost_calculator.py b/litellm/llms/azure_ai/cost_calculator.py index 5934525eca3..55bc2fd47c1 100644 --- a/litellm/llms/azure_ai/cost_calculator.py +++ b/litellm/llms/azure_ai/cost_calculator.py @@ -6,7 +6,7 @@ Handles Azure AI Foundry Model Router flat cost and other Azure AI specific pric from typing import Final from litellm._logging import verbose_logger -from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text from litellm.types.utils import Usage from litellm.utils import get_model_info @@ -65,9 +65,7 @@ def calculate_azure_model_router_flat_cost(model: str, prompt_tokens: int) -> fl def _response_model_cost(model: str, usage: Usage, service_tier: str | None) -> tuple[float, float]: try: - return generic_cost_per_token( - model=model, usage=usage, custom_llm_provider="azure_ai", service_tier=service_tier - ) + return cost_per_text(model=model, usage=usage, custom_llm_provider="azure_ai", service_tier=service_tier) except Exception as e: if not is_azure_model_router(model): raise diff --git a/litellm/llms/azure_ai/image_generation/cost_calculator.py b/litellm/llms/azure_ai/image_generation/cost_calculator.py index 53fb755be80..f5958564d9c 100644 --- a/litellm/llms/azure_ai/image_generation/cost_calculator.py +++ b/litellm/llms/azure_ai/image_generation/cost_calculator.py @@ -4,22 +4,23 @@ from typing import Final import litellm from litellm.litellm_core_utils.llm_cost_calc.utils import ( calculate_image_response_cost_from_usage, - get_cost_per_unit, + image_cost, resolve_image_model_info, ) from litellm.types.utils import ImageResponse, ModelInfo -def _input_cost_per_pixel(resolved: ModelInfo) -> float: - deployment_price: Final = get_cost_per_unit(resolved, "input_cost_per_pixel", default_value=None) - if deployment_price is not None: - return deployment_price - model_cost_key: Final = resolved.get("key") - shared_entry: Final = litellm.model_cost.get(model_cost_key) if model_cost_key is not None else None - if shared_entry is None: - return 0.0 - shared_price: Final = shared_entry.get("input_cost_per_pixel") - return float(shared_price) if isinstance(shared_price, (int, float)) else 0.0 +def _requested_dimensions(optional_params: Mapping[str, object] | None, size: object) -> tuple[int, int] | None: + width: Final = optional_params.get("width") if optional_params else None + height: Final = optional_params.get("height") if optional_params else None + if type(width) is int and type(height) is int and width > 0 and height > 0: + return width, height + if not isinstance(size, str): + return None + parts: Final = size.replace("-x-", "x").split("x") + if len(parts) != 2 or not all(part.isdigit() for part in parts): + return None + return int(parts[0]), int(parts[1]) def cost_calculator( @@ -50,27 +51,10 @@ def cost_calculator( return token_based_cost num_images: Final = n if n is not None else len(image_response.data or ()) - output_cost_per_image: Final[float] = _model_info.get("output_cost_per_image") or 0.0 - if output_cost_per_image: - return output_cost_per_image * num_images - - if _input_cost_per_pixel(_model_info): - from litellm.cost_calculator import default_image_cost_calculator - - width: Final = optional_params.get("width") if optional_params else None - height: Final = optional_params.get("height") if optional_params else None - pixel_size: Final = ( - f"{width}x{height}" - if type(width) is int and type(height) is int and width > 0 and height > 0 - else size or image_response.size - ) - return default_image_cost_calculator( - model=_model_info.get("key", model), - custom_llm_provider=litellm.LlmProviders.AZURE_AI.value, - size=pixel_size, - n=num_images, - model_info=model_info, - ) - return 0.0 + width, height = _requested_dimensions(optional_params, size or image_response.size) or (1024, 1024) + key: Final = _model_info.get("key") + catalog_row: Final = litellm.model_cost.get(key) if isinstance(key, str) else None + row: Final = {**_model_info, **(catalog_row or {}), **(model_info or {})} + return image_cost(row, num_images, width=width, height=height) or 0.0 raise ValueError(f"image_response must be of type ImageResponse got type={type(image_response)}") diff --git a/litellm/llms/bedrock/cost_calculation.py b/litellm/llms/bedrock/cost_calculation.py index 1b45531dda4..af625c35d7f 100644 --- a/litellm/llms/bedrock/cost_calculation.py +++ b/litellm/llms/bedrock/cost_calculation.py @@ -3,13 +3,9 @@ Helper util for handling bedrock-specific cost calculation - e.g.: prompt caching """ -from typing import TYPE_CHECKING, Final +from typing import TYPE_CHECKING -import litellm -from litellm.litellm_core_utils.llm_cost_calc.utils import ( - generic_cost_per_token, - get_provider_specific_geo_multiplier, -) +from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text if TYPE_CHECKING: from litellm.types.utils import Usage @@ -21,13 +17,4 @@ def cost_per_token(model: str, usage: "Usage", service_tier: str | None = None) Follows the same logic as Anthropic's cost per token calculation. """ - model_info: Final = litellm.get_model_info(model=model, custom_llm_provider="bedrock") - prompt_cost, completion_cost = generic_cost_per_token( - model=model, - usage=usage, - custom_llm_provider="bedrock", - service_tier=service_tier, - model_info=model_info, - ) - geo_multiplier: Final = get_provider_specific_geo_multiplier(model_info=model_info, usage=usage) - return prompt_cost * geo_multiplier, completion_cost * geo_multiplier + return cost_per_text(model=model, usage=usage, custom_llm_provider="bedrock", service_tier=service_tier) diff --git a/litellm/llms/bedrock/image_generation/amazon_nova_canvas_transformation.py b/litellm/llms/bedrock/image_generation/amazon_nova_canvas_transformation.py index 6f972986e56..ce176a56205 100644 --- a/litellm/llms/bedrock/image_generation/amazon_nova_canvas_transformation.py +++ b/litellm/llms/bedrock/image_generation/amazon_nova_canvas_transformation.py @@ -4,6 +4,7 @@ from typing import Any, Final from openai.types.image import Image +from litellm.litellm_core_utils.llm_cost_calc.utils import image_cost from litellm.llms.bedrock.common_utils import get_cached_model_info from litellm.types.llms.bedrock import ( AmazonNovaCanvasColorGuidedGenerationParams, @@ -206,9 +207,7 @@ class AmazonNovaCanvasConfig: model=model, custom_llm_provider="bedrock", ) - - output_cost_per_image: Final[float] = model_info.get("output_cost_per_image") or 0.0 num_images: int = 0 if image_response.data: num_images = len(image_response.data) - return output_cost_per_image * num_images + return image_cost(model_info, num_images) or 0.0 diff --git a/litellm/llms/bedrock/image_generation/amazon_stability1_transformation.py b/litellm/llms/bedrock/image_generation/amazon_stability1_transformation.py index 3ebb128e89c..ab18549bf64 100644 --- a/litellm/llms/bedrock/image_generation/amazon_stability1_transformation.py +++ b/litellm/llms/bedrock/image_generation/amazon_stability1_transformation.py @@ -6,6 +6,7 @@ from typing import Final from openai.types.image import Image +from litellm.litellm_core_utils.llm_cost_calc.utils import image_cost from litellm.llms.bedrock.common_utils import get_cached_model_info from litellm.types.utils import ImageResponse @@ -156,9 +157,7 @@ class AmazonStabilityConfig: model=model, custom_llm_provider="bedrock", ) - - output_cost_per_image: Final[float] = model_info.get("output_cost_per_image") or 0.0 num_images: int = 0 if image_response.data: num_images = len(image_response.data) - return output_cost_per_image * num_images + return image_cost(model_info, num_images) or 0.0 diff --git a/litellm/llms/bedrock/image_generation/amazon_stability3_transformation.py b/litellm/llms/bedrock/image_generation/amazon_stability3_transformation.py index e5c5edba750..9e34c5d3122 100644 --- a/litellm/llms/bedrock/image_generation/amazon_stability3_transformation.py +++ b/litellm/llms/bedrock/image_generation/amazon_stability3_transformation.py @@ -4,6 +4,7 @@ from typing import Final from openai.types.image import Image +from litellm.litellm_core_utils.llm_cost_calc.utils import image_cost from litellm.llms.bedrock.common_utils import BedrockError, get_cached_model_info from litellm.types.llms.bedrock import ( AmazonStability3TextToImageRequest, @@ -126,9 +127,7 @@ class AmazonStability3Config: model=model, custom_llm_provider="bedrock", ) - - output_cost_per_image: Final[float] = model_info.get("output_cost_per_image") or 0.0 num_images: int = 0 if image_response.data: num_images = len(image_response.data) - return output_cost_per_image * num_images + return image_cost(model_info, num_images) or 0.0 diff --git a/litellm/llms/bedrock/image_generation/amazon_titan_transformation.py b/litellm/llms/bedrock/image_generation/amazon_titan_transformation.py index 5731541da56..4d7bb3b40c3 100644 --- a/litellm/llms/bedrock/image_generation/amazon_titan_transformation.py +++ b/litellm/llms/bedrock/image_generation/amazon_titan_transformation.py @@ -1,3 +1,5 @@ +from litellm.litellm_core_utils.llm_cost_calc.utils import image_cost + """ Transformation logic for Amazon Titan Image Generation. """ @@ -152,8 +154,7 @@ class AmazonTitanImageGenerationConfig: optional_params: dict | None = None, ) -> float: model_info: Final = get_model_info(model=model) - output_cost_per_image: Final = model_info.get("output_cost_per_image") or 0.0 if not image_response.data: return 0.0 num_images: Final = len(image_response.data) - return output_cost_per_image * num_images + return image_cost(model_info, num_images) or 0.0 diff --git a/litellm/llms/cometapi/image_generation/cost_calculator.py b/litellm/llms/cometapi/image_generation/cost_calculator.py index d9248eac598..adc1ef18003 100644 --- a/litellm/llms/cometapi/image_generation/cost_calculator.py +++ b/litellm/llms/cometapi/image_generation/cost_calculator.py @@ -1,7 +1,7 @@ from typing import Final import litellm -from litellm.litellm_core_utils.llm_cost_calc.utils import resolve_image_model_info +from litellm.litellm_core_utils.llm_cost_calc.utils import image_cost, resolve_image_model_info from litellm.types.utils import ImageResponse, ModelInfo @@ -18,11 +18,10 @@ def cost_calculator( custom_llm_provider=litellm.LlmProviders.COMETAPI.value, model_info=model_info, ) - output_cost_per_image: Final[float] = _model_info.get("output_cost_per_image") or 0.0 num_images: int = 0 if isinstance(image_response, ImageResponse): if image_response.data: num_images = len(image_response.data) - return output_cost_per_image * num_images + return image_cost(_model_info, num_images) or 0.0 else: raise ValueError(f"image_response must be of type ImageResponse got type={type(image_response)}") diff --git a/litellm/llms/databricks/cost_calculator.py b/litellm/llms/databricks/cost_calculator.py index 2bb5b99f0ad..9952965ef9b 100644 --- a/litellm/llms/databricks/cost_calculator.py +++ b/litellm/llms/databricks/cost_calculator.py @@ -6,7 +6,7 @@ Helper util for handling databricks-specific cost calculation from types import MappingProxyType from typing import Final -from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text from litellm.types.utils import Usage _LEGACY_ENDPOINT_NAMES: Final = MappingProxyType( @@ -41,7 +41,7 @@ def cost_per_token(model: str, usage: Usage, service_tier: str | None = None) -> Returns: Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd """ - return generic_cost_per_token( + return cost_per_text( model=_registry_key(model), usage=usage, custom_llm_provider="databricks", diff --git a/litellm/llms/deepseek/cost_calculator.py b/litellm/llms/deepseek/cost_calculator.py index 5a0c065f0ee..b8332b0562b 100644 --- a/litellm/llms/deepseek/cost_calculator.py +++ b/litellm/llms/deepseek/cost_calculator.py @@ -4,7 +4,7 @@ Cost calculator for DeepSeek Chat models. Handles prompt caching scenario. """ -from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text from litellm.types.utils import Usage @@ -14,4 +14,4 @@ def cost_per_token(model: str, usage: Usage) -> tuple[float, float]: Follows the same logic as Anthropic's cost per token calculation. """ - return generic_cost_per_token(model=model, usage=usage, custom_llm_provider="deepseek") + return cost_per_text(model=model, usage=usage, custom_llm_provider="deepseek") diff --git a/litellm/llms/fal_ai/cost_calculator.py b/litellm/llms/fal_ai/cost_calculator.py index 2acf1c69594..4e8049d0617 100644 --- a/litellm/llms/fal_ai/cost_calculator.py +++ b/litellm/llms/fal_ai/cost_calculator.py @@ -7,7 +7,8 @@ from typing import Final from pydantic import TypeAdapter import litellm -from litellm.litellm_core_utils.llm_cost_calc.utils import deployment_pricing, resolve_image_model_info +from litellm._catalog import CATALOG +from litellm.litellm_core_utils.llm_cost_calc.utils import deployment_pricing, image_cost, resolve_image_model_info from litellm.types.utils import ImageObject, ImageResponse, ModelInfo FAL_KEYED_PRICING_DEFAULT_QUALITY: Final[str] = "high" @@ -82,38 +83,20 @@ def _parse_keyed_dimensions(size: str | None) -> tuple[int, int] | None: return (width, height) if width > 0 and height > 0 else None -def _keyed_rows(model: str, quality: str) -> tuple[tuple[int, int, float], ...]: - prefix: Final = f"fal_ai/{quality}/" - suffix: Final = f"/{model}" - return tuple( - (width, height, float(raw_cost)) - for key in (*litellm.model_cost,) - if isinstance(key, str) and key.startswith(prefix) and key.endswith(suffix) - for size in (key[len(prefix) : -len(suffix)],) - for dimensions in (_parse_keyed_dimensions(size),) - if dimensions is not None - for entry in (_entry(key),) - if entry is not None - for raw_cost in (entry.get("output_cost_per_image"),) - if isinstance(raw_cost, (int, float)) - for width, height in (dimensions,) - ) - - def _keyed_cost_per_image( model: str, image: object, optional_params: Mapping[str, object], ) -> float | None: - quality: Final = _keyed_quality(optional_params) - rows: Final = _keyed_rows(model, quality) - if not rows: - return None - target_dimensions: Final = ( + """The per-image price of the size-keyed row nearest the image's size""" + target: Final = ( _image_dimensions(image) or _parse_keyed_dimensions(_keyed_size(optional_params)) or _DEFAULT_KEYED_DIMENSIONS ) - target_pixels: Final = target_dimensions[0] * target_dimensions[1] - return min(rows, key=lambda row: (abs(row[0] * row[1] - target_pixels), row[0] * row[1]))[2] + key: Final = CATALOG.nearest_image_key( + model, litellm.LlmProviders.FAL_AI.value, quality=_keyed_quality(optional_params), size=target + ) + entry: Final = _entry(key) if key is not None else None + return image_cost(entry, 1) if entry is not None else None def _flat_cost_per_image( diff --git a/litellm/llms/fireworks_ai/cache_pricing.py b/litellm/llms/fireworks_ai/cache_pricing.py deleted file mode 100644 index f5e49cad01a..00000000000 --- a/litellm/llms/fireworks_ai/cache_pricing.py +++ /dev/null @@ -1,42 +0,0 @@ -from typing import ( - Final, - cast, # noqa: TID251 # the derived entry is a dict copy of a ReadOnly TypedDict; no cast-free way to retype it -) - -from litellm.constants import FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO -from litellm.types.utils import ModelInfo - - -def _as_rate(value: object) -> float | None: - if isinstance(value, bool) or not isinstance(value, (int, float, str)): - return None - try: - return float(value) - except ValueError: - return None - - -def with_default_cache_read_rate(model_info: ModelInfo) -> ModelInfo: - input_rate: Final = _as_rate(model_info.get("input_cost_per_token")) - if model_info.get("cache_read_input_token_cost") is not None or input_rate is None: - return model_info - cache_read_rate: Final = input_rate * FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO - off_peak: Final = model_info.get("off_peak_pricing") - if off_peak is None or "cache_read_input_token_cost" in off_peak: - return cast(ModelInfo, {**model_info, "cache_read_input_token_cost": cache_read_rate}) - off_peak_input_rate: Final = _as_rate(off_peak.get("input_cost_per_token")) - return cast( - ModelInfo, - { - **model_info, - "cache_read_input_token_cost": cache_read_rate, - "off_peak_pricing": { - **off_peak, - "cache_read_input_token_cost": ( - off_peak_input_rate * FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO - if off_peak_input_rate is not None - else cache_read_rate - ), - }, - }, - ) diff --git a/litellm/llms/fireworks_ai/cost_calculator.py b/litellm/llms/fireworks_ai/cost_calculator.py index d42b4ddbbf7..f498d3183c3 100644 --- a/litellm/llms/fireworks_ai/cost_calculator.py +++ b/litellm/llms/fireworks_ai/cost_calculator.py @@ -11,7 +11,7 @@ from litellm.constants import ( FIREWORKS_AI_56_B_MOE, FIREWORKS_AI_176_B_MOE, ) -from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text from litellm.types.utils import ModelInfo, Usage from litellm.utils import get_model_info @@ -85,7 +85,7 @@ def cost_per_token(model: str, usage: Usage, current_time: datetime | None = Non Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd """ model_info: Final = _resolve_model_info(model) - return generic_cost_per_token( + return cost_per_text( model=model, usage=usage, custom_llm_provider=model_info["litellm_provider"], diff --git a/litellm/llms/gemini/cost_calculator.py b/litellm/llms/gemini/cost_calculator.py index b82103b0ff8..dce6f3db62c 100644 --- a/litellm/llms/gemini/cost_calculator.py +++ b/litellm/llms/gemini/cost_calculator.py @@ -16,9 +16,9 @@ def cost_per_token(model: str, usage: "Usage", service_tier: str | None = None) Follows the same logic as Anthropic's cost per token calculation. """ - from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token + from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text - return generic_cost_per_token( + return cost_per_text( model=model, usage=usage, custom_llm_provider="gemini", @@ -26,52 +26,36 @@ def cost_per_token(model: str, usage: "Usage", service_tier: str | None = None) ) -def cost_per_web_search_request(usage: "Usage", model_info: "ModelInfo") -> float: - """ - Calculates the cost of web search (grounding with Google Search). - - Billing mode is determined by ``web_search_billing_unit`` in model_info: - - ``"per_query"``: charged per individual search query (Gemini 3.x). - - ``"per_prompt"`` (default): charged per grounded prompt (Gemini 2.x), - regardless of how many queries were executed internally. - - Reads the per-request cost from ``search_context_cost_per_query`` in - ``model_info`` when available, falling back to $0.035 for models not - yet updated in the pricing JSON. - """ - from litellm.litellm_core_utils.llm_cost_calc.utils import ( - get_web_search_requests_from_usage, - ) +def web_search_requests(usage: "Usage | None") -> int: + """Grounded searches, Gemini's ``prompt_tokens_details.web_search_requests`` else ``server_tool_use``""" + from litellm.litellm_core_utils.llm_cost_calc.utils import get_web_search_requests_from_usage from litellm.types.utils import PromptTokensDetailsWrapper - _DEFAULT_COST: Final = 35e-3 - search_costs: Final = model_info.get("search_context_cost_per_query") or {} - _cost: Final = search_costs.get("search_context_size_medium", _DEFAULT_COST) - - requests_from_prompt_details: Final = ( - usage.prompt_tokens_details.web_search_requests - if ( - usage is not None - and usage.prompt_tokens_details is not None - and isinstance(usage.prompt_tokens_details, PromptTokensDetailsWrapper) - and hasattr(usage.prompt_tokens_details, "web_search_requests") - and usage.prompt_tokens_details.web_search_requests is not None - ) + details: Final = usage.prompt_tokens_details if usage is not None else None + from_details: Final = ( + details.web_search_requests + if isinstance(details, PromptTokensDetailsWrapper) and getattr(details, "web_search_requests", None) else None ) - requests_from_server_tool_use: Final = get_web_search_requests_from_usage(usage) - number_of_web_search_requests: Final = requests_from_prompt_details or requests_from_server_tool_use or 0 + return from_details or (get_web_search_requests_from_usage(usage) if usage is not None else None) or 0 - billing_mode: Final = model_info.get("web_search_billing_unit") or "per_prompt" - billable_requests: Final = ( - 1 if (number_of_web_search_requests > 0 and billing_mode == "per_prompt") else number_of_web_search_requests + +def cost_per_web_search_request(usage: "Usage", model_info: "ModelInfo") -> float: + """Grounding with Google Search, per query or once per prompt as the row's ``web_search_billing_unit`` says""" + from litellm._cost import Usage as TokenUsage + from litellm.litellm_core_utils.llm_cost_calc.utils import search_cost + + return search_cost( + _as_google_row(model_info), + TokenUsage(prompt_tokens=0, text=0, output_text=0, web_search_requests=web_search_requests(usage)), ) - return _cost * billable_requests - -GOOGLE_MAPS_GROUNDING_DEFAULT_COST_PER_QUERY: Final = 14e-3 -GOOGLE_MAPS_GROUNDING_DEFAULT_COST_PER_PROMPT: Final = 25e-3 +def _as_google_row(model_info: "ModelInfo") -> dict[str, object]: + """The row with a Google provider, so its missing search and Maps prices take Gemini's defaults""" + provider: Final = model_info.get("litellm_provider") + google: Final = isinstance(provider, str) and (provider == "gemini" or provider.startswith("vertex_ai")) + return {**model_info, "litellm_provider": provider if google else "gemini"} def google_maps_grounding_requests(usage: "Usage | None") -> int | None: @@ -84,26 +68,16 @@ def google_maps_grounding_requests(usage: "Usage | None") -> int | None: def cost_per_google_maps_grounding_request(usage: "Usage", model_info: "ModelInfo") -> float: - """ - Calculates the cost of Grounding with Google Maps. + """Grounding with Google Maps, under the same billing unit as Google Search grounding""" + from litellm._cost import Usage as TokenUsage + from litellm.litellm_core_utils.llm_cost_calc.utils import search_cost - Billing follows ``web_search_billing_unit`` in model_info the same way Google Search grounding - does: ``"per_query"`` (Gemini 3.x) multiplies the executed Maps queries, ``"per_prompt"`` - (default, Gemini 2.x) charges one flat fee per grounded prompt. - - The rate comes from ``google_maps_grounding_cost_per_query`` in ``model_info``, falling back - to Google's list price for that billing unit when the pricing JSON has no entry yet. - """ - requests: Final = google_maps_grounding_requests(usage) - if not requests or requests <= 0: - return 0.0 - billing_mode: Final = model_info.get("web_search_billing_unit") or "per_prompt" - default_cost: Final = ( - GOOGLE_MAPS_GROUNDING_DEFAULT_COST_PER_QUERY - if billing_mode == "per_query" - else GOOGLE_MAPS_GROUNDING_DEFAULT_COST_PER_PROMPT + return search_cost( + _as_google_row(model_info), + TokenUsage( + prompt_tokens=0, + text=0, + output_text=0, + maps_grounding_requests=max(google_maps_grounding_requests(usage) or 0, 0), + ), ) - configured_cost: Final = model_info.get("google_maps_grounding_cost_per_query") - cost: Final = default_cost if configured_cost is None else configured_cost - billed_requests: Final = requests if billing_mode == "per_query" else 1 - return cost * billed_requests diff --git a/litellm/llms/groq/cost_calculator.py b/litellm/llms/groq/cost_calculator.py index 19d87c44940..247f5d63438 100644 --- a/litellm/llms/groq/cost_calculator.py +++ b/litellm/llms/groq/cost_calculator.py @@ -16,15 +16,22 @@ if TYPE_CHECKING: def cost_per_web_search_request(usage: Usage, model_info: "ModelInfo") -> float: - search_costs: Final = model_info.get("search_context_cost_per_query") - cost_per_search: Final = search_costs.get("search_context_size_medium", 0.0) if search_costs else 0.0 + from litellm._cost import Usage as TokenUsage + from litellm.litellm_core_utils.llm_cost_calc.utils import search_cost + try: server_tool_use: Final = usage.server_tool_use except AttributeError: return 0.0 if server_tool_use is None: return 0.0 - searches: Final = server_tool_use.web_search_requests or 0 - opens: Final = server_tool_use.browser_open_requests or 0 - cost_per_open: Final = model_info.get("browser_open_cost_per_request") or 0.0 - return searches * cost_per_search + opens * cost_per_open + return search_cost( + model_info, + TokenUsage( + prompt_tokens=0, + text=0, + output_text=0, + web_search_requests=server_tool_use.web_search_requests or 0, + browser_opens=server_tool_use.browser_open_requests or 0, + ), + ) diff --git a/litellm/llms/openai/cost_calculation.py b/litellm/llms/openai/cost_calculation.py index 63be44e83b9..e7424c490c6 100644 --- a/litellm/llms/openai/cost_calculation.py +++ b/litellm/llms/openai/cost_calculation.py @@ -5,7 +5,7 @@ Helper util for handling openai-specific cost calculation from typing import Literal -from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text from litellm.types.utils import CallTypes, Usage @@ -35,7 +35,7 @@ def cost_per_token( Returns: Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd """ - return generic_cost_per_token( + return cost_per_text( model=model, usage=usage, custom_llm_provider="openai", diff --git a/litellm/llms/openai/image_generation/cost_calculator.py b/litellm/llms/openai/image_generation/cost_calculator.py index 350fdbaee2d..a5da2dd2207 100644 --- a/litellm/llms/openai/image_generation/cost_calculator.py +++ b/litellm/llms/openai/image_generation/cost_calculator.py @@ -9,8 +9,8 @@ from typing import Final from litellm import verbose_logger from litellm.litellm_core_utils.llm_cost_calc.utils import ( calculate_image_response_cost_from_usage, + cost_per_text, flat_image_cost, - generic_cost_per_token, resolve_image_model_info, ) from litellm.types.utils import ImageResponse, ModelInfo, Usage @@ -35,9 +35,9 @@ def cost_calculator( verbose_logger.debug("No usage data available for %s, cannot calculate token-based cost", model) return flat_image_cost(price_table, image_response) - # A chat Usage with an explicit output breakdown: cost via generic_cost_per_token. + # A chat Usage with an explicit output breakdown: cost via cost_per_text. if isinstance(usage, Usage) and usage.completion_tokens_details is not None: - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model, usage=usage, custom_llm_provider=provider, model_info=price_table ) return prompt_cost + completion_cost @@ -54,9 +54,9 @@ def cost_calculator( return token_based_cost # Fallback: a Usage with no output breakdown that the image helper can't read — - # cost via generic_cost_per_token (text rate) instead of returning 0.0. + # cost via cost_per_text (text rate) instead of returning 0.0. if isinstance(usage, Usage): - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model, usage=usage, custom_llm_provider=provider, model_info=price_table ) return prompt_cost + completion_cost diff --git a/litellm/llms/parallel_ai/search/cost_calculator.py b/litellm/llms/parallel_ai/search/cost_calculator.py index 477db6d66a9..30dff0e94e6 100644 --- a/litellm/llms/parallel_ai/search/cost_calculator.py +++ b/litellm/llms/parallel_ai/search/cost_calculator.py @@ -4,7 +4,9 @@ from typing import Final from pydantic import TypeAdapter, ValidationError -from litellm.utils import get_model_info +from litellm._cost import Usage as TokenUsage +from litellm._cost import as_main_cost +from litellm.litellm_core_utils.llm_cost_calc.utils import price_usage, pricing_row PARALLEL_AI_DEFAULT_RESULTS: Final = 10 PARALLEL_AI_USAGE_PARAM: Final = "_parallel_ai_usage" @@ -60,16 +62,6 @@ def _effective_max_results(optional_params: Mapping[str, object]) -> int: return max_results if max_results is not None else PARALLEL_AI_DEFAULT_RESULTS -def _request_cost(mode: str) -> tuple[float, float]: - """The per-request price and the price of each result past the included ones""" - pricing_model: Final = PARALLEL_AI_PRICING_MODEL_BY_MODE.get(mode, PARALLEL_AI_STANDARD_SEARCH_MODEL) - model_info: Final = get_model_info(model=pricing_model, custom_llm_provider="parallel_ai") - return ( - float(model_info.get("input_cost_per_query") or 0.0), - float(model_info.get("cost_per_additional_result") or 0.0), - ) - - def _additional_results( optional_params: Mapping[str, object], usage: Sequence[Mapping[str, object]] | None, @@ -86,8 +78,19 @@ def parallel_ai_search_cost( optional_params: Mapping[str, object], usage: Sequence[Mapping[str, object]] | None, ) -> float: - request_cost, additional_result_cost = _request_cost(_effective_mode(optional_params)) + mode: Final = _effective_mode(optional_params) + pricing_model: Final = PARALLEL_AI_PRICING_MODEL_BY_MODE.get(mode, PARALLEL_AI_STANDARD_SEARCH_MODEL) request_count_from_usage: Final = _usage_count(usage, "sku_search") if usage is not None else None - request_count: Final = request_count_from_usage if request_count_from_usage is not None else 1 - additional_results: Final = _additional_results(optional_params, usage) - return request_count * request_cost + additional_results * additional_result_cost + prompt_cost, completion_cost = as_main_cost( + price_usage( + pricing_row(pricing_model, "parallel_ai", None), + TokenUsage( + prompt_tokens=0, + text=0, + output_text=0, + search_queries=request_count_from_usage if request_count_from_usage is not None else 1, + additional_results=_additional_results(optional_params, usage), + ), + ) + ) + return prompt_cost + completion_cost diff --git a/litellm/llms/recraft/cost_calculator.py b/litellm/llms/recraft/cost_calculator.py index 052e09a30c4..75ddeb7fe44 100644 --- a/litellm/llms/recraft/cost_calculator.py +++ b/litellm/llms/recraft/cost_calculator.py @@ -1,7 +1,7 @@ from typing import Final import litellm -from litellm.litellm_core_utils.llm_cost_calc.utils import resolve_image_model_info +from litellm.litellm_core_utils.llm_cost_calc.utils import image_cost, resolve_image_model_info from litellm.types.utils import ImageResponse, ModelInfo @@ -18,11 +18,10 @@ def cost_calculator( custom_llm_provider=litellm.LlmProviders.RECRAFT.value, model_info=model_info, ) - output_cost_per_image: Final[float] = _model_info.get("output_cost_per_image") or 0.0 num_images: int = 0 if isinstance(image_response, ImageResponse): if image_response.data: num_images = len(image_response.data) - return output_cost_per_image * num_images + return image_cost(_model_info, num_images) or 0.0 else: raise ValueError(f"image_response must be of type ImageResponse got type={type(image_response)}") diff --git a/litellm/llms/runwayml/cost_calculator.py b/litellm/llms/runwayml/cost_calculator.py index 729ca516213..f10dcd5f69b 100644 --- a/litellm/llms/runwayml/cost_calculator.py +++ b/litellm/llms/runwayml/cost_calculator.py @@ -1,7 +1,7 @@ from typing import Final import litellm -from litellm.litellm_core_utils.llm_cost_calc.utils import resolve_image_model_info +from litellm.litellm_core_utils.llm_cost_calc.utils import image_cost, resolve_image_model_info from litellm.types.utils import ImageResponse, ModelInfo @@ -21,11 +21,10 @@ def cost_calculator( custom_llm_provider=litellm.LlmProviders.RUNWAYML.value, model_info=model_info, ) - output_cost_per_image: Final[float] = _model_info.get("output_cost_per_image") or 0.0 num_images: int = 0 if isinstance(image_response, ImageResponse): if image_response.data: num_images = len(image_response.data) - return output_cost_per_image * num_images + return image_cost(_model_info, num_images) or 0.0 else: raise ValueError(f"image_response must be of type ImageResponse, got type={type(image_response)}") diff --git a/litellm/llms/tencent/cost_calculator.py b/litellm/llms/tencent/cost_calculator.py index d9aebdc3284..6347ef24298 100644 --- a/litellm/llms/tencent/cost_calculator.py +++ b/litellm/llms/tencent/cost_calculator.py @@ -1,6 +1,6 @@ -from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text from litellm.types.utils import Usage def cost_per_token(model: str, usage: Usage) -> tuple[float, float]: - return generic_cost_per_token(model=model, usage=usage, custom_llm_provider="tencent") + return cost_per_text(model=model, usage=usage, custom_llm_provider="tencent") diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index d0453b72701..80584f2445d 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -80,18 +80,30 @@ def _vertex_ai_lyria_model_key(model: str) -> str: return model if model.startswith("vertex_ai/") else f"vertex_ai/{model}" -def _vertex_ai_lyria_generation_cost(model_info: VertexAILyriaModelInfo | None) -> float | None: - return None if model_info is None else model_info.get("output_cost_per_image") - - def get_vertex_ai_lyria_model_info(model: str) -> VertexAILyriaModelInfo | None: model_key: Final = _vertex_ai_lyria_model_key(model) return _validate_vertex_ai_lyria_model_info(litellm.model_cost.get(model_key)) -def get_vertex_ai_lyria_generation_cost(model: str) -> float | None: +def get_vertex_ai_lyria_generation_cost(model: str, generations: int = 1) -> float | None: + """``generations`` Lyria generations at the row's per-generation price, None when the model is not a priced Lyria + row""" + from litellm._cost import ImageOut, Priced + from litellm._cost import Usage as TokenUsage + from litellm.litellm_core_utils.llm_cost_calc.utils import price_usage + model_key: Final = _vertex_ai_lyria_model_key(model) - return _vertex_ai_lyria_generation_cost(_validate_vertex_ai_lyria_model_info(litellm.model_cost.get(model_key))) + row: Final = litellm.model_cost.get(model_key) + if row is None or _validate_vertex_ai_lyria_model_info(row) is None: + return None + priced: Final = price_usage( + row, + TokenUsage(prompt_tokens=0, text=0, output_text=0, images_out=[ImageOut()] * max(generations, 0)), + key=model_key, + ) + if not isinstance(priced, Priced) or (generations > 0 and not priced.lines): + return None + return priced.total class VertexAIError(BaseLLMException): diff --git a/litellm/llms/vertex_ai/cost_calculator.py b/litellm/llms/vertex_ai/cost_calculator.py index 8ec034e7a5e..0cc13c2dee4 100644 --- a/litellm/llms/vertex_ai/cost_calculator.py +++ b/litellm/llms/vertex_ai/cost_calculator.py @@ -1,14 +1,8 @@ # What is this? ## Cost calculation for Google AI Studio / Vertex AI models -from typing import Final -import litellm -from litellm.litellm_core_utils.llm_cost_calc.utils import ( - generic_cost_per_token, - get_vertex_regional_endpoint_uplift, - is_above_128k, -) -from litellm.types.utils import ModelInfo, Usage +from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text +from litellm.types.utils import Usage """ Gemini pricing covers: @@ -19,32 +13,6 @@ Gemini pricing covers: """ -def _handle_128k_pricing( - model_info: ModelInfo, - usage: Usage, -) -> tuple[float, float]: - ## CALCULATE INPUT COST - input_cost_per_token_above_128k_tokens: Final = model_info.get("input_cost_per_token_above_128k_tokens") - output_cost_per_token_above_128k_tokens = model_info.get("output_cost_per_token_above_128k_tokens") - - prompt_tokens: Final = usage.prompt_tokens - completion_tokens: Final = usage.completion_tokens - - if is_above_128k(tokens=prompt_tokens) and input_cost_per_token_above_128k_tokens is not None: - prompt_cost = prompt_tokens * input_cost_per_token_above_128k_tokens - else: - prompt_cost = prompt_tokens * (model_info["input_cost_per_token"] or 0.0) - - ## CALCULATE OUTPUT COST - output_cost_per_token_above_128k_tokens = model_info.get("output_cost_per_token_above_128k_tokens") - if is_above_128k(tokens=completion_tokens) and output_cost_per_token_above_128k_tokens is not None: - completion_cost = completion_tokens * output_cost_per_token_above_128k_tokens - else: - completion_cost = completion_tokens * (model_info["output_cost_per_token"] or 0.0) - - return prompt_cost, completion_cost - - def cost_per_token( model: str, custom_llm_provider: str, @@ -67,26 +35,9 @@ def cost_per_token( Returns: Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd - - Raises: - Exception if model requires >128k pricing, but model cost not mapped """ - ## GET MODEL INFO - model_info: Final = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider) - - ## HANDLE 128k+ PRICING - input_cost_per_token_above_128k_tokens: Final = model_info.get("input_cost_per_token_above_128k_tokens") - output_cost_per_token_above_128k_tokens: Final = model_info.get("output_cost_per_token_above_128k_tokens") - if input_cost_per_token_above_128k_tokens is not None or output_cost_per_token_above_128k_tokens is not None: - prompt_cost_128k, completion_cost_128k = _handle_128k_pricing( - model_info=model_info, - usage=usage, - ) - vertex_uplift: Final = get_vertex_regional_endpoint_uplift(model_info, vertex_location) - return prompt_cost_128k * vertex_uplift, completion_cost_128k * vertex_uplift - - return generic_cost_per_token( + return cost_per_text( model=model, custom_llm_provider=custom_llm_provider, usage=usage, diff --git a/litellm/llms/vertex_ai/image_edit/cost_calculator.py b/litellm/llms/vertex_ai/image_edit/cost_calculator.py index 7d1c2c1fc51..d6b3c245bd4 100644 --- a/litellm/llms/vertex_ai/image_edit/cost_calculator.py +++ b/litellm/llms/vertex_ai/image_edit/cost_calculator.py @@ -5,6 +5,7 @@ Vertex AI Image Edit Cost Calculator from typing import Final import litellm +from litellm.litellm_core_utils.llm_cost_calc.utils import image_cost from litellm.types.utils import ImageResponse @@ -23,10 +24,8 @@ def cost_calculator( custom_llm_provider="vertex_ai", ) - output_cost_per_image: Final[float] = model_info.get("output_cost_per_image") or 0.0 - if not isinstance(image_response, ImageResponse): raise ValueError(f"image_response must be of type ImageResponse got type={type(image_response)}") num_images: Final = len(image_response.data or []) - return output_cost_per_image * num_images + return image_cost(model_info, num_images) or 0.0 diff --git a/litellm/llms/xai/cost_calculator.py b/litellm/llms/xai/cost_calculator.py index aec535c2086..6405535f086 100644 --- a/litellm/llms/xai/cost_calculator.py +++ b/litellm/llms/xai/cost_calculator.py @@ -9,7 +9,7 @@ import math from collections.abc import Mapping from typing import TYPE_CHECKING, Final -from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text from litellm.types.utils import PromptTokensDetailsWrapper, Usage if TYPE_CHECKING: @@ -53,7 +53,7 @@ def _split_reported_cost(model: str, usage: Usage, reported_cost: float) -> tupl """xAI reports one total, so attribute it to input and output in the ratio our own token math gives.""" try: priced_input, priced_output = _token_math_cost_per_token(model, usage) - except Exception: # noqa: BLE001 # generic_cost_per_token raises bare Exception for unmapped models + except Exception: # noqa: BLE001 # cost_per_text raises bare Exception for unmapped models return 0.0, reported_cost priced_total: Final = priced_input + priced_output if priced_total <= 0: @@ -105,7 +105,7 @@ def _token_math_cost_per_token(model: str, usage: Usage) -> tuple[float, float]: completion_tokens_details=None, ) - prompt_cost, completion_cost = generic_cost_per_token(model=model, usage=modified_usage, custom_llm_provider="xai") + prompt_cost, completion_cost = cost_per_text(model=model, usage=modified_usage, custom_llm_provider="xai") return prompt_cost, completion_cost diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py index 5010de46fcb..ea05f1cb97a 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py @@ -442,7 +442,7 @@ class VertexPassthroughLoggingHandler: prediction_count: Final = VertexPassthroughLoggingHandler._get_audio_prediction_count( json_response=json_response ) - response_cost: Final = (get_vertex_ai_lyria_generation_cost(model=model) or 0.0) * prediction_count + response_cost: Final = get_vertex_ai_lyria_generation_cost(model=model, generations=prediction_count) or 0.0 logging_obj.model = model # rebind-ok: passthrough attribution records the resolved Vertex model logging_obj.model_call_details[ # rebind-ok: passthrough attribution enriches callback metadata diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py index 604a7a389e6..589fe238799 100644 --- a/litellm/proxy/spend_tracking/savings.py +++ b/litellm/proxy/spend_tracking/savings.py @@ -22,7 +22,7 @@ from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY from litellm.litellm_core_utils.get_llm_provider_logic import resolve_model_and_provider_for_metadata from litellm.litellm_core_utils.llm_cost_calc.utils import ( calculate_prompt_caching_savings, - generic_cost_per_token, + cost_per_text, get_cost_per_unit, ) from litellm.types.integrations.anthropic_cache_control_hook import ( @@ -195,7 +195,7 @@ def _pricing_basis(cost_breakdown: Mapping[str, object] | None) -> PricingBasis: they price at standard rates, which is what they already did. These values survive a JSON round trip on the way here, so none is guaranteed to be - a string. `generic_cost_per_token` calls `.lower()` on them without a type check, and + a string. `cost_per_text` calls `.lower()` on them without a type check, and the resulting `AttributeError` would be swallowed into a silent zero by the caller's `except`, so anything that is not a string is dropped here instead. """ @@ -215,7 +215,7 @@ def _recorded_token_cost(cost_breakdown: Mapping[str, object] | None) -> float | """What the biller charged for this request's tokens, or ``None`` when unrecorded. ``input_cost`` already carries the cache buckets, so it and ``output_cost`` sum to - exactly what `generic_cost_per_token` returns for the same request; the separate + exactly what `cost_per_text` returns for the same request; the separate ``cache_read_cost`` and ``cache_creation_cost`` entries decompose that sum rather than adding to it, and including them would charge those tokens twice. @@ -251,7 +251,7 @@ def _cost_of_usage( model_info=model_info, ) else: - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model.model, usage=usage, custom_llm_provider=model.provider, diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 0ddfc5ce62f..65206086aa0 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -161,7 +161,7 @@ from litellm.litellm_core_utils.core_helpers import ( from litellm.litellm_core_utils.duration_parser import get_budget_window_start from litellm.litellm_core_utils.env_utils import get_env_float, get_env_int from litellm.litellm_core_utils.litellm_logging import BilledRoundsDispatch, Logging, deployment_pricing_model_info -from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.litellm_core_utils.served_output_texts import ( @@ -1289,12 +1289,12 @@ def _deployment_priced_input_cost( model: str, custom_llm_provider: str | None, prompt_tokens: int, deployment_pricing: ModelInfo ) -> float: """Input price for the estimated prompt at the deployment's registered rates, read through - ``deployment_pricing_model_info`` and priced by ``generic_cost_per_token`` the way the success + ``deployment_pricing_model_info`` and priced by ``cost_per_text`` the way the success path prices them, so a long-context tier the deployment declares (``input_cost_per_token_above_200k_tokens``) is honoured instead of flattened to the standard rate. An entry it cannot price keeps the row at $0.""" usage: Final = Usage(prompt_tokens=prompt_tokens, completion_tokens=0, total_tokens=prompt_tokens) try: - prompt_cost, _ = generic_cost_per_token( + prompt_cost, _ = cost_per_text( model=model, usage=usage, custom_llm_provider=custom_llm_provider or str(deployment_pricing.get("litellm_provider") or ""), diff --git a/litellm/router_strategy/savings_baseline.py b/litellm/router_strategy/savings_baseline.py index add0f78fd3a..f4ed6d8f72e 100644 --- a/litellm/router_strategy/savings_baseline.py +++ b/litellm/router_strategy/savings_baseline.py @@ -110,14 +110,14 @@ def _priced(router: "Router", candidate: Baseline) -> tuple[float, Baseline] | N dimension to that engine. A candidate that prices to nothing there cannot stand in for what the traffic would have cost. """ - from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token + from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text provider, _, model_name = candidate.model.partition("/") try: info: Final = router.cached_deployment_model_info(candidate.deployment_id or "", candidate.model) if info is None: return None - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model_name or candidate.model, usage=_REFERENCE_REQUEST, custom_llm_provider=provider, diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi index de8c1ebc3ec..c27694e7b56 100644 --- a/litellm/rust_bridge/_native.pyi +++ b/litellm/rust_bridge/_native.pyi @@ -156,8 +156,23 @@ class Usage: output_video_duration: timedelta | None = None, video_resolution: str | None = None, text_messages: int = 0, + images_in: int = 0, + images_out: Sequence[ImageOut] = (), + pixels_in: int = 0, + search_queries: int = 0, + max_results: int | None = None, + web_search_requests: int = 0, + web_search_context: Literal["low", "medium", "high"] = "medium", + maps_grounding_requests: int = 0, + citation_tokens: int = 0, + browser_opens: int = 0, + additional_results: int = 0, ) -> None: ... +@final +class ImageOut: + def __init__(self, *, size: str | None = None, width: int | None = None, height: int | None = None) -> None: ... + @final class CallContext: @@ -170,6 +185,8 @@ class CallContext: vertex_location: str | None, at: datetime, custom_pricing: Mapping[str, object] | None, + speed: str | None = None, + inference_geo: str | None = None, ) -> None: ... @final @@ -208,6 +225,15 @@ class Catalog: def bound_to(module: ModuleType, attr: str) -> Catalog: ... def bundled(self) -> Fetched: ... def fetch(self, source: Remote | File | Bundled, *, timeout: float, attempts: int) -> Fetched | Unavailable: ... + def resolve_image( + self, + model: str, + provider: str | None = None, + *, + size: tuple[int, int] | None = None, + quality: str | None = None, + ) -> Resolved | Missing | Invalid: ... + def nearest_image_key(self, model: str, provider: str, *, quality: str, size: tuple[int, int]) -> str | None: ... def reload(self, source: Remote | File | Bundled) -> Installed | Unavailable: ... def provenance(self) -> Provenance | None: ... def resolve(self, model: str, provider: str | None = None) -> Resolved | Missing | Invalid: ... diff --git a/litellm/search/cost_calculator.py b/litellm/search/cost_calculator.py index 15755128136..8a0e14797d7 100644 --- a/litellm/search/cost_calculator.py +++ b/litellm/search/cost_calculator.py @@ -9,8 +9,10 @@ from typing import Final from pydantic import TypeAdapter, ValidationError import litellm +from litellm._cost import Usage as TokenUsage +from litellm._cost import as_main_cost from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap -from litellm.utils import get_model_info +from litellm.litellm_core_utils.llm_cost_calc.utils import price_usage, pricing_row PROVIDER_USAGE_ADAPTER: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] = TypeAdapter( tuple[Mapping[str, object], ...] @@ -93,28 +95,16 @@ def search_provider_cost_per_query( ) return (input_cost, 0.0) - model_info: Final = get_model_info( - model=_pricing_model(model, custom_llm_provider, optional_params), - custom_llm_provider=custom_llm_provider, + max_results: Final = (optional_params or {}).get("max_results") + return as_main_cost( + price_usage( + pricing_row(_pricing_model(model, custom_llm_provider, optional_params), custom_llm_provider, None), + TokenUsage( + prompt_tokens=0, + text=0, + output_text=0, + search_queries=number_of_queries, + max_results=max_results if isinstance(max_results, int) and not isinstance(max_results, bool) else None, + ), + ) ) - - # Check for tiered pricing (e.g., Exa AI based on max_results) - tiered_pricing: Final = model_info.get("tiered_pricing") - if tiered_pricing and isinstance(tiered_pricing, list): - max_results: Final = (optional_params or {}).get("max_results", 10) # default 10 results - cost_per_query = 0.0 - - for tier in tiered_pricing: - range_min, range_max = tier["max_results_range"] - if range_min <= max_results <= range_max: - cost_per_query = tier["input_cost_per_query"] - break - else: - # Fallback to highest tier if out of range - cost_per_query = tiered_pricing[-1]["input_cost_per_query"] - else: - # Simple flat rate - cost_per_query = float(model_info.get("input_cost_per_query") or 0.0) - - total_cost: Final = number_of_queries * cost_per_query - return (total_cost, 0.0) # (input_cost, output_cost) diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index e36ef2a11f5..3103258e4fd 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -1,4 +1,3 @@ -import json from collections.abc import Mapping from datetime import datetime, timezone from typing import Final, cast @@ -11,19 +10,14 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import ( BilledTokenRates, CostCalculatorUtils, TokenRates, - _get_token_base_cost, _is_off_peak, _is_within_off_peak_window, apply_off_peak_pricing, - apply_provider_cache_read_default, calculate_cache_writing_cost, - generic_cost_per_token, + cost_per_text, get_billed_token_rates, get_token_type_cost_breakdown, ) -from litellm.llms.gemini.image_generation.cost_calculator import ( - cost_calculator as gemini_image_generation_cost_calculator, -) from litellm.llms.vertex_ai.image_generation.cost_calculator import ( cost_calculator as vertex_image_generation_cost_calculator, ) @@ -46,28 +40,7 @@ def _local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) -@pytest.mark.parametrize("prompt_tokens", [100, 200000, 200001]) -@pytest.mark.parametrize("read_rate", [None, 0.0, 0.25e-6]) -@pytest.mark.parametrize("service_tier", [None, "priority"]) -def test_missing_cache_read_rate_resolves_to_input_rate(prompt_tokens, read_rate, service_tier): - info = { - "input_cost_per_token": 3e-6, - "input_cost_per_token_priority": 4e-6, - "input_cost_per_token_above_200k_tokens": 6e-6, - "input_cost_per_token_above_200k_tokens_priority": 8e-6, - "output_cost_per_token": 1e-6, - "cache_read_input_token_cost": read_rate, - } - usage = Usage(prompt_tokens=prompt_tokens, prompt_tokens_details={"cached_tokens": 100}) - billed = _get_token_base_cost(info, usage, service_tier=service_tier) - prompt_cost, _ = generic_cost_per_token( - "policy-fixture", usage, "openai", service_tier=service_tier, model_info=info - ) - assert billed.cache_read_rate == pytest.approx(read_rate if read_rate is not None else billed.input_rate) - assert prompt_cost == pytest.approx((prompt_tokens - 100) * billed.input_rate + 100 * billed.cache_read_rate) - - -def test_generic_cost_per_token_bills_cache_reads_at_input_rate_when_no_cache_read_rate() -> None: +def test_cost_per_text_bills_cache_reads_at_input_rate_when_no_cache_read_rate() -> None: model_info: ModelInfo = { "key": "bare-model", "max_tokens": None, @@ -86,7 +59,7 @@ def test_generic_cost_per_token_bills_cache_reads_at_input_rate_when_no_cache_re prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=12288), ) - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model="bare-model", usage=usage, custom_llm_provider="bedrock", @@ -110,7 +83,7 @@ def test_generic_cost_per_token_bills_cache_reads_at_input_rate_when_no_cache_re "token_priced_model_ignores_the_character_count", ], ) -def test_generic_cost_per_token_bills_a_character_counted_prompt_under_one_unit( +def test_cost_per_text_bills_a_character_counted_prompt_under_one_unit( model_info: ModelInfo, expected_prompt_cost: float ) -> None: usage = Usage( @@ -120,27 +93,14 @@ def test_generic_cost_per_token_bills_a_character_counted_prompt_under_one_unit( prompt_tokens_details=PromptTokensDetailsWrapper(character_count=35), ) - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model="embedding-fixture", usage=usage, custom_llm_provider="vertex_ai", model_info=model_info ) assert (prompt_cost, completion_cost) == (pytest.approx(expected_prompt_cost), 0.0) -def test_apply_provider_cache_read_default_only_derives_a_rate_for_fireworks() -> None: - openai_info: ModelInfo = {"input_cost_per_token": 2e-6} - fireworks_info: ModelInfo = {"input_cost_per_token": 2e-6} - - assert apply_provider_cache_read_default(openai_info, "openai") is openai_info - assert apply_provider_cache_read_default(openai_info, None) is openai_info - - processed_fireworks_info = apply_provider_cache_read_default(fireworks_info, "fireworks_ai") - - assert processed_fireworks_info is not fireworks_info - assert processed_fireworks_info["cache_read_input_token_cost"] == pytest.approx(2e-6 * 0.5) - - -def test_generic_cost_per_token_prefers_audio_per_second_rate() -> None: +def test_cost_per_text_prefers_audio_per_second_rate() -> None: model_info: ModelInfo = { "key": "gemini-embedding-2", "max_tokens": None, @@ -164,7 +124,7 @@ def test_generic_cost_per_token_prefers_audio_per_second_rate() -> None: ), ) - prompt_cost, _ = generic_cost_per_token( + prompt_cost, _ = cost_per_text( model="gemini-embedding-2", usage=usage, custom_llm_provider="vertex_ai", @@ -174,7 +134,7 @@ def test_generic_cost_per_token_prefers_audio_per_second_rate() -> None: assert prompt_cost == pytest.approx(2 * 0.00016) -def test_generic_cost_per_token_prefers_image_per_image_rate() -> None: +def test_cost_per_text_prefers_image_per_image_rate() -> None: model_info: ModelInfo = { "key": "gemini-embedding-2", "max_tokens": None, @@ -198,7 +158,7 @@ def test_generic_cost_per_token_prefers_image_per_image_rate() -> None: ), ) - prompt_cost, _ = generic_cost_per_token( + prompt_cost, _ = cost_per_text( model="gemini-embedding-2", usage=usage, custom_llm_provider="vertex_ai", @@ -208,7 +168,7 @@ def test_generic_cost_per_token_prefers_image_per_image_rate() -> None: assert prompt_cost == pytest.approx(0.00012) -def test_generic_cost_per_token_prefers_video_per_second_rate() -> None: +def test_cost_per_text_prefers_video_per_second_rate() -> None: model_info: ModelInfo = { "key": "gemini-embedding-2", "max_tokens": None, @@ -232,7 +192,7 @@ def test_generic_cost_per_token_prefers_video_per_second_rate() -> None: ), ) - prompt_cost, _ = generic_cost_per_token( + prompt_cost, _ = cost_per_text( model="gemini-embedding-2", usage=usage, custom_llm_provider="vertex_ai", @@ -242,18 +202,6 @@ def test_generic_cost_per_token_prefers_video_per_second_rate() -> None: assert prompt_cost == pytest.approx(2 * 0.00079) -def test_missing_cache_read_uses_off_peak_input_rate(): - from datetime import datetime, timezone - - info = { - "input_cost_per_token": 3e-6, - "off_peak_pricing": {"hours_utc": "00:00-23:59", "input_cost_per_token": 5e-6}, - } - when = datetime(2026, 9, 7, 12, tzinfo=timezone.utc) - billed = _get_token_base_cost(info, Usage(prompt_tokens=100), current_time=when) - assert billed.input_rate == billed.cache_read_rate == 5e-6 - - def test_reasoning_tokens_no_price_set(_local_model_cost_map): # Use o1 - o1-mini was deprecated/renamed; o1 has same reasoning-token semantics # (no separate output_cost_per_reasoning_token, so all completion tokens use output_cost_per_token) @@ -274,7 +222,7 @@ def test_reasoning_tokens_no_price_set(_local_model_cost_map): audio_tokens=None, cached_tokens=None, text_tokens=17, image_tokens=None ), ) - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model, usage=usage, custom_llm_provider="openai", @@ -310,7 +258,7 @@ def test_reasoning_tokens_gemini(_local_model_cost_map): ), ) model_cost_map = litellm.model_cost[model] - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model, usage=usage, custom_llm_provider=custom_llm_provider, @@ -359,7 +307,7 @@ def test_image_tokens_with_custom_pricing(): "litellm.litellm_core_utils.llm_cost_calc.utils.get_model_info", return_value=mock_model_info, ): - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model="test-model", usage=usage, custom_llm_provider="gemini" ) @@ -405,7 +353,7 @@ def test_image_tokens_fallback_to_base_cost(): "litellm.litellm_core_utils.llm_cost_calc.utils.get_model_info", return_value=mock_model_info, ): - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model="test-model", usage=usage, custom_llm_provider="gemini" ) @@ -432,7 +380,7 @@ def test_video_input_tokens_gemini_omni_flash_preview(_local_model_cost_map): ) model_cost_map = litellm.model_cost[f"gemini/{model}"] - prompt_cost, _ = generic_cost_per_token( + prompt_cost, _ = cost_per_text( model=model, usage=usage, custom_llm_provider="gemini", @@ -468,7 +416,7 @@ def test_video_tokens_fallback_to_base_cost(): "litellm.litellm_core_utils.llm_cost_calc.utils.get_model_info", return_value=mock_model_info, ): - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model="test-model", usage=usage, custom_llm_provider="gemini" ) @@ -499,7 +447,7 @@ def test_prompt_audio_and_video_tokens_without_modality_rates_bill_at_tiered_inp "litellm.litellm_core_utils.llm_cost_calc.utils.get_model_info", return_value=mock_model_info, ): - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model="test-model", usage=usage, custom_llm_provider="vertex_ai" ) @@ -531,12 +479,12 @@ def test_prompt_modality_rates_still_win_over_tiered_input_rate(): "litellm.litellm_core_utils.llm_cost_calc.utils.get_model_info", return_value=mock_model_info, ): - prompt_cost, _ = generic_cost_per_token(model="test-model", usage=usage, custom_llm_provider="vertex_ai") + prompt_cost, _ = cost_per_text(model="test-model", usage=usage, custom_llm_provider="vertex_ai") assert round(prompt_cost, 9) == round(100000 * 4e-6 + 100000 * 3e-6 + 100000 * 1e-6, 9) -def test_generic_cost_per_token_above_200k_tokens(_local_model_cost_map): +def test_cost_per_text_above_200k_tokens(_local_model_cost_map): # gemini-2.5-pro-exp-03-25 was removed; gemini-2.5-pro has same above-200k pricing model = "gemini-2.5-pro" custom_llm_provider = "vertex_ai" @@ -549,7 +497,7 @@ def test_generic_cost_per_token_above_200k_tokens(_local_model_cost_map): completion_tokens=completion_tokens, total_tokens=prompt_tokens + completion_tokens, ) - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model, usage=usage, custom_llm_provider=custom_llm_provider, @@ -564,26 +512,6 @@ def test_generic_cost_per_token_above_200k_tokens(_local_model_cost_map): ) -def test_get_token_base_cost_picks_highest_crossed_tier(): - """Regression test for #30345. - - With graduated tiers at 90k and 128k whose keys have different digit lengths, a request - crossing both must be billed at the highest tier it crosses (128k), not the lower one that - happens to sort first lexicographically. - """ - model_info = { - "input_cost_per_token": 1e-6, - "output_cost_per_token": 2e-6, - "input_cost_per_token_above_90k_tokens": 5e-6, - "input_cost_per_token_above_128k_tokens": 9e-6, - } - usage = Usage(prompt_tokens=150_000, completion_tokens=10, total_tokens=150_010) - - prompt_base_cost = _get_token_base_cost(model_info, usage).input_rate - - assert prompt_base_cost == 9e-6 - - def test_is_within_off_peak_window_same_day(): from datetime import datetime, timezone @@ -754,152 +682,6 @@ def test_is_off_peak_flat_hours_and_windows_are_a_union(): assert _is_off_peak(block, datetime(2026, 8, 28, 20, 0, tzinfo=timezone.utc)) is False -def test_get_token_base_cost_weekend_only_off_peak_rate(): - from datetime import datetime, timezone - from typing import cast - - from litellm.types.utils import ModelInfo - - model_info = cast( - ModelInfo, - { - "input_cost_per_token": 1e-6, - "output_cost_per_token": 2e-6, - "off_peak_pricing": { - "windows": [ - {"hours_utc": ["00:00-01:00", "04:00-06:00", "10:00-00:00"], "weekdays": [1, 2, 3, 4, 5]}, - {"hours_utc": "00:00-00:00", "weekdays": [6, 7]}, - ], - "input_cost_per_token": 5e-7, - "output_cost_per_token": 1e-6, - }, - }, - ) - usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) - - saturday_peak_hours = _get_token_base_cost( - model_info, usage, current_time=datetime(2026, 8, 29, 2, 0, tzinfo=timezone.utc) - ) - assert (saturday_peak_hours.input_rate, saturday_peak_hours.output_rate) == (5e-7, 1e-6) - - monday_same_hours = _get_token_base_cost( - model_info, usage, current_time=datetime(2026, 8, 24, 2, 0, tzinfo=timezone.utc) - ) - assert (monday_same_hours.input_rate, monday_same_hours.output_rate) == (1e-6, 2e-6) - - -def test_get_token_base_cost_applies_off_peak_pricing(): - from datetime import datetime, timezone - from typing import cast - - from litellm.types.utils import ModelInfo - - model_info = cast( - ModelInfo, - { - "input_cost_per_token": 1e-6, - "output_cost_per_token": 2e-6, - "cache_read_input_token_cost": 1e-7, - "off_peak_pricing": { - "hours_utc": "16:30-00:30", - "input_cost_per_token": 5e-7, - "output_cost_per_token": 1e-6, - "cache_read_input_token_cost": 5e-8, - }, - }, - ) - usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) - - off_peak = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc)) - assert off_peak.input_rate == 5e-7 - assert off_peak.output_rate == 1e-6 - assert off_peak.cache_read_rate == 5e-8 - - peak = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc)) - assert peak.input_rate == 1e-6 - assert peak.output_rate == 2e-6 - assert peak.cache_read_rate == 1e-7 - - -def test_get_token_base_cost_non_mapping_off_peak_block_bills_standard_rates(): - """A truthy non-mapping off_peak_pricing value (a bare string or a list in - YAML) must bill standard rates rather than raising, matching how every - other malformed piece of the block behaves. - """ - from datetime import datetime, timezone - from typing import cast - - from litellm.types.utils import ModelInfo - - usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) - when = datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc) - - for malformed_block in ("16:00-19:00", ["16:00-19:00"], 5e-7, True): - model_info = cast( - ModelInfo, - { - "input_cost_per_token": 1e-6, - "output_cost_per_token": 2e-6, - "off_peak_pricing": malformed_block, - }, - ) - result = _get_token_base_cost(model_info, usage, current_time=when) - assert result.input_rate == 1e-6 - assert result.output_rate == 2e-6 - - -def test_get_token_base_cost_off_peak_falls_back_to_standard_when_unset(): - from datetime import datetime, timezone - from typing import cast - - from litellm.types.utils import ModelInfo - - model_info = cast( - ModelInfo, - { - "input_cost_per_token": 1e-6, - "output_cost_per_token": 2e-6, - "off_peak_pricing": {"hours_utc": "16:30-00:30", "input_cost_per_token": 5e-7}, - }, - ) - usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) - - result = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc)) - assert result.input_rate == 5e-7 - assert result.output_rate == 2e-6 - - -def test_get_token_base_cost_off_peak_wins_over_threshold(): - from datetime import datetime, timezone - from typing import cast - - from litellm.types.utils import ModelInfo - - model_info = cast( - ModelInfo, - { - "input_cost_per_token": 1e-6, - "output_cost_per_token": 2e-6, - "input_cost_per_token_above_200k_tokens": 3e-6, - "output_cost_per_token_above_200k_tokens": 4e-6, - "off_peak_pricing": { - "hours_utc": "16:30-00:30", - "input_cost_per_token": 5e-7, - "output_cost_per_token": 1e-6, - }, - }, - ) - usage = Usage(prompt_tokens=250000, completion_tokens=250000, total_tokens=500000) - - off_peak = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc)) - assert off_peak.input_rate == 5e-7 - assert off_peak.output_rate == 1e-6 - - peak = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc)) - assert peak.input_rate == 3e-6 - assert peak.output_rate == 4e-6 - - def test_get_model_info_propagates_off_peak_fields(): model_name = "test-off-peak-model" off_peak_pricing = { @@ -923,38 +705,6 @@ def test_get_model_info_propagates_off_peak_fields(): assert info["off_peak_pricing"] == off_peak_pricing -def test_get_token_base_cost_off_peak_wins_over_tiered_pricing(): - """Tiered pricing resolves base rates on its own path and returns early, so off-peak has to - be applied there too or a model carrying both would silently bill the tier rate all day.""" - from datetime import datetime, timezone - - model_name = "litellm-test-off-peak-tiered" - litellm.register_model( - { - model_name: { - "litellm_provider": "openai", - "mode": "chat", - "tiered_pricing": [ - {"range": [0, 128000], "input_cost_per_token": 3e-6, "output_cost_per_token": 6e-6}, - ], - "off_peak_pricing": { - "hours_utc": "16:30-00:30", - "input_cost_per_token": 5e-7, - "output_cost_per_token": 1e-6, - }, - } - } - ) - info = litellm.get_model_info(model=model_name) - usage = Usage(prompt_tokens=1_000, completion_tokens=100, total_tokens=1_100) - - inside = _get_token_base_cost(info, usage, current_time=datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc)) - assert (inside.input_rate, inside.output_rate) == (5e-7, 1e-6) - - outside = _get_token_base_cost(info, usage, current_time=datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc)) - assert (outside.input_rate, outside.output_rate) == (3e-6, 6e-6) - - def _register_off_peak_reasoning_model( model_name: str, off_peak_pricing: dict, reasoning_rate: float | None = 4e-6, **service_tier_rates: float ) -> None: @@ -985,7 +735,7 @@ def _off_peak_reasoning_usage() -> Usage: ) -def test_generic_cost_per_token_off_peak_reasoning_rate(): +def test_cost_per_text_off_peak_reasoning_rate(): from datetime import datetime, timezone model_name = "litellm-test-off-peak-reasoning" @@ -994,7 +744,7 @@ def test_generic_cost_per_token_off_peak_reasoning_rate(): {"hours_utc": "16:30-00:30", "output_cost_per_token": 1e-6, "output_cost_per_reasoning_token": 5e-7}, ) - _, inside = generic_cost_per_token( + _, inside = cost_per_text( model=model_name, usage=_off_peak_reasoning_usage(), custom_llm_provider="openai", @@ -1002,7 +752,7 @@ def test_generic_cost_per_token_off_peak_reasoning_rate(): ) assert inside == pytest.approx(50 * 1e-6 + 30 * 5e-7) - _, outside = generic_cost_per_token( + _, outside = cost_per_text( model=model_name, usage=_off_peak_reasoning_usage(), custom_llm_provider="openai", @@ -1011,14 +761,14 @@ def test_generic_cost_per_token_off_peak_reasoning_rate(): assert outside == pytest.approx(50 * 2e-6 + 30 * 4e-6) -def test_generic_cost_per_token_off_peak_block_without_reasoning_rate(): +def test_cost_per_text_off_peak_block_without_reasoning_rate(): from datetime import datetime, timezone inside_window = datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc) block = {"hours_utc": "16:30-00:30", "output_cost_per_token": 1e-6} _register_off_peak_reasoning_model("litellm-test-off-peak-model-reasoning-rate", block) - _, with_model_rate = generic_cost_per_token( + _, with_model_rate = cost_per_text( model="litellm-test-off-peak-model-reasoning-rate", usage=_off_peak_reasoning_usage(), custom_llm_provider="openai", @@ -1027,7 +777,7 @@ def test_generic_cost_per_token_off_peak_block_without_reasoning_rate(): assert with_model_rate == pytest.approx(50 * 1e-6 + 30 * 4e-6) _register_off_peak_reasoning_model("litellm-test-off-peak-no-reasoning-rate", block, reasoning_rate=None) - _, without_model_rate = generic_cost_per_token( + _, without_model_rate = cost_per_text( model="litellm-test-off-peak-no-reasoning-rate", usage=_off_peak_reasoning_usage(), custom_llm_provider="openai", @@ -1036,7 +786,7 @@ def test_generic_cost_per_token_off_peak_block_without_reasoning_rate(): assert without_model_rate == pytest.approx(80 * 1e-6) -def test_generic_cost_per_token_off_peak_reasoning_rate_wins_over_the_tier(): +def test_cost_per_text_off_peak_reasoning_rate_wins_over_the_tier(): from datetime import datetime, timezone model_name = "litellm-test-off-peak-tiered-reasoning" @@ -1062,7 +812,7 @@ def test_generic_cost_per_token_off_peak_reasoning_rate_wins_over_the_tier(): } ) - _, inside = generic_cost_per_token( + _, inside = cost_per_text( model=model_name, usage=_off_peak_reasoning_usage(), custom_llm_provider="openai", @@ -1070,7 +820,7 @@ def test_generic_cost_per_token_off_peak_reasoning_rate_wins_over_the_tier(): ) assert inside == pytest.approx(50 * 1e-6 + 30 * 5e-7) - _, outside = generic_cost_per_token( + _, outside = cost_per_text( model=model_name, usage=_off_peak_reasoning_usage(), custom_llm_provider="openai", @@ -1079,7 +829,7 @@ def test_generic_cost_per_token_off_peak_reasoning_rate_wins_over_the_tier(): assert outside == pytest.approx(50 * 6e-6 + 30 * 8e-6) -def test_generic_cost_per_token_off_peak_reasoning_rate_wins_over_the_service_tier(): +def test_cost_per_text_off_peak_reasoning_rate_wins_over_the_service_tier(): from datetime import datetime, timezone model_name = "litellm-test-off-peak-reasoning-service-tier" @@ -1090,7 +840,7 @@ def test_generic_cost_per_token_off_peak_reasoning_rate_wins_over_the_service_ti output_cost_per_reasoning_token_priority=6e-6, ) - _, inside = generic_cost_per_token( + _, inside = cost_per_text( model=model_name, usage=_off_peak_reasoning_usage(), custom_llm_provider="openai", @@ -1099,7 +849,7 @@ def test_generic_cost_per_token_off_peak_reasoning_rate_wins_over_the_service_ti ) assert inside == pytest.approx(50 * 1e-6 + 30 * 5e-7) - _, outside = generic_cost_per_token( + _, outside = cost_per_text( model=model_name, usage=_off_peak_reasoning_usage(), custom_llm_provider="openai", @@ -1134,39 +884,6 @@ def test_apply_off_peak_pricing_treats_bool_as_unset_and_parses_strings(): assert rates.reasoning_rate == 5e-7 -def test_get_token_base_cost_off_peak_cache_creation_rate(): - from datetime import datetime, timezone - from typing import cast - - from litellm.types.utils import ModelInfo - - model_info = cast( - ModelInfo, - { - "input_cost_per_token": 1e-6, - "output_cost_per_token": 2e-6, - "cache_creation_input_token_cost": 1.25e-6, - "cache_creation_input_token_cost_above_1hr": 2e-6, - "off_peak_pricing": {"hours_utc": "16:30-00:30", "cache_creation_input_token_cost": 5e-7}, - }, - ) - usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) - inside_window = datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc) - - inside = _get_token_base_cost(model_info, usage, current_time=inside_window) - assert inside.cache_creation_rate == 5e-7 - assert inside.cache_creation_rates_by_ttl["1hr"] == 2e-6 - - outside = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc)) - assert outside.cache_creation_rate == 1.25e-6 - - without_key = cast( - ModelInfo, - {**model_info, "off_peak_pricing": {"hours_utc": "16:30-00:30", "input_cost_per_token": 5e-7}}, - ) - assert _get_token_base_cost(without_key, usage, current_time=inside_window).cache_creation_rate == 1.25e-6 - - def test_get_token_type_cost_breakdown_reflects_off_peak_reasoning_and_cache_creation_rates(): from datetime import datetime, timezone @@ -1207,7 +924,7 @@ def test_get_token_type_cost_breakdown_reflects_off_peak_reasoning_and_cache_cre assert outside.cache_creation_cost == pytest.approx(400 * 1.25e-6) -def test_generic_cost_per_token_gpt54_above_272k_tokens(_local_model_cost_map): +def test_cost_per_text_gpt54_above_272k_tokens(_local_model_cost_map): """GPT-5.4/5.4-pro: prompts >272K input tokens priced at 2x input, 1.5x output.""" model = "gpt-5.4" custom_llm_provider = "openai" @@ -1220,7 +937,7 @@ def test_generic_cost_per_token_gpt54_above_272k_tokens(_local_model_cost_map): completion_tokens=completion_tokens, total_tokens=prompt_tokens + completion_tokens, ) - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model, usage=usage, custom_llm_provider=custom_llm_provider, @@ -1238,7 +955,7 @@ def test_generic_cost_per_token_gpt54_above_272k_tokens(_local_model_cost_map): (300_000, 2.4e-05, 2.4e-06, 9e-05), ], ) -def test_generic_cost_per_token_azure_eu_gpt_6_astra_tiers( +def test_cost_per_text_azure_eu_gpt_6_astra_tiers( _local_model_cost_map, prompt_tokens, input_rate, cache_read_rate, output_rate ): """azure/eu/gpt-6-astra bills Azure's Data Zone rates, doubling input and cache read past 272K.""" @@ -1250,7 +967,7 @@ def test_generic_cost_per_token_azure_eu_gpt_6_astra_tiers( total_tokens=prompt_tokens + completion_tokens, prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=cached_tokens), ) - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model="azure/eu/gpt-6-astra", usage=usage, custom_llm_provider="azure", @@ -1260,7 +977,7 @@ def test_generic_cost_per_token_azure_eu_gpt_6_astra_tiers( assert completion_cost == pytest.approx(completion_tokens * output_rate) -def test_generic_cost_per_token_minimax_m3_above_512k_tokens(_local_model_cost_map): +def test_cost_per_text_minimax_m3_above_512k_tokens(_local_model_cost_map): """MiniMax-M3: prompts >512K input tokens priced at 2x input, output, and cache read.""" model = "minimax/MiniMax-M3" custom_llm_provider = "minimax" @@ -1275,7 +992,7 @@ def test_generic_cost_per_token_minimax_m3_above_512k_tokens(_local_model_cost_m total_tokens=prompt_tokens + completion_tokens, prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=cached_tokens), ) - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model, usage=usage, custom_llm_provider=custom_llm_provider, @@ -1289,7 +1006,7 @@ def test_generic_cost_per_token_minimax_m3_above_512k_tokens(_local_model_cost_m assert round(completion_cost, 10) == round(expected_completion, 10) -def test_generic_cost_per_token_honors_non_standard_above_threshold(): +def test_cost_per_text_honors_non_standard_above_threshold(): """Regression for #30344: get_model_info must keep arbitrary input/output_cost_per_token_above__tokens thresholds, not only the hard-coded 128k/200k/272k/512k set, so a custom tier boundary is applied past its limit.""" @@ -1316,7 +1033,7 @@ def test_generic_cost_per_token_honors_non_standard_above_threshold(): completion_tokens=completion_tokens, total_tokens=prompt_tokens + completion_tokens, ) - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model, usage=usage, custom_llm_provider=custom_llm_provider, @@ -1327,7 +1044,7 @@ def test_generic_cost_per_token_honors_non_standard_above_threshold(): litellm.model_cost.pop(model, None) -def test_generic_cost_per_token_tiered_pricing_charges_cache_creation_at_tier_rate(): +def test_cost_per_text_tiered_pricing_charges_cache_creation_at_tier_rate(): """Regression for LIT-4375: a tier's cache_creation_input_token_cost must be billed on the generic (provider-agnostic) path, not silently dropped.""" model = "litellm-test-tiered-cache-creation" @@ -1364,7 +1081,7 @@ def test_generic_cost_per_token_tiered_pricing_charges_cache_creation_at_tier_ra total_tokens=301000, prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=40000, cache_creation_tokens=60000), ) - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model, usage=usage, custom_llm_provider=custom_llm_provider, @@ -1377,7 +1094,7 @@ def test_generic_cost_per_token_tiered_pricing_charges_cache_creation_at_tier_ra litellm.model_cost.pop(model, None) -def test_generic_cost_per_token_tiered_pricing_is_all_or_nothing(): +def test_cost_per_text_tiered_pricing_is_all_or_nothing(): """Tiered pricing bills the whole request at the tier picked from its input tokens, for any provider, and falls back to flat pricing when no tier matches.""" model = "litellm-test-tiered-all-or-nothing" @@ -1407,7 +1124,7 @@ def test_generic_cost_per_token_tiered_pricing_is_all_or_nothing(): try: usage = Usage(prompt_tokens=40000, completion_tokens=1000, total_tokens=41000) - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model, usage=usage, custom_llm_provider=custom_llm_provider, @@ -1416,7 +1133,7 @@ def test_generic_cost_per_token_tiered_pricing_is_all_or_nothing(): assert round(completion_cost, 10) == round(1000 * 3.5e-06, 10) boundary_usage = Usage(prompt_tokens=32000, completion_tokens=10, total_tokens=32010) - boundary_prompt_cost, _ = generic_cost_per_token( + boundary_prompt_cost, _ = cost_per_text( model=model, usage=boundary_usage, custom_llm_provider=custom_llm_provider, @@ -1424,7 +1141,7 @@ def test_generic_cost_per_token_tiered_pricing_is_all_or_nothing(): assert round(boundary_prompt_cost, 10) == round(32000 * 4.6e-07, 10) empty_prompt_usage = Usage(prompt_tokens=0, completion_tokens=100, total_tokens=100) - empty_prompt_cost, empty_completion_cost = generic_cost_per_token( + empty_prompt_cost, empty_completion_cost = cost_per_text( model=model, usage=empty_prompt_usage, custom_llm_provider=custom_llm_provider, @@ -1435,7 +1152,7 @@ def test_generic_cost_per_token_tiered_pricing_is_all_or_nothing(): litellm.model_cost.pop(model, None) -def test_generic_cost_per_token_tier_without_an_output_rate_bills_the_model_rate(): +def test_cost_per_text_tier_without_an_output_rate_bills_the_model_rate(): """Regression: a tier table that spells out only input rates served every completion for free, since a tier's missing output rate has no tier-level fallback to stand in for it.""" model = "litellm-test-tiered-input-only" @@ -1459,7 +1176,7 @@ def test_generic_cost_per_token_tier_without_an_output_rate_bills_the_model_rate total_tokens=195, completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=100), ) - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model, usage=usage, custom_llm_provider=custom_llm_provider, @@ -1470,7 +1187,7 @@ def test_generic_cost_per_token_tier_without_an_output_rate_bills_the_model_rate litellm.model_cost.pop(model, None) -def test_generic_cost_per_token_tier_without_cache_rates_bills_cache_at_the_tier_input_rate(): +def test_cost_per_text_tier_without_cache_rates_bills_cache_at_the_tier_input_rate(): model = "litellm-test-tiered-no-cache-rates" custom_llm_provider = "openrouter" litellm.register_model( @@ -1504,12 +1221,12 @@ def test_generic_cost_per_token_tier_without_cache_rates_bills_cache_at_the_tier total_tokens=40100, prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=5000, cache_creation_tokens=15000), ) - uncached_prompt_cost, _ = generic_cost_per_token( + uncached_prompt_cost, _ = cost_per_text( model=model, usage=uncached, custom_llm_provider=custom_llm_provider, ) - cached_prompt_cost, cached_completion_cost = generic_cost_per_token( + cached_prompt_cost, cached_completion_cost = cost_per_text( model=model, usage=cached, custom_llm_provider=custom_llm_provider, @@ -1547,7 +1264,7 @@ def test_generic_cost_per_token_tier_without_cache_rates_bills_cache_at_the_tier ], ) @pytest.mark.parametrize("one_hour_tokens", [10, 20]) -def test_generic_cost_per_token_tier_1hr_cache_rate_preserves_zero_and_fallback( +def test_cost_per_text_tier_1hr_cache_rate_preserves_zero_and_fallback( cache_rates: Mapping[str, float | None], expected_write_rate: float, one_hour_tokens: int ) -> None: model: Final = "litellm-test-tiered-1hr-cache-rate" @@ -1583,7 +1300,7 @@ def test_generic_cost_per_token_tier_1hr_cache_rate_preserves_zero_and_fallback( ), ), ) - costs: Final = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="openrouter") + costs: Final = cost_per_text(model=model, usage=usage, custom_llm_provider="openrouter") breakdown: Final = get_token_type_cost_breakdown(model=model, usage=usage, custom_llm_provider="openrouter") ordinary_rate: Final = cache_rates.get("cache_creation_input_token_cost") @@ -1596,7 +1313,7 @@ def test_generic_cost_per_token_tier_1hr_cache_rate_preserves_zero_and_fallback( litellm.model_cost.pop(model, None) -def test_generic_cost_per_token_tier_without_an_input_rate_is_not_a_priced_tier(): +def test_cost_per_text_tier_without_an_input_rate_is_not_a_priced_tier(): model = "litellm-test-tiered-no-input-rate" custom_llm_provider = "openrouter" litellm.register_model( @@ -1613,7 +1330,7 @@ def test_generic_cost_per_token_tier_without_an_input_rate_is_not_a_priced_tier( try: usage = Usage(prompt_tokens=1000, completion_tokens=100, total_tokens=1100) - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model, usage=usage, custom_llm_provider=custom_llm_provider, @@ -1652,7 +1369,7 @@ def test_router_deployment_with_input_only_tiers_bills_completions_at_the_backen try: usage = Usage(prompt_tokens=21, completion_tokens=4, total_tokens=25) - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model_id, usage=usage, custom_llm_provider="anthropic", @@ -1664,7 +1381,7 @@ def test_router_deployment_with_input_only_tiers_bills_completions_at_the_backen litellm.model_cost.pop(model_id, None) -def test_generic_cost_per_token_tiered_pricing_bills_reasoning_at_tier_rate(): +def test_cost_per_text_tiered_pricing_bills_reasoning_at_tier_rate(): """Regression: a tier's output_cost_per_reasoning_token must price reasoning tokens on the generic path and in the logged breakdown, not the tier's plain output rate.""" model = "litellm-test-tiered-reasoning" @@ -1699,7 +1416,7 @@ def test_generic_cost_per_token_tiered_pricing_bills_reasoning_at_tier_rate(): total_tokens=1500, completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=400), ) - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model, usage=usage, custom_llm_provider=custom_llm_provider, @@ -1793,7 +1510,7 @@ def test_string_cost_values(): "litellm.litellm_core_utils.llm_cost_calc.utils.get_model_info", return_value=mock_model_info, ): - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model="test-model", usage=usage, custom_llm_provider="test-provider" ) @@ -1814,7 +1531,7 @@ def test_string_cost_values(): assert round(completion_cost, 12) == round(expected_completion_cost, 12) -def test_generic_cost_per_token_overlapping_cached_and_image_tokens(): +def test_cost_per_text_overlapping_cached_and_image_tokens(): """Some providers report cached_tokens and image_tokens as overlapping subsets of prompt_tokens. Billing each in full charged the overlap twice, once at the cache rate and again at the input rate.""" @@ -1837,14 +1554,14 @@ def test_generic_cost_per_token_overlapping_cached_and_image_tokens(): prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=None, cached_tokens=90, image_tokens=80), ) - prompt_cost, completion_cost = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="openai") + prompt_cost, completion_cost = cost_per_text(model=model, usage=usage, custom_llm_provider="openai") # 90 cached at 1e-7, the remaining 10 uncached tokens once at 1e-6 assert prompt_cost == pytest.approx(90 * 1e-7 + 10 * 1e-6) assert completion_cost == pytest.approx(10 * 2e-6) -def test_generic_cost_per_token_warm_prefix_cache_spanning_text_and_image_tokens(): +def test_cost_per_text_warm_prefix_cache_spanning_text_and_image_tokens(): """xAI reports text_tokens + image_tokens = prompt_tokens with cached_tokens overlapping both, so a warm prefix cache covering the whole image exceeds the text-only count. Observed live on grok-4.6 (issue #37281): the image tokens were billed a second time at @@ -1869,7 +1586,7 @@ def test_generic_cost_per_token_warm_prefix_cache_spanning_text_and_image_tokens prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=1319, cached_tokens=2432, image_tokens=1142), ) - prompt_cost, completion_cost = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="openai") + prompt_cost, completion_cost = cost_per_text(model=model, usage=usage, custom_llm_provider="openai") # 2432 cached at the cache-read rate, the 29 uncached tokens once at the input rate assert prompt_cost == pytest.approx(2432 * 5e-7 + 29 * 2e-6) @@ -1953,7 +1670,7 @@ def test_string_cost_values_edge_cases(): "litellm.litellm_core_utils.llm_cost_calc.utils.get_model_info", return_value=mock_model_info, ): - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model="test-model", usage=usage, custom_llm_provider="test-provider" ) @@ -1983,7 +1700,7 @@ def test_string_cost_values_with_threshold(): "litellm.litellm_core_utils.llm_cost_calc.utils.get_model_info", return_value=mock_model_info, ): - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model="test-model", usage=usage, custom_llm_provider="test-provider" ) @@ -2103,7 +1820,7 @@ def test_calculate_cache_writing_cost_bills_writes_of_an_unknown_lifetime_at_the (300_000, "priority", 6e-6, 12e-6), ], ) -def test_generic_cost_per_token_prices_12h_cache_writes_at_prompt_threshold( +def test_cost_per_text_prices_12h_cache_writes_at_prompt_threshold( prompt_tokens: int, service_tier: str, write_rate: float, twelve_hour_rate: float ) -> None: model_info: Final[ModelInfo] = { @@ -2134,7 +1851,7 @@ def test_generic_cost_per_token_prices_12h_cache_writes_at_prompt_threshold( ), ) - result: Final = generic_cost_per_token( + result: Final = cost_per_text( model="test-12h-cache-write", usage=usage, custom_llm_provider="openai", @@ -2149,7 +1866,7 @@ def test_generic_cost_per_token_prices_12h_cache_writes_at_prompt_threshold( ) -def test_generic_cost_per_token_falls_back_to_default_for_missing_12h_rate() -> None: +def test_cost_per_text_falls_back_to_default_for_missing_12h_rate() -> None: model_info: Final[ModelInfo] = { "input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6, @@ -2165,7 +1882,7 @@ def test_generic_cost_per_token_falls_back_to_default_for_missing_12h_rate() -> ), ) - result: Final = generic_cost_per_token( + result: Final = cost_per_text( model="test-12h-cache-write-fallback", usage=usage, custom_llm_provider="openai", @@ -2176,7 +1893,7 @@ def test_generic_cost_per_token_falls_back_to_default_for_missing_12h_rate() -> @pytest.mark.parametrize(("prompt_tokens", "twelve_hour_rate"), [(10_000, 7e-6), (300_000, 14e-6)]) -def test_generic_cost_per_token_prices_ultrafast_12h_cache_writes_at_the_ultrafast_rate( +def test_cost_per_text_prices_ultrafast_12h_cache_writes_at_the_ultrafast_rate( prompt_tokens: int, twelve_hour_rate: float ) -> None: model_info: Final[ModelInfo] = { @@ -2202,7 +1919,7 @@ def test_generic_cost_per_token_prices_ultrafast_12h_cache_writes_at_the_ultrafa ), ) - prompt_cost, _ = generic_cost_per_token( + prompt_cost, _ = cost_per_text( model="test-ultrafast-12h-cache-write", usage=usage, custom_llm_provider="openai", @@ -2238,7 +1955,7 @@ def test_cache_writing_cost_with_zero_creation_tokens_and_ephemeral_details(): ), ) - prompt_cost, _ = generic_cost_per_token( + prompt_cost, _ = cost_per_text( model="litellm-test-ephemeral-cache-writes", usage=usage, custom_llm_provider="anthropic", @@ -2285,7 +2002,7 @@ def test_service_tier_ultrafast_pricing(): "cache_read_input_token_cost_ultrafast": 5e-06, } - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model="gpt-5.6-sol", usage=usage, custom_llm_provider="openai", @@ -2308,13 +2025,13 @@ def test_service_tier_ultrafast_fallback_pricing(_local_model_cost_map): usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) - std_prompt_cost, std_completion_cost = generic_cost_per_token( + std_prompt_cost, std_completion_cost = cost_per_text( model="gpt-5.6-sol", usage=usage, custom_llm_provider="openai", service_tier=None, ) - ultrafast_prompt_cost, ultrafast_completion_cost = generic_cost_per_token( + ultrafast_prompt_cost, ultrafast_completion_cost = cost_per_text( model="gpt-5.6-sol", usage=usage, custom_llm_provider="openai", @@ -2365,8 +2082,7 @@ def test_gemini_image_generation_cost_with_zero_text_tokens(_local_model_cost_ma ), ) - model_cost_map = litellm.model_cost[model] - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model, usage=usage, custom_llm_provider=custom_llm_provider, @@ -2378,8 +2094,9 @@ def test_gemini_image_generation_cost_with_zero_text_tokens(_local_model_cost_ma # - reasoning_tokens: 225 * output_cost_per_token # Total completion should include both image + reasoning costs. - output_cost_per_image_token = model_cost_map.get("output_cost_per_image_token", 0) - output_cost_per_token = model_cost_map.get("output_cost_per_token", 0) + model_info = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider) + output_cost_per_image_token = model_info.get("output_cost_per_image_token") or 0.0 + output_cost_per_token = model_info.get("output_cost_per_token") or 0.0 expected_image_cost = 1120 * output_cost_per_image_token expected_reasoning_cost = 225 * output_cost_per_token # reasoning uses base token cost @@ -2465,7 +2182,7 @@ def test_query_count_is_free_without_a_per_query_price(_local_model_cost_map): prompt_tokens_details=PromptTokensDetailsWrapper(query_count=1), ) - prompt_cost, _ = generic_cost_per_token(model="text-embedding-3-small", usage=usage, custom_llm_provider="openai") + prompt_cost, _ = cost_per_text(model="text-embedding-3-small", usage=usage, custom_llm_provider="openai") assert prompt_cost == 0.0 @@ -2485,12 +2202,12 @@ def test_data_residency_applies_uplift(data_residency, model, _local_model_cost_ usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) - base = generic_cost_per_token( + base = cost_per_text( model=model, usage=usage, custom_llm_provider="openai", ) - regional = generic_cost_per_token( + regional = cost_per_text( model=model, usage=usage, custom_llm_provider="openai", @@ -2513,8 +2230,8 @@ def test_data_residency_no_uplift_for_pre_march_2026_models(model, _local_model_ usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) - base = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="openai") - regional = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="openai", data_residency="eu") + base = cost_per_text(model=model, usage=usage, custom_llm_provider="openai") + regional = cost_per_text(model=model, usage=usage, custom_llm_provider="openai", data_residency="eu") assert base == regional, f"{model} should not have a regional uplift, but cost changed with data_residency" @@ -2526,12 +2243,12 @@ def test_data_residency_no_uplift_for_unmarked_model(_local_model_cost_map): usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) - base = generic_cost_per_token( + base = cost_per_text( model="gpt-3.5-turbo", usage=usage, custom_llm_provider="openai", ) - with_residency = generic_cost_per_token( + with_residency = cost_per_text( model="gpt-3.5-turbo", usage=usage, custom_llm_provider="openai", @@ -2547,12 +2264,12 @@ def test_data_residency_none_no_uplift(_local_model_cost_map): usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) - base = generic_cost_per_token( + base = cost_per_text( model="gpt-5.4", usage=usage, custom_llm_provider="openai", ) - explicit_none = generic_cost_per_token( + explicit_none = cost_per_text( model="gpt-5.4", usage=usage, custom_llm_provider="openai", @@ -2568,13 +2285,13 @@ def test_data_residency_composes_with_service_tier(_local_model_cost_map): usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) - priority_base = generic_cost_per_token( + priority_base = cost_per_text( model="gpt-5.4", usage=usage, custom_llm_provider="openai", service_tier="priority", ) - priority_eu = generic_cost_per_token( + priority_eu = cost_per_text( model="gpt-5.4", usage=usage, custom_llm_provider="openai", @@ -2599,8 +2316,8 @@ def test_vertex_regional_location_applies_uplift(vertex_location, model, _local_ usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) - base = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="vertex_ai") - regional = generic_cost_per_token( + base = cost_per_text(model=model, usage=usage, custom_llm_provider="vertex_ai") + regional = cost_per_text( model=model, usage=usage, custom_llm_provider="vertex_ai", @@ -2624,8 +2341,8 @@ def test_vertex_global_or_absent_location_no_uplift(vertex_location, _local_mode usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) - base = generic_cost_per_token(model="claude-haiku-4-5@20251001", usage=usage, custom_llm_provider="vertex_ai") - located = generic_cost_per_token( + base = cost_per_text(model="claude-haiku-4-5@20251001", usage=usage, custom_llm_provider="vertex_ai") + located = cost_per_text( model="claude-haiku-4-5@20251001", usage=usage, custom_llm_provider="vertex_ai", @@ -2669,89 +2386,6 @@ def test_get_cost_per_unit_falls_back_from_service_tier_key_to_base(): assert get_cost_per_unit(model_info_direct, "input_cost_per_token_priority") == 5e-6 -@pytest.mark.parametrize( - "field", ("output_cost_per_token", "cache_read_input_token_cost", "cache_creation_input_token_cost") -) -def test_flat_input_price_keeps_other_token_thresholds(field: str) -> None: - from typing import Final, cast - from litellm.litellm_core_utils.llm_cost_calc.utils import _get_token_base_cost - - info: Final = cast( - ModelInfo, - { - "input_cost_per_token": 0.001, - "output_cost_per_token": 0.002, - "cache_read_input_token_cost": 0.003, - "cache_creation_input_token_cost": 0.004, - f"{field}_above_300k_tokens": 0.005, - }, - ) - rates: Final = _get_token_base_cost( - model_info=info, - usage=Usage(prompt_tokens=300_001, completion_tokens=3, total_tokens=300_004), - ) - assert rates.input_rate == 0.001 - assert rates.output_rate == (0.005 if field == "output_cost_per_token" else 0.002) - assert rates.cache_creation_rate == (0.005 if field == "cache_creation_input_token_cost" else 0.004) - assert rates.cache_read_rate == (0.005 if field == "cache_read_input_token_cost" else 0.003) - - -@pytest.mark.parametrize("prompt_tokens", (200_000, 200_001, 512_000, 512_001)) -@pytest.mark.parametrize("higher_rate", (0.0, 0.009)) -def test_mixed_token_thresholds_preserve_each_crossed_rate(prompt_tokens: int, higher_rate: float) -> None: - from typing import Final, cast - from litellm.litellm_core_utils.llm_cost_calc.utils import _get_token_base_cost - - info: Final = cast( - ModelInfo, - { - "input_cost_per_token": 0.001, - "output_cost_per_token": 0.002, - "cache_read_input_token_cost": 0.003, - "cache_creation_input_token_cost": 0.004, - "input_cost_per_token_above_200k_tokens": 0.005, - "output_cost_per_token_above_200k_tokens": 0.006, - "cache_creation_input_token_cost_above_200k_tokens": 0.007, - "cache_read_input_token_cost_above_512k_tokens": higher_rate, - "output_cost_per_token_above_512k_tokens": higher_rate, - }, - ) - rates: Final = _get_token_base_cost( - model_info=info, - usage=Usage(prompt_tokens=prompt_tokens, completion_tokens=3, total_tokens=prompt_tokens + 3), - ) - assert rates.input_rate == (0.005 if prompt_tokens > 200_000 else 0.001) - assert rates.output_rate == ( - higher_rate if prompt_tokens > 512_000 else 0.006 if prompt_tokens > 200_000 else 0.002 - ) - assert rates.cache_creation_rate == (0.007 if prompt_tokens > 200_000 else 0.004) - assert rates.cache_read_rate == (higher_rate if prompt_tokens > 512_000 else 0.003) - - -def test_threshold_keys_exclude_service_tier_variants(): - from typing import cast - - from litellm.litellm_core_utils.llm_cost_calc.utils import _get_token_base_cost - from litellm.types.utils import ModelInfo, Usage - - # The service-tier-suffixed above-threshold key must be excluded from - # threshold detection. The _priority variant has a higher threshold (300k), - # so if it were not excluded it would sort first and drive a 9e-6 rate for - # this non-tier request. With the exclusion only the standard 200k key - # applies, giving 3e-6. - model_info = cast( - ModelInfo, - { - "input_cost_per_token": 1e-6, - "input_cost_per_token_above_200k_tokens": 3e-6, - "input_cost_per_token_above_300k_tokens_priority": 9e-6, - "output_cost_per_token": 2e-6, - }, - ) - usage = Usage(prompt_tokens=350_000, completion_tokens=1_000, total_tokens=351_000) - assert _get_token_base_cost(model_info=model_info, usage=usage).input_rate == 3e-6 - - @pytest.mark.parametrize( "model,custom_llm_provider,reasoning_tokens,cached_tokens", [ @@ -2769,7 +2403,7 @@ def test_token_type_cost_breakdown_is_provider_agnostic( """ Reasoning and cache-read costs must be surfaced for every provider that reports those tokens, regardless of which cost calculator the provider routes through - (Perplexity, Cerebras, Dashscope bypass generic_cost_per_token entirely). + (Perplexity, Cerebras, Dashscope bypass cost_per_text entirely). Cache tokens always land in prompt_tokens_details.cached_tokens, so reading from there - not the top-level cache_read_input_tokens attribute the old breakdown code @@ -2839,7 +2473,7 @@ def test_token_type_cost_breakdown_reads_cache_write_tokens(_local_model_cost_ma assert breakdown.cache_creation_cost == pytest.approx(300 * model_info["cache_creation_input_token_cost"]) -def test_generic_cost_per_token_openai_cache_write_tokens_gpt_5_6(_local_model_cost_map): +def test_cost_per_text_openai_cache_write_tokens_gpt_5_6(_local_model_cost_map): """ Regression: OpenAI gpt-5.6 reports cache-write tokens under prompt_tokens_details.cache_write_tokens (not the Anthropic cache_creation_tokens @@ -2859,7 +2493,7 @@ def test_generic_cost_per_token_openai_cache_write_tokens_gpt_5_6(_local_model_c assert usage.prompt_tokens_details.cache_write_tokens == 800 assert usage.prompt_tokens_details.cache_creation_tokens == 800 - prompt_cost, _ = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="openai") + prompt_cost, _ = cost_per_text(model=model, usage=usage, custom_llm_provider="openai") info = litellm.get_model_info(model=model, custom_llm_provider="openai") expected_prompt = (1000 - 800) * info["input_cost_per_token"] + 800 * info["cache_creation_input_token_cost"] @@ -2868,7 +2502,7 @@ def test_generic_cost_per_token_openai_cache_write_tokens_gpt_5_6(_local_model_c assert prompt_cost > 1000 * info["input_cost_per_token"] -def test_generic_cost_per_token_backs_out_cache_write_tokens_from_text_tokens(_local_model_cost_map): +def test_cost_per_text_backs_out_cache_write_tokens_from_text_tokens(_local_model_cost_map): """ Regression for #34801: when a provider reports text_tokens covering the whole prompt alongside cache-write tokens (and no cache reads), the cache-write tokens @@ -2883,7 +2517,7 @@ def test_generic_cost_per_token_backs_out_cache_write_tokens_from_text_tokens(_l prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=0, cache_write_tokens=800, text_tokens=1000), ) - prompt_cost, _ = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="openai") + prompt_cost, _ = cost_per_text(model=model, usage=usage, custom_llm_provider="openai") info = litellm.get_model_info(model=model, custom_llm_provider="openai") expected_prompt = 200 * info["input_cost_per_token"] + 800 * info["cache_creation_input_token_cost"] @@ -2894,7 +2528,7 @@ def test_token_type_cost_breakdown_reconciles_with_generic_total(_local_model_co """ Both-ways check: the reasoning subset must sum with the remaining (text) output cost to exactly the completion total, and the cache-read subset with the remaining - input cost to exactly the prompt total, as computed by generic_cost_per_token. + input cost to exactly the prompt total, as computed by cost_per_text. A mismatch here would mean the breakdown misrepresents what was actually billed. """ @@ -2908,7 +2542,7 @@ def test_token_type_cost_breakdown_reconciles_with_generic_total(_local_model_co prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=300, text_tokens=700), ) - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model, usage=usage, custom_llm_provider=custom_llm_provider ) breakdown = get_token_type_cost_breakdown(model=model, custom_llm_provider=custom_llm_provider, usage=usage) @@ -3049,12 +2683,12 @@ def test_a_pinned_billing_time_prices_the_totals_and_the_reported_rates_at_one_m usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) with pinned_billing_time(datetime(2026, 1, 1, 2, 30, tzinfo=timezone.utc)): - off_peak_prompt_cost, off_peak_completion_cost = generic_cost_per_token( + off_peak_prompt_cost, off_peak_completion_cost = cost_per_text( model="off-peak-model", usage=usage, custom_llm_provider="openai" ) off_peak_rates = get_billed_token_rates(model="off-peak-model", custom_llm_provider="openai", usage=usage) with pinned_billing_time(datetime(2026, 1, 1, 12, 30, tzinfo=timezone.utc)): - peak_prompt_cost, peak_completion_cost = generic_cost_per_token( + peak_prompt_cost, peak_completion_cost = cost_per_text( model="off-peak-model", usage=usage, custom_llm_provider="openai" ) peak_rates = get_billed_token_rates(model="off-peak-model", custom_llm_provider="openai", usage=usage) @@ -3219,14 +2853,17 @@ def test_token_type_cost_breakdown_applies_regional_uplift(_local_model_cost_map assert eu.cache_read_cost == pytest.approx(base.cache_read_cost * uplift) # The uplifted breakdown must still reconcile with the uplifted totals. - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model, usage=usage, custom_llm_provider=custom_llm_provider, data_residency="eu", ) - text_output_cost = 300 * model_info["output_cost_per_token"] * uplift - text_input_cost = 600 * model_info["input_cost_per_token"] * uplift + output_rate = model_info["output_cost_per_token"] + input_rate = model_info["input_cost_per_token"] + assert output_rate is not None and input_rate is not None + text_output_cost = 300 * output_rate * uplift + text_input_cost = 600 * input_rate * uplift assert text_output_cost + eu.reasoning_cost == pytest.approx(completion_cost) assert text_input_cost + eu.cache_read_cost == pytest.approx(prompt_cost) @@ -3264,7 +2901,7 @@ def test_token_type_cost_breakdown_applies_vertex_regional_uplift(_local_model_c assert regional.cache_read_cost == pytest.approx(base.cache_read_cost * uplift) # The uplifted breakdown must still reconcile with the uplifted totals. - prompt_cost, _completion_cost = generic_cost_per_token( + prompt_cost, _completion_cost = cost_per_text( model=model, usage=usage, custom_llm_provider=custom_llm_provider, @@ -3420,9 +3057,9 @@ def test_fast_service_tier_is_case_insensitive(_local_model_cost_map): usage = Usage(prompt_tokens=1_000, completion_tokens=500) - assert generic_cost_per_token( + assert cost_per_text( model="gpt-5.6-sol", usage=usage, custom_llm_provider="openai", service_tier="FAST" - ) == generic_cost_per_token(model="gpt-5.6-sol", usage=usage, custom_llm_provider="openai", service_tier="fast") + ) == cost_per_text(model="gpt-5.6-sol", usage=usage, custom_llm_provider="openai", service_tier="fast") def test_priority_reasoning_tokens_bill_at_the_priority_output_rate(_local_model_cost_map): @@ -3444,13 +3081,13 @@ def test_priority_reasoning_tokens_bill_at_the_priority_output_rate(_local_model assert priority_output_rate is not None assert priority_output_rate != standard_reasoning_rate - standard = generic_cost_per_token( + standard = cost_per_text( model="gemini-3.5-flash", usage=usage, custom_llm_provider="gemini", service_tier=None ) - priority = generic_cost_per_token( + priority = cost_per_text( model="gemini-3.5-flash", usage=usage, custom_llm_provider="gemini", service_tier="priority" ) - fast = generic_cost_per_token( + fast = cost_per_text( model="gemini-3.5-flash", usage=usage, custom_llm_provider="gemini", service_tier="fast" ) @@ -3476,7 +3113,7 @@ def test_explicit_tier_reasoning_key_wins_over_the_tier_output_rate(): completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=600), ) - _, completion_cost = generic_cost_per_token( + _, completion_cost = cost_per_text( model="synthetic-model", usage=usage, custom_llm_provider="openai", @@ -3506,7 +3143,7 @@ def test_null_tier_reasoning_key_falls_back_to_the_tier_output_rate(): completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=600), ) - _, completion_cost = generic_cost_per_token( + _, completion_cost = cost_per_text( model="synthetic-model", usage=usage, custom_llm_provider="openai", @@ -3531,7 +3168,7 @@ def test_tier_request_without_tier_pricing_keeps_the_standard_reasoning_rate(): completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=600), ) - _, completion_cost = generic_cost_per_token( + _, completion_cost = cost_per_text( model="synthetic-model", usage=usage, custom_llm_provider="openai", @@ -3661,7 +3298,7 @@ def test_route_image_generation_cost_falls_back_to_requested_size(monkeypatch, r assert cost == expected_cost -def test_generic_cost_per_token_bills_reasoning_nested_in_text_tokens_once(_local_model_cost_map: None) -> None: +def test_cost_per_text_bills_reasoning_nested_in_text_tokens_once(_local_model_cost_map: None) -> None: """ Realtime usage (OpenAI and Azure) reports output_tokens == text_tokens + audio_tokens with reasoning_tokens already counted inside text_tokens, so reasoning must not be billed on top. @@ -3678,7 +3315,7 @@ def test_generic_cost_per_token_bills_reasoning_nested_in_text_tokens_once(_loca completion_tokens_details=CompletionTokensDetailsWrapper(text_tokens=29, audio_tokens=0, reasoning_tokens=19), ) - prompt_cost, completion_cost = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="openai") + prompt_cost, completion_cost = cost_per_text(model=model, usage=usage, custom_llm_provider="openai") breakdown = get_token_type_cost_breakdown(model=model, custom_llm_provider="openai", usage=usage) info = litellm.get_model_info(model=model, custom_llm_provider="openai") @@ -3691,7 +3328,7 @@ def test_generic_cost_per_token_bills_reasoning_nested_in_text_tokens_once(_loca ) -def test_generic_cost_per_token_keeps_billing_reasoning_reported_beside_text_tokens( +def test_cost_per_text_keeps_billing_reasoning_reported_beside_text_tokens( _local_model_cost_map: None, ) -> None: """Providers whose text_tokens exclude reasoning (text + reasoning == completion) stay billed in full.""" @@ -3704,13 +3341,13 @@ def test_generic_cost_per_token_keeps_billing_reasoning_reported_beside_text_tok completion_tokens_details=CompletionTokensDetailsWrapper(text_tokens=25, audio_tokens=0, reasoning_tokens=19), ) - _, completion_cost = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="openai") + _, completion_cost = cost_per_text(model=model, usage=usage, custom_llm_provider="openai") info = litellm.get_model_info(model=model, custom_llm_provider="openai") assert completion_cost == pytest.approx(44 * info["output_cost_per_token"]) -def test_generic_cost_per_token_strips_only_the_reasoning_share_when_text_over_reports( +def test_cost_per_text_strips_only_the_reasoning_share_when_text_over_reports( _local_model_cost_map: None, ) -> None: """Text over-reported past the reasoning share keeps its extra tokens billed; only the nested reasoning is netted out.""" @@ -3723,7 +3360,7 @@ def test_generic_cost_per_token_strips_only_the_reasoning_share_when_text_over_r completion_tokens_details=CompletionTokensDetailsWrapper(text_tokens=100, audio_tokens=70, reasoning_tokens=10), ) - _, completion_cost = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="openai") + _, completion_cost = cost_per_text(model=model, usage=usage, custom_llm_provider="openai") breakdown = get_token_type_cost_breakdown(model=model, custom_llm_provider="openai", usage=usage) info = litellm.get_model_info(model=model, custom_llm_provider="openai") @@ -3733,7 +3370,7 @@ def test_generic_cost_per_token_strips_only_the_reasoning_share_when_text_over_r ) -def test_generic_cost_per_token_bills_nested_reasoning_once_beside_audio_output(_local_model_cost_map: None) -> None: +def test_cost_per_text_bills_nested_reasoning_once_beside_audio_output(_local_model_cost_map: None) -> None: """Audio-output realtime usage nests reasoning inside text_tokens next to audio_tokens; text is billed net of it.""" model = "gpt-realtime-2.1-mini" @@ -3744,7 +3381,7 @@ def test_generic_cost_per_token_bills_nested_reasoning_once_beside_audio_output( completion_tokens_details=CompletionTokensDetailsWrapper(text_tokens=30, audio_tokens=70, reasoning_tokens=20), ) - _, completion_cost = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="openai") + _, completion_cost = cost_per_text(model=model, usage=usage, custom_llm_provider="openai") breakdown = get_token_type_cost_breakdown(model=model, custom_llm_provider="openai", usage=usage) info = litellm.get_model_info(model=model, custom_llm_provider="openai") @@ -3772,7 +3409,7 @@ def test_cached_audio_tokens_fall_back_to_cache_read_input_token_cost() -> None: ), ) - prompt_cost, _ = generic_cost_per_token( + prompt_cost, _ = cost_per_text( model="some-realtime-model", usage=usage, custom_llm_provider="openai", @@ -3782,7 +3419,7 @@ def test_cached_audio_tokens_fall_back_to_cache_read_input_token_cost() -> None: assert prompt_cost == pytest.approx(expected) -def test_generic_cost_per_token_bills_cache_creation_at_the_input_rate_without_a_write_price(): +def test_cost_per_text_bills_cache_creation_at_the_input_rate_without_a_write_price(): """Azure and OpenAI publish no cache-write price and bill cache writes as ordinary input. A deployment priced with only input, output, and cache-read rates must bill the creation tokens the provider reports at the input rate, never at 0. The numbers are a cold 7,336-token @@ -3799,7 +3436,7 @@ def test_generic_cost_per_token_bills_cache_creation_at_the_input_rate_without_a prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=0, cache_creation_tokens=7333), ) - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model="custom-priced-deployment", usage=usage, custom_llm_provider="azure", model_info=model_info ) @@ -3807,75 +3444,6 @@ def test_generic_cost_per_token_bills_cache_creation_at_the_input_rate_without_a assert completion_cost == pytest.approx(23 * 1.25e-6) -@pytest.mark.parametrize( - ("cache_rates", "current_time", "expected_creation", "expected_creation_1h"), - ( - pytest.param({}, None, 2e-7, 2e-7, id="no-write-price-uses-the-input-rate"), - pytest.param( - {"cache_creation_input_token_cost": 2.5e-7}, None, 2.5e-7, 2.5e-7, id="no-1h-price-uses-the-write-price" - ), - pytest.param({"cache_creation_input_token_cost": 0.0}, None, 0.0, 0.0, id="explicit-zero-stays-zero"), - pytest.param( - {"off_peak_pricing": {"hours_utc": "00:00-23:59", "input_cost_per_token": 1e-7}}, - datetime(2026, 9, 14, 12, tzinfo=timezone.utc), - 1e-7, - 1e-7, - id="no-write-price-uses-the-off-peak-input-rate", - ), - pytest.param( - { - "off_peak_pricing": { - "hours_utc": "00:00-23:59", - "input_cost_per_token": 1e-7, - "cache_creation_input_token_cost": 3e-7, - } - }, - datetime(2026, 9, 14, 12, tzinfo=timezone.utc), - 3e-7, - 3e-7, - id="no-1h-price-uses-the-off-peak-write-price", - ), - pytest.param( - { - "tiered_pricing": [ - { - "range": [0, 128000], - "input_cost_per_token": 2e-7, - "output_cost_per_token": 1.25e-6, - "cache_creation_input_token_cost": 2.5e-7, - } - ], - "off_peak_pricing": {"hours_utc": "00:00-23:59", "cache_creation_input_token_cost": 3e-7}, - }, - datetime(2026, 9, 14, 12, tzinfo=timezone.utc), - 3e-7, - 3e-7, - id="tiered-no-1h-price-uses-the-off-peak-write-price", - ), - ), -) -def test_get_token_base_cost_resolves_missing_cache_write_rates_like_the_tiered_path( - cache_rates: Mapping[str, float | Mapping[str, float | str]], - current_time: datetime | None, - expected_creation: float, - expected_creation_1h: float, -) -> None: - model_info = {"input_cost_per_token": 2e-7, "output_cost_per_token": 1.25e-6, **cache_rates} - usage = Usage(prompt_tokens=10, completion_tokens=1, total_tokens=11) - - rates = _get_token_base_cost(model_info, usage, current_time=current_time) - - assert rates.cache_creation_rate == pytest.approx(expected_creation) - assert calculate_cache_writing_cost( - cache_creation_tokens=3, - cache_creation_token_details=CacheCreationTokenDetails( - ephemeral_1h_input_tokens=1, ephemeral_12h_input_tokens=2 - ), - cache_creation_cost=rates.cache_creation_rate, - cache_creation_cost_by_ttl=rates.cache_creation_rates_by_ttl, - ) == pytest.approx(3 * expected_creation_1h) - - def _image_response(num_images: int = 1, usage: ImageUsage | None = None) -> ImageResponse: return ImageResponse( data=[ImageObject(url="https://example.com/img.png") for _ in range(num_images)], @@ -4211,7 +3779,7 @@ def test_azure_gpt_5_6_alias_matches_sol_pricing(_local_model_cost_map, region_p ("vertex_ai/claude-haiku-5-5", "vertex_ai", 100_001, 5e-07, 5e-08, 2.5e-06), ], ) -def test_generic_cost_per_token_claude_haiku_5_5_prompt_length_tiers( +def test_cost_per_text_claude_haiku_5_5_prompt_length_tiers( _local_model_cost_map: None, model: str, custom_llm_provider: str, @@ -4230,7 +3798,7 @@ def test_generic_cost_per_token_claude_haiku_5_5_prompt_length_tiers( prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=cached_tokens), ) - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model, usage=usage, custom_llm_provider=custom_llm_provider, @@ -4256,7 +3824,7 @@ def test_vertex_regional_endpoint_uplift_scales_claude_haiku_5_5_over_100k_rates prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=cached_tokens), ) - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model="vertex_ai/claude-haiku-5-5", usage=usage, custom_llm_provider="vertex_ai", @@ -4394,7 +3962,7 @@ def test_token_breakdown_uses_supplied_pricing_without_catalog_lookup(rate: floa prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=5), cache_creation_input_tokens=5, ) - prompt_cost, output_cost = generic_cost_per_token( + prompt_cost, output_cost = cost_per_text( model="supplied-pricing-test", usage=usage, custom_llm_provider="bedrock", diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_openai_cache_write_cost.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_openai_cache_write_cost.py index 6c333a83d14..8dd57d4aa1f 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_openai_cache_write_cost.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_openai_cache_write_cost.py @@ -13,7 +13,7 @@ discarded by ``get_model_info``. import pytest import litellm -from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text from litellm.responses.utils import ResponseAPILoggingUtils from litellm.types.utils import Usage @@ -46,7 +46,7 @@ def test_openai_cache_write_tokens_billed_at_the_cache_creation_rate(local_model fresh_tokens = prompt_tokens - cache_write_tokens completion_tokens = 5 - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=MODEL, usage=_openai_chat_usage(prompt_tokens, cache_write_tokens, completion_tokens), custom_llm_provider="openai", @@ -77,12 +77,12 @@ def test_responses_api_cache_write_costs_the_same_as_chat(local_model_cost_map): ) assert responses_usage.prompt_tokens_details.cache_write_tokens == cache_write_tokens - responses_prompt_cost, responses_completion_cost = generic_cost_per_token( + responses_prompt_cost, responses_completion_cost = cost_per_text( model=MODEL, usage=responses_usage, custom_llm_provider="openai", ) - chat_prompt_cost, chat_completion_cost = generic_cost_per_token( + chat_prompt_cost, chat_completion_cost = cost_per_text( model=MODEL, usage=_openai_chat_usage(prompt_tokens, cache_write_tokens, completion_tokens), custom_llm_provider="openai", @@ -126,7 +126,7 @@ def test_tiered_cache_creation_rates_are_registered_and_billed( }[rate_key] input_rate = litellm.model_cost[MODEL][input_rate_key] - prompt_cost, _ = generic_cost_per_token( + prompt_cost, _ = cost_per_text( model=MODEL, usage=_openai_chat_usage(prompt_tokens, cache_write_tokens, completion_tokens=100), custom_llm_provider="openai", diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index ef002a73e52..3047d447c30 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -1,7 +1,7 @@ import copy import json from itertools import chain -from typing import Final +from typing import Final, cast from unittest.mock import MagicMock, patch import httpx @@ -39,7 +39,7 @@ from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation im ) from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES, ANTHROPIC_TOOL_SEARCH_TOOL_TYPES from litellm.types.llms.openai import ChatCompletionToolCallFunctionChunk -from litellm.types.utils import ChatCompletionToolCallChunk, ModelResponse, ServerToolUse, Usage +from litellm.types.utils import ChatCompletionToolCallChunk, ModelInfo, ModelResponse, ServerToolUse, Usage from tests.unit.llms.anthropic.anthropic_docs_refusal import ANTHROPIC_DOCS_REFUSAL_MESSAGE @@ -4468,86 +4468,37 @@ def test_fast_mode_usage_calculation(): assert usage.speed == "fast" -def test_fast_mode_cost_calculation(): - """ - Test that fast mode applies the 'fast' multiplier from provider_specific_entry - on top of the base model cost (1.1x for claude-opus-4-6). - """ +_FAST_MODE_ROW: Final = cast( + ModelInfo, + { + "litellm_provider": "anthropic", + "input_cost_per_token": 5e-06, + "output_cost_per_token": 2.5e-05, + "provider_specific_entry": {"fast": 1.1, "us": 1.1}, + }, +) + +@pytest.mark.parametrize( + ("speed", "inference_geo", "multiplier"), + [ + pytest.param("fast", None, 1.1, id="fast"), + pytest.param("fast", "us", 1.1 * 1.1, id="fast-and-us"), + pytest.param(None, "global", 1.0, id="global-geo-is-no-uplift"), + ], +) +def test_fast_mode_and_inference_geo_multiply_the_base_cost( + speed: str | None, inference_geo: str | None, multiplier: float +) -> None: from litellm.llms.anthropic.cost_calculation import cost_per_token from litellm.types.utils import Usage - base_prompt = 0.005 - base_completion = 0.025 + usage = Usage(prompt_tokens=1000, completion_tokens=1000, speed=speed, inference_geo=inference_geo) - with ( - patch("litellm.llms.anthropic.cost_calculation.generic_cost_per_token") as mock_cost, - patch("litellm.get_model_info") as mock_info, - ): - mock_cost.return_value = (base_prompt, base_completion) - mock_info.return_value = {"provider_specific_entry": {"fast": 1.1, "us": 1.1}} + prompt_cost, completion_cost = cost_per_token(model="claude-opus-4-6", usage=usage, model_info=_FAST_MODE_ROW) - usage_fast = Usage( - prompt_tokens=1000, - completion_tokens=1000, - speed="fast", - ) - - prompt_cost, completion_cost = cost_per_token( - model="claude-opus-4-6", - usage=usage_fast, - ) - - # generic_cost_per_token called with the plain base model name - mock_cost.assert_called_once() - assert mock_cost.call_args[1]["model"] == "claude-opus-4-6" - assert mock_cost.call_args[1]["custom_llm_provider"] == "anthropic" - - # 1.1x multiplier applied - assert abs(prompt_cost - base_prompt * 1.1) < 1e-10 - assert abs(completion_cost - base_completion * 1.1) < 1e-10 - - -def test_fast_mode_with_inference_geo(): - """ - Test that fast mode + inference_geo both apply their multipliers from - provider_specific_entry (1.1 * 1.1 = 1.21x for claude-opus-4-6). - """ - - from litellm.llms.anthropic.cost_calculation import cost_per_token - from litellm.types.utils import Usage - - base_prompt = 0.005 - base_completion = 0.025 - - with ( - patch("litellm.llms.anthropic.cost_calculation.generic_cost_per_token") as mock_cost, - patch("litellm.get_model_info") as mock_info, - ): - mock_cost.return_value = (base_prompt, base_completion) - mock_info.return_value = {"provider_specific_entry": {"fast": 1.1, "us": 1.1}} - - usage = Usage( - prompt_tokens=1000, - completion_tokens=1000, - speed="fast", - inference_geo="us", - ) - - prompt_cost, completion_cost = cost_per_token( - model="claude-opus-4-6", - usage=usage, - ) - - # generic_cost_per_token called with the plain base model name - mock_cost.assert_called_once() - assert mock_cost.call_args[1]["model"] == "claude-opus-4-6" - assert mock_cost.call_args[1]["custom_llm_provider"] == "anthropic" - - # 1.1 (fast) * 1.1 (us) = 1.21x multiplier applied - expected_multiplier = 1.1 * 1.1 - assert abs(prompt_cost - base_prompt * expected_multiplier) < 1e-10 - assert abs(completion_cost - base_completion * expected_multiplier) < 1e-10 + assert prompt_cost == pytest.approx(1000 * 5e-06 * multiplier) + assert completion_cost == pytest.approx(1000 * 2.5e-05 * multiplier) def test_calculate_usage_captures_service_tier(): diff --git a/tests/unit/llms/dashscope/test_dashscope_cost_calculator.py b/tests/unit/llms/dashscope/test_dashscope_cost_calculator.py index 551caab7153..7f4379368d1 100644 --- a/tests/unit/llms/dashscope/test_dashscope_cost_calculator.py +++ b/tests/unit/llms/dashscope/test_dashscope_cost_calculator.py @@ -19,7 +19,7 @@ import litellm from litellm.litellm_core_utils.llm_cost_calc.utils import ( resolve_explicit_cache_read_cost_rate, calculate_prompt_caching_savings, - generic_cost_per_token, + cost_per_text, get_token_type_cost_breakdown, ) from litellm.llms.dashscope.cost_calculator import ( @@ -475,7 +475,7 @@ class TestDashscopeCostCalculator: prompt_tokens_details={"cached_tokens": 600, "cache_type": "ephemeral"}, ) - prompt_cost, _ = generic_cost_per_token( + prompt_cost, _ = cost_per_text( model="custom-qwen", usage=usage, custom_llm_provider="openai", diff --git a/tests/unit/llms/fireworks_ai/test_fireworks_ai_cache_pricing.py b/tests/unit/llms/fireworks_ai/test_fireworks_ai_cache_pricing.py deleted file mode 100644 index c21943cfe75..00000000000 --- a/tests/unit/llms/fireworks_ai/test_fireworks_ai_cache_pricing.py +++ /dev/null @@ -1,65 +0,0 @@ -from copy import deepcopy - -import pytest - -from litellm.constants import FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO -from litellm.llms.fireworks_ai.cache_pricing import with_default_cache_read_rate -from litellm.types.utils import ModelInfo - - -def test_explicit_cache_read_rate_and_missing_input_rate_keep_the_entry_untouched() -> None: - explicit_info: ModelInfo = {"input_cost_per_token": 2e-6, "cache_read_input_token_cost": 1e-6} - no_input_rate_info: ModelInfo = {"output_cost_per_token": 3e-6} - - assert with_default_cache_read_rate(explicit_info) is explicit_info - assert with_default_cache_read_rate(no_input_rate_info) is no_input_rate_info - - -def test_missing_cache_read_rate_is_derived_for_standard_and_off_peak_without_mutating_the_entry() -> None: - model_info: ModelInfo = { - "input_cost_per_token": 2e-6, - "off_peak_pricing": { - "hours_utc": "14:00-00:00", - "input_cost_per_token": 1e-6, - "output_cost_per_token": 3e-6, - }, - } - original: ModelInfo = deepcopy(model_info) - - derived = with_default_cache_read_rate(model_info) - - assert model_info == original - assert derived is not model_info - assert derived["cache_read_input_token_cost"] == pytest.approx(2e-6 * FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO) - assert derived["off_peak_pricing"] == { - "hours_utc": "14:00-00:00", - "input_cost_per_token": 1e-6, - "output_cost_per_token": 3e-6, - "cache_read_input_token_cost": 1e-6 * FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO, - } - - -def test_off_peak_window_without_its_own_input_rate_reuses_the_standard_derived_rate() -> None: - model_info: ModelInfo = { - "input_cost_per_token": 2e-6, - "off_peak_pricing": {"hours_utc": "14:00-00:00", "output_cost_per_token": 3e-6}, - } - - derived = with_default_cache_read_rate(model_info) - - assert derived["off_peak_pricing"]["cache_read_input_token_cost"] == derived["cache_read_input_token_cost"] - - -def test_string_rates_from_config_are_coerced_before_the_discount_is_applied() -> None: - model_info: ModelInfo = { - "input_cost_per_token": "2e-6", - "off_peak_pricing": { - "hours_utc": "14:00-00:00", - "input_cost_per_token": "1e-6", - }, - } - - derived = with_default_cache_read_rate(model_info) - - assert derived["cache_read_input_token_cost"] == pytest.approx(1e-6) - assert derived["off_peak_pricing"]["cache_read_input_token_cost"] == pytest.approx(5e-7) diff --git a/tests/unit/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py b/tests/unit/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py index c3aec979aa6..aa5990ae0b3 100644 --- a/tests/unit/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py +++ b/tests/unit/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py @@ -9,7 +9,7 @@ import pytest import litellm from litellm.litellm_core_utils.llm_cost_calc.utils import ( calculate_prompt_caching_savings, - generic_cost_per_token, + cost_per_text, get_token_type_cost_breakdown, ) from litellm.llms.fireworks_ai.cost_calculator import cost_per_token @@ -213,7 +213,7 @@ def test_fireworks_cache_read_rates_match_breakdown_and_caching_savings(): assert math.isclose(savings, 300 * INPUT_COST * 0.5, rel_tol=1e-10) -def test_generic_cost_per_token_applies_fireworks_cache_read_default_with_or_without_model_info(): +def test_cost_per_text_applies_fireworks_cache_read_default_with_or_without_model_info(): model = "accounts/fireworks/models/generic-cache-read-test" litellm.model_cost = { # test-quality-ok: the restore_model_cost fixture returns litellm.model_cost to the original object after each test **litellm.model_cost, @@ -227,12 +227,12 @@ def test_generic_cost_per_token_applies_fireworks_cache_read_default_with_or_wit usage = _usage(prompt_tokens=1000, cached_tokens=300, completion_tokens=200) expected_prompt_cost = (700 * INPUT_COST) + (300 * INPUT_COST * 0.5) - implicit_model_info_cost, _ = generic_cost_per_token( + implicit_model_info_cost, _ = cost_per_text( model=model, usage=usage, custom_llm_provider="fireworks_ai", ) - explicit_model_info_cost, _ = generic_cost_per_token( + explicit_model_info_cost, _ = cost_per_text( model=model, usage=usage, custom_llm_provider="fireworks_ai", @@ -392,6 +392,6 @@ def test_a_routed_xai_model_keeps_xais_inclusive_token_threshold(): routed_prompt_cost, routed_completion_cost = cost_per_token(model=f"fireworks_ai/{key}", usage=usage) - assert (routed_prompt_cost, routed_completion_cost) == generic_cost_per_token( + assert (routed_prompt_cost, routed_completion_cost) == cost_per_text( model=key, usage=usage, custom_llm_provider="xai" ) diff --git a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 0f5c21d08c7..833f12fac63 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -19,7 +19,7 @@ from litellm._logging import verbose_logger from litellm.cost_calculator import batch_cost_calculator from litellm.litellm_core_utils.exception_mapping_utils import exception_type from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap -from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token, parse_prompt_tokens_details +from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text, parse_prompt_tokens_details from litellm.llms.anthropic.messages.native import handler as anthropic_messages_handler from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.gemini.chat.transformation import GoogleAIStudioGeminiConfig @@ -1620,7 +1620,7 @@ def test_gemini_raw_usage_metadata_prices_cached_audio_at_its_standard_rate() -> + 5 * model_info["cache_read_input_audio_token_cost"] ) - prompt_cost, _ = generic_cost_per_token( + prompt_cost, _ = cost_per_text( model="gemini-audio-standard-test-model", usage=usage, custom_llm_provider="gemini", @@ -1640,13 +1640,13 @@ def test_gemini_usage_totals_are_subtracted_once_and_audio_cache_rate_matches_au cached_usage: Final = _gemini_usage_with_cached_audio(5) uncached_usage: Final = _gemini_usage_with_cached_audio(0) cached_details: Final = parse_prompt_tokens_details(cached_usage) - cached_prompt_cost, _ = generic_cost_per_token( + cached_prompt_cost, _ = cost_per_text( model="gemini-audio-standard-test-model", usage=cached_usage, custom_llm_provider="gemini", model_info=model_info, ) - uncached_prompt_cost, _ = generic_cost_per_token( + uncached_prompt_cost, _ = cost_per_text( model="gemini-audio-standard-test-model", usage=uncached_usage, custom_llm_provider="gemini", @@ -6299,7 +6299,7 @@ def test_vertex_ai_usage_metadata_with_document_tokens_cached(): "input_cost_per_token": 2e-6, "cache_read_input_token_cost": 5e-7, } - prompt_cost, _ = generic_cost_per_token( + prompt_cost, _ = cost_per_text( model="gemini-document-cache-test-model", usage=result, custom_llm_provider="gemini", diff --git a/tests/unit/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py b/tests/unit/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py index d7d921ab35c..efa13e5ccad 100644 --- a/tests/unit/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py +++ b/tests/unit/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py @@ -18,7 +18,7 @@ import respx import litellm from litellm.exceptions import BadRequestError -from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text from litellm.llms.vertex_ai.gemini_embeddings.batch_embed_content_transformation import ( _build_part_for_input, file_reference_name, @@ -348,7 +348,7 @@ class TestProcessEmbedContentResponseUsage: assert result.usage.total_tokens == 258 assert result.usage.prompt_tokens_details.image_tokens == 258 - prompt_cost, _ = generic_cost_per_token( + prompt_cost, _ = cost_per_text( model=self.MODEL, usage=result.usage, custom_llm_provider="vertex_ai", @@ -373,7 +373,7 @@ class TestProcessEmbedContentResponseUsage: assert result.usage.prompt_tokens == 12 assert result.usage.prompt_tokens_details.text_tokens == 12 - prompt_cost, _ = generic_cost_per_token( + prompt_cost, _ = cost_per_text( model=self.MODEL, usage=result.usage, custom_llm_provider="vertex_ai", diff --git a/tests/unit/proxy/management_endpoints/test_ptu_model_settings.py b/tests/unit/proxy/management_endpoints/test_ptu_model_settings.py index efb84ee8f3a..b01cd0b08ed 100644 --- a/tests/unit/proxy/management_endpoints/test_ptu_model_settings.py +++ b/tests/unit/proxy/management_endpoints/test_ptu_model_settings.py @@ -11,7 +11,7 @@ from unittest.mock import patch as patch_ctx import pytest from fastapi import HTTPException -from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text from litellm.litellm_core_utils.ptu_pricing import ptu_terms from litellm.llms.gemini.cost_calculator import cost_per_web_search_request from litellm.proxy._types import ( @@ -1089,7 +1089,7 @@ class TestPtuDeploymentsAreNotBilledPerToken: registered = router.get_deployment_model_info(model_id="dep-ptu", model_name="dashscope/qwen-flash") assert registered is not None assert registered["tiered_pricing"] == [] - assert generic_cost_per_token( + assert cost_per_text( model="dashscope/qwen-flash", usage=Usage(prompt_tokens=1000, completion_tokens=100, total_tokens=1100), custom_llm_provider="dashscope", diff --git a/tests/unit/proxy/pass_through_endpoints/test_vertex_ai_live_passthrough.py b/tests/unit/proxy/pass_through_endpoints/test_vertex_ai_live_passthrough.py index 9c372380749..dbbc4801d24 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_vertex_ai_live_passthrough.py +++ b/tests/unit/proxy/pass_through_endpoints/test_vertex_ai_live_passthrough.py @@ -574,7 +574,7 @@ class TestVertexAILivePassthroughLoggingHandler: def test_reporting_tool_use_tokens_does_not_move_the_bill(self, handler, mock_logging_obj): """Deliberate boundary: these tokens are reported here, and priced nowhere. - generic_cost_per_token reads the input bill out of prompt_tokens_details, and falls + cost_per_text reads the input bill out of prompt_tokens_details, and falls back to prompt_tokens only when the details carry no text or a cache hit overlaps them, so adding tool-use tokens to prompt_tokens is worth nothing on an ordinary Live turn and over-charges against the cache-overlap correction when it is not. Pricing them belongs diff --git a/tests/unit/proxy/spend_tracking/test_savings.py b/tests/unit/proxy/spend_tracking/test_savings.py index f9566fd1ba0..1adcd6b0fa9 100644 --- a/tests/unit/proxy/spend_tracking/test_savings.py +++ b/tests/unit/proxy/spend_tracking/test_savings.py @@ -3,7 +3,7 @@ from typing import Final, Literal import pytest import litellm -from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text from litellm.llms.anthropic.cost_calculation import cost_per_token as anthropic_cost_per_token from litellm.proxy.spend_tracking.savings import ( _baseline_usage, @@ -108,7 +108,7 @@ def _cached_usage_object() -> dict: def _cost_on(model: str, usage_object: dict) -> float: - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model, usage=Usage(**usage_object), custom_llm_provider="anthropic" ) return prompt_cost + completion_cost @@ -159,7 +159,7 @@ def _net_caching_savings_against_biller(usage_object: dict, model: str = "claude Bills the request as it happened, then bills the same token total with nothing cached, and returns the difference. Deriving the expectation from - ``generic_cost_per_token`` rather than restating the formula is what makes these + ``cost_per_text`` rather than restating the formula is what makes these tests able to fail: a wrong formula in savings.py cannot also be wrong here. """ prompt_tokens = usage_object["prompt_tokens"] @@ -245,7 +245,7 @@ def test_caching_savings_agree_with_biller_on_the_request_pricing_basis( ) costs: Final = tuple( sum( - generic_cost_per_token( + cost_per_text( model=model, usage=arm, custom_llm_provider=provider, @@ -486,8 +486,8 @@ def test_supplied_baseline_usage_is_priced_independently( baseline: str, selected: str, actual: Usage, modeled: Usage, loses_money: bool, ) -> None: result: Final = _savings(baseline, selected, actual, modeled) - expected: Final = sum(generic_cost_per_token(model=baseline, usage=modeled, custom_llm_provider="anthropic")) - sum( - generic_cost_per_token(model=selected, usage=actual, custom_llm_provider="anthropic") + expected: Final = sum(cost_per_text(model=baseline, usage=modeled, custom_llm_provider="anthropic")) - sum( + cost_per_text(model=selected, usage=actual, custom_llm_provider="anthropic") ) assert result == pytest.approx(expected) assert result is not None and (result < 0) is loses_money diff --git a/tests/unit/responses/test_responses_utils.py b/tests/unit/responses/test_responses_utils.py index 0863abd1f1e..37d876495bf 100644 --- a/tests/unit/responses/test_responses_utils.py +++ b/tests/unit/responses/test_responses_utils.py @@ -8,7 +8,7 @@ import pytest import litellm from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig -from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text from litellm.responses.litellm_completion_transformation.transformation import LiteLLMCompletionResponsesConfig from litellm.responses.utils import ResponseAPILoggingUtils, ResponsesAPIRequestUtils from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIOptionalRequestParams, ResponsesAPIResponse @@ -440,7 +440,7 @@ class TestResponseAPILoggingUtils: assert usage.prompt_tokens_details.cache_creation_token_details is not None assert usage.prompt_tokens_details.cache_creation_token_details.ephemeral_12h_input_tokens == 2_000 assert usage.prompt_tokens_details.cache_creation_token_details.ephemeral_30m_input_tokens == 3_000 - cost: Final = generic_cost_per_token( + cost: Final = cost_per_text( model="test-12h-cache-write", usage=usage, custom_llm_provider="openai", diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 7ad78c956d4..6ebfacdf539 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -1927,7 +1927,7 @@ def test_gemini_25_explicit_caching_cost_direct_usage(): This test reproduces the issue from #11156 where cached tokens should receive a 75% discount. """ - from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token + from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text from litellm.types.utils import ( CompletionTokensDetailsWrapper, PromptTokensDetailsWrapper, @@ -1953,7 +1953,7 @@ def test_gemini_25_explicit_caching_cost_direct_usage(): ), ) - input_cost, output_cost = generic_cost_per_token( + input_cost, output_cost = cost_per_text( model="gemini/gemini-2.5-pro", usage=usage, custom_llm_provider="gemini", @@ -1987,13 +1987,13 @@ def test_gemini_25_explicit_caching_cost_direct_usage(): def test_azure_ai_cache_cost_calculation(_local_model_cost_map): """ - Test that azure_ai provider correctly calculates cache costs using generic_cost_per_token. + Test that azure_ai provider correctly calculates cache costs using cost_per_text. This verifies that azure_ai models with custom cache pricing in model_info will have their cache_creation_input_token_cost and cache_read_input_token_cost applied correctly. """ - from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token + from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text from litellm.types.utils import PromptTokensDetailsWrapper, Usage # Register a custom azure_ai model with cache pricing @@ -2023,7 +2023,7 @@ def test_azure_ai_cache_cost_calculation(_local_model_cost_map): cache_creation_input_tokens=100, # 100 cache creation tokens ) - input_cost, output_cost = generic_cost_per_token( + input_cost, output_cost = cost_per_text( model=test_model_id, usage=usage, custom_llm_provider="azure_ai", @@ -2887,7 +2887,7 @@ def test_completion_cost_service_tier_for_anthropic(_local_model_cost_map): Regression for LIT-3771: the Anthropic cost route dropped ``service_tier``, so priority requests (whose tier is reported on the response usage) were always billed at the standard rate. The tier is captured by the - transformation and must flow through to ``generic_cost_per_token``. + transformation and must flow through to ``cost_per_text``. """ from litellm import completion_cost from litellm.llms.anthropic.chat.transformation import AnthropicConfig diff --git a/tests/unit/test_openai_service_tier_long_context_pricing.py b/tests/unit/test_openai_service_tier_long_context_pricing.py index 9777af1af70..e315aa330f9 100644 --- a/tests/unit/test_openai_service_tier_long_context_pricing.py +++ b/tests/unit/test_openai_service_tier_long_context_pricing.py @@ -6,7 +6,7 @@ from typing import Final import pytest import litellm -from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text from litellm.types.utils import PromptTokensDetailsWrapper, Usage REPO_ROOT = Path(__file__).parents[2] @@ -151,7 +151,7 @@ def test_tiered_long_context_cost_uses_catalog_rates( completion_tokens=COMPLETION_TOKENS, total_tokens=LONG_CONTEXT_PROMPT_TOKENS + COMPLETION_TOKENS, ) - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model, usage=usage, custom_llm_provider="openai", @@ -169,13 +169,13 @@ def test_gpt_6_astra_ultrafast_long_context_costs_and_controls() -> None: total_tokens=301_000, prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=100, cache_creation_tokens=200), ) - ultrafast_prompt_cost, ultrafast_completion_cost = generic_cost_per_token( + ultrafast_prompt_cost, ultrafast_completion_cost = cost_per_text( model="gpt-6-astra", usage=ultrafast_usage, custom_llm_provider="openai", service_tier="ultrafast", ) - standard_prompt_cost, standard_completion_cost = generic_cost_per_token( + standard_prompt_cost, standard_completion_cost = cost_per_text( model="gpt-6-astra", usage=ultrafast_usage, custom_llm_provider="openai", @@ -186,7 +186,7 @@ def test_gpt_6_astra_ultrafast_long_context_costs_and_controls() -> None: total_tokens=272_000, prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=100, cache_creation_tokens=200), ) - below_threshold_prompt_cost, below_threshold_completion_cost = generic_cost_per_token( + below_threshold_prompt_cost, below_threshold_completion_cost = cost_per_text( model="gpt-6-astra", usage=below_threshold_usage, custom_llm_provider="openai", diff --git a/tests/unit/test_private_usage_aliases.py b/tests/unit/test_private_usage_aliases.py index d4d5e6478d8..92ae90190e0 100644 --- a/tests/unit/test_private_usage_aliases.py +++ b/tests/unit/test_private_usage_aliases.py @@ -1480,7 +1480,6 @@ ALIAS_CASES: Final = ( "get_service_tier_cost_key", False, ), - ("litellm.litellm_core_utils.llm_cost_calc.utils", "", "_is_above_128k", "is_above_128k", False), ( "litellm.litellm_core_utils.llm_request_utils", "", diff --git a/tests/unit/test_register_model_custom_pricing.py b/tests/unit/test_register_model_custom_pricing.py index f125ec14bed..b63d76905a5 100644 --- a/tests/unit/test_register_model_custom_pricing.py +++ b/tests/unit/test_register_model_custom_pricing.py @@ -121,7 +121,7 @@ def test_register_model_inherits_builtin_cache_pricing_for_unmapped_key(monkeypa Regression for the cache-pricing dropout under partial overrides. """ - from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token + from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text from litellm.types.utils import PromptTokensDetailsWrapper, Usage original_model_cost = litellm.model_cost @@ -168,7 +168,7 @@ def test_register_model_inherits_builtin_cache_pricing_for_unmapped_key(monkeypa cache_creation_input_tokens=200, ) - input_cost, output_cost = generic_cost_per_token( + input_cost, output_cost = cost_per_text( model=registered_key, usage=usage, custom_llm_provider="bedrock", diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 914589be55d..86d8a7a372d 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -5069,17 +5069,15 @@ class _EntryReadDuringARegistration(dict[str, object]): return super().__contains__(key) -@pytest.mark.parametrize("walk", ["fal_ai", "ollama", "gpt-5"]) +@pytest.mark.parametrize("walk", ["ollama", "gpt-5"]) def test_request_path_cost_map_walks_survive_a_registration_mid_walk( monkeypatch: pytest.MonkeyPatch, walk: str ) -> None: - from litellm.llms.fal_ai.cost_calculator import _keyed_rows from litellm.llms.ollama.common_utils import OllamaModelInfo from litellm.llms.openai.chat.gpt_reasoning_transformation import _catalogue_declares_default_effort from litellm.utils import _invalidate_model_cost_lowercase_map walks: Final[Mapping[str, Callable[[], object]]] = { - "fal_ai": lambda: _keyed_rows("lit-5853-model", "standard"), "ollama": lambda: OllamaModelInfo._is_static_ollama_model("ollama/lit-5853-model"), "gpt-5": _catalogue_declares_default_effort, }