From 6a486c3b43ae8a2d395b4c4677bfbeb515342171 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Wed, 28 Oct 2026 03:24:45 -0700 Subject: [PATCH] refactor(cost): price images, search and TTS with the Rust cost core [10/12] cost::calculate gains image, pixel, query tier, web search, Maps grounding, citation and feature arms, with Gemini's search defaults as fallback lines and the Anthropic speed and geo multipliers as uplifts. The generic_cost_per_token shim is gone, and search, Lyria, per-image and web search pricing go through Prices.price Co-Authored-By: Claude Opus 5.5 --- litellm-rust/crates/cost/src/calculate.rs | 420 +++++++++-- litellm-rust/crates/cost/src/price.rs | 14 +- .../crates/model-catalog/src/lower.rs | 79 +- .../crates/model-catalog/src/resolve/image.rs | 158 ++++ litellm-rust/crates/python-bridge/src/cost.rs | 115 ++- litellm-rust/crates/python-bridge/src/lib.rs | 2 +- .../python-bridge/src/model_registry.rs | 63 ++ .../src/model_registry/resolve.rs | 72 +- litellm/_cost/__init__.py | 3 +- litellm/constants.py | 1 - litellm/cost_calculator.py | 84 +-- .../llm_cost_calc/tool_call_cost_tracking.py | 26 +- .../litellm_core_utils/llm_cost_calc/utils.py | 576 +++----------- .../aiml/image_generation/cost_calculator.py | 5 +- litellm/llms/amazon_nova/cost_calculation.py | 4 +- litellm/llms/anthropic/cost_calculation.py | 55 +- litellm/llms/azure/cost_calculation.py | 4 +- litellm/llms/azure_ai/cost_calculator.py | 6 +- .../image_generation/cost_calculator.py | 50 +- litellm/llms/bedrock/cost_calculation.py | 19 +- .../amazon_nova_canvas_transformation.py | 5 +- .../amazon_stability1_transformation.py | 5 +- .../amazon_stability3_transformation.py | 5 +- .../amazon_titan_transformation.py | 5 +- .../image_generation/cost_calculator.py | 5 +- litellm/llms/databricks/cost_calculator.py | 4 +- litellm/llms/deepseek/cost_calculator.py | 4 +- litellm/llms/fal_ai/cost_calculator.py | 35 +- litellm/llms/fireworks_ai/cache_pricing.py | 42 -- litellm/llms/fireworks_ai/cost_calculator.py | 4 +- litellm/llms/gemini/cost_calculator.py | 96 +-- litellm/llms/groq/cost_calculator.py | 19 +- litellm/llms/openai/cost_calculation.py | 4 +- .../image_generation/cost_calculator.py | 10 +- .../parallel_ai/search/cost_calculator.py | 33 +- litellm/llms/recraft/cost_calculator.py | 5 +- litellm/llms/runwayml/cost_calculator.py | 5 +- litellm/llms/tencent/cost_calculator.py | 4 +- litellm/llms/vertex_ai/common_utils.py | 24 +- litellm/llms/vertex_ai/cost_calculator.py | 55 +- .../vertex_ai/image_edit/cost_calculator.py | 5 +- litellm/llms/xai/cost_calculator.py | 6 +- .../vertex_passthrough_logging_handler.py | 2 +- litellm/proxy/spend_tracking/savings.py | 8 +- litellm/proxy/utils.py | 6 +- litellm/router_strategy/savings_baseline.py | 4 +- litellm/rust_bridge/_native.pyi | 26 + litellm/search/cost_calculator.py | 40 +- .../llm_cost_calc/test_llm_cost_calc_utils.py | 700 ++++-------------- .../test_openai_cache_write_cost.py | 10 +- .../test_anthropic_chat_transformation.py | 103 +-- .../test_dashscope_cost_calculator.py | 4 +- .../test_fireworks_ai_cache_pricing.py | 65 -- .../test_fireworks_ai_cost_calculator.py | 10 +- ...test_vertex_and_google_ai_studio_gemini.py | 10 +- ...test_batch_embed_content_transformation.py | 6 +- .../test_ptu_model_settings.py | 4 +- .../test_vertex_ai_live_passthrough.py | 2 +- .../unit/proxy/spend_tracking/test_savings.py | 12 +- tests/unit/responses/test_responses_utils.py | 4 +- tests/unit/test_cost_calculator.py | 12 +- ...penai_service_tier_long_context_pricing.py | 10 +- tests/unit/test_private_usage_aliases.py | 1 - .../test_register_model_custom_pricing.py | 4 +- tests/unit/test_utils.py | 4 +- 65 files changed, 1416 insertions(+), 1767 deletions(-) delete mode 100644 litellm/llms/fireworks_ai/cache_pricing.py delete mode 100644 tests/unit/llms/fireworks_ai/test_fireworks_ai_cache_pricing.py diff --git a/litellm-rust/crates/cost/src/calculate.rs b/litellm-rust/crates/cost/src/calculate.rs index abcd7e87e76..3d7a3429078 100644 --- a/litellm-rust/crates/cost/src/calculate.rs +++ b/litellm-rust/crates/cost/src/calculate.rs @@ -5,9 +5,11 @@ use crate::price::{ Meter, OffPeakRates, OutputUnit, Price, Prices, ServiceTier, TierScope, TokenKind, Uplift, When, }; -use crate::price::{Resolution, TextUnit, WallClock}; +use crate::price::{Resolution, SearchDefaults, TextUnit, WallClock, WebSearchUnit}; use crate::priced::{Line, Outcome, Priced, Quantity, Side}; -use crate::usage::{DurationMeter, Usage}; +use std::num::NonZeroU64; + +use crate::usage::{DurationMeter, ImageOut, Usage}; const TEXT_INPUT: Meter = Meter::Input(InputUnit::Token(TokenKind::Text)); const TEXT_OUTPUT: Meter = Meter::Output(OutputUnit::Token(TokenKind::Text)); @@ -35,6 +37,8 @@ pub fn calculate( let billed: Vec = token_meters(&rates, usage) .chain(character_meters(&rates, usage)) .chain(second_meters(&rates, usage)) + .chain(image_meters(&rates, usage)) + .chain(search_meters(&rates, usage)) .chain([( Meter::Input(InputUnit::TextMessage), rates.banded(&Meter::Input(InputUnit::TextMessage)), @@ -55,7 +59,6 @@ pub fn calculate( let missing: Vec = billed .into_iter() .filter_map(|(meter, rate, ..)| rate.is_none().then_some(meter)) - .chain(without_an_arm(usage)) .collect(); if lines.is_empty() && !missing.is_empty() { return Outcome::Unpriced { missing }; @@ -437,6 +440,38 @@ impl<'a> Rates<'a> { ) } + /// `search_provider_cost_per_query`'s tier walk. The tier whose range holds `max_results`, past the last range the + /// highest tier, without tiers the flat query price + fn query_rate(&self, max_results: u64) -> (Meter, Option) { + let tiers: Vec<(Option, UnitPrice)> = self + .prices + .iter() + .filter_map(|(price, rate)| match price.meter { + Meter::Input(InputUnit::Query { over_results }) + if price.channel == Channel::Online => + { + Some((over_results, rate)) + } + _ => None, + }) + .collect(); + let floor = |over: Option| over.map_or(0, NonZeroU64::get); + let held = tiers + .iter() + .filter(|(over, _)| max_results > floor(*over)) + .max_by_key(|(over, _)| floor(*over)) + .or_else(|| tiers.iter().max_by_key(|(over, _)| floor(*over))); + match held { + Some((over, rate)) => ( + Meter::Input(InputUnit::Query { + over_results: *over, + }), + Some(Rate::own(*rate)), + ), + None => (Meter::Input(InputUnit::Query { over_results: None }), None), + } + } + /// The first of `meters` with a price, borrowed when it is not the first fn first_of(&self, meters: &[Meter]) -> Option { let (own, rest) = meters.split_first()?; @@ -556,47 +591,142 @@ impl<'a> Rates<'a> { } } -/// Meters with usage that no arm prices yet, the per-unit meters 10 adds -fn without_an_arm(usage: &Usage) -> impl Iterator + '_ { - let counted = [ - (usage.images.input, Meter::Input(InputUnit::Image)), - (usage.images.input_pixels, Meter::Input(InputUnit::Pixel)), - ( - usage.images.output.len() as u64, - Meter::Output(OutputUnit::Image(None)), - ), - ( - usage.search.queries, - Meter::Input(InputUnit::Query { over_results: None }), - ), - ( - usage.search.maps_grounding, - Meter::Feature(Feature::GoogleMapsGroundingQuery), - ), - ( - usage.search.browser_opens, - Meter::Feature(Feature::BrowserOpen), - ), - ( - usage.search.additional_results, - Meter::Feature(Feature::AdditionalResult), - ), - ( - usage.tokens.citation, - Meter::Feature(Feature::CitationToken), - ), - ]; - let web_search = usage - .search - .web_search - .filter(|search| search.requests > 0) - .map(|search| Meter::WebSearch(search.context)); - counted - .into_iter() - .filter_map(|(count, meter)| (count > 0).then_some(meter)) - .chain(web_search) +const MEGAPIXEL: u64 = 1_048_576; + +/// main's `default_image_cost_calculator` and fal's image rules. A requested size with its own price bills that +/// price, then the first of the input per-image price, the output per-image price and the input per-pixel price +/// times the image's pixels, then fal's output per-pixel price per started megapixel +fn image_meters(rates: &Rates<'_>, usage: &Usage) -> Vec { + let per_image = |image: &ImageOut| -> Billed { + let pixels = u64::from(image.width.unwrap_or(0)) * u64::from(image.height.unwrap_or(0)); + let sized = image.size.clone().and_then(|size| { + let meter = Meter::Output(OutputUnit::Image(Some(size))); + Some((meter.clone(), rates.banded(&meter)?)) + }); + let candidates = [ + (Meter::Input(InputUnit::Image), 1), + (Meter::Output(OutputUnit::Image(None)), 1), + (Meter::Input(InputUnit::Pixel), pixels), + ( + Meter::Output(OutputUnit::Pixel), + MEGAPIXEL * pixels.div_ceil(MEGAPIXEL), + ), + ]; + let flat = || { + candidates + .into_iter() + .filter(|(_, quantity)| *quantity > 0) + .find_map(|(meter, quantity)| { + Some((meter.clone(), rates.banded(&meter)?, quantity)) + }) + }; + match sized.map(|(meter, rate)| (meter, rate, 1)).or_else(flat) { + Some((meter, rate, quantity)) => ( + meter, + Some(rate), + Quantity::Count(quantity), + Side::Completion, + ), + None => ( + Meter::Output(OutputUnit::Image(None)), + None, + Quantity::Count(1), + Side::Completion, + ), + } + }; + let input_image = Meter::Input(InputUnit::Image); + let input_pixel = Meter::Input(InputUnit::Pixel); + usage + .images + .output + .iter() + .map(per_image) + .chain([ + ( + input_image.clone(), + rates.banded(&input_image), + Quantity::Count(usage.images.input), + Side::Prompt, + ), + ( + input_pixel.clone(), + rates.banded(&input_pixel), + Quantity::Count(usage.images.input_pixels), + Side::Prompt, + ), + ]) + .collect() } +/// Search query tiers, web search, Maps grounding, citation tokens, Groq browser opens and Parallel AI's extra +/// results. Web search and Maps bill once per prompt unless the row bills them per query +fn search_meters(rates: &Rates<'_>, usage: &Usage) -> Vec { + let search = &usage.search; + let per_prompt = rates.prices.billing().web_search == WebSearchUnit::PerPrompt; + let billed_requests = |requests: u64| match per_prompt { + true => requests.min(1), + false => requests, + }; + let google = rates.prices.billing().search_defaults == SearchDefaults::Google; + let default = |price: f64| google.then(|| UnitPrice::new(price)).flatten(); + let web_search = search.web_search.map(|web_search| { + let meter = Meter::WebSearch(web_search.context); + let rate = rates.banded(&meter).or_else(|| { + default(GOOGLE_WEB_SEARCH_DEFAULT).map(|price| Rate::own(price).borrowed_from(&meter)) + }); + ( + meter, + rate, + Quantity::Count(billed_requests(web_search.requests)), + Side::Prompt, + ) + }); + let maps = Meter::Feature(Feature::GoogleMapsGroundingQuery); + let maps_default = match per_prompt { + true => GOOGLE_MAPS_DEFAULT_PER_PROMPT, + false => GOOGLE_MAPS_DEFAULT_PER_QUERY, + }; + let maps_line = ( + maps.clone(), + rates + .banded(&maps) + .or_else(|| default(maps_default).map(|price| Rate::own(price).borrowed_from(&maps))), + Quantity::Count(billed_requests(search.maps_grounding)), + Side::Prompt, + ); + let feature = |feature: Feature, count: u64| { + let meter = Meter::Feature(feature); + ( + meter.clone(), + rates.banded(&meter), + Quantity::Count(count), + Side::Prompt, + ) + }; + let query = (search.queries > 0).then(|| { + let (meter, rate) = rates.query_rate(search.max_results.unwrap_or(DEFAULT_MAX_RESULTS)); + (meter, rate, Quantity::Count(search.queries), Side::Prompt) + }); + query + .into_iter() + .chain(web_search) + .chain([ + maps_line, + feature(Feature::CitationToken, usage.tokens.citation), + feature(Feature::BrowserOpen, search.browser_opens), + feature(Feature::AdditionalResult, search.additional_results), + ]) + .collect() +} + +/// Gemini's defaults for a Google row with no web search or Maps grounding price +const GOOGLE_WEB_SEARCH_DEFAULT: f64 = 0.035; +const GOOGLE_MAPS_DEFAULT_PER_QUERY: f64 = 0.014; +const GOOGLE_MAPS_DEFAULT_PER_PROMPT: f64 = 0.025; +/// The results a search returns when the request names no `max_results`, main's default +const DEFAULT_MAX_RESULTS: u64 = 10; + fn uplifts(adjustments: &Adjustments, call: &CallContext) -> Vec { let residency = call .residency @@ -605,7 +735,19 @@ fn uplifts(adjustments: &Adjustments, call: &CallContext) -> Vec { .non_global_endpoint .then(|| adjustments.get(&Uplift::NonGlobalEndpoint)) .flatten(); - residency.into_iter().chain(endpoint).collect() + let speed = call + .speed + .and_then(|speed| adjustments.get(&Uplift::Speed(speed))); + let geo = call + .inference_geo + .clone() + .and_then(|geo| adjustments.get(&Uplift::InferenceGeo(geo))); + residency + .into_iter() + .chain(endpoint) + .chain(speed) + .chain(geo) + .collect() } #[cfg(test)] @@ -616,7 +758,7 @@ mod tests { use rstest::rstest; use super::*; - use crate::price::{Billing, Residency}; + use crate::price::{Billing, Residency, SearchContextSize}; fn price( meter: &Meter, @@ -1152,4 +1294,194 @@ mod tests { let expected = if prompt > 32_000 { rate } else { 0.1e-6 }; close(total(&prices, &usage, &call()), 100.0 * expected); } + + fn images(images: Vec) -> Usage { + Usage { + images: crate::usage::Images { + output: images, + ..crate::usage::Images::default() + }, + ..Usage::default() + } + } + + fn image(size: Option<&str>, width: u32, height: u32) -> ImageOut { + ImageOut { + size: size.and_then(crate::price::ImageSize::new), + width: Some(width), + height: Some(height), + } + } + + #[rstest] + #[case::input_price_wins(Some(0.04), Some(0.08), None, 0.04)] + #[case::output_price(None, Some(0.08), None, 0.08)] + #[case::input_pixels(None, None, Some(1e-8), 1e-8 * 1024.0 * 1024.0)] + fn an_image_bills_the_first_per_image_price_in_mains_order( + #[case] input: Option, + #[case] output: Option, + #[case] per_pixel: Option, + #[case] expected: f64, + ) { + let prices = row(input + .map(|rate| standard(&Meter::Input(InputUnit::Image), rate)) + .into_iter() + .chain(output.map(|rate| standard(&Meter::Output(OutputUnit::Image(None)), rate))) + .chain(per_pixel.map(|rate| standard(&Meter::Input(InputUnit::Pixel), rate)))); + close( + total( + &prices, + &images(vec![image(None, 1024, 1024), image(None, 1024, 1024)]), + &call(), + ), + 2.0 * expected, + ); + } + + #[rstest] + fn a_requested_size_with_its_own_price_bills_that_price() { + let size = crate::price::ImageSize::new("1K").unwrap(); + let prices = row([ + standard(&Meter::Output(OutputUnit::Image(None)), 0.08), + standard(&Meter::Output(OutputUnit::Image(Some(size))), 0.03), + ]); + close( + total( + &prices, + &images(vec![image(Some("1K"), 1024, 1024)]), + &call(), + ), + 0.03, + ); + } + + #[rstest] + fn output_pixels_bill_per_started_megapixel() { + let prices = row([standard(&Meter::Output(OutputUnit::Pixel), 1e-8)]); + let outcome = priced(calculate( + &prices, + &Adjustments::default(), + &images(vec![image(None, 1500, 1000)]), + &call(), + )); + assert_eq!(outcome.lines[0].quantity, Quantity::Count(2 * MEGAPIXEL)); + } + + fn queries(count: u64, max_results: Option) -> Usage { + Usage { + search: crate::usage::Search { + queries: count, + max_results, + ..crate::usage::Search::default() + }, + ..Usage::default() + } + } + + #[rstest] + #[case::first_tier(Some(10), 0.005)] + #[case::second_tier(Some(50), 0.025)] + #[case::past_the_last_range(Some(500), 0.025)] + #[case::default_max_results(None, 0.005)] + fn a_query_bills_the_tier_its_max_results_falls_in( + #[case] max_results: Option, + #[case] rate: f64, + ) { + let query = |over: u64| { + Meter::Input(InputUnit::Query { + over_results: NonZeroU64::new(over), + }) + }; + let prices = row([standard(&query(0), 0.005), standard(&query(25), 0.025)]); + close( + total(&prices, &queries(2, max_results), &call()), + 2.0 * rate, + ); + } + + fn web_searches(requests: u64) -> Usage { + Usage { + search: crate::usage::Search { + web_search: Some(crate::usage::WebSearch { + requests, + context: SearchContextSize::Medium, + }), + ..crate::usage::Search::default() + }, + ..Usage::default() + } + } + + fn billed(rates: impl IntoIterator, billing: Billing) -> Prices { + Prices::new(rates.into_iter().collect(), None, billing) + } + + #[rstest] + #[case::per_prompt(WebSearchUnit::PerPrompt, 0.01)] + #[case::per_query(WebSearchUnit::PerQuery, 0.03)] + fn web_search_bills_once_per_prompt_unless_the_row_bills_per_query( + #[case] web_search: WebSearchUnit, + #[case] expected: f64, + ) { + let prices = billed( + [standard(&Meter::WebSearch(SearchContextSize::Medium), 0.01)], + Billing { + web_search, + ..Billing::default() + }, + ); + close(total(&prices, &web_searches(3), &call()), expected); + } + + #[rstest] + #[case::google_row(SearchDefaults::Google, Some(0.035))] + #[case::other_row(SearchDefaults::None, None)] + fn a_google_row_without_a_web_search_price_bills_geminis_default( + #[case] search_defaults: SearchDefaults, + #[case] expected: Option, + ) { + let prices = billed( + [standard(&TEXT_INPUT, 1e-6)], + Billing { + search_defaults, + web_search: WebSearchUnit::PerPrompt, + ..Billing::default() + }, + ); + let outcome = calculate(&prices, &Adjustments::default(), &web_searches(3), &call()); + let web_search_total = match outcome { + Outcome::Priced(priced) => Some(priced.total.get()), + Outcome::Unpriced { .. } => None, + }; + assert_eq!(web_search_total, expected); + } + + #[rstest] + #[case::fast(Some(crate::call::Speed::Fast), None, 2.0)] + #[case::geo(None, Some("US"), 1.1)] + #[case::neither(None, None, 1.0)] + fn speed_and_geo_uplifts_multiply_every_line( + #[case] speed: Option, + #[case] geo: Option<&str>, + #[case] factor: f64, + ) { + let prices = row([standard(&TEXT_INPUT, 1e-6), standard(&TEXT_OUTPUT, 2e-6)]); + let adjustments = Adjustments::new([ + ( + Uplift::Speed(crate::call::Speed::Fast), + Multiplier::new(2.0).unwrap(), + ), + ( + Uplift::InferenceGeo(crate::call::InferenceGeo::new("us")), + Multiplier::new(1.1).unwrap(), + ), + ]); + let call = CallContext { + speed, + inference_geo: geo.map(crate::call::InferenceGeo::new), + ..call() + }; + let outcome = priced(calculate(&prices, &adjustments, &usage(100, 0, 10), &call)); + close(outcome.total.get(), (100.0 * 1e-6 + 10.0 * 2e-6) * factor); + } } diff --git a/litellm-rust/crates/cost/src/price.rs b/litellm-rust/crates/cost/src/price.rs index 50a485127ae..afe6e0c82c9 100644 --- a/litellm-rust/crates/cost/src/price.rs +++ b/litellm-rust/crates/cost/src/price.rs @@ -65,8 +65,11 @@ pub struct Billing { pub cache_read_fallback: CacheReadFallback, /// Whether a row with no token price is priced by wall clock, from its `mode` pub wall_clock: WallClock, - /// From `web_search_billing_unit`, whether web search and Maps grounding bill per query or once per prompt + /// From `web_search_billing_unit`, whether web search and Maps grounding bill per query or once per prompt. A + /// Google row without one bills per prompt, Gemini's default, and every other row per query pub web_search: WebSearchUnit, + /// Prices a Google row bills web search and Maps grounding at when it sets none, Gemini's defaults on main + pub search_defaults: SearchDefaults, /// Whether `tiered_pricing` bills each meter in its own tier or the whole call at the tier the input picks. /// A prompt-token tier table bills the whole call, main's `_get_tiered_base_costs` pub tier_scope: TierScope, @@ -105,11 +108,18 @@ pub enum WallClock { #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub enum WebSearchUnit { - PerQuery, #[default] + PerQuery, PerPrompt, } +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum SearchDefaults { + #[default] + None, + Google, +} + #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub enum TierScope { #[default] diff --git a/litellm-rust/crates/model-catalog/src/lower.rs b/litellm-rust/crates/model-catalog/src/lower.rs index 379e5978c15..4cd0772b6b7 100644 --- a/litellm-rust/crates/model-catalog/src/lower.rs +++ b/litellm-rust/crates/model-catalog/src/lower.rs @@ -2,11 +2,13 @@ use std::collections::BTreeMap; use std::collections::btree_map::Entry; use std::num::NonZeroU64; -use litellm_cost::money::UnitPrice; +use litellm_cost::call::{InferenceGeo, Speed}; +use litellm_cost::money::{Multiplier, UnitPrice}; use litellm_cost::price::{ Adjustments, Band, BandEdge, Billing, CacheReadFallback, CacheReadMode, CacheTtl, GuardrailUnit, InputUnit, Meter, OffPeak, OffPeakRates, OutputUnit, Price, Prices, Residency, - SearchContextSize, TextUnit, TierScope, TokenKind, Uplift, WallClock, WebSearchUnit, When, + SearchContextSize, SearchDefaults, TextUnit, TierScope, TokenKind, Uplift, WallClock, + WebSearchUnit, When, }; use litellm_cost::schedule::{Schedule, Window}; @@ -92,6 +94,10 @@ impl TryFrom<&EntryWire> for Prices { fn billing(wire: &EntryWire, rates: &BTreeMap) -> Billing { let provider = wire.attributes.litellm_provider.as_deref(); + let google = provider.is_some_and(|provider| { + provider == "gemini" + || (provider.starts_with("vertex_ai") && !provider.contains("anthropic")) + }); let prices_meter = |priced: fn(&Meter) -> bool| rates.keys().any(|price| priced(&price.meter)); let prices_tokens = prices_meter(|meter| { matches!( @@ -137,9 +143,14 @@ fn billing(wire: &EntryWire, rates: &BTreeMap) -> Billing { Some(Mode::AudioTranscription) => WallClock::AudioFile, Some(_) => WallClock::Never, }, - web_search: match wire.attributes.web_search_billing_unit { - Some(WebSearchBillingUnit::PerQuery) => WebSearchUnit::PerQuery, - Some(WebSearchBillingUnit::PerPrompt) | None => WebSearchUnit::PerPrompt, + web_search: match (wire.attributes.web_search_billing_unit, google) { + (Some(WebSearchBillingUnit::PerQuery), _) | (None, false) => WebSearchUnit::PerQuery, + (Some(WebSearchBillingUnit::PerPrompt), _) | (None, true) => WebSearchUnit::PerPrompt, + }, + search_defaults: if google { + SearchDefaults::Google + } else { + SearchDefaults::None }, tier_scope: if wire .tier_rows @@ -155,7 +166,21 @@ fn billing(wire: &EntryWire, rates: &BTreeMap) -> Billing { } impl From<&EntryWire> for Adjustments { + /// `provider_specific_entry.fast` is Anthropic's speed uplift and every other number in it a geo uplift, as + /// main's `get_provider_specific_speed_multiplier` and `get_provider_specific_geo_multiplier` read it fn from(wire: &EntryWire) -> Self { + let provider_specific = wire + .attributes + .provider_specific_entry + .iter() + .flatten() + .filter_map(|(key, value)| { + let uplift = match key.as_str() { + "fast" => Uplift::Speed(Speed::Fast), + geo => Uplift::InferenceGeo(InferenceGeo::new(geo)), + }; + Some((uplift, Multiplier::new(value.as_f64()?)?)) + }); Adjustments::new( [ (Uplift::NonGlobalEndpoint, wire.regional_endpoint_uplift), @@ -163,7 +188,8 @@ impl From<&EntryWire> for Adjustments { (Uplift::Residency(Residency::Us), wire.residency_uplift_us), ] .into_iter() - .filter_map(|(uplift, factor)| Some((uplift, factor?))), + .filter_map(|(uplift, factor)| Some((uplift, factor?))) + .chain(provider_specific), ) } } @@ -929,11 +955,16 @@ mod tests { #[rstest] #[case::tokens_win_over_characters( json!({"litellm_provider": "vertex_ai-language-models", "input_cost_per_character": 1e-6, "input_cost_per_token": 1e-6}), - Billing::default() + Billing { web_search: WebSearchUnit::PerPrompt, search_defaults: SearchDefaults::Google, ..Billing::default() } )] #[case::characters_only( json!({"litellm_provider": "vertex_ai-language-models", "input_cost_per_character": 1e-6}), - Billing { text_unit: TextUnit::Character, ..Billing::default() } + Billing { + text_unit: TextUnit::Character, + web_search: WebSearchUnit::PerPrompt, + search_defaults: SearchDefaults::Google, + ..Billing::default() + } )] #[case::xai_band_is_inclusive( json!({"litellm_provider": "xai", "input_cost_per_token": 1e-6}), @@ -961,7 +992,15 @@ mod tests { )] #[case::gemini_per_query_search( json!({"litellm_provider": "gemini", "web_search_billing_unit": "per_query"}), - Billing { web_search: WebSearchUnit::PerQuery, ..Billing::default() } + Billing { web_search: WebSearchUnit::PerQuery, search_defaults: SearchDefaults::Google, ..Billing::default() } + )] + #[case::vertex_rows_take_googles_search_defaults( + json!({"litellm_provider": "vertex_ai-language-models"}), + Billing { web_search: WebSearchUnit::PerPrompt, search_defaults: SearchDefaults::Google, ..Billing::default() } + )] + #[case::vertex_claude_rows_bill_search_like_anthropic( + json!({"litellm_provider": "vertex_ai-anthropic_models"}), + Billing::default() )] #[case::token_tiers_bill_the_whole_call( json!({"litellm_provider": "openai", "tiered_pricing": [{"range": [0, 32000], "input_cost_per_token": 1e-6}]}), @@ -977,4 +1016,26 @@ mod tests { ) { assert_eq!(prices(entry).unwrap().billing(), expected); } + + #[rstest] + fn provider_specific_entry_lowers_to_speed_and_geo_uplifts() { + let wire = EntryWire::read( + json!({"litellm_provider": "anthropic", "provider_specific_entry": {"fast": 6.0, "us": 1.1}}) + .as_object() + .unwrap(), + ); + let adjustments = Adjustments::from(&wire); + assert_eq!( + adjustments + .get(&Uplift::Speed(Speed::Fast)) + .map(Multiplier::get), + Some(6.0) + ); + assert_eq!( + adjustments + .get(&Uplift::InferenceGeo(InferenceGeo::new("US"))) + .map(Multiplier::get), + Some(1.1) + ); + } } diff --git a/litellm-rust/crates/model-catalog/src/resolve/image.rs b/litellm-rust/crates/model-catalog/src/resolve/image.rs index d24e170b32c..d2686a6897b 100644 --- a/litellm-rust/crates/model-catalog/src/resolve/image.rs +++ b/litellm-rust/crates/model-catalog/src/resolve/image.rs @@ -16,3 +16,161 @@ pub struct ImageKey { pub height: u32, pub model: String, } + +impl ImageCandidates { + /// main's `default_image_cost_calculator` key spellings, in its order + pub fn spellings(&self) -> Vec { + let (first, second) = self.size.unwrap_or((1024, 1024)); + let size = format!("{first}-x-{second}"); + let provider_prefix = self + .provider + .as_deref() + .map(|provider| format!("{provider}/")); + let without_provider = provider_prefix + .as_deref() + .and_then(|prefix| self.model.strip_prefix(prefix)) + .map(|rest| rest.replace(provider_prefix.as_deref().unwrap_or_default(), "")); + let base = match (&self.provider, &without_provider) { + (Some(provider), Some(rest)) => format!("{provider}/{size}/{rest}"), + _ => format!("{size}/{}", self.model), + }; + let quality = self + .quality + .as_deref() + .filter(|quality| !quality.is_empty()); + let with_quality = |name: &str| { + quality.map_or_else(|| name.to_owned(), |quality| format!("{quality}/{name}")) + }; + let provider_first = quality + .zip(self.provider.as_deref()) + .map(|(quality, provider)| { + format!( + "{provider}/{quality}/{size}/{}", + without_provider.as_deref().unwrap_or(&self.model) + ) + }); + let last_segment = self.model.rsplit('/').next().unwrap_or(&self.model); + let model_without_provider = format!("{size}/{last_segment}"); + [ + Some(with_quality(&base)), + provider_first, + Some(base.clone()), + Some(format!("high/{base}")), + Some(with_quality(&model_without_provider)), + Some(model_without_provider.clone()), + Some(self.model.clone()), + without_provider, + ] + .into_iter() + .flatten() + .collect() + } +} + +impl ImageKey { + /// A `{provider}/{quality}/{W}-x-{H}/{model}` key, `None` for any other shape + pub fn parse(key: &str, provider: &str) -> Option { + let rest = key.strip_prefix(provider)?.strip_prefix('/')?; + let (quality, rest) = rest.split_once('/')?; + let (size, model) = rest.split_once('/')?; + let (width, height) = size.split_once("-x-")?; + let width: u32 = width.parse().ok().filter(|width| *width > 0)?; + let height: u32 = height.parse().ok().filter(|height| *height > 0)?; + Some(Self { + quality: Some(quality.to_owned()), + width, + height, + model: model.to_owned(), + }) + } + + fn pixels(&self) -> u64 { + u64::from(self.width) * u64::from(self.height) + } +} + +/// fal's nearest size match. Of the `{provider}/{quality}/{W}-x-{H}/{model}` keys the row filter keeps, the one whose +/// pixel count is closest to the target's, the smaller one on a tie +pub fn nearest_image_key<'a>( + keys: impl IntoIterator, + provider: &str, + model: &str, + quality: &str, + target: (u32, u32), +) -> Option<&'a str> { + let target_pixels = u64::from(target.0) * u64::from(target.1); + keys.into_iter() + .filter_map(|key| Some((key, ImageKey::parse(key, provider)?))) + .filter(|(_, parsed)| parsed.model == model && parsed.quality.as_deref() == Some(quality)) + .min_by_key(|(_, parsed)| (parsed.pixels().abs_diff(target_pixels), parsed.pixels())) + .map(|(key, _)| key) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[rstest] + fn the_spellings_follow_mains_order() { + let candidates = ImageCandidates { + model: "azure/dall-e-3".into(), + provider: Some("azure".into()), + size: Some((1024, 1792)), + quality: Some("hd".into()), + }; + assert_eq!( + candidates.spellings(), + [ + "hd/azure/1024-x-1792/dall-e-3", + "azure/hd/1024-x-1792/dall-e-3", + "azure/1024-x-1792/dall-e-3", + "high/azure/1024-x-1792/dall-e-3", + "hd/1024-x-1792/dall-e-3", + "1024-x-1792/dall-e-3", + "azure/dall-e-3", + "dall-e-3", + ] + ); + } + + #[rstest] + fn without_a_provider_prefix_or_quality_the_spellings_are_the_bare_ones() { + let candidates = ImageCandidates { + model: "dall-e-2".into(), + provider: Some("openai".into()), + size: None, + quality: None, + }; + assert_eq!( + candidates.spellings(), + [ + "1024-x-1024/dall-e-2", + "1024-x-1024/dall-e-2", + "high/1024-x-1024/dall-e-2", + "1024-x-1024/dall-e-2", + "1024-x-1024/dall-e-2", + "dall-e-2", + ] + ); + } + + #[rstest] + #[case::exact((1024, 1024), "fal_ai/high/1024-x-1024/flux")] + #[case::nearest((1100, 1100), "fal_ai/high/1024-x-1024/flux")] + #[case::tie_takes_the_smaller((768, 768), "fal_ai/high/512-x-512/flux")] + fn the_nearest_size_wins(#[case] target: (u32, u32), #[case] expected: &str) { + let keys = [ + "fal_ai/high/512-x-512/flux", + "fal_ai/high/1024-x-1024/flux", + "fal_ai/low/1024-x-1024/flux", + "fal_ai/high/1024-x-1024/other", + "fal_ai/flux", + ]; + assert_eq!( + nearest_image_key(keys, "fal_ai", "flux", "high", target), + Some(expected) + ); + } +} diff --git a/litellm-rust/crates/python-bridge/src/cost.rs b/litellm-rust/crates/python-bridge/src/cost.rs index 93343c0210d..554cb56a8bb 100644 --- a/litellm-rust/crates/python-bridge/src/cost.rs +++ b/litellm-rust/crates/python-bridge/src/cost.rs @@ -4,12 +4,17 @@ use std::sync::Arc; use std::time::Duration; use litellm_cost::call::CallContext as CostCallContext; +use litellm_cost::call::{InferenceGeo, Speed}; use litellm_cost::price::{ - Adjustments, CacheReadMode, CacheTtl, Channel, InputUnit, Meter, OutputUnit, - Prices as CostPrices, Residency, Resolution, ServiceTier, TextUnit, TokenKind, + Adjustments, CacheReadMode, CacheTtl, Channel, Feature, ImageSize, InputUnit, Meter, + OutputUnit, Prices as CostPrices, Residency, Resolution, SearchContextSize, ServiceTier, + TextUnit, TokenKind, }; use litellm_cost::priced::{Outcome, Quantity, Side}; -use litellm_cost::usage::{Characters, DurationMeter, Usage as CostUsage}; +use litellm_cost::usage::{ + Characters, DurationMeter, ImageOut as CostImageOut, Images, Search, Usage as CostUsage, + WebSearch, +}; use litellm_host_python::from_py; use litellm_model_catalog::rejects_row; use litellm_model_catalog::wire::{EntryWire, Fields, FlatKey}; @@ -190,7 +195,9 @@ impl Usage { cache_read_explicit = 0, cache_write_5m = 0, cache_write_1h = 0, cache_write_12h = 0, output_audio = 0, output_image = 0, output_video = 0, reasoning = 0, prompt_characters = 0, completion_characters = 0, wall_clock = None, input_audio = None, live_voice = None, output_video_duration = None, - video_resolution = None, text_messages = 0, + video_resolution = None, text_messages = 0, images_in = 0, images_out = Vec::new(), pixels_in = 0, + search_queries = 0, max_results = None, web_search_requests = 0, web_search_context = "medium", + maps_grounding_requests = 0, citation_tokens = 0, browser_opens = 0, additional_results = 0, ))] #[expect( clippy::too_many_arguments, @@ -221,7 +228,18 @@ impl Usage { output_video_duration: Option, video_resolution: Option<&str>, text_messages: u64, - ) -> Self { + images_in: u64, + images_out: Vec>, + pixels_in: u64, + search_queries: u64, + max_results: Option, + web_search_requests: u64, + web_search_context: &str, + maps_grounding_requests: u64, + citation_tokens: u64, + browser_opens: u64, + additional_results: u64, + ) -> PyResult { let mut usage = CostUsage { prompt_tokens, ..CostUsage::default() @@ -276,7 +294,34 @@ impl Usage { .filter_map(|(meter, duration)| Some((meter, duration?))) .collect(); usage.text_messages = text_messages; - Self(usage) + usage.tokens.citation = citation_tokens; + usage.images = Images { + input: images_in, + input_pixels: pixels_in, + output: images_out.iter().map(|image| image.0.clone()).collect(), + }; + let context = match web_search_context { + "low" => SearchContextSize::Low, + "medium" => SearchContextSize::Medium, + "high" => SearchContextSize::High, + other => { + return Err(PyValueError::new_err(format!( + "unknown search context size {other:?}" + ))); + } + }; + usage.search = Search { + queries: search_queries, + max_results, + web_search: (web_search_requests > 0).then_some(WebSearch { + requests: web_search_requests, + context, + }), + maps_grounding: maps_grounding_requests, + browser_opens, + additional_results, + }; + Ok(Self(usage)) } /// The provider-reported prompt size, which picks the band @@ -286,10 +331,34 @@ impl Usage { } } +/// One returned image. `size` picks a size-keyed price, `width` and `height` the per-pixel ones +#[pyclass(frozen, module = "litellm.rust_bridge._native")] +pub(crate) struct ImageOut(CostImageOut); + +#[pymethods] +impl ImageOut { + #[new] + #[pyo3(signature = (*, size = None, width = None, height = None))] + fn new(size: Option<&str>, width: Option, height: Option) -> Self { + Self(CostImageOut { + size: size.and_then(ImageSize::new), + width, + height, + }) + } +} + #[pymethods] impl CallContext { #[new] - #[pyo3(signature = (*, service_tier, channel, data_residency, vertex_location, at, custom_pricing))] + #[pyo3(signature = ( + *, service_tier, channel, data_residency, vertex_location, at, custom_pricing, speed = None, + inference_geo = None, + ))] + #[expect( + clippy::too_many_arguments, + reason = "one keyword argument per call fact, the Python signature" + )] fn new( service_tier: Option<&str>, channel: &str, @@ -297,6 +366,8 @@ impl CallContext { vertex_location: Option<&str>, at: &Bound<'_, PyAny>, custom_pricing: Option<&Bound<'_, PyDict>>, + speed: Option<&str>, + inference_geo: Option<&str>, ) -> PyResult { let channel = match channel { "online" => Channel::Online, @@ -328,8 +399,12 @@ impl CallContext { residency: data_residency.and_then(residency), non_global_endpoint: vertex_location .is_some_and(|location| !location.eq_ignore_ascii_case("global")), - speed: None, - inference_geo: None, + speed: (speed == Some("fast")).then_some(Speed::Fast), + inference_geo: inference_geo + .filter(|geo| { + !matches!(geo.to_lowercase().as_str(), "global" | "not_available") + }) + .map(InferenceGeo::new), at: timestamp(at)?, overlay, }, @@ -518,6 +593,28 @@ fn meter_name(meter: &Meter) -> String { Meter::Input(InputUnit::AudioSecond) => "input_cost_per_audio_per_second".to_owned(), Meter::Output(OutputUnit::VideoSecond) => "output_cost_per_video_per_second".to_owned(), Meter::Input(InputUnit::TextMessage) => "input_cost_per_text_message".to_owned(), + Meter::Input(InputUnit::Image) => "input_cost_per_image".to_owned(), + Meter::Output(OutputUnit::Image(None)) => "output_cost_per_image".to_owned(), + Meter::Output(OutputUnit::Image(Some(size))) => { + format!("output_cost_per_image_{}", size.as_str()) + } + Meter::Input(InputUnit::Pixel) => "input_cost_per_pixel".to_owned(), + Meter::Output(OutputUnit::Pixel) => "output_cost_per_pixel".to_owned(), + Meter::Input(InputUnit::Query { .. }) => "input_cost_per_query".to_owned(), + Meter::WebSearch(context) => format!( + "search_context_cost_per_query.search_context_size_{}", + match context { + SearchContextSize::Low => "low", + SearchContextSize::Medium => "medium", + SearchContextSize::High => "high", + } + ), + Meter::Feature(Feature::GoogleMapsGroundingQuery) => { + "google_maps_grounding_cost_per_query".to_owned() + } + Meter::Feature(Feature::CitationToken) => "citation_cost_per_token".to_owned(), + Meter::Feature(Feature::BrowserOpen) => "browser_open_cost_per_request".to_owned(), + Meter::Feature(Feature::AdditionalResult) => "cost_per_additional_result".to_owned(), other => format!("{other:?}"), } } diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index 1c7f69ad25d..69b5415c6e5 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -26,7 +26,7 @@ mod _native { }; #[pymodule_export] - use crate::cost::{CallContext, Line, Priced, PricesHandle, Unpriced, Usage}; + use crate::cost::{CallContext, ImageOut, Line, Priced, PricesHandle, Unpriced, Usage}; #[cfg(feature = "panic-test")] #[pymodule_export] use crate::diagnostics::_panic_for_test; diff --git a/litellm-rust/crates/python-bridge/src/model_registry.rs b/litellm-rust/crates/python-bridge/src/model_registry.rs index 6a7ec87a7ae..778afd17fe4 100644 --- a/litellm-rust/crates/python-bridge/src/model_registry.rs +++ b/litellm-rust/crates/python-bridge/src/model_registry.rs @@ -245,6 +245,69 @@ impl CatalogHandle { }) } + /// The row for an image call, through main's size and quality key spellings + #[pyo3(signature = (model, provider = None, *, size = None, quality = None))] + fn resolve_image( + &self, + py: Python<'_>, + model: &str, + provider: Option<&str>, + size: Option<(u32, u32)>, + quality: Option<&str>, + ) -> PyResult> { + let dict = self.bound_dict(py)?; + let keys = resolve::LiveKeys { + dict: &dict, + index: &self.index, + }; + let candidates = litellm_model_catalog::ImageCandidates { + model: model.to_owned(), + provider: provider.map(str::to_owned), + size, + quality: quality.map(str::to_owned), + }; + resolve::resolve_image(py, &keys, &candidates) + } + + /// fal's nearest size row, `{provider}/{quality}/{W}-x-{H}/{model}` with an `output_cost_per_image` price, from the + /// provider's keys in the index + #[pyo3(signature = (model, provider, *, quality, size))] + fn nearest_image_key( + &self, + py: Python<'_>, + model: &str, + provider: &str, + quality: &str, + size: (u32, u32), + ) -> PyResult> { + let dict = self.bound_dict(py)?; + let keys = resolve::LiveKeys { + dict: &dict, + index: &self.index, + }; + let priced: Vec = keys + .provider_keys(provider) + .into_iter() + .filter(|key| { + keys.fields_of(key) + .and_then(|fields| { + fields + .get("output_cost_per_image") + .map(serde_json::Value::is_number) + }) + .unwrap_or(false) + }) + .collect(); + Ok(litellm_model_catalog::nearest_image_key( + priced.iter().map(String::as_str), + provider, + model, + quality, + size, + ) + .map(str::to_owned)) + } + /// The keys whose row names this `litellm_provider`, from the index fn provider_keys(&self, py: Python<'_>, provider: &str) -> PyResult> { let dict = self.bound_dict(py)?; diff --git a/litellm-rust/crates/python-bridge/src/model_registry/resolve.rs b/litellm-rust/crates/python-bridge/src/model_registry/resolve.rs index e2313679150..e86b87c3877 100644 --- a/litellm-rust/crates/python-bridge/src/model_registry/resolve.rs +++ b/litellm-rust/crates/python-bridge/src/model_registry/resolve.rs @@ -4,7 +4,7 @@ use std::sync::{Arc, Mutex, PoisonError}; use litellm_host_python::{from_py, to_py}; use litellm_model_catalog::wire::Fields; use litellm_model_catalog::{ - Candidates, Catalog, Keys, Resolution, RowRef, model_info, rejects_row, + Candidates, Catalog, ImageCandidates, Keys, Resolution, RowRef, model_info, rejects_row, }; use pyo3::prelude::*; use pyo3::types::{PyDict, PyTuple}; @@ -167,34 +167,50 @@ pub(crate) fn resolve( Resolution::Missing { tried } => { Ok(Missing { tried }.into_pyobject(py)?.into_any().unbind()) } - Resolution::Found { key, .. } => { - let (fields, converted) = keys.fields(&key).unwrap_or_default(); - let errors: Vec = if converted { - Catalog::row_errors(&key, &fields) - .iter() - .filter(|error| rejects_row(&error.error)) - .map(ToString::to_string) - .collect() - } else { - vec![format!("{key} must contain JSON-compatible values")] - }; - let fields = Arc::new(fields); - if errors.is_empty() { - Ok(Resolved { key, fields } - .into_pyobject(py)? - .into_any() - .unbind()) - } else { - Ok(Invalid { - key, - errors, - fields, - } - .into_pyobject(py)? - .into_any() - .unbind()) - } + Resolution::Found { key, .. } => found(py, keys, key), + } +} + +/// The row at `key` as `Resolved`, or `Invalid` when it fails validation +pub(crate) fn found(py: Python<'_>, keys: &LiveKeys<'_, '_>, key: String) -> PyResult> { + let (fields, converted) = keys.fields(&key).unwrap_or_default(); + let errors: Vec = if converted { + Catalog::row_errors(&key, &fields) + .iter() + .filter(|error| rejects_row(&error.error)) + .map(ToString::to_string) + .collect() + } else { + vec![format!("{key} must contain JSON-compatible values")] + }; + let fields = Arc::new(fields); + if errors.is_empty() { + Ok(Resolved { key, fields } + .into_pyobject(py)? + .into_any() + .unbind()) + } else { + Ok(Invalid { + key, + errors, + fields, } + .into_pyobject(py)? + .into_any() + .unbind()) + } +} + +/// main's `default_image_cost_calculator` lookup, the first of its eight key spellings in the live dict +pub(crate) fn resolve_image( + py: Python<'_>, + keys: &LiveKeys<'_, '_>, + candidates: &ImageCandidates, +) -> PyResult> { + let tried = candidates.spellings(); + match tried.iter().find(|key| keys.contains(key)) { + Some(key) => found(py, keys, key.clone()), + None => Ok(Missing { tried }.into_pyobject(py)?.into_any().unbind()), } } diff --git a/litellm/_cost/__init__.py b/litellm/_cost/__init__.py index 25e5dccbd7a..7615392ba22 100644 --- a/litellm/_cost/__init__.py +++ b/litellm/_cost/__init__.py @@ -6,7 +6,7 @@ from contextvars import ContextVar from typing import Final, Literal from litellm._catalog import Invalid, report_invalid -from litellm.rust_bridge._native import CallContext, Line, Priced, Prices, Unpriced, Usage +from litellm.rust_bridge._native import CallContext, ImageOut, Line, Priced, Prices, Unpriced, Usage Channel = Literal["online", "batch", "decisions"] @@ -45,6 +45,7 @@ def as_main_cost(outcome: Priced | Unpriced | Invalid) -> tuple[float, float]: __all__ = [ "CallContext", "Channel", + "ImageOut", "Line", "Priced", "Prices", diff --git a/litellm/constants.py b/litellm/constants.py index e8bba6dd762..aba2c63a134 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -693,7 +693,6 @@ FIREWORKS_AI_16_B: Final = get_env_int("FIREWORKS_AI_16_B", 16) FIREWORKS_AI_80_B: Final = get_env_int("FIREWORKS_AI_80_B", 80) # https://docs.fireworks.ai/guides/prompt-caching (accessed 2026-09-19): serverless cached prompt tokens # default to a 50% discount off the input rate -FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO: Final = 0.5 #### Logging callback constants #### REDACTED_BY_LITELM_STRING: Final = "REDACTED_BY_LITELM" MAX_LANGFUSE_INITIALIZED_CLIENTS: Final = get_env_int("MAX_LANGFUSE_INITIALIZED_CLIENTS", 50) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 5f1b7cf8c4b..e10d58bcdb5 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -16,6 +16,7 @@ from typing_extensions import ReadOnly, TypedDict, assert_never import litellm import litellm._logging from litellm import verbose_logger +from litellm._catalog import CATALOG, Missing from litellm._cost import Priced, as_main_cost, call_custom_pricing, custom_pricing_for_call from litellm._cost import Usage as TokenUsage from litellm.constants import ( @@ -43,13 +44,13 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import ( calculate_cache_writing_cost, calculate_cost_component, cost_per_text, - generic_cost_per_token, get_batch_cost_rates, get_billable_input_tokens, get_regional_uplift_multiplier, get_service_tier_cost_key, get_token_type_cost_breakdown, get_vertex_regional_endpoint_uplift, + image_cost, parse_prompt_tokens_details, price_usage, pricing_row, @@ -111,7 +112,6 @@ from litellm.types.extraction import ExtractionResponse, ExtractionUsage from litellm.types.llms.base import CachedTokensDetails, LiteLLMBaseModel from litellm.types.llms.openai import ( HttpxBinaryResponseContent, - ImageGenerationRequestQuality, OpenAIModerationResponse, OpenAIRealtimeStreamList, OpenAIRealtimeStreamResponseBaseObject, @@ -436,7 +436,7 @@ def cost_per_token( if custom_cost_per_token is not None: with custom_pricing_for_call(dict(custom_cost_per_token)): - return generic_cost_per_token( + return cost_per_text( model=model, usage=_flat_custom_pricing_usage( usage_object, prompt_tokens, completion_tokens, cache_read_input_tokens, cache_creation_input_tokens @@ -514,7 +514,7 @@ def cost_per_token( litellm.model_cost.get(decisions_model_info["key"], {}) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # model cost map is dynamically loaded ) if decisions_overrides: - return generic_cost_per_token( + return cost_per_text( model=model, usage=usage_block, custom_llm_provider=custom_llm_provider, @@ -580,7 +580,7 @@ def cost_per_token( ) elif call_type == "atranscription" or call_type == "transcription": if _transcription_usage_has_token_details(usage_block): - return generic_cost_per_token( + return cost_per_text( model=model_without_prefix, usage=usage_block, custom_llm_provider=custom_llm_provider, @@ -613,7 +613,7 @@ def cost_per_token( ) is not None: return per_second_cost elif call_custom_pricing() is not None and _cost_map_model_info(model, custom_llm_provider) is None: - return generic_cost_per_token( + return cost_per_text( model=model, usage=usage_block, custom_llm_provider=custom_llm_provider, @@ -646,7 +646,7 @@ def cost_per_token( elif custom_llm_provider == "bedrock": resolved_bedrock_info: Final = bedrock_model_info or _bedrock_pricing_info(model, None, "bedrock", False) if resolved_bedrock_info is not None: - return generic_cost_per_token( + return cost_per_text( model=model, usage=usage_block, custom_llm_provider="bedrock", @@ -699,7 +699,7 @@ def cost_per_token( service_tier=service_tier, ) else: - return generic_cost_per_token( + return cost_per_text( model=model, usage=usage_block, custom_llm_provider=custom_llm_provider, @@ -2259,15 +2259,15 @@ def _video_input_image_cost( ) -> float: match usage: case {"billable_input_image_count": int(billable_input_images)} if billable_input_images > 0: - deployment_rate: Final = model_info.get("input_cost_per_image") if model_info is not None else None - image_rate: Final = ( - deployment_rate - if deployment_rate is not None - else litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider).get( - "input_cost_per_image" - ) + row: Final = ( + model_info + if model_info is not None and model_info.get("input_cost_per_image") is not None + else pricing_row(model, custom_llm_provider, None) ) - return billable_input_images * image_rate if isinstance(image_rate, (int, float)) else 0.0 + prompt_cost, completion_cost = as_main_cost( + price_usage(row, TokenUsage(prompt_tokens=0, text=0, output_text=0, images_in=billable_input_images)) + ) + return prompt_cost + completion_cost case _: return 0.0 @@ -2682,57 +2682,19 @@ def default_image_cost_calculator( # Parse dimensions height, width = map(int, size_str.split("-x-")) - # Build model names for cost lookup - base_model_name = f"{size_str}/{model}" - model_name_without_custom_llm_provider: str | None = None - if custom_llm_provider and model.startswith(f"{custom_llm_provider}/"): - model_name_without_custom_llm_provider = model.replace(f"{custom_llm_provider}/", "") - base_model_name = f"{custom_llm_provider}/{size_str}/{model_name_without_custom_llm_provider}" - model_name_with_quality: Final = f"{quality}/{base_model_name}" if quality else base_model_name - provider_first_model_name_with_quality: Final = ( - f"{custom_llm_provider}/{quality}/{size_str}/{model_name_without_custom_llm_provider or model}" - if quality and custom_llm_provider - else None - ) - - # gpt-image-1 models use low, medium, high quality. If user did not specify quality, use medium fot gpt-image-1 model family - model_name_with_v2_quality: Final = f"{ImageGenerationRequestQuality.HIGH.value}/{base_model_name}" - - verbose_logger.debug("Looking up cost for models: %s, %s", model_name_with_quality, base_model_name) - - model_without_provider: Final = f"{size_str}/{model.split('/')[-1]}" - model_with_quality_without_provider = f"{quality}/{model_without_provider}" if quality else model_without_provider - - models_to_check: Final = ( - model_name_with_quality, - provider_first_model_name_with_quality, - base_model_name, - model_name_with_v2_quality, - model_with_quality_without_provider, - model_without_provider, - model, - model_name_without_custom_llm_provider, - ) - matched_model: Final = next( - (_model for _model in models_to_check if _model is not None and _model in litellm.model_cost), None - ) - if matched_model is None and model_info is None: + resolution: Final = CATALOG.resolve_image(model, custom_llm_provider, size=(height, width), quality=quality or None) + models_to_check: Final = tuple(resolution.tried) if isinstance(resolution, Missing) else (resolution.key,) + if isinstance(resolution, Missing) and model_info is None: raise Exception(f"Model not found in cost map. Tried checking {models_to_check}") - shared_cost_info: Final = litellm.model_cost[matched_model] if matched_model is not None else None + shared_cost_info: Final = None if isinstance(resolution, Missing) else litellm.model_cost[resolution.key] price_tables: Final = tuple(table for table in (model_info, shared_cost_info) if table is not None) image_count: Final = n if n is not None else 1 - unit_counts: Final = ( - ("input_cost_per_image", image_count), - ("output_cost_per_image", image_count), - ("input_cost_per_pixel", height * width * image_count), - ) cost: Final = next( ( - price * units + table_cost for price_table in price_tables - for cost_key, units in unit_counts - if isinstance(price := price_table.get(cost_key), (int, float)) + if (table_cost := image_cost(price_table, image_count, width=int(width), height=int(height))) is not None ), None, ) @@ -3382,7 +3344,7 @@ def _candidate_realtime_token_costs( data_residency: str | None, ) -> tuple[float, float] | None: try: - return generic_cost_per_token( + return cost_per_text( model=model_name, usage=combined_usage_object, custom_llm_provider=custom_llm_provider, diff --git a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py index 46d0ea1c6bf..817f3c9fce2 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py +++ b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py @@ -281,21 +281,33 @@ class StandardBuiltInToolCostTracking: tool_usage: Final = _responses_tool_usage(response_object) if tool_usage is None or tool_usage.image_gen is None or tool_usage.image_gen.total_tokens <= 0: return None + from litellm._cost import Usage as TokenUsage + from litellm._cost import as_main_cost + from litellm.litellm_core_utils.llm_cost_calc.utils import price_usage, pricing_row + try: - model_info: Final = litellm.get_model_info(model=tool_model, custom_llm_provider=custom_llm_provider) + row: Final = pricing_row(tool_model, custom_llm_provider, None) except Exception as e: # noqa: BLE001 # get_model_info raises bare Exception for unmapped models; fall back verbose_logger.debug("Could not resolve pricing for image tool model %s: %s", tool_model, e) return None image_gen: Final = tool_usage.image_gen input_details: Final = image_gen.input_tokens_details output_details: Final = image_gen.output_tokens_details - token_cost: Final = ( - (input_details.text_tokens if input_details else 0) * (model_info.get("input_cost_per_token") or 0) - + (input_details.image_tokens if input_details else 0) * (model_info.get("input_cost_per_image_token") or 0) - + (output_details.image_tokens if output_details else 0) - * (model_info.get("output_cost_per_image_token") or 0) - + (output_details.text_tokens if output_details else 0) * (model_info.get("output_cost_per_token") or 0) + text_in: Final = input_details.text_tokens if input_details else 0 + image_in: Final = input_details.image_tokens if input_details else 0 + prompt_cost, completion_cost = as_main_cost( + price_usage( + row, + TokenUsage( + prompt_tokens=text_in + image_in, + text=text_in, + image=image_in, + output_text=output_details.text_tokens if output_details else 0, + output_image=output_details.image_tokens if output_details else 0, + ), + ) ) + token_cost: Final = prompt_cost + completion_cost return token_cost if token_cost > 0 else None @staticmethod diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index be53ddae460..c1e61a8aef6 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -13,7 +13,16 @@ from typing_extensions import ReadOnly import litellm from litellm._catalog import Invalid -from litellm._cost import CallContext, Channel, Priced, Prices, Unpriced, as_main_cost, call_custom_pricing +from litellm._cost import ( + CallContext, + Channel, + ImageOut, + Priced, + Prices, + Unpriced, + as_main_cost, + call_custom_pricing, +) from litellm._cost import Usage as TokenUsage from litellm._internal_context import current_billing_time from litellm._logging import verbose_logger @@ -21,7 +30,6 @@ from litellm.litellm_core_utils.llm_cost_calc.tiered_pricing import ( select_tier_for_input, tier_rate, ) -from litellm.llms.fireworks_ai.cache_pricing import with_default_cache_read_rate from litellm.types.utils import ( CacheCreationTokenDetails, CallTypes, @@ -109,19 +117,10 @@ _BATCH_RATE_PREFIXES: Final = ( ) _BATCH_TIER_PREFIXES: Final = (*_BATCH_RATE_PREFIXES, *_TTL_PRICED_CACHE_WRITE_RATE_KEYS.values()) _BATCH_TIER_KEY: Final = re.compile(rf"^({'|'.join(_BATCH_TIER_PREFIXES)})_above_(\d+k?)_tokens{_BATCH_KEY_SUFFIX}$") -_NON_STANDARD_THRESHOLD_SUFFIXES: Final = (*_SERVICE_TIER_SUFFIXES, _BATCH_KEY_SUFFIX) - - def _uses_inclusive_token_thresholds(custom_llm_provider: str | None) -> bool: return custom_llm_provider in _INCLUSIVE_THRESHOLD_PROVIDERS -def apply_provider_cache_read_default(model_info: ModelInfo, custom_llm_provider: str | None) -> ModelInfo: - if custom_llm_provider == "fireworks_ai": - return with_default_cache_read_rate(model_info) - return model_info - - def _get_token_detail_value(details: object, key: str) -> int | None: if isinstance(details, dict): value = details.get(key) @@ -173,15 +172,6 @@ def get_web_search_requests_from_usage(usage: Usage) -> int | None: return get_web_search_requests(getattr(usage, "server_tool_use", None)) -def is_above_128k(tokens: float) -> bool: - if tokens > 128000: - return True - return False - - -_is_above_128k = is_above_128k - - def get_billable_input_tokens(usage: Usage) -> int: """ Returns the number of billable input tokens. @@ -219,10 +209,6 @@ def _parse_token_threshold(threshold: str) -> float: return float(threshold.replace("k", "")) * (1000 if "k" in threshold else 1) -def _parse_above_token_threshold(key: str) -> float: - return _parse_token_threshold(key.rsplit("_above_", 1)[1].split("_tokens")[0]) - - def _prompt_exceeds_threshold(prompt_tokens: int, threshold: float, inclusive: bool) -> bool: return prompt_tokens > threshold or (inclusive and prompt_tokens == threshold) @@ -324,46 +310,6 @@ def _select_priced_tier(model_info: ModelInfo, usage: Usage) -> Mapping[str, obj return tier -def _get_tiered_reasoning_rate(model_info: ModelInfo, usage: Usage) -> float | None: - tier: Final = _select_priced_tier(model_info=model_info, usage=usage) - if tier is None: - return None - if "output_cost_per_reasoning_token" not in tier and "output_cost_per_token" not in tier: - return None - return tier_rate(tier, "output_cost_per_reasoning_token", "output_cost_per_token") - - -def _get_tiered_base_costs(model_info: ModelInfo, usage: Usage) -> tuple[TokenRates, Mapping[str, object]] | None: - """ - Resolve the base rates from a model's ``tiered_pricing`` table, if it has one. - - Tiered pricing is all-or-nothing: one tier is picked from the request's input tokens - and every token of the request is billed at that tier's rate. Rates the tier does not - declare fall back to the tier's input rate, so a request never mixes tiers. - - An output rate is the exception: a tier table that spells out only input rates would - otherwise serve every completion for free, so the model's own output rate stands in. - """ - tier: Final = _select_priced_tier(model_info=model_info, usage=usage) - if tier is None: - return None - - completion_cost: Final = ( - tier_rate(tier, "output_cost_per_token") - if "output_cost_per_token" in tier - else get_cost_per_unit(model_info, "output_cost_per_token") or 0.0 - ) - cache_creation_rate: Final = tier_rate(tier, "cache_creation_input_token_cost", "input_cost_per_token") - rates: Final = TokenRates( - input_rate=tier_rate(tier, "input_cost_per_token"), - output_rate=completion_cost, - cache_read_rate=tier_rate(tier, "cache_read_input_token_cost", "input_cost_per_token"), - cache_creation_rate=cache_creation_rate, - reasoning_rate=None, - ) - return rates, tier - - def _is_within_off_peak_window(off_peak_hours_utc: str | Sequence[str], current_time: datetime | None = None) -> bool: """Return True if current_time (UTC, defaulting to now) falls inside any off-peak window. @@ -601,215 +547,6 @@ def apply_off_peak_pricing(model_info: ModelInfo, current_time: datetime | None, ) -def _apply_off_peak_to_base_costs( - model_info: ModelInfo, - current_time: datetime | None, - base_rates: TokenRates, - tier: Mapping[str, object], -) -> TokenRates: - """Apply off-peak rates to the base costs resolved from a pricing tier. off_peak_pricing has no - field for the long-TTL cache-creation rates, so a tier that declares one keeps it, and a tier - that does not bills that TTL at the off-peak-adjusted write rate. Reasoning is left to - _resolve_billed_reasoning_rate. - """ - rates: Final = apply_off_peak_pricing(model_info, current_time, base_rates) - return replace( - rates, - cache_creation_rates_by_ttl=MappingProxyType( - { - ttl: rates.cache_creation_rate if tier.get(key) is None else tier_rate(tier, key) - for ttl, key in _TTL_PRICED_CACHE_WRITE_RATE_KEYS.items() - } - ), - ) - - -def _first_present_key(model_info: ModelInfo, preferred_key: str, fallback_key: str) -> str: - return preferred_key if model_info.get(preferred_key) is not None else fallback_key - - -def _get_token_base_cost( - model_info: ModelInfo, - usage: Usage, - service_tier: str | None = None, - current_time: datetime | None = None, - *, - threshold_is_inclusive: bool = False, -) -> TokenRates: - """ - Return prompt cost, completion cost, and cache costs for a given model and usage. - - If input_tokens > threshold and `input_cost_per_token_above_[x]k_tokens` or `input_cost_per_token_above_[x]_tokens` is set, - then we use the corresponding threshold cost for all token types. - - `threshold_is_inclusive` switches that comparison to >=, for providers such as xAI - that bill the higher tier once the prompt reaches the threshold. - - An absent cache-creation or cache-read rate always resolves to the resolved input - rate, the way the tiered table and custom deployment pricing already do, since a - provider that publishes no cache price bills cached tokens as ordinary input. An - absent long-TTL write rate resolves to the cache-creation rate, off-peak included. An - explicit 0.0 stays a real price for all of them. - """ - tiered_base_costs: Final = _get_tiered_base_costs(model_info=model_info, usage=usage) - if tiered_base_costs is not None: - return _apply_off_peak_to_base_costs(model_info, current_time, *tiered_base_costs) - - # Get service tier aware cost keys - input_cost_key: Final = get_service_tier_cost_key("input_cost_per_token", service_tier) - output_cost_key: Final = get_service_tier_cost_key("output_cost_per_token", service_tier) - cache_creation_cost_key: Final = get_service_tier_cost_key("cache_creation_input_token_cost", service_tier) - cache_read_cost_key: Final = get_service_tier_cost_key("cache_read_input_token_cost", service_tier) - - prompt_base_cost = cast( # cast-ok: model pricing data is external - float, get_cost_per_unit(model_info, input_cost_key) - ) - completion_base_cost = cast( # cast-ok: model pricing data is external - float, get_cost_per_unit(model_info, output_cost_key) - ) - - # For image generation models that don't have output_cost_per_token, - # use output_cost_per_image_token as the base cost (all output tokens are image tokens) - if completion_base_cost == 0.0: - output_image_cost: Final = get_cost_per_unit(model_info, "output_cost_per_image_token", None) - if output_image_cost is not None: - completion_base_cost = output_image_cost - cache_creation_cost = get_cost_per_unit(model_info, cache_creation_cost_key, default_value=None) - cache_creation_keys_by_ttl: Mapping[str, str] = MappingProxyType( - {ttl: get_service_tier_cost_key(key, service_tier) for ttl, key in _TTL_PRICED_CACHE_WRITE_RATE_KEYS.items()} - ) - cache_read_cost = get_cost_per_unit(model_info, cache_read_cost_key, default_value=None) - - ## CHECK IF ABOVE THRESHOLD - # Optimization: collect threshold keys first to avoid sorting all model_info keys. - # Standard thresholds and thresholds suffixed for this request's service tier both count. - tier_key_suffix: Final = get_service_tier_cost_key("", service_tier) - threshold_prefixes: Final = tuple(f"{prefix}_above_" for prefix in _BATCH_RATE_PREFIXES) - threshold_keys: Final = [ - k - for k in model_info - if k.startswith(threshold_prefixes) - and "_tokens" in k - and ( - not k.endswith(_NON_STANDARD_THRESHOLD_SUFFIXES) or (tier_key_suffix != "" and k.endswith(tier_key_suffix)) - ) - ] - - # Only sort the threshold keys (typically 1-2 keys instead of 66+) - threshold: float | None = None - for key in sorted(threshold_keys, key=_parse_above_token_threshold): - value = model_info.get(key) - if value is not None: - try: - # Handle both formats: _above_128k_tokens and _above_128_tokens - threshold_str = key.rsplit("_above_", 1)[1].split("_tokens")[0] - threshold = _parse_above_token_threshold(key) - if _prompt_exceeds_threshold(usage.prompt_tokens, threshold, threshold_is_inclusive): - # Prefer a service_tier-specific above-threshold key when available, - # e.g. input_cost_per_token_priority_above_200k_tokens for Gemini - # ON_DEMAND_PRIORITY. Falls back to the standard key automatically - # via _get_cost_per_unit's service_tier fallback logic. - tiered_input_key = ( - get_service_tier_cost_key( - f"input_cost_per_token_above_{threshold_str}_tokens", - service_tier, - ) - if service_tier - else f"input_cost_per_token_above_{threshold_str}_tokens" - ) - prompt_base_cost = cast( - float, - get_cost_per_unit(model_info, tiered_input_key, prompt_base_cost), - ) - tiered_output_key = ( - get_service_tier_cost_key( - f"output_cost_per_token_above_{threshold_str}_tokens", - service_tier, - ) - if service_tier - else f"output_cost_per_token_above_{threshold_str}_tokens" - ) - completion_base_cost = cast( - float, - get_cost_per_unit( - model_info, - tiered_output_key, - completion_base_cost, - ), - ) - - # Apply tiered pricing to cache costs - cache_creation_tiered_key = ( - get_service_tier_cost_key( - f"cache_creation_input_token_cost_above_{threshold_str}_tokens", - service_tier, - ) - if service_tier - else f"cache_creation_input_token_cost_above_{threshold_str}_tokens" - ) - cache_read_tiered_key = ( - get_service_tier_cost_key( - f"cache_read_input_token_cost_above_{threshold_str}_tokens", - service_tier, - ) - if service_tier - else f"cache_read_input_token_cost_above_{threshold_str}_tokens" - ) - - cache_creation_cost = get_cost_per_unit(model_info, cache_creation_tiered_key, cache_creation_cost) - - cache_creation_keys_by_ttl = MappingProxyType( - { - ttl: _first_present_key( - model_info, - get_service_tier_cost_key( - f"{_TTL_PRICED_CACHE_WRITE_RATE_KEYS[ttl]}_above_{threshold_str}_tokens", - service_tier, - ), - key, - ) - for ttl, key in cache_creation_keys_by_ttl.items() - } - ) - cache_read_cost = get_cost_per_unit(model_info, cache_read_tiered_key, cache_read_cost) - except (IndexError, ValueError): - continue - except Exception: - continue - - input_rate_for_missing_cache_rates: Final = _off_peak_rate( - _open_off_peak_block(model_info, current_time) or MappingProxyType({}), - "input_cost_per_token", - prompt_base_cost, - ) - resolved_cache_read_cost: Final = input_rate_for_missing_cache_rates if cache_read_cost is None else cache_read_cost - resolved_cache_creation_cost: Final = ( - input_rate_for_missing_cache_rates if cache_creation_cost is None else cache_creation_cost - ) - base_rates: Final = apply_off_peak_pricing( - model_info, - current_time, - TokenRates( - input_rate=prompt_base_cost, - output_rate=completion_base_cost, - cache_read_rate=resolved_cache_read_cost, - cache_creation_rate=resolved_cache_creation_cost, - reasoning_rate=None, - ), - ) - return replace( - base_rates, - cache_creation_rates_by_ttl=MappingProxyType( - { - ttl: base_rates.cache_creation_rate - if (rate := get_cost_per_unit(model_info, key, default_value=None)) is None - else rate - for ttl, key in cache_creation_keys_by_ttl.items() - } - ), - ) - - def calculate_cost_component(model_info: Mapping[str, object], cost_key: str, usage_value: float | None) -> float: """ Generic cost calculator for any usage component @@ -1164,111 +901,6 @@ def get_vertex_regional_endpoint_uplift(model_info: Mapping[str, object], vertex return 1.0 -def get_provider_specific_geo_multiplier(model_info: ModelInfo, usage: Usage) -> float: - """ - Resolve the provider-specific regional pricing multiplier for the geo the - request was served from (``usage.inference_geo``), e.g. Anthropic's ``us: 1.1`` - stored under ``provider_specific_entry``. The regional surcharge applies to - every token type, so per-type cost breakdowns must scale by it too. - - Returns 1.0 when the request was served globally or the model carries no - multiplier for the geo. - """ - inference_geo: Final = getattr(usage, "inference_geo", None) - if not isinstance(inference_geo, str) or inference_geo.lower() in ("global", "not_available"): - return 1.0 - provider_specific_entry: Final[dict[str, float]] = model_info.get("provider_specific_entry") or {} - return float(provider_specific_entry.get(inference_geo.lower(), 1.0)) - - -def get_provider_specific_speed_multiplier(model_info: ModelInfo, usage: Usage) -> float: - provider_specific_entry: Final = model_info.get("provider_specific_entry") - if getattr(usage, "speed", None) != "fast" or not provider_specific_entry: - return 1.0 - return float(provider_specific_entry.get("fast", 1.0)) - - -def _resolve_reasoning_token_cost( - model_info: ModelInfo, - service_tier: str | None, - completion_base_cost: float, -) -> float: - tier_reasoning_key: Final = get_service_tier_cost_key("output_cost_per_reasoning_token", service_tier) - if model_info.get(tier_reasoning_key) is not None: - tier_reasoning_cost: Final = get_cost_per_unit(model_info, tier_reasoning_key, None) - if tier_reasoning_cost is not None: - return tier_reasoning_cost - tier_output_key: Final = get_service_tier_cost_key("output_cost_per_token", service_tier) - if tier_output_key != "output_cost_per_token" and model_info.get(tier_output_key) is not None: - return completion_base_cost - standard_reasoning_cost: Final = get_cost_per_unit(model_info, "output_cost_per_reasoning_token", None) - return standard_reasoning_cost if standard_reasoning_cost is not None else completion_base_cost - - -def _resolve_billed_reasoning_rate( - model_info: ModelInfo, - usage: Usage, - service_tier: str | None, - completion_base_cost: float, - current_time: datetime | None, -) -> float: - off_peak: Final = _open_off_peak_block(model_info, current_time) - off_peak_reasoning_rate: Final = ( - None if off_peak is None else _parse_off_peak_rate(off_peak.get("output_cost_per_reasoning_token")) - ) - if off_peak_reasoning_rate is not None: - return off_peak_reasoning_rate - tiered_reasoning_rate: Final = _get_tiered_reasoning_rate(model_info=model_info, usage=usage) - if tiered_reasoning_rate is not None: - return tiered_reasoning_rate - return _resolve_reasoning_token_cost( - model_info=model_info, - service_tier=service_tier, - completion_base_cost=completion_base_cost, - ) - - -def generic_cost_per_token( - model: str, - usage: Usage, - custom_llm_provider: str, - service_tier: str | None = None, - data_residency: str | None = None, - model_info: ModelInfo | None = None, - vertex_location: str | None = None, - current_time: datetime | None = None, -) -> tuple[float, float]: - """ - Calculates the cost per token for a given model, prompt tokens, and completion tokens. - - Handles context caching as well. - - Input: - - model: str, the model name without provider prefix - - usage: LiteLLM Usage block, containing anthropic caching information - - data_residency: optional OpenAI data-residency region (e.g. "eu", "us"), - used to apply the per-model regional-processing uplift multiplier. - - vertex_location: optional Vertex AI location the request was served from - (e.g. "us-east5", "global"), used to apply the per-model - regional-endpoint uplift multiplier when non-global. - - current_time: the moment the request is billed at, for off_peak_pricing; defaults to now, UTC - - Returns: - Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd - """ - - return cost_per_text( - model, - usage, - custom_llm_provider, - service_tier=service_tier, - data_residency=data_residency, - model_info=model_info, - vertex_location=vertex_location, - current_time=current_time, - ) - - def cost_per_text( model: str, usage: Usage, @@ -1293,6 +925,8 @@ def cost_per_text( data_residency=data_residency, vertex_location=vertex_location, current_time=current_time, + speed=_usage_str(usage, "speed"), + inference_geo=_usage_str(usage, "inference_geo"), key=model, ) ) @@ -1311,6 +945,8 @@ def price_usage( vertex_location: str | None = None, current_time: datetime | None = None, channel: Channel = "online", + speed: str | None = None, + inference_geo: str | None = None, key: str = "model_info", ) -> Priced | Unpriced | Invalid: """Prices ``usage`` over ``row`` with the call's custom pricing over it, the one place a cost is computed""" @@ -1329,10 +965,40 @@ def price_usage( vertex_location=vertex_location, at=current_time if current_time is not None else current_billing_time(), custom_pricing=None if custom_pricing is None else dict(custom_pricing), + speed=speed, + inference_geo=inference_geo, ), ) +def image_cost( + row: Mapping[str, object], images: int, width: int | None = None, height: int | None = None +) -> float | None: + """``images`` images priced per image or per pixel over ``row``, None when it prices neither""" + priced: Final = price_usage( + row, + TokenUsage( + prompt_tokens=0, + text=0, + output_text=0, + images_out=[ImageOut(width=width, height=height)] * max(images, 0), + ), + ) + return priced.total if isinstance(priced, Priced) and priced.lines else None + + +def search_cost(model_info: Mapping[str, object], usage: TokenUsage) -> float: + """The search, grounding and tool meters of ``usage`` priced over ``model_info``""" + prompt_cost, completion_cost = as_main_cost(price_usage(model_info, usage)) + return prompt_cost + completion_cost + + +def _usage_str(usage: Usage, name: str) -> str | None: + """Anthropic's ``speed`` and ``inference_geo``, which pick the row's speed and geo uplifts""" + value: Final = getattr(usage, name, None) + return value if isinstance(value, str) else None + + def pricing_row(model: str, custom_llm_provider: str | None, model_info: ModelInfo | None) -> Mapping[str, object]: """The row main priced, as written rather than `get_model_info`'s copy, which writes `0` for a missing token price. A call with custom pricing prices an unmapped model from its overlay alone""" @@ -1356,8 +1022,13 @@ def bills_characters(row: Mapping[str, object]) -> bool: def _without_tiers(row: Mapping[str, object]) -> dict[str, object]: - """Main picks no tier for an empty prompt and bills the row's flat prices""" - return {name: value for name, value in row.items() if name != "tiered_pricing"} + """Main picks no prompt-token tier for an empty prompt and bills the row's flat prices. A search row's result + tiers stay""" + tiers: Final = row.get("tiered_pricing") + token_tiers: Final = isinstance(tiers, list) and any( + isinstance(tier, Mapping) and "range" in tier for tier in tiers + ) + return {name: value for name, value in row.items() if not (token_tiers and name == "tiered_pricing")} def _billable_prompt_details(usage: Usage) -> PromptTokensDetailsResult: @@ -1426,9 +1097,8 @@ def _token_usage( ) -> TokenUsage: """The meters main billed. A prompt billed per image or second of media does not bill the same tokens again, and a row that bills characters bills no text tokens (decision 32)""" - cache_type: Final = getattr(usage.prompt_tokens_details, "cache_type", None) cached_text: Final = details["cache_hit_tokens"] - details["cache_hit_audio_tokens"] - explicit_cache: Final = cache_type == "ephemeral" + explicit_cache: Final = _reads_explicit_cache(usage) cache_write_5m, cache_write_1h, cache_write_12h = _cache_write_tokens(details) completion: Final = _completion_token_counts(usage) return TokenUsage( @@ -1571,6 +1241,10 @@ def _custom_pricing_rates(custom_cost_per_token: CostPerToken) -> BilledTokenRat ) +def _reads_explicit_cache(usage: Usage) -> bool: + return getattr(usage.prompt_tokens_details, "cache_type", None) == "ephemeral" + + def _cost_map_billed_rates( model_info: ModelInfo, usage: Usage, @@ -1579,54 +1253,55 @@ def _cost_map_billed_rates( data_residency: str | None, vertex_location: str | None, current_time: datetime | None, + *, + provider_multipliers: bool = True, ) -> BilledTokenRates: - billing_time: Final = current_time if current_time is not None else current_billing_time() - base_costs: Final = _get_token_base_cost( - model_info=model_info, - usage=usage, - service_tier=service_tier, - current_time=billing_time, - threshold_is_inclusive=_uses_inclusive_token_thresholds(custom_llm_provider), - ) - provider_cache_read_cost_rate: Final = resolve_explicit_cache_read_cost_rate( - model_info=model_info, - usage=usage, - current_time=billing_time, - service_tier=service_tier, - ) - effective_cache_read_cost_rate: Final = ( - base_costs.cache_read_rate if provider_cache_read_cost_rate is None else provider_cache_read_cost_rate - ) - - reasoning_rate: Final = _resolve_billed_reasoning_rate( - model_info=model_info, - usage=usage, - service_tier=service_tier, - completion_base_cost=base_costs.output_rate, - current_time=billing_time, - ) - audio_cache_read_rate: Final = get_cost_per_unit( + """The rate of every token meter this call would bill, read off one priced probe: one unit of each meter at the + call's prompt size, so the band, tier, off-peak window and uplifts are the ones the call itself billed at""" + explicit_cache: Final = _reads_explicit_cache(usage) + priced: Final = price_usage( model_info, - get_service_tier_cost_key("cache_read_input_audio_token_cost", service_tier), - None, - ) - multiplier: Final = ( - get_regional_uplift_multiplier(model_info, data_residency) - * get_vertex_regional_endpoint_uplift(model_info, vertex_location) - * get_provider_specific_geo_multiplier(model_info=model_info, usage=usage) - * get_provider_specific_speed_multiplier(model_info=model_info, usage=usage) - ) - return BilledTokenRates( - input_cost_per_token=base_costs.input_rate, - output_cost_per_token=base_costs.output_rate, - cache_read_input_token_cost=effective_cache_read_cost_rate, - cache_read_input_audio_token_cost=( - audio_cache_read_rate if audio_cache_read_rate is not None else effective_cache_read_cost_rate + TokenUsage( + prompt_tokens=usage.prompt_tokens, + text=1, + output_text=1, + cache_read=0 if explicit_cache else 1, + cache_read_explicit=1 if explicit_cache else 0, + cache_read_audio=1, + cache_write_5m=1, + cache_write_1h=1, + cache_write_12h=1, + reasoning=1, ), - cache_creation_input_token_cost=base_costs.cache_creation_rate, - cache_creation_input_token_cost_by_ttl=base_costs.cache_creation_rates_by_ttl, - output_cost_per_reasoning_token=reasoning_rate, - ).scaled(multiplier) + service_tier=service_tier, + data_residency=data_residency, + vertex_location=vertex_location, + current_time=current_time, + speed=_usage_str(usage, "speed") if provider_multipliers else None, + inference_geo=_usage_str(usage, "inference_geo") if provider_multipliers else None, + key=str(model_info.get("key") or custom_llm_provider or "model_info"), + ) + rates: Final = MappingProxyType( + {line.meter: line.amount for line in priced.lines} if isinstance(priced, Priced) else {} + ) + cache_read: Final = rates.get( + "explicit_cache_read_input_token_cost" if explicit_cache else "cache_read_input_token_cost", 0.0 + ) + cache_write: Final = rates.get("cache_creation_input_token_cost", 0.0) + return BilledTokenRates( + input_cost_per_token=rates.get("input_cost_per_token", 0.0), + output_cost_per_token=rates.get("output_cost_per_token", 0.0), + cache_read_input_token_cost=cache_read, + cache_read_input_audio_token_cost=rates.get("cache_read_input_audio_token_cost", cache_read), + cache_creation_input_token_cost=cache_write, + cache_creation_input_token_cost_by_ttl=MappingProxyType( + { + "1hr": rates.get("cache_creation_input_token_cost_above_1hr", cache_write), + "12hr": rates.get("cache_creation_input_token_cost_above_12hr", cache_write), + } + ), + output_cost_per_reasoning_token=rates.get("output_cost_per_reasoning_token", 0.0), + ) def get_billed_token_rates( @@ -1645,16 +1320,11 @@ def get_billed_token_rates( if custom_cost_per_token is not None: return _custom_pricing_rates(custom_cost_per_token) try: - resolved_model_info: Final = apply_provider_cache_read_default( - model_info - if model_info is not None - else get_model_info(model=model, custom_llm_provider=custom_llm_provider), - custom_llm_provider, - ) + resolved_model_info: Final = pricing_row(model, custom_llm_provider, model_info) except Exception: # noqa: BLE001 # get_model_info raises a bare Exception for an unmapped model: no rates return None return _cost_map_billed_rates( - model_info=resolved_model_info, + model_info=cast(ModelInfo, resolved_model_info), # cast-ok: the row as written, priced by its own meters usage=usage, custom_llm_provider=custom_llm_provider, service_tier=service_tier, @@ -1680,7 +1350,7 @@ def get_token_type_cost_breakdown( object and model pricing alone. This works for every provider, including Perplexity/Cerebras/Dashscope whose - cost calculators bypass ``generic_cost_per_token``, because cache tokens always + cost calculators bypass ``cost_per_text``, because cache tokens always land on ``prompt_tokens_details`` (via the Usage constructor and provider transformations) and reasoning tokens on ``completion_tokens_details``. It reuses the same rate resolution as the total-cost path (``get_billed_token_rates``) so the @@ -1743,25 +1413,18 @@ def calculate_prompt_caching_savings( ``billed_at`` is the request's completion time, so off-peak windows resolve as the biller saw them rather than at the later spend write. """ - model_info_with_cache_read_default: Final = apply_provider_cache_read_default(model_info, custom_llm_provider) - base_costs: Final = _get_token_base_cost( - model_info=model_info_with_cache_read_default, - usage=usage, - service_tier=service_tier, - current_time=billed_at, - threshold_is_inclusive=_uses_inclusive_token_thresholds(custom_llm_provider), - ) - provider_cache_read_cost: Final = resolve_explicit_cache_read_cost_rate( + rates: Final = _cost_map_billed_rates( model_info=model_info, usage=usage, - current_time=billed_at, + custom_llm_provider=custom_llm_provider, service_tier=service_tier, + data_residency=data_residency, + vertex_location=vertex_location, + current_time=billed_at, + provider_multipliers=False, ) - prompt_base_cost: Final = base_costs.input_rate - effective_cache_read_cost: Final = ( - base_costs.cache_read_rate if provider_cache_read_cost is None else provider_cache_read_cost - ) - write_rate: Final = base_costs.cache_creation_rate or prompt_base_cost + input_rate: Final = rates.input_cost_per_token + write_rate: Final = rates.cache_creation_input_token_cost or input_rate prompt_tokens_details: Final = parse_prompt_tokens_details(usage) cache_read_tokens: Final = max(prompt_tokens_details["cache_hit_tokens"], 0) cache_creation_tokens: Final = max(prompt_tokens_details["cache_creation_tokens"], 0) @@ -1776,19 +1439,16 @@ def calculate_prompt_caching_savings( if details is not None else None ) - read_discount: Final = cache_read_tokens * max(prompt_base_cost - effective_cache_read_cost, 0.0) + read_discount: Final = cache_read_tokens * max(input_rate - rates.cache_read_input_token_cost, 0.0) write_premium: Final = calculate_cache_writing_cost( cache_creation_tokens=cache_creation_tokens, cache_creation_token_details=cache_creation_details, - cache_creation_cost=write_rate - prompt_base_cost, + cache_creation_cost=write_rate - input_rate, cache_creation_cost_by_ttl=MappingProxyType( - {ttl: rate - prompt_base_cost for ttl, rate in base_costs.cache_creation_rates_by_ttl.items()} + {ttl: rate - input_rate for ttl, rate in rates.cache_creation_input_token_cost_by_ttl.items()} ), ) - uplift: Final = get_regional_uplift_multiplier(model_info, data_residency) * get_vertex_regional_endpoint_uplift( - model_info, vertex_location - ) - return (read_discount - write_premium) * uplift + return read_discount - write_premium def calculate_image_response_cost_from_usage( @@ -1875,7 +1535,7 @@ def calculate_image_response_cost_from_usage( completion_tokens_details=completion_tokens_details, ) - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model, usage=normalized_usage, custom_llm_provider=custom_llm_provider, diff --git a/litellm/llms/aiml/image_generation/cost_calculator.py b/litellm/llms/aiml/image_generation/cost_calculator.py index ff49052dc25..e74cc162541 100644 --- a/litellm/llms/aiml/image_generation/cost_calculator.py +++ b/litellm/llms/aiml/image_generation/cost_calculator.py @@ -1,7 +1,7 @@ from typing import Final import litellm -from litellm.litellm_core_utils.llm_cost_calc.utils import resolve_image_model_info +from litellm.litellm_core_utils.llm_cost_calc.utils import image_cost, resolve_image_model_info from litellm.types.utils import ImageResponse, ModelInfo @@ -18,11 +18,10 @@ def cost_calculator( custom_llm_provider=litellm.LlmProviders.AIML.value, model_info=model_info, ) - output_cost_per_image: Final[float] = _model_info.get("output_cost_per_image") or 0.0 num_images: int = 0 if isinstance(image_response, ImageResponse): if image_response.data: num_images = len(image_response.data) - return output_cost_per_image * num_images + return image_cost(_model_info, num_images) or 0.0 else: raise ValueError(f"image_response must be of type ImageResponse got type={type(image_response)}") diff --git a/litellm/llms/amazon_nova/cost_calculation.py b/litellm/llms/amazon_nova/cost_calculation.py index 6e691cf279e..6aca46b601b 100644 --- a/litellm/llms/amazon_nova/cost_calculation.py +++ b/litellm/llms/amazon_nova/cost_calculation.py @@ -5,7 +5,7 @@ Helper util for handling amazon nova cost calculation from typing import TYPE_CHECKING -from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text if TYPE_CHECKING: from litellm.types.utils import Usage @@ -16,4 +16,4 @@ def cost_per_token(model: str, usage: "Usage") -> tuple[float, float]: Calculates the cost per token for a given model, prompt tokens, and completion tokens. Follows the same logic as Anthropic's cost per token calculation. """ - return generic_cost_per_token(model=model, usage=usage, custom_llm_provider="amazon_nova") + return cost_per_text(model=model, usage=usage, custom_llm_provider="amazon_nova") diff --git a/litellm/llms/anthropic/cost_calculation.py b/litellm/llms/anthropic/cost_calculation.py index 94c8149eff3..2d8bdd4ad34 100644 --- a/litellm/llms/anthropic/cost_calculation.py +++ b/litellm/llms/anthropic/cost_calculation.py @@ -8,16 +8,13 @@ from typing import TYPE_CHECKING, Final, Optional from pydantic import ValidationError from litellm.litellm_core_utils.llm_cost_calc.utils import ( - generic_cost_per_token, - get_provider_specific_geo_multiplier, - get_provider_specific_speed_multiplier, + cost_per_text, get_web_search_requests_from_usage, ) from litellm.types.llms.base import LiteLLMBaseModel if TYPE_CHECKING: from litellm.types.utils import ModelInfo, Usage -import litellm def cost_per_token( @@ -36,7 +33,7 @@ def cost_per_token( Returns: Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd """ - prompt_cost, completion_cost = generic_cost_per_token( + return cost_per_text( model=model, usage=usage, custom_llm_provider="anthropic", @@ -44,28 +41,6 @@ def cost_per_token( model_info=model_info, ) - # Apply provider_specific_entry multipliers for geo/speed routing - try: - effective_info: Final = ( - model_info - if model_info is not None - else litellm.get_model_info(model=model, custom_llm_provider="anthropic") - ) - geo_multiplier: Final = get_provider_specific_geo_multiplier(model_info=effective_info, usage=usage) - speed_multiplier: Final = get_provider_specific_speed_multiplier(model_info=effective_info, usage=usage) - - if speed_multiplier != 1.0: - prompt_cost *= speed_multiplier - completion_cost *= speed_multiplier - - if geo_multiplier != 1.0: - prompt_cost *= geo_multiplier - completion_cost *= geo_multiplier - except Exception: - pass - - return prompt_cost, completion_cost - class _AnthropicServerToolUseProbe(LiteLLMBaseModel): web_search_requests: int | None = None @@ -99,29 +74,15 @@ def get_cost_for_anthropic_web_search( model_info: Optional["ModelInfo"] = None, usage: Optional["Usage"] = None, ) -> float: - """ - Get the cost of using a web search tool for Anthropic. - """ - from litellm.types.utils import SearchContextCostPerQuery + """``server_tool_use.web_search_requests`` at the row's medium search context price""" + from litellm._cost import Usage as TokenUsage + from litellm.litellm_core_utils.llm_cost_calc.utils import search_cost - ## Check if web search requests are in the usage object - if model_info is None: - return 0.0 - - if usage is None: + if model_info is None or usage is None: return 0.0 web_search_requests: Final = get_web_search_requests_from_usage(usage) if web_search_requests is None: return 0.0 - - ## Get the cost per web search request - search_context_pricing: Final[SearchContextCostPerQuery] = ( - model_info.get("search_context_cost_per_query") or SearchContextCostPerQuery() + return search_cost( + model_info, TokenUsage(prompt_tokens=0, text=0, output_text=0, web_search_requests=web_search_requests) ) - cost_per_web_search_request: Final = search_context_pricing.get("search_context_size_medium", 0.0) - if cost_per_web_search_request is None or cost_per_web_search_request == 0.0: - return 0.0 - - ## Calculate the total cost - total_cost: Final = cost_per_web_search_request * web_search_requests - return total_cost diff --git a/litellm/llms/azure/cost_calculation.py b/litellm/llms/azure/cost_calculation.py index 057e9dbb9d9..ba5fbc3e39b 100644 --- a/litellm/llms/azure/cost_calculation.py +++ b/litellm/llms/azure/cost_calculation.py @@ -3,7 +3,7 @@ Helper util for handling azure openai-specific cost calculation - e.g.: prompt caching, audio tokens """ -from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text from litellm.types.utils import Usage @@ -25,7 +25,7 @@ def cost_per_token( """ ## Use generic cost calculator for all other cases ## This properly handles: text tokens, audio tokens, cached tokens, reasoning tokens, etc. - return generic_cost_per_token( + return cost_per_text( model=model, usage=usage, custom_llm_provider="azure", diff --git a/litellm/llms/azure_ai/cost_calculator.py b/litellm/llms/azure_ai/cost_calculator.py index 5934525eca3..55bc2fd47c1 100644 --- a/litellm/llms/azure_ai/cost_calculator.py +++ b/litellm/llms/azure_ai/cost_calculator.py @@ -6,7 +6,7 @@ Handles Azure AI Foundry Model Router flat cost and other Azure AI specific pric from typing import Final from litellm._logging import verbose_logger -from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text from litellm.types.utils import Usage from litellm.utils import get_model_info @@ -65,9 +65,7 @@ def calculate_azure_model_router_flat_cost(model: str, prompt_tokens: int) -> fl def _response_model_cost(model: str, usage: Usage, service_tier: str | None) -> tuple[float, float]: try: - return generic_cost_per_token( - model=model, usage=usage, custom_llm_provider="azure_ai", service_tier=service_tier - ) + return cost_per_text(model=model, usage=usage, custom_llm_provider="azure_ai", service_tier=service_tier) except Exception as e: if not is_azure_model_router(model): raise diff --git a/litellm/llms/azure_ai/image_generation/cost_calculator.py b/litellm/llms/azure_ai/image_generation/cost_calculator.py index 53fb755be80..f5958564d9c 100644 --- a/litellm/llms/azure_ai/image_generation/cost_calculator.py +++ b/litellm/llms/azure_ai/image_generation/cost_calculator.py @@ -4,22 +4,23 @@ from typing import Final import litellm from litellm.litellm_core_utils.llm_cost_calc.utils import ( calculate_image_response_cost_from_usage, - get_cost_per_unit, + image_cost, resolve_image_model_info, ) from litellm.types.utils import ImageResponse, ModelInfo -def _input_cost_per_pixel(resolved: ModelInfo) -> float: - deployment_price: Final = get_cost_per_unit(resolved, "input_cost_per_pixel", default_value=None) - if deployment_price is not None: - return deployment_price - model_cost_key: Final = resolved.get("key") - shared_entry: Final = litellm.model_cost.get(model_cost_key) if model_cost_key is not None else None - if shared_entry is None: - return 0.0 - shared_price: Final = shared_entry.get("input_cost_per_pixel") - return float(shared_price) if isinstance(shared_price, (int, float)) else 0.0 +def _requested_dimensions(optional_params: Mapping[str, object] | None, size: object) -> tuple[int, int] | None: + width: Final = optional_params.get("width") if optional_params else None + height: Final = optional_params.get("height") if optional_params else None + if type(width) is int and type(height) is int and width > 0 and height > 0: + return width, height + if not isinstance(size, str): + return None + parts: Final = size.replace("-x-", "x").split("x") + if len(parts) != 2 or not all(part.isdigit() for part in parts): + return None + return int(parts[0]), int(parts[1]) def cost_calculator( @@ -50,27 +51,10 @@ def cost_calculator( return token_based_cost num_images: Final = n if n is not None else len(image_response.data or ()) - output_cost_per_image: Final[float] = _model_info.get("output_cost_per_image") or 0.0 - if output_cost_per_image: - return output_cost_per_image * num_images - - if _input_cost_per_pixel(_model_info): - from litellm.cost_calculator import default_image_cost_calculator - - width: Final = optional_params.get("width") if optional_params else None - height: Final = optional_params.get("height") if optional_params else None - pixel_size: Final = ( - f"{width}x{height}" - if type(width) is int and type(height) is int and width > 0 and height > 0 - else size or image_response.size - ) - return default_image_cost_calculator( - model=_model_info.get("key", model), - custom_llm_provider=litellm.LlmProviders.AZURE_AI.value, - size=pixel_size, - n=num_images, - model_info=model_info, - ) - return 0.0 + width, height = _requested_dimensions(optional_params, size or image_response.size) or (1024, 1024) + key: Final = _model_info.get("key") + catalog_row: Final = litellm.model_cost.get(key) if isinstance(key, str) else None + row: Final = {**_model_info, **(catalog_row or {}), **(model_info or {})} + return image_cost(row, num_images, width=width, height=height) or 0.0 raise ValueError(f"image_response must be of type ImageResponse got type={type(image_response)}") diff --git a/litellm/llms/bedrock/cost_calculation.py b/litellm/llms/bedrock/cost_calculation.py index 1b45531dda4..af625c35d7f 100644 --- a/litellm/llms/bedrock/cost_calculation.py +++ b/litellm/llms/bedrock/cost_calculation.py @@ -3,13 +3,9 @@ Helper util for handling bedrock-specific cost calculation - e.g.: prompt caching """ -from typing import TYPE_CHECKING, Final +from typing import TYPE_CHECKING -import litellm -from litellm.litellm_core_utils.llm_cost_calc.utils import ( - generic_cost_per_token, - get_provider_specific_geo_multiplier, -) +from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text if TYPE_CHECKING: from litellm.types.utils import Usage @@ -21,13 +17,4 @@ def cost_per_token(model: str, usage: "Usage", service_tier: str | None = None) Follows the same logic as Anthropic's cost per token calculation. """ - model_info: Final = litellm.get_model_info(model=model, custom_llm_provider="bedrock") - prompt_cost, completion_cost = generic_cost_per_token( - model=model, - usage=usage, - custom_llm_provider="bedrock", - service_tier=service_tier, - model_info=model_info, - ) - geo_multiplier: Final = get_provider_specific_geo_multiplier(model_info=model_info, usage=usage) - return prompt_cost * geo_multiplier, completion_cost * geo_multiplier + return cost_per_text(model=model, usage=usage, custom_llm_provider="bedrock", service_tier=service_tier) diff --git a/litellm/llms/bedrock/image_generation/amazon_nova_canvas_transformation.py b/litellm/llms/bedrock/image_generation/amazon_nova_canvas_transformation.py index 6f972986e56..ce176a56205 100644 --- a/litellm/llms/bedrock/image_generation/amazon_nova_canvas_transformation.py +++ b/litellm/llms/bedrock/image_generation/amazon_nova_canvas_transformation.py @@ -4,6 +4,7 @@ from typing import Any, Final from openai.types.image import Image +from litellm.litellm_core_utils.llm_cost_calc.utils import image_cost from litellm.llms.bedrock.common_utils import get_cached_model_info from litellm.types.llms.bedrock import ( AmazonNovaCanvasColorGuidedGenerationParams, @@ -206,9 +207,7 @@ class AmazonNovaCanvasConfig: model=model, custom_llm_provider="bedrock", ) - - output_cost_per_image: Final[float] = model_info.get("output_cost_per_image") or 0.0 num_images: int = 0 if image_response.data: num_images = len(image_response.data) - return output_cost_per_image * num_images + return image_cost(model_info, num_images) or 0.0 diff --git a/litellm/llms/bedrock/image_generation/amazon_stability1_transformation.py b/litellm/llms/bedrock/image_generation/amazon_stability1_transformation.py index 3ebb128e89c..ab18549bf64 100644 --- a/litellm/llms/bedrock/image_generation/amazon_stability1_transformation.py +++ b/litellm/llms/bedrock/image_generation/amazon_stability1_transformation.py @@ -6,6 +6,7 @@ from typing import Final from openai.types.image import Image +from litellm.litellm_core_utils.llm_cost_calc.utils import image_cost from litellm.llms.bedrock.common_utils import get_cached_model_info from litellm.types.utils import ImageResponse @@ -156,9 +157,7 @@ class AmazonStabilityConfig: model=model, custom_llm_provider="bedrock", ) - - output_cost_per_image: Final[float] = model_info.get("output_cost_per_image") or 0.0 num_images: int = 0 if image_response.data: num_images = len(image_response.data) - return output_cost_per_image * num_images + return image_cost(model_info, num_images) or 0.0 diff --git a/litellm/llms/bedrock/image_generation/amazon_stability3_transformation.py b/litellm/llms/bedrock/image_generation/amazon_stability3_transformation.py index e5c5edba750..9e34c5d3122 100644 --- a/litellm/llms/bedrock/image_generation/amazon_stability3_transformation.py +++ b/litellm/llms/bedrock/image_generation/amazon_stability3_transformation.py @@ -4,6 +4,7 @@ from typing import Final from openai.types.image import Image +from litellm.litellm_core_utils.llm_cost_calc.utils import image_cost from litellm.llms.bedrock.common_utils import BedrockError, get_cached_model_info from litellm.types.llms.bedrock import ( AmazonStability3TextToImageRequest, @@ -126,9 +127,7 @@ class AmazonStability3Config: model=model, custom_llm_provider="bedrock", ) - - output_cost_per_image: Final[float] = model_info.get("output_cost_per_image") or 0.0 num_images: int = 0 if image_response.data: num_images = len(image_response.data) - return output_cost_per_image * num_images + return image_cost(model_info, num_images) or 0.0 diff --git a/litellm/llms/bedrock/image_generation/amazon_titan_transformation.py b/litellm/llms/bedrock/image_generation/amazon_titan_transformation.py index 5731541da56..4d7bb3b40c3 100644 --- a/litellm/llms/bedrock/image_generation/amazon_titan_transformation.py +++ b/litellm/llms/bedrock/image_generation/amazon_titan_transformation.py @@ -1,3 +1,5 @@ +from litellm.litellm_core_utils.llm_cost_calc.utils import image_cost + """ Transformation logic for Amazon Titan Image Generation. """ @@ -152,8 +154,7 @@ class AmazonTitanImageGenerationConfig: optional_params: dict | None = None, ) -> float: model_info: Final = get_model_info(model=model) - output_cost_per_image: Final = model_info.get("output_cost_per_image") or 0.0 if not image_response.data: return 0.0 num_images: Final = len(image_response.data) - return output_cost_per_image * num_images + return image_cost(model_info, num_images) or 0.0 diff --git a/litellm/llms/cometapi/image_generation/cost_calculator.py b/litellm/llms/cometapi/image_generation/cost_calculator.py index d9248eac598..adc1ef18003 100644 --- a/litellm/llms/cometapi/image_generation/cost_calculator.py +++ b/litellm/llms/cometapi/image_generation/cost_calculator.py @@ -1,7 +1,7 @@ from typing import Final import litellm -from litellm.litellm_core_utils.llm_cost_calc.utils import resolve_image_model_info +from litellm.litellm_core_utils.llm_cost_calc.utils import image_cost, resolve_image_model_info from litellm.types.utils import ImageResponse, ModelInfo @@ -18,11 +18,10 @@ def cost_calculator( custom_llm_provider=litellm.LlmProviders.COMETAPI.value, model_info=model_info, ) - output_cost_per_image: Final[float] = _model_info.get("output_cost_per_image") or 0.0 num_images: int = 0 if isinstance(image_response, ImageResponse): if image_response.data: num_images = len(image_response.data) - return output_cost_per_image * num_images + return image_cost(_model_info, num_images) or 0.0 else: raise ValueError(f"image_response must be of type ImageResponse got type={type(image_response)}") diff --git a/litellm/llms/databricks/cost_calculator.py b/litellm/llms/databricks/cost_calculator.py index 2bb5b99f0ad..9952965ef9b 100644 --- a/litellm/llms/databricks/cost_calculator.py +++ b/litellm/llms/databricks/cost_calculator.py @@ -6,7 +6,7 @@ Helper util for handling databricks-specific cost calculation from types import MappingProxyType from typing import Final -from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text from litellm.types.utils import Usage _LEGACY_ENDPOINT_NAMES: Final = MappingProxyType( @@ -41,7 +41,7 @@ def cost_per_token(model: str, usage: Usage, service_tier: str | None = None) -> Returns: Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd """ - return generic_cost_per_token( + return cost_per_text( model=_registry_key(model), usage=usage, custom_llm_provider="databricks", diff --git a/litellm/llms/deepseek/cost_calculator.py b/litellm/llms/deepseek/cost_calculator.py index 5a0c065f0ee..b8332b0562b 100644 --- a/litellm/llms/deepseek/cost_calculator.py +++ b/litellm/llms/deepseek/cost_calculator.py @@ -4,7 +4,7 @@ Cost calculator for DeepSeek Chat models. Handles prompt caching scenario. """ -from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text from litellm.types.utils import Usage @@ -14,4 +14,4 @@ def cost_per_token(model: str, usage: Usage) -> tuple[float, float]: Follows the same logic as Anthropic's cost per token calculation. """ - return generic_cost_per_token(model=model, usage=usage, custom_llm_provider="deepseek") + return cost_per_text(model=model, usage=usage, custom_llm_provider="deepseek") diff --git a/litellm/llms/fal_ai/cost_calculator.py b/litellm/llms/fal_ai/cost_calculator.py index 2acf1c69594..4e8049d0617 100644 --- a/litellm/llms/fal_ai/cost_calculator.py +++ b/litellm/llms/fal_ai/cost_calculator.py @@ -7,7 +7,8 @@ from typing import Final from pydantic import TypeAdapter import litellm -from litellm.litellm_core_utils.llm_cost_calc.utils import deployment_pricing, resolve_image_model_info +from litellm._catalog import CATALOG +from litellm.litellm_core_utils.llm_cost_calc.utils import deployment_pricing, image_cost, resolve_image_model_info from litellm.types.utils import ImageObject, ImageResponse, ModelInfo FAL_KEYED_PRICING_DEFAULT_QUALITY: Final[str] = "high" @@ -82,38 +83,20 @@ def _parse_keyed_dimensions(size: str | None) -> tuple[int, int] | None: return (width, height) if width > 0 and height > 0 else None -def _keyed_rows(model: str, quality: str) -> tuple[tuple[int, int, float], ...]: - prefix: Final = f"fal_ai/{quality}/" - suffix: Final = f"/{model}" - return tuple( - (width, height, float(raw_cost)) - for key in (*litellm.model_cost,) - if isinstance(key, str) and key.startswith(prefix) and key.endswith(suffix) - for size in (key[len(prefix) : -len(suffix)],) - for dimensions in (_parse_keyed_dimensions(size),) - if dimensions is not None - for entry in (_entry(key),) - if entry is not None - for raw_cost in (entry.get("output_cost_per_image"),) - if isinstance(raw_cost, (int, float)) - for width, height in (dimensions,) - ) - - def _keyed_cost_per_image( model: str, image: object, optional_params: Mapping[str, object], ) -> float | None: - quality: Final = _keyed_quality(optional_params) - rows: Final = _keyed_rows(model, quality) - if not rows: - return None - target_dimensions: Final = ( + """The per-image price of the size-keyed row nearest the image's size""" + target: Final = ( _image_dimensions(image) or _parse_keyed_dimensions(_keyed_size(optional_params)) or _DEFAULT_KEYED_DIMENSIONS ) - target_pixels: Final = target_dimensions[0] * target_dimensions[1] - return min(rows, key=lambda row: (abs(row[0] * row[1] - target_pixels), row[0] * row[1]))[2] + key: Final = CATALOG.nearest_image_key( + model, litellm.LlmProviders.FAL_AI.value, quality=_keyed_quality(optional_params), size=target + ) + entry: Final = _entry(key) if key is not None else None + return image_cost(entry, 1) if entry is not None else None def _flat_cost_per_image( diff --git a/litellm/llms/fireworks_ai/cache_pricing.py b/litellm/llms/fireworks_ai/cache_pricing.py deleted file mode 100644 index f5e49cad01a..00000000000 --- a/litellm/llms/fireworks_ai/cache_pricing.py +++ /dev/null @@ -1,42 +0,0 @@ -from typing import ( - Final, - cast, # noqa: TID251 # the derived entry is a dict copy of a ReadOnly TypedDict; no cast-free way to retype it -) - -from litellm.constants import FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO -from litellm.types.utils import ModelInfo - - -def _as_rate(value: object) -> float | None: - if isinstance(value, bool) or not isinstance(value, (int, float, str)): - return None - try: - return float(value) - except ValueError: - return None - - -def with_default_cache_read_rate(model_info: ModelInfo) -> ModelInfo: - input_rate: Final = _as_rate(model_info.get("input_cost_per_token")) - if model_info.get("cache_read_input_token_cost") is not None or input_rate is None: - return model_info - cache_read_rate: Final = input_rate * FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO - off_peak: Final = model_info.get("off_peak_pricing") - if off_peak is None or "cache_read_input_token_cost" in off_peak: - return cast(ModelInfo, {**model_info, "cache_read_input_token_cost": cache_read_rate}) - off_peak_input_rate: Final = _as_rate(off_peak.get("input_cost_per_token")) - return cast( - ModelInfo, - { - **model_info, - "cache_read_input_token_cost": cache_read_rate, - "off_peak_pricing": { - **off_peak, - "cache_read_input_token_cost": ( - off_peak_input_rate * FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO - if off_peak_input_rate is not None - else cache_read_rate - ), - }, - }, - ) diff --git a/litellm/llms/fireworks_ai/cost_calculator.py b/litellm/llms/fireworks_ai/cost_calculator.py index d42b4ddbbf7..f498d3183c3 100644 --- a/litellm/llms/fireworks_ai/cost_calculator.py +++ b/litellm/llms/fireworks_ai/cost_calculator.py @@ -11,7 +11,7 @@ from litellm.constants import ( FIREWORKS_AI_56_B_MOE, FIREWORKS_AI_176_B_MOE, ) -from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text from litellm.types.utils import ModelInfo, Usage from litellm.utils import get_model_info @@ -85,7 +85,7 @@ def cost_per_token(model: str, usage: Usage, current_time: datetime | None = Non Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd """ model_info: Final = _resolve_model_info(model) - return generic_cost_per_token( + return cost_per_text( model=model, usage=usage, custom_llm_provider=model_info["litellm_provider"], diff --git a/litellm/llms/gemini/cost_calculator.py b/litellm/llms/gemini/cost_calculator.py index b82103b0ff8..dce6f3db62c 100644 --- a/litellm/llms/gemini/cost_calculator.py +++ b/litellm/llms/gemini/cost_calculator.py @@ -16,9 +16,9 @@ def cost_per_token(model: str, usage: "Usage", service_tier: str | None = None) Follows the same logic as Anthropic's cost per token calculation. """ - from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token + from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text - return generic_cost_per_token( + return cost_per_text( model=model, usage=usage, custom_llm_provider="gemini", @@ -26,52 +26,36 @@ def cost_per_token(model: str, usage: "Usage", service_tier: str | None = None) ) -def cost_per_web_search_request(usage: "Usage", model_info: "ModelInfo") -> float: - """ - Calculates the cost of web search (grounding with Google Search). - - Billing mode is determined by ``web_search_billing_unit`` in model_info: - - ``"per_query"``: charged per individual search query (Gemini 3.x). - - ``"per_prompt"`` (default): charged per grounded prompt (Gemini 2.x), - regardless of how many queries were executed internally. - - Reads the per-request cost from ``search_context_cost_per_query`` in - ``model_info`` when available, falling back to $0.035 for models not - yet updated in the pricing JSON. - """ - from litellm.litellm_core_utils.llm_cost_calc.utils import ( - get_web_search_requests_from_usage, - ) +def web_search_requests(usage: "Usage | None") -> int: + """Grounded searches, Gemini's ``prompt_tokens_details.web_search_requests`` else ``server_tool_use``""" + from litellm.litellm_core_utils.llm_cost_calc.utils import get_web_search_requests_from_usage from litellm.types.utils import PromptTokensDetailsWrapper - _DEFAULT_COST: Final = 35e-3 - search_costs: Final = model_info.get("search_context_cost_per_query") or {} - _cost: Final = search_costs.get("search_context_size_medium", _DEFAULT_COST) - - requests_from_prompt_details: Final = ( - usage.prompt_tokens_details.web_search_requests - if ( - usage is not None - and usage.prompt_tokens_details is not None - and isinstance(usage.prompt_tokens_details, PromptTokensDetailsWrapper) - and hasattr(usage.prompt_tokens_details, "web_search_requests") - and usage.prompt_tokens_details.web_search_requests is not None - ) + details: Final = usage.prompt_tokens_details if usage is not None else None + from_details: Final = ( + details.web_search_requests + if isinstance(details, PromptTokensDetailsWrapper) and getattr(details, "web_search_requests", None) else None ) - requests_from_server_tool_use: Final = get_web_search_requests_from_usage(usage) - number_of_web_search_requests: Final = requests_from_prompt_details or requests_from_server_tool_use or 0 + return from_details or (get_web_search_requests_from_usage(usage) if usage is not None else None) or 0 - billing_mode: Final = model_info.get("web_search_billing_unit") or "per_prompt" - billable_requests: Final = ( - 1 if (number_of_web_search_requests > 0 and billing_mode == "per_prompt") else number_of_web_search_requests + +def cost_per_web_search_request(usage: "Usage", model_info: "ModelInfo") -> float: + """Grounding with Google Search, per query or once per prompt as the row's ``web_search_billing_unit`` says""" + from litellm._cost import Usage as TokenUsage + from litellm.litellm_core_utils.llm_cost_calc.utils import search_cost + + return search_cost( + _as_google_row(model_info), + TokenUsage(prompt_tokens=0, text=0, output_text=0, web_search_requests=web_search_requests(usage)), ) - return _cost * billable_requests - -GOOGLE_MAPS_GROUNDING_DEFAULT_COST_PER_QUERY: Final = 14e-3 -GOOGLE_MAPS_GROUNDING_DEFAULT_COST_PER_PROMPT: Final = 25e-3 +def _as_google_row(model_info: "ModelInfo") -> dict[str, object]: + """The row with a Google provider, so its missing search and Maps prices take Gemini's defaults""" + provider: Final = model_info.get("litellm_provider") + google: Final = isinstance(provider, str) and (provider == "gemini" or provider.startswith("vertex_ai")) + return {**model_info, "litellm_provider": provider if google else "gemini"} def google_maps_grounding_requests(usage: "Usage | None") -> int | None: @@ -84,26 +68,16 @@ def google_maps_grounding_requests(usage: "Usage | None") -> int | None: def cost_per_google_maps_grounding_request(usage: "Usage", model_info: "ModelInfo") -> float: - """ - Calculates the cost of Grounding with Google Maps. + """Grounding with Google Maps, under the same billing unit as Google Search grounding""" + from litellm._cost import Usage as TokenUsage + from litellm.litellm_core_utils.llm_cost_calc.utils import search_cost - Billing follows ``web_search_billing_unit`` in model_info the same way Google Search grounding - does: ``"per_query"`` (Gemini 3.x) multiplies the executed Maps queries, ``"per_prompt"`` - (default, Gemini 2.x) charges one flat fee per grounded prompt. - - The rate comes from ``google_maps_grounding_cost_per_query`` in ``model_info``, falling back - to Google's list price for that billing unit when the pricing JSON has no entry yet. - """ - requests: Final = google_maps_grounding_requests(usage) - if not requests or requests <= 0: - return 0.0 - billing_mode: Final = model_info.get("web_search_billing_unit") or "per_prompt" - default_cost: Final = ( - GOOGLE_MAPS_GROUNDING_DEFAULT_COST_PER_QUERY - if billing_mode == "per_query" - else GOOGLE_MAPS_GROUNDING_DEFAULT_COST_PER_PROMPT + return search_cost( + _as_google_row(model_info), + TokenUsage( + prompt_tokens=0, + text=0, + output_text=0, + maps_grounding_requests=max(google_maps_grounding_requests(usage) or 0, 0), + ), ) - configured_cost: Final = model_info.get("google_maps_grounding_cost_per_query") - cost: Final = default_cost if configured_cost is None else configured_cost - billed_requests: Final = requests if billing_mode == "per_query" else 1 - return cost * billed_requests diff --git a/litellm/llms/groq/cost_calculator.py b/litellm/llms/groq/cost_calculator.py index 19d87c44940..247f5d63438 100644 --- a/litellm/llms/groq/cost_calculator.py +++ b/litellm/llms/groq/cost_calculator.py @@ -16,15 +16,22 @@ if TYPE_CHECKING: def cost_per_web_search_request(usage: Usage, model_info: "ModelInfo") -> float: - search_costs: Final = model_info.get("search_context_cost_per_query") - cost_per_search: Final = search_costs.get("search_context_size_medium", 0.0) if search_costs else 0.0 + from litellm._cost import Usage as TokenUsage + from litellm.litellm_core_utils.llm_cost_calc.utils import search_cost + try: server_tool_use: Final = usage.server_tool_use except AttributeError: return 0.0 if server_tool_use is None: return 0.0 - searches: Final = server_tool_use.web_search_requests or 0 - opens: Final = server_tool_use.browser_open_requests or 0 - cost_per_open: Final = model_info.get("browser_open_cost_per_request") or 0.0 - return searches * cost_per_search + opens * cost_per_open + return search_cost( + model_info, + TokenUsage( + prompt_tokens=0, + text=0, + output_text=0, + web_search_requests=server_tool_use.web_search_requests or 0, + browser_opens=server_tool_use.browser_open_requests or 0, + ), + ) diff --git a/litellm/llms/openai/cost_calculation.py b/litellm/llms/openai/cost_calculation.py index 63be44e83b9..e7424c490c6 100644 --- a/litellm/llms/openai/cost_calculation.py +++ b/litellm/llms/openai/cost_calculation.py @@ -5,7 +5,7 @@ Helper util for handling openai-specific cost calculation from typing import Literal -from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text from litellm.types.utils import CallTypes, Usage @@ -35,7 +35,7 @@ def cost_per_token( Returns: Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd """ - return generic_cost_per_token( + return cost_per_text( model=model, usage=usage, custom_llm_provider="openai", diff --git a/litellm/llms/openai/image_generation/cost_calculator.py b/litellm/llms/openai/image_generation/cost_calculator.py index 350fdbaee2d..a5da2dd2207 100644 --- a/litellm/llms/openai/image_generation/cost_calculator.py +++ b/litellm/llms/openai/image_generation/cost_calculator.py @@ -9,8 +9,8 @@ from typing import Final from litellm import verbose_logger from litellm.litellm_core_utils.llm_cost_calc.utils import ( calculate_image_response_cost_from_usage, + cost_per_text, flat_image_cost, - generic_cost_per_token, resolve_image_model_info, ) from litellm.types.utils import ImageResponse, ModelInfo, Usage @@ -35,9 +35,9 @@ def cost_calculator( verbose_logger.debug("No usage data available for %s, cannot calculate token-based cost", model) return flat_image_cost(price_table, image_response) - # A chat Usage with an explicit output breakdown: cost via generic_cost_per_token. + # A chat Usage with an explicit output breakdown: cost via cost_per_text. if isinstance(usage, Usage) and usage.completion_tokens_details is not None: - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model, usage=usage, custom_llm_provider=provider, model_info=price_table ) return prompt_cost + completion_cost @@ -54,9 +54,9 @@ def cost_calculator( return token_based_cost # Fallback: a Usage with no output breakdown that the image helper can't read — - # cost via generic_cost_per_token (text rate) instead of returning 0.0. + # cost via cost_per_text (text rate) instead of returning 0.0. if isinstance(usage, Usage): - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model, usage=usage, custom_llm_provider=provider, model_info=price_table ) return prompt_cost + completion_cost diff --git a/litellm/llms/parallel_ai/search/cost_calculator.py b/litellm/llms/parallel_ai/search/cost_calculator.py index 477db6d66a9..30dff0e94e6 100644 --- a/litellm/llms/parallel_ai/search/cost_calculator.py +++ b/litellm/llms/parallel_ai/search/cost_calculator.py @@ -4,7 +4,9 @@ from typing import Final from pydantic import TypeAdapter, ValidationError -from litellm.utils import get_model_info +from litellm._cost import Usage as TokenUsage +from litellm._cost import as_main_cost +from litellm.litellm_core_utils.llm_cost_calc.utils import price_usage, pricing_row PARALLEL_AI_DEFAULT_RESULTS: Final = 10 PARALLEL_AI_USAGE_PARAM: Final = "_parallel_ai_usage" @@ -60,16 +62,6 @@ def _effective_max_results(optional_params: Mapping[str, object]) -> int: return max_results if max_results is not None else PARALLEL_AI_DEFAULT_RESULTS -def _request_cost(mode: str) -> tuple[float, float]: - """The per-request price and the price of each result past the included ones""" - pricing_model: Final = PARALLEL_AI_PRICING_MODEL_BY_MODE.get(mode, PARALLEL_AI_STANDARD_SEARCH_MODEL) - model_info: Final = get_model_info(model=pricing_model, custom_llm_provider="parallel_ai") - return ( - float(model_info.get("input_cost_per_query") or 0.0), - float(model_info.get("cost_per_additional_result") or 0.0), - ) - - def _additional_results( optional_params: Mapping[str, object], usage: Sequence[Mapping[str, object]] | None, @@ -86,8 +78,19 @@ def parallel_ai_search_cost( optional_params: Mapping[str, object], usage: Sequence[Mapping[str, object]] | None, ) -> float: - request_cost, additional_result_cost = _request_cost(_effective_mode(optional_params)) + mode: Final = _effective_mode(optional_params) + pricing_model: Final = PARALLEL_AI_PRICING_MODEL_BY_MODE.get(mode, PARALLEL_AI_STANDARD_SEARCH_MODEL) request_count_from_usage: Final = _usage_count(usage, "sku_search") if usage is not None else None - request_count: Final = request_count_from_usage if request_count_from_usage is not None else 1 - additional_results: Final = _additional_results(optional_params, usage) - return request_count * request_cost + additional_results * additional_result_cost + prompt_cost, completion_cost = as_main_cost( + price_usage( + pricing_row(pricing_model, "parallel_ai", None), + TokenUsage( + prompt_tokens=0, + text=0, + output_text=0, + search_queries=request_count_from_usage if request_count_from_usage is not None else 1, + additional_results=_additional_results(optional_params, usage), + ), + ) + ) + return prompt_cost + completion_cost diff --git a/litellm/llms/recraft/cost_calculator.py b/litellm/llms/recraft/cost_calculator.py index 052e09a30c4..75ddeb7fe44 100644 --- a/litellm/llms/recraft/cost_calculator.py +++ b/litellm/llms/recraft/cost_calculator.py @@ -1,7 +1,7 @@ from typing import Final import litellm -from litellm.litellm_core_utils.llm_cost_calc.utils import resolve_image_model_info +from litellm.litellm_core_utils.llm_cost_calc.utils import image_cost, resolve_image_model_info from litellm.types.utils import ImageResponse, ModelInfo @@ -18,11 +18,10 @@ def cost_calculator( custom_llm_provider=litellm.LlmProviders.RECRAFT.value, model_info=model_info, ) - output_cost_per_image: Final[float] = _model_info.get("output_cost_per_image") or 0.0 num_images: int = 0 if isinstance(image_response, ImageResponse): if image_response.data: num_images = len(image_response.data) - return output_cost_per_image * num_images + return image_cost(_model_info, num_images) or 0.0 else: raise ValueError(f"image_response must be of type ImageResponse got type={type(image_response)}") diff --git a/litellm/llms/runwayml/cost_calculator.py b/litellm/llms/runwayml/cost_calculator.py index 729ca516213..f10dcd5f69b 100644 --- a/litellm/llms/runwayml/cost_calculator.py +++ b/litellm/llms/runwayml/cost_calculator.py @@ -1,7 +1,7 @@ from typing import Final import litellm -from litellm.litellm_core_utils.llm_cost_calc.utils import resolve_image_model_info +from litellm.litellm_core_utils.llm_cost_calc.utils import image_cost, resolve_image_model_info from litellm.types.utils import ImageResponse, ModelInfo @@ -21,11 +21,10 @@ def cost_calculator( custom_llm_provider=litellm.LlmProviders.RUNWAYML.value, model_info=model_info, ) - output_cost_per_image: Final[float] = _model_info.get("output_cost_per_image") or 0.0 num_images: int = 0 if isinstance(image_response, ImageResponse): if image_response.data: num_images = len(image_response.data) - return output_cost_per_image * num_images + return image_cost(_model_info, num_images) or 0.0 else: raise ValueError(f"image_response must be of type ImageResponse, got type={type(image_response)}") diff --git a/litellm/llms/tencent/cost_calculator.py b/litellm/llms/tencent/cost_calculator.py index d9aebdc3284..6347ef24298 100644 --- a/litellm/llms/tencent/cost_calculator.py +++ b/litellm/llms/tencent/cost_calculator.py @@ -1,6 +1,6 @@ -from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text from litellm.types.utils import Usage def cost_per_token(model: str, usage: Usage) -> tuple[float, float]: - return generic_cost_per_token(model=model, usage=usage, custom_llm_provider="tencent") + return cost_per_text(model=model, usage=usage, custom_llm_provider="tencent") diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index d0453b72701..80584f2445d 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -80,18 +80,30 @@ def _vertex_ai_lyria_model_key(model: str) -> str: return model if model.startswith("vertex_ai/") else f"vertex_ai/{model}" -def _vertex_ai_lyria_generation_cost(model_info: VertexAILyriaModelInfo | None) -> float | None: - return None if model_info is None else model_info.get("output_cost_per_image") - - def get_vertex_ai_lyria_model_info(model: str) -> VertexAILyriaModelInfo | None: model_key: Final = _vertex_ai_lyria_model_key(model) return _validate_vertex_ai_lyria_model_info(litellm.model_cost.get(model_key)) -def get_vertex_ai_lyria_generation_cost(model: str) -> float | None: +def get_vertex_ai_lyria_generation_cost(model: str, generations: int = 1) -> float | None: + """``generations`` Lyria generations at the row's per-generation price, None when the model is not a priced Lyria + row""" + from litellm._cost import ImageOut, Priced + from litellm._cost import Usage as TokenUsage + from litellm.litellm_core_utils.llm_cost_calc.utils import price_usage + model_key: Final = _vertex_ai_lyria_model_key(model) - return _vertex_ai_lyria_generation_cost(_validate_vertex_ai_lyria_model_info(litellm.model_cost.get(model_key))) + row: Final = litellm.model_cost.get(model_key) + if row is None or _validate_vertex_ai_lyria_model_info(row) is None: + return None + priced: Final = price_usage( + row, + TokenUsage(prompt_tokens=0, text=0, output_text=0, images_out=[ImageOut()] * max(generations, 0)), + key=model_key, + ) + if not isinstance(priced, Priced) or (generations > 0 and not priced.lines): + return None + return priced.total class VertexAIError(BaseLLMException): diff --git a/litellm/llms/vertex_ai/cost_calculator.py b/litellm/llms/vertex_ai/cost_calculator.py index 8ec034e7a5e..0cc13c2dee4 100644 --- a/litellm/llms/vertex_ai/cost_calculator.py +++ b/litellm/llms/vertex_ai/cost_calculator.py @@ -1,14 +1,8 @@ # What is this? ## Cost calculation for Google AI Studio / Vertex AI models -from typing import Final -import litellm -from litellm.litellm_core_utils.llm_cost_calc.utils import ( - generic_cost_per_token, - get_vertex_regional_endpoint_uplift, - is_above_128k, -) -from litellm.types.utils import ModelInfo, Usage +from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text +from litellm.types.utils import Usage """ Gemini pricing covers: @@ -19,32 +13,6 @@ Gemini pricing covers: """ -def _handle_128k_pricing( - model_info: ModelInfo, - usage: Usage, -) -> tuple[float, float]: - ## CALCULATE INPUT COST - input_cost_per_token_above_128k_tokens: Final = model_info.get("input_cost_per_token_above_128k_tokens") - output_cost_per_token_above_128k_tokens = model_info.get("output_cost_per_token_above_128k_tokens") - - prompt_tokens: Final = usage.prompt_tokens - completion_tokens: Final = usage.completion_tokens - - if is_above_128k(tokens=prompt_tokens) and input_cost_per_token_above_128k_tokens is not None: - prompt_cost = prompt_tokens * input_cost_per_token_above_128k_tokens - else: - prompt_cost = prompt_tokens * (model_info["input_cost_per_token"] or 0.0) - - ## CALCULATE OUTPUT COST - output_cost_per_token_above_128k_tokens = model_info.get("output_cost_per_token_above_128k_tokens") - if is_above_128k(tokens=completion_tokens) and output_cost_per_token_above_128k_tokens is not None: - completion_cost = completion_tokens * output_cost_per_token_above_128k_tokens - else: - completion_cost = completion_tokens * (model_info["output_cost_per_token"] or 0.0) - - return prompt_cost, completion_cost - - def cost_per_token( model: str, custom_llm_provider: str, @@ -67,26 +35,9 @@ def cost_per_token( Returns: Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd - - Raises: - Exception if model requires >128k pricing, but model cost not mapped """ - ## GET MODEL INFO - model_info: Final = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider) - - ## HANDLE 128k+ PRICING - input_cost_per_token_above_128k_tokens: Final = model_info.get("input_cost_per_token_above_128k_tokens") - output_cost_per_token_above_128k_tokens: Final = model_info.get("output_cost_per_token_above_128k_tokens") - if input_cost_per_token_above_128k_tokens is not None or output_cost_per_token_above_128k_tokens is not None: - prompt_cost_128k, completion_cost_128k = _handle_128k_pricing( - model_info=model_info, - usage=usage, - ) - vertex_uplift: Final = get_vertex_regional_endpoint_uplift(model_info, vertex_location) - return prompt_cost_128k * vertex_uplift, completion_cost_128k * vertex_uplift - - return generic_cost_per_token( + return cost_per_text( model=model, custom_llm_provider=custom_llm_provider, usage=usage, diff --git a/litellm/llms/vertex_ai/image_edit/cost_calculator.py b/litellm/llms/vertex_ai/image_edit/cost_calculator.py index 7d1c2c1fc51..d6b3c245bd4 100644 --- a/litellm/llms/vertex_ai/image_edit/cost_calculator.py +++ b/litellm/llms/vertex_ai/image_edit/cost_calculator.py @@ -5,6 +5,7 @@ Vertex AI Image Edit Cost Calculator from typing import Final import litellm +from litellm.litellm_core_utils.llm_cost_calc.utils import image_cost from litellm.types.utils import ImageResponse @@ -23,10 +24,8 @@ def cost_calculator( custom_llm_provider="vertex_ai", ) - output_cost_per_image: Final[float] = model_info.get("output_cost_per_image") or 0.0 - if not isinstance(image_response, ImageResponse): raise ValueError(f"image_response must be of type ImageResponse got type={type(image_response)}") num_images: Final = len(image_response.data or []) - return output_cost_per_image * num_images + return image_cost(model_info, num_images) or 0.0 diff --git a/litellm/llms/xai/cost_calculator.py b/litellm/llms/xai/cost_calculator.py index aec535c2086..6405535f086 100644 --- a/litellm/llms/xai/cost_calculator.py +++ b/litellm/llms/xai/cost_calculator.py @@ -9,7 +9,7 @@ import math from collections.abc import Mapping from typing import TYPE_CHECKING, Final -from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text from litellm.types.utils import PromptTokensDetailsWrapper, Usage if TYPE_CHECKING: @@ -53,7 +53,7 @@ def _split_reported_cost(model: str, usage: Usage, reported_cost: float) -> tupl """xAI reports one total, so attribute it to input and output in the ratio our own token math gives.""" try: priced_input, priced_output = _token_math_cost_per_token(model, usage) - except Exception: # noqa: BLE001 # generic_cost_per_token raises bare Exception for unmapped models + except Exception: # noqa: BLE001 # cost_per_text raises bare Exception for unmapped models return 0.0, reported_cost priced_total: Final = priced_input + priced_output if priced_total <= 0: @@ -105,7 +105,7 @@ def _token_math_cost_per_token(model: str, usage: Usage) -> tuple[float, float]: completion_tokens_details=None, ) - prompt_cost, completion_cost = generic_cost_per_token(model=model, usage=modified_usage, custom_llm_provider="xai") + prompt_cost, completion_cost = cost_per_text(model=model, usage=modified_usage, custom_llm_provider="xai") return prompt_cost, completion_cost diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py index 5010de46fcb..ea05f1cb97a 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py @@ -442,7 +442,7 @@ class VertexPassthroughLoggingHandler: prediction_count: Final = VertexPassthroughLoggingHandler._get_audio_prediction_count( json_response=json_response ) - response_cost: Final = (get_vertex_ai_lyria_generation_cost(model=model) or 0.0) * prediction_count + response_cost: Final = get_vertex_ai_lyria_generation_cost(model=model, generations=prediction_count) or 0.0 logging_obj.model = model # rebind-ok: passthrough attribution records the resolved Vertex model logging_obj.model_call_details[ # rebind-ok: passthrough attribution enriches callback metadata diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py index 604a7a389e6..589fe238799 100644 --- a/litellm/proxy/spend_tracking/savings.py +++ b/litellm/proxy/spend_tracking/savings.py @@ -22,7 +22,7 @@ from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY from litellm.litellm_core_utils.get_llm_provider_logic import resolve_model_and_provider_for_metadata from litellm.litellm_core_utils.llm_cost_calc.utils import ( calculate_prompt_caching_savings, - generic_cost_per_token, + cost_per_text, get_cost_per_unit, ) from litellm.types.integrations.anthropic_cache_control_hook import ( @@ -195,7 +195,7 @@ def _pricing_basis(cost_breakdown: Mapping[str, object] | None) -> PricingBasis: they price at standard rates, which is what they already did. These values survive a JSON round trip on the way here, so none is guaranteed to be - a string. `generic_cost_per_token` calls `.lower()` on them without a type check, and + a string. `cost_per_text` calls `.lower()` on them without a type check, and the resulting `AttributeError` would be swallowed into a silent zero by the caller's `except`, so anything that is not a string is dropped here instead. """ @@ -215,7 +215,7 @@ def _recorded_token_cost(cost_breakdown: Mapping[str, object] | None) -> float | """What the biller charged for this request's tokens, or ``None`` when unrecorded. ``input_cost`` already carries the cache buckets, so it and ``output_cost`` sum to - exactly what `generic_cost_per_token` returns for the same request; the separate + exactly what `cost_per_text` returns for the same request; the separate ``cache_read_cost`` and ``cache_creation_cost`` entries decompose that sum rather than adding to it, and including them would charge those tokens twice. @@ -251,7 +251,7 @@ def _cost_of_usage( model_info=model_info, ) else: - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model.model, usage=usage, custom_llm_provider=model.provider, diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 0ddfc5ce62f..65206086aa0 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -161,7 +161,7 @@ from litellm.litellm_core_utils.core_helpers import ( from litellm.litellm_core_utils.duration_parser import get_budget_window_start from litellm.litellm_core_utils.env_utils import get_env_float, get_env_int from litellm.litellm_core_utils.litellm_logging import BilledRoundsDispatch, Logging, deployment_pricing_model_info -from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.litellm_core_utils.served_output_texts import ( @@ -1289,12 +1289,12 @@ def _deployment_priced_input_cost( model: str, custom_llm_provider: str | None, prompt_tokens: int, deployment_pricing: ModelInfo ) -> float: """Input price for the estimated prompt at the deployment's registered rates, read through - ``deployment_pricing_model_info`` and priced by ``generic_cost_per_token`` the way the success + ``deployment_pricing_model_info`` and priced by ``cost_per_text`` the way the success path prices them, so a long-context tier the deployment declares (``input_cost_per_token_above_200k_tokens``) is honoured instead of flattened to the standard rate. An entry it cannot price keeps the row at $0.""" usage: Final = Usage(prompt_tokens=prompt_tokens, completion_tokens=0, total_tokens=prompt_tokens) try: - prompt_cost, _ = generic_cost_per_token( + prompt_cost, _ = cost_per_text( model=model, usage=usage, custom_llm_provider=custom_llm_provider or str(deployment_pricing.get("litellm_provider") or ""), diff --git a/litellm/router_strategy/savings_baseline.py b/litellm/router_strategy/savings_baseline.py index add0f78fd3a..f4ed6d8f72e 100644 --- a/litellm/router_strategy/savings_baseline.py +++ b/litellm/router_strategy/savings_baseline.py @@ -110,14 +110,14 @@ def _priced(router: "Router", candidate: Baseline) -> tuple[float, Baseline] | N dimension to that engine. A candidate that prices to nothing there cannot stand in for what the traffic would have cost. """ - from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token + from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text provider, _, model_name = candidate.model.partition("/") try: info: Final = router.cached_deployment_model_info(candidate.deployment_id or "", candidate.model) if info is None: return None - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model_name or candidate.model, usage=_REFERENCE_REQUEST, custom_llm_provider=provider, diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi index de8c1ebc3ec..c27694e7b56 100644 --- a/litellm/rust_bridge/_native.pyi +++ b/litellm/rust_bridge/_native.pyi @@ -156,8 +156,23 @@ class Usage: output_video_duration: timedelta | None = None, video_resolution: str | None = None, text_messages: int = 0, + images_in: int = 0, + images_out: Sequence[ImageOut] = (), + pixels_in: int = 0, + search_queries: int = 0, + max_results: int | None = None, + web_search_requests: int = 0, + web_search_context: Literal["low", "medium", "high"] = "medium", + maps_grounding_requests: int = 0, + citation_tokens: int = 0, + browser_opens: int = 0, + additional_results: int = 0, ) -> None: ... +@final +class ImageOut: + def __init__(self, *, size: str | None = None, width: int | None = None, height: int | None = None) -> None: ... + @final class CallContext: @@ -170,6 +185,8 @@ class CallContext: vertex_location: str | None, at: datetime, custom_pricing: Mapping[str, object] | None, + speed: str | None = None, + inference_geo: str | None = None, ) -> None: ... @final @@ -208,6 +225,15 @@ class Catalog: def bound_to(module: ModuleType, attr: str) -> Catalog: ... def bundled(self) -> Fetched: ... def fetch(self, source: Remote | File | Bundled, *, timeout: float, attempts: int) -> Fetched | Unavailable: ... + def resolve_image( + self, + model: str, + provider: str | None = None, + *, + size: tuple[int, int] | None = None, + quality: str | None = None, + ) -> Resolved | Missing | Invalid: ... + def nearest_image_key(self, model: str, provider: str, *, quality: str, size: tuple[int, int]) -> str | None: ... def reload(self, source: Remote | File | Bundled) -> Installed | Unavailable: ... def provenance(self) -> Provenance | None: ... def resolve(self, model: str, provider: str | None = None) -> Resolved | Missing | Invalid: ... diff --git a/litellm/search/cost_calculator.py b/litellm/search/cost_calculator.py index 15755128136..8a0e14797d7 100644 --- a/litellm/search/cost_calculator.py +++ b/litellm/search/cost_calculator.py @@ -9,8 +9,10 @@ from typing import Final from pydantic import TypeAdapter, ValidationError import litellm +from litellm._cost import Usage as TokenUsage +from litellm._cost import as_main_cost from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap -from litellm.utils import get_model_info +from litellm.litellm_core_utils.llm_cost_calc.utils import price_usage, pricing_row PROVIDER_USAGE_ADAPTER: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] = TypeAdapter( tuple[Mapping[str, object], ...] @@ -93,28 +95,16 @@ def search_provider_cost_per_query( ) return (input_cost, 0.0) - model_info: Final = get_model_info( - model=_pricing_model(model, custom_llm_provider, optional_params), - custom_llm_provider=custom_llm_provider, + max_results: Final = (optional_params or {}).get("max_results") + return as_main_cost( + price_usage( + pricing_row(_pricing_model(model, custom_llm_provider, optional_params), custom_llm_provider, None), + TokenUsage( + prompt_tokens=0, + text=0, + output_text=0, + search_queries=number_of_queries, + max_results=max_results if isinstance(max_results, int) and not isinstance(max_results, bool) else None, + ), + ) ) - - # Check for tiered pricing (e.g., Exa AI based on max_results) - tiered_pricing: Final = model_info.get("tiered_pricing") - if tiered_pricing and isinstance(tiered_pricing, list): - max_results: Final = (optional_params or {}).get("max_results", 10) # default 10 results - cost_per_query = 0.0 - - for tier in tiered_pricing: - range_min, range_max = tier["max_results_range"] - if range_min <= max_results <= range_max: - cost_per_query = tier["input_cost_per_query"] - break - else: - # Fallback to highest tier if out of range - cost_per_query = tiered_pricing[-1]["input_cost_per_query"] - else: - # Simple flat rate - cost_per_query = float(model_info.get("input_cost_per_query") or 0.0) - - total_cost: Final = number_of_queries * cost_per_query - return (total_cost, 0.0) # (input_cost, output_cost) diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index e36ef2a11f5..3103258e4fd 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -1,4 +1,3 @@ -import json from collections.abc import Mapping from datetime import datetime, timezone from typing import Final, cast @@ -11,19 +10,14 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import ( BilledTokenRates, CostCalculatorUtils, TokenRates, - _get_token_base_cost, _is_off_peak, _is_within_off_peak_window, apply_off_peak_pricing, - apply_provider_cache_read_default, calculate_cache_writing_cost, - generic_cost_per_token, + cost_per_text, get_billed_token_rates, get_token_type_cost_breakdown, ) -from litellm.llms.gemini.image_generation.cost_calculator import ( - cost_calculator as gemini_image_generation_cost_calculator, -) from litellm.llms.vertex_ai.image_generation.cost_calculator import ( cost_calculator as vertex_image_generation_cost_calculator, ) @@ -46,28 +40,7 @@ def _local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) -@pytest.mark.parametrize("prompt_tokens", [100, 200000, 200001]) -@pytest.mark.parametrize("read_rate", [None, 0.0, 0.25e-6]) -@pytest.mark.parametrize("service_tier", [None, "priority"]) -def test_missing_cache_read_rate_resolves_to_input_rate(prompt_tokens, read_rate, service_tier): - info = { - "input_cost_per_token": 3e-6, - "input_cost_per_token_priority": 4e-6, - "input_cost_per_token_above_200k_tokens": 6e-6, - "input_cost_per_token_above_200k_tokens_priority": 8e-6, - "output_cost_per_token": 1e-6, - "cache_read_input_token_cost": read_rate, - } - usage = Usage(prompt_tokens=prompt_tokens, prompt_tokens_details={"cached_tokens": 100}) - billed = _get_token_base_cost(info, usage, service_tier=service_tier) - prompt_cost, _ = generic_cost_per_token( - "policy-fixture", usage, "openai", service_tier=service_tier, model_info=info - ) - assert billed.cache_read_rate == pytest.approx(read_rate if read_rate is not None else billed.input_rate) - assert prompt_cost == pytest.approx((prompt_tokens - 100) * billed.input_rate + 100 * billed.cache_read_rate) - - -def test_generic_cost_per_token_bills_cache_reads_at_input_rate_when_no_cache_read_rate() -> None: +def test_cost_per_text_bills_cache_reads_at_input_rate_when_no_cache_read_rate() -> None: model_info: ModelInfo = { "key": "bare-model", "max_tokens": None, @@ -86,7 +59,7 @@ def test_generic_cost_per_token_bills_cache_reads_at_input_rate_when_no_cache_re prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=12288), ) - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model="bare-model", usage=usage, custom_llm_provider="bedrock", @@ -110,7 +83,7 @@ def test_generic_cost_per_token_bills_cache_reads_at_input_rate_when_no_cache_re "token_priced_model_ignores_the_character_count", ], ) -def test_generic_cost_per_token_bills_a_character_counted_prompt_under_one_unit( +def test_cost_per_text_bills_a_character_counted_prompt_under_one_unit( model_info: ModelInfo, expected_prompt_cost: float ) -> None: usage = Usage( @@ -120,27 +93,14 @@ def test_generic_cost_per_token_bills_a_character_counted_prompt_under_one_unit( prompt_tokens_details=PromptTokensDetailsWrapper(character_count=35), ) - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model="embedding-fixture", usage=usage, custom_llm_provider="vertex_ai", model_info=model_info ) assert (prompt_cost, completion_cost) == (pytest.approx(expected_prompt_cost), 0.0) -def test_apply_provider_cache_read_default_only_derives_a_rate_for_fireworks() -> None: - openai_info: ModelInfo = {"input_cost_per_token": 2e-6} - fireworks_info: ModelInfo = {"input_cost_per_token": 2e-6} - - assert apply_provider_cache_read_default(openai_info, "openai") is openai_info - assert apply_provider_cache_read_default(openai_info, None) is openai_info - - processed_fireworks_info = apply_provider_cache_read_default(fireworks_info, "fireworks_ai") - - assert processed_fireworks_info is not fireworks_info - assert processed_fireworks_info["cache_read_input_token_cost"] == pytest.approx(2e-6 * 0.5) - - -def test_generic_cost_per_token_prefers_audio_per_second_rate() -> None: +def test_cost_per_text_prefers_audio_per_second_rate() -> None: model_info: ModelInfo = { "key": "gemini-embedding-2", "max_tokens": None, @@ -164,7 +124,7 @@ def test_generic_cost_per_token_prefers_audio_per_second_rate() -> None: ), ) - prompt_cost, _ = generic_cost_per_token( + prompt_cost, _ = cost_per_text( model="gemini-embedding-2", usage=usage, custom_llm_provider="vertex_ai", @@ -174,7 +134,7 @@ def test_generic_cost_per_token_prefers_audio_per_second_rate() -> None: assert prompt_cost == pytest.approx(2 * 0.00016) -def test_generic_cost_per_token_prefers_image_per_image_rate() -> None: +def test_cost_per_text_prefers_image_per_image_rate() -> None: model_info: ModelInfo = { "key": "gemini-embedding-2", "max_tokens": None, @@ -198,7 +158,7 @@ def test_generic_cost_per_token_prefers_image_per_image_rate() -> None: ), ) - prompt_cost, _ = generic_cost_per_token( + prompt_cost, _ = cost_per_text( model="gemini-embedding-2", usage=usage, custom_llm_provider="vertex_ai", @@ -208,7 +168,7 @@ def test_generic_cost_per_token_prefers_image_per_image_rate() -> None: assert prompt_cost == pytest.approx(0.00012) -def test_generic_cost_per_token_prefers_video_per_second_rate() -> None: +def test_cost_per_text_prefers_video_per_second_rate() -> None: model_info: ModelInfo = { "key": "gemini-embedding-2", "max_tokens": None, @@ -232,7 +192,7 @@ def test_generic_cost_per_token_prefers_video_per_second_rate() -> None: ), ) - prompt_cost, _ = generic_cost_per_token( + prompt_cost, _ = cost_per_text( model="gemini-embedding-2", usage=usage, custom_llm_provider="vertex_ai", @@ -242,18 +202,6 @@ def test_generic_cost_per_token_prefers_video_per_second_rate() -> None: assert prompt_cost == pytest.approx(2 * 0.00079) -def test_missing_cache_read_uses_off_peak_input_rate(): - from datetime import datetime, timezone - - info = { - "input_cost_per_token": 3e-6, - "off_peak_pricing": {"hours_utc": "00:00-23:59", "input_cost_per_token": 5e-6}, - } - when = datetime(2026, 9, 7, 12, tzinfo=timezone.utc) - billed = _get_token_base_cost(info, Usage(prompt_tokens=100), current_time=when) - assert billed.input_rate == billed.cache_read_rate == 5e-6 - - def test_reasoning_tokens_no_price_set(_local_model_cost_map): # Use o1 - o1-mini was deprecated/renamed; o1 has same reasoning-token semantics # (no separate output_cost_per_reasoning_token, so all completion tokens use output_cost_per_token) @@ -274,7 +222,7 @@ def test_reasoning_tokens_no_price_set(_local_model_cost_map): audio_tokens=None, cached_tokens=None, text_tokens=17, image_tokens=None ), ) - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model, usage=usage, custom_llm_provider="openai", @@ -310,7 +258,7 @@ def test_reasoning_tokens_gemini(_local_model_cost_map): ), ) model_cost_map = litellm.model_cost[model] - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model, usage=usage, custom_llm_provider=custom_llm_provider, @@ -359,7 +307,7 @@ def test_image_tokens_with_custom_pricing(): "litellm.litellm_core_utils.llm_cost_calc.utils.get_model_info", return_value=mock_model_info, ): - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model="test-model", usage=usage, custom_llm_provider="gemini" ) @@ -405,7 +353,7 @@ def test_image_tokens_fallback_to_base_cost(): "litellm.litellm_core_utils.llm_cost_calc.utils.get_model_info", return_value=mock_model_info, ): - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model="test-model", usage=usage, custom_llm_provider="gemini" ) @@ -432,7 +380,7 @@ def test_video_input_tokens_gemini_omni_flash_preview(_local_model_cost_map): ) model_cost_map = litellm.model_cost[f"gemini/{model}"] - prompt_cost, _ = generic_cost_per_token( + prompt_cost, _ = cost_per_text( model=model, usage=usage, custom_llm_provider="gemini", @@ -468,7 +416,7 @@ def test_video_tokens_fallback_to_base_cost(): "litellm.litellm_core_utils.llm_cost_calc.utils.get_model_info", return_value=mock_model_info, ): - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model="test-model", usage=usage, custom_llm_provider="gemini" ) @@ -499,7 +447,7 @@ def test_prompt_audio_and_video_tokens_without_modality_rates_bill_at_tiered_inp "litellm.litellm_core_utils.llm_cost_calc.utils.get_model_info", return_value=mock_model_info, ): - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model="test-model", usage=usage, custom_llm_provider="vertex_ai" ) @@ -531,12 +479,12 @@ def test_prompt_modality_rates_still_win_over_tiered_input_rate(): "litellm.litellm_core_utils.llm_cost_calc.utils.get_model_info", return_value=mock_model_info, ): - prompt_cost, _ = generic_cost_per_token(model="test-model", usage=usage, custom_llm_provider="vertex_ai") + prompt_cost, _ = cost_per_text(model="test-model", usage=usage, custom_llm_provider="vertex_ai") assert round(prompt_cost, 9) == round(100000 * 4e-6 + 100000 * 3e-6 + 100000 * 1e-6, 9) -def test_generic_cost_per_token_above_200k_tokens(_local_model_cost_map): +def test_cost_per_text_above_200k_tokens(_local_model_cost_map): # gemini-2.5-pro-exp-03-25 was removed; gemini-2.5-pro has same above-200k pricing model = "gemini-2.5-pro" custom_llm_provider = "vertex_ai" @@ -549,7 +497,7 @@ def test_generic_cost_per_token_above_200k_tokens(_local_model_cost_map): completion_tokens=completion_tokens, total_tokens=prompt_tokens + completion_tokens, ) - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model, usage=usage, custom_llm_provider=custom_llm_provider, @@ -564,26 +512,6 @@ def test_generic_cost_per_token_above_200k_tokens(_local_model_cost_map): ) -def test_get_token_base_cost_picks_highest_crossed_tier(): - """Regression test for #30345. - - With graduated tiers at 90k and 128k whose keys have different digit lengths, a request - crossing both must be billed at the highest tier it crosses (128k), not the lower one that - happens to sort first lexicographically. - """ - model_info = { - "input_cost_per_token": 1e-6, - "output_cost_per_token": 2e-6, - "input_cost_per_token_above_90k_tokens": 5e-6, - "input_cost_per_token_above_128k_tokens": 9e-6, - } - usage = Usage(prompt_tokens=150_000, completion_tokens=10, total_tokens=150_010) - - prompt_base_cost = _get_token_base_cost(model_info, usage).input_rate - - assert prompt_base_cost == 9e-6 - - def test_is_within_off_peak_window_same_day(): from datetime import datetime, timezone @@ -754,152 +682,6 @@ def test_is_off_peak_flat_hours_and_windows_are_a_union(): assert _is_off_peak(block, datetime(2026, 8, 28, 20, 0, tzinfo=timezone.utc)) is False -def test_get_token_base_cost_weekend_only_off_peak_rate(): - from datetime import datetime, timezone - from typing import cast - - from litellm.types.utils import ModelInfo - - model_info = cast( - ModelInfo, - { - "input_cost_per_token": 1e-6, - "output_cost_per_token": 2e-6, - "off_peak_pricing": { - "windows": [ - {"hours_utc": ["00:00-01:00", "04:00-06:00", "10:00-00:00"], "weekdays": [1, 2, 3, 4, 5]}, - {"hours_utc": "00:00-00:00", "weekdays": [6, 7]}, - ], - "input_cost_per_token": 5e-7, - "output_cost_per_token": 1e-6, - }, - }, - ) - usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) - - saturday_peak_hours = _get_token_base_cost( - model_info, usage, current_time=datetime(2026, 8, 29, 2, 0, tzinfo=timezone.utc) - ) - assert (saturday_peak_hours.input_rate, saturday_peak_hours.output_rate) == (5e-7, 1e-6) - - monday_same_hours = _get_token_base_cost( - model_info, usage, current_time=datetime(2026, 8, 24, 2, 0, tzinfo=timezone.utc) - ) - assert (monday_same_hours.input_rate, monday_same_hours.output_rate) == (1e-6, 2e-6) - - -def test_get_token_base_cost_applies_off_peak_pricing(): - from datetime import datetime, timezone - from typing import cast - - from litellm.types.utils import ModelInfo - - model_info = cast( - ModelInfo, - { - "input_cost_per_token": 1e-6, - "output_cost_per_token": 2e-6, - "cache_read_input_token_cost": 1e-7, - "off_peak_pricing": { - "hours_utc": "16:30-00:30", - "input_cost_per_token": 5e-7, - "output_cost_per_token": 1e-6, - "cache_read_input_token_cost": 5e-8, - }, - }, - ) - usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) - - off_peak = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc)) - assert off_peak.input_rate == 5e-7 - assert off_peak.output_rate == 1e-6 - assert off_peak.cache_read_rate == 5e-8 - - peak = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc)) - assert peak.input_rate == 1e-6 - assert peak.output_rate == 2e-6 - assert peak.cache_read_rate == 1e-7 - - -def test_get_token_base_cost_non_mapping_off_peak_block_bills_standard_rates(): - """A truthy non-mapping off_peak_pricing value (a bare string or a list in - YAML) must bill standard rates rather than raising, matching how every - other malformed piece of the block behaves. - """ - from datetime import datetime, timezone - from typing import cast - - from litellm.types.utils import ModelInfo - - usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) - when = datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc) - - for malformed_block in ("16:00-19:00", ["16:00-19:00"], 5e-7, True): - model_info = cast( - ModelInfo, - { - "input_cost_per_token": 1e-6, - "output_cost_per_token": 2e-6, - "off_peak_pricing": malformed_block, - }, - ) - result = _get_token_base_cost(model_info, usage, current_time=when) - assert result.input_rate == 1e-6 - assert result.output_rate == 2e-6 - - -def test_get_token_base_cost_off_peak_falls_back_to_standard_when_unset(): - from datetime import datetime, timezone - from typing import cast - - from litellm.types.utils import ModelInfo - - model_info = cast( - ModelInfo, - { - "input_cost_per_token": 1e-6, - "output_cost_per_token": 2e-6, - "off_peak_pricing": {"hours_utc": "16:30-00:30", "input_cost_per_token": 5e-7}, - }, - ) - usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) - - result = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc)) - assert result.input_rate == 5e-7 - assert result.output_rate == 2e-6 - - -def test_get_token_base_cost_off_peak_wins_over_threshold(): - from datetime import datetime, timezone - from typing import cast - - from litellm.types.utils import ModelInfo - - model_info = cast( - ModelInfo, - { - "input_cost_per_token": 1e-6, - "output_cost_per_token": 2e-6, - "input_cost_per_token_above_200k_tokens": 3e-6, - "output_cost_per_token_above_200k_tokens": 4e-6, - "off_peak_pricing": { - "hours_utc": "16:30-00:30", - "input_cost_per_token": 5e-7, - "output_cost_per_token": 1e-6, - }, - }, - ) - usage = Usage(prompt_tokens=250000, completion_tokens=250000, total_tokens=500000) - - off_peak = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc)) - assert off_peak.input_rate == 5e-7 - assert off_peak.output_rate == 1e-6 - - peak = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc)) - assert peak.input_rate == 3e-6 - assert peak.output_rate == 4e-6 - - def test_get_model_info_propagates_off_peak_fields(): model_name = "test-off-peak-model" off_peak_pricing = { @@ -923,38 +705,6 @@ def test_get_model_info_propagates_off_peak_fields(): assert info["off_peak_pricing"] == off_peak_pricing -def test_get_token_base_cost_off_peak_wins_over_tiered_pricing(): - """Tiered pricing resolves base rates on its own path and returns early, so off-peak has to - be applied there too or a model carrying both would silently bill the tier rate all day.""" - from datetime import datetime, timezone - - model_name = "litellm-test-off-peak-tiered" - litellm.register_model( - { - model_name: { - "litellm_provider": "openai", - "mode": "chat", - "tiered_pricing": [ - {"range": [0, 128000], "input_cost_per_token": 3e-6, "output_cost_per_token": 6e-6}, - ], - "off_peak_pricing": { - "hours_utc": "16:30-00:30", - "input_cost_per_token": 5e-7, - "output_cost_per_token": 1e-6, - }, - } - } - ) - info = litellm.get_model_info(model=model_name) - usage = Usage(prompt_tokens=1_000, completion_tokens=100, total_tokens=1_100) - - inside = _get_token_base_cost(info, usage, current_time=datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc)) - assert (inside.input_rate, inside.output_rate) == (5e-7, 1e-6) - - outside = _get_token_base_cost(info, usage, current_time=datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc)) - assert (outside.input_rate, outside.output_rate) == (3e-6, 6e-6) - - def _register_off_peak_reasoning_model( model_name: str, off_peak_pricing: dict, reasoning_rate: float | None = 4e-6, **service_tier_rates: float ) -> None: @@ -985,7 +735,7 @@ def _off_peak_reasoning_usage() -> Usage: ) -def test_generic_cost_per_token_off_peak_reasoning_rate(): +def test_cost_per_text_off_peak_reasoning_rate(): from datetime import datetime, timezone model_name = "litellm-test-off-peak-reasoning" @@ -994,7 +744,7 @@ def test_generic_cost_per_token_off_peak_reasoning_rate(): {"hours_utc": "16:30-00:30", "output_cost_per_token": 1e-6, "output_cost_per_reasoning_token": 5e-7}, ) - _, inside = generic_cost_per_token( + _, inside = cost_per_text( model=model_name, usage=_off_peak_reasoning_usage(), custom_llm_provider="openai", @@ -1002,7 +752,7 @@ def test_generic_cost_per_token_off_peak_reasoning_rate(): ) assert inside == pytest.approx(50 * 1e-6 + 30 * 5e-7) - _, outside = generic_cost_per_token( + _, outside = cost_per_text( model=model_name, usage=_off_peak_reasoning_usage(), custom_llm_provider="openai", @@ -1011,14 +761,14 @@ def test_generic_cost_per_token_off_peak_reasoning_rate(): assert outside == pytest.approx(50 * 2e-6 + 30 * 4e-6) -def test_generic_cost_per_token_off_peak_block_without_reasoning_rate(): +def test_cost_per_text_off_peak_block_without_reasoning_rate(): from datetime import datetime, timezone inside_window = datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc) block = {"hours_utc": "16:30-00:30", "output_cost_per_token": 1e-6} _register_off_peak_reasoning_model("litellm-test-off-peak-model-reasoning-rate", block) - _, with_model_rate = generic_cost_per_token( + _, with_model_rate = cost_per_text( model="litellm-test-off-peak-model-reasoning-rate", usage=_off_peak_reasoning_usage(), custom_llm_provider="openai", @@ -1027,7 +777,7 @@ def test_generic_cost_per_token_off_peak_block_without_reasoning_rate(): assert with_model_rate == pytest.approx(50 * 1e-6 + 30 * 4e-6) _register_off_peak_reasoning_model("litellm-test-off-peak-no-reasoning-rate", block, reasoning_rate=None) - _, without_model_rate = generic_cost_per_token( + _, without_model_rate = cost_per_text( model="litellm-test-off-peak-no-reasoning-rate", usage=_off_peak_reasoning_usage(), custom_llm_provider="openai", @@ -1036,7 +786,7 @@ def test_generic_cost_per_token_off_peak_block_without_reasoning_rate(): assert without_model_rate == pytest.approx(80 * 1e-6) -def test_generic_cost_per_token_off_peak_reasoning_rate_wins_over_the_tier(): +def test_cost_per_text_off_peak_reasoning_rate_wins_over_the_tier(): from datetime import datetime, timezone model_name = "litellm-test-off-peak-tiered-reasoning" @@ -1062,7 +812,7 @@ def test_generic_cost_per_token_off_peak_reasoning_rate_wins_over_the_tier(): } ) - _, inside = generic_cost_per_token( + _, inside = cost_per_text( model=model_name, usage=_off_peak_reasoning_usage(), custom_llm_provider="openai", @@ -1070,7 +820,7 @@ def test_generic_cost_per_token_off_peak_reasoning_rate_wins_over_the_tier(): ) assert inside == pytest.approx(50 * 1e-6 + 30 * 5e-7) - _, outside = generic_cost_per_token( + _, outside = cost_per_text( model=model_name, usage=_off_peak_reasoning_usage(), custom_llm_provider="openai", @@ -1079,7 +829,7 @@ def test_generic_cost_per_token_off_peak_reasoning_rate_wins_over_the_tier(): assert outside == pytest.approx(50 * 6e-6 + 30 * 8e-6) -def test_generic_cost_per_token_off_peak_reasoning_rate_wins_over_the_service_tier(): +def test_cost_per_text_off_peak_reasoning_rate_wins_over_the_service_tier(): from datetime import datetime, timezone model_name = "litellm-test-off-peak-reasoning-service-tier" @@ -1090,7 +840,7 @@ def test_generic_cost_per_token_off_peak_reasoning_rate_wins_over_the_service_ti output_cost_per_reasoning_token_priority=6e-6, ) - _, inside = generic_cost_per_token( + _, inside = cost_per_text( model=model_name, usage=_off_peak_reasoning_usage(), custom_llm_provider="openai", @@ -1099,7 +849,7 @@ def test_generic_cost_per_token_off_peak_reasoning_rate_wins_over_the_service_ti ) assert inside == pytest.approx(50 * 1e-6 + 30 * 5e-7) - _, outside = generic_cost_per_token( + _, outside = cost_per_text( model=model_name, usage=_off_peak_reasoning_usage(), custom_llm_provider="openai", @@ -1134,39 +884,6 @@ def test_apply_off_peak_pricing_treats_bool_as_unset_and_parses_strings(): assert rates.reasoning_rate == 5e-7 -def test_get_token_base_cost_off_peak_cache_creation_rate(): - from datetime import datetime, timezone - from typing import cast - - from litellm.types.utils import ModelInfo - - model_info = cast( - ModelInfo, - { - "input_cost_per_token": 1e-6, - "output_cost_per_token": 2e-6, - "cache_creation_input_token_cost": 1.25e-6, - "cache_creation_input_token_cost_above_1hr": 2e-6, - "off_peak_pricing": {"hours_utc": "16:30-00:30", "cache_creation_input_token_cost": 5e-7}, - }, - ) - usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) - inside_window = datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc) - - inside = _get_token_base_cost(model_info, usage, current_time=inside_window) - assert inside.cache_creation_rate == 5e-7 - assert inside.cache_creation_rates_by_ttl["1hr"] == 2e-6 - - outside = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc)) - assert outside.cache_creation_rate == 1.25e-6 - - without_key = cast( - ModelInfo, - {**model_info, "off_peak_pricing": {"hours_utc": "16:30-00:30", "input_cost_per_token": 5e-7}}, - ) - assert _get_token_base_cost(without_key, usage, current_time=inside_window).cache_creation_rate == 1.25e-6 - - def test_get_token_type_cost_breakdown_reflects_off_peak_reasoning_and_cache_creation_rates(): from datetime import datetime, timezone @@ -1207,7 +924,7 @@ def test_get_token_type_cost_breakdown_reflects_off_peak_reasoning_and_cache_cre assert outside.cache_creation_cost == pytest.approx(400 * 1.25e-6) -def test_generic_cost_per_token_gpt54_above_272k_tokens(_local_model_cost_map): +def test_cost_per_text_gpt54_above_272k_tokens(_local_model_cost_map): """GPT-5.4/5.4-pro: prompts >272K input tokens priced at 2x input, 1.5x output.""" model = "gpt-5.4" custom_llm_provider = "openai" @@ -1220,7 +937,7 @@ def test_generic_cost_per_token_gpt54_above_272k_tokens(_local_model_cost_map): completion_tokens=completion_tokens, total_tokens=prompt_tokens + completion_tokens, ) - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model, usage=usage, custom_llm_provider=custom_llm_provider, @@ -1238,7 +955,7 @@ def test_generic_cost_per_token_gpt54_above_272k_tokens(_local_model_cost_map): (300_000, 2.4e-05, 2.4e-06, 9e-05), ], ) -def test_generic_cost_per_token_azure_eu_gpt_6_astra_tiers( +def test_cost_per_text_azure_eu_gpt_6_astra_tiers( _local_model_cost_map, prompt_tokens, input_rate, cache_read_rate, output_rate ): """azure/eu/gpt-6-astra bills Azure's Data Zone rates, doubling input and cache read past 272K.""" @@ -1250,7 +967,7 @@ def test_generic_cost_per_token_azure_eu_gpt_6_astra_tiers( total_tokens=prompt_tokens + completion_tokens, prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=cached_tokens), ) - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model="azure/eu/gpt-6-astra", usage=usage, custom_llm_provider="azure", @@ -1260,7 +977,7 @@ def test_generic_cost_per_token_azure_eu_gpt_6_astra_tiers( assert completion_cost == pytest.approx(completion_tokens * output_rate) -def test_generic_cost_per_token_minimax_m3_above_512k_tokens(_local_model_cost_map): +def test_cost_per_text_minimax_m3_above_512k_tokens(_local_model_cost_map): """MiniMax-M3: prompts >512K input tokens priced at 2x input, output, and cache read.""" model = "minimax/MiniMax-M3" custom_llm_provider = "minimax" @@ -1275,7 +992,7 @@ def test_generic_cost_per_token_minimax_m3_above_512k_tokens(_local_model_cost_m total_tokens=prompt_tokens + completion_tokens, prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=cached_tokens), ) - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model, usage=usage, custom_llm_provider=custom_llm_provider, @@ -1289,7 +1006,7 @@ def test_generic_cost_per_token_minimax_m3_above_512k_tokens(_local_model_cost_m assert round(completion_cost, 10) == round(expected_completion, 10) -def test_generic_cost_per_token_honors_non_standard_above_threshold(): +def test_cost_per_text_honors_non_standard_above_threshold(): """Regression for #30344: get_model_info must keep arbitrary input/output_cost_per_token_above__tokens thresholds, not only the hard-coded 128k/200k/272k/512k set, so a custom tier boundary is applied past its limit.""" @@ -1316,7 +1033,7 @@ def test_generic_cost_per_token_honors_non_standard_above_threshold(): completion_tokens=completion_tokens, total_tokens=prompt_tokens + completion_tokens, ) - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model, usage=usage, custom_llm_provider=custom_llm_provider, @@ -1327,7 +1044,7 @@ def test_generic_cost_per_token_honors_non_standard_above_threshold(): litellm.model_cost.pop(model, None) -def test_generic_cost_per_token_tiered_pricing_charges_cache_creation_at_tier_rate(): +def test_cost_per_text_tiered_pricing_charges_cache_creation_at_tier_rate(): """Regression for LIT-4375: a tier's cache_creation_input_token_cost must be billed on the generic (provider-agnostic) path, not silently dropped.""" model = "litellm-test-tiered-cache-creation" @@ -1364,7 +1081,7 @@ def test_generic_cost_per_token_tiered_pricing_charges_cache_creation_at_tier_ra total_tokens=301000, prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=40000, cache_creation_tokens=60000), ) - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model, usage=usage, custom_llm_provider=custom_llm_provider, @@ -1377,7 +1094,7 @@ def test_generic_cost_per_token_tiered_pricing_charges_cache_creation_at_tier_ra litellm.model_cost.pop(model, None) -def test_generic_cost_per_token_tiered_pricing_is_all_or_nothing(): +def test_cost_per_text_tiered_pricing_is_all_or_nothing(): """Tiered pricing bills the whole request at the tier picked from its input tokens, for any provider, and falls back to flat pricing when no tier matches.""" model = "litellm-test-tiered-all-or-nothing" @@ -1407,7 +1124,7 @@ def test_generic_cost_per_token_tiered_pricing_is_all_or_nothing(): try: usage = Usage(prompt_tokens=40000, completion_tokens=1000, total_tokens=41000) - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model, usage=usage, custom_llm_provider=custom_llm_provider, @@ -1416,7 +1133,7 @@ def test_generic_cost_per_token_tiered_pricing_is_all_or_nothing(): assert round(completion_cost, 10) == round(1000 * 3.5e-06, 10) boundary_usage = Usage(prompt_tokens=32000, completion_tokens=10, total_tokens=32010) - boundary_prompt_cost, _ = generic_cost_per_token( + boundary_prompt_cost, _ = cost_per_text( model=model, usage=boundary_usage, custom_llm_provider=custom_llm_provider, @@ -1424,7 +1141,7 @@ def test_generic_cost_per_token_tiered_pricing_is_all_or_nothing(): assert round(boundary_prompt_cost, 10) == round(32000 * 4.6e-07, 10) empty_prompt_usage = Usage(prompt_tokens=0, completion_tokens=100, total_tokens=100) - empty_prompt_cost, empty_completion_cost = generic_cost_per_token( + empty_prompt_cost, empty_completion_cost = cost_per_text( model=model, usage=empty_prompt_usage, custom_llm_provider=custom_llm_provider, @@ -1435,7 +1152,7 @@ def test_generic_cost_per_token_tiered_pricing_is_all_or_nothing(): litellm.model_cost.pop(model, None) -def test_generic_cost_per_token_tier_without_an_output_rate_bills_the_model_rate(): +def test_cost_per_text_tier_without_an_output_rate_bills_the_model_rate(): """Regression: a tier table that spells out only input rates served every completion for free, since a tier's missing output rate has no tier-level fallback to stand in for it.""" model = "litellm-test-tiered-input-only" @@ -1459,7 +1176,7 @@ def test_generic_cost_per_token_tier_without_an_output_rate_bills_the_model_rate total_tokens=195, completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=100), ) - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model, usage=usage, custom_llm_provider=custom_llm_provider, @@ -1470,7 +1187,7 @@ def test_generic_cost_per_token_tier_without_an_output_rate_bills_the_model_rate litellm.model_cost.pop(model, None) -def test_generic_cost_per_token_tier_without_cache_rates_bills_cache_at_the_tier_input_rate(): +def test_cost_per_text_tier_without_cache_rates_bills_cache_at_the_tier_input_rate(): model = "litellm-test-tiered-no-cache-rates" custom_llm_provider = "openrouter" litellm.register_model( @@ -1504,12 +1221,12 @@ def test_generic_cost_per_token_tier_without_cache_rates_bills_cache_at_the_tier total_tokens=40100, prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=5000, cache_creation_tokens=15000), ) - uncached_prompt_cost, _ = generic_cost_per_token( + uncached_prompt_cost, _ = cost_per_text( model=model, usage=uncached, custom_llm_provider=custom_llm_provider, ) - cached_prompt_cost, cached_completion_cost = generic_cost_per_token( + cached_prompt_cost, cached_completion_cost = cost_per_text( model=model, usage=cached, custom_llm_provider=custom_llm_provider, @@ -1547,7 +1264,7 @@ def test_generic_cost_per_token_tier_without_cache_rates_bills_cache_at_the_tier ], ) @pytest.mark.parametrize("one_hour_tokens", [10, 20]) -def test_generic_cost_per_token_tier_1hr_cache_rate_preserves_zero_and_fallback( +def test_cost_per_text_tier_1hr_cache_rate_preserves_zero_and_fallback( cache_rates: Mapping[str, float | None], expected_write_rate: float, one_hour_tokens: int ) -> None: model: Final = "litellm-test-tiered-1hr-cache-rate" @@ -1583,7 +1300,7 @@ def test_generic_cost_per_token_tier_1hr_cache_rate_preserves_zero_and_fallback( ), ), ) - costs: Final = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="openrouter") + costs: Final = cost_per_text(model=model, usage=usage, custom_llm_provider="openrouter") breakdown: Final = get_token_type_cost_breakdown(model=model, usage=usage, custom_llm_provider="openrouter") ordinary_rate: Final = cache_rates.get("cache_creation_input_token_cost") @@ -1596,7 +1313,7 @@ def test_generic_cost_per_token_tier_1hr_cache_rate_preserves_zero_and_fallback( litellm.model_cost.pop(model, None) -def test_generic_cost_per_token_tier_without_an_input_rate_is_not_a_priced_tier(): +def test_cost_per_text_tier_without_an_input_rate_is_not_a_priced_tier(): model = "litellm-test-tiered-no-input-rate" custom_llm_provider = "openrouter" litellm.register_model( @@ -1613,7 +1330,7 @@ def test_generic_cost_per_token_tier_without_an_input_rate_is_not_a_priced_tier( try: usage = Usage(prompt_tokens=1000, completion_tokens=100, total_tokens=1100) - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model, usage=usage, custom_llm_provider=custom_llm_provider, @@ -1652,7 +1369,7 @@ def test_router_deployment_with_input_only_tiers_bills_completions_at_the_backen try: usage = Usage(prompt_tokens=21, completion_tokens=4, total_tokens=25) - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model_id, usage=usage, custom_llm_provider="anthropic", @@ -1664,7 +1381,7 @@ def test_router_deployment_with_input_only_tiers_bills_completions_at_the_backen litellm.model_cost.pop(model_id, None) -def test_generic_cost_per_token_tiered_pricing_bills_reasoning_at_tier_rate(): +def test_cost_per_text_tiered_pricing_bills_reasoning_at_tier_rate(): """Regression: a tier's output_cost_per_reasoning_token must price reasoning tokens on the generic path and in the logged breakdown, not the tier's plain output rate.""" model = "litellm-test-tiered-reasoning" @@ -1699,7 +1416,7 @@ def test_generic_cost_per_token_tiered_pricing_bills_reasoning_at_tier_rate(): total_tokens=1500, completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=400), ) - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model, usage=usage, custom_llm_provider=custom_llm_provider, @@ -1793,7 +1510,7 @@ def test_string_cost_values(): "litellm.litellm_core_utils.llm_cost_calc.utils.get_model_info", return_value=mock_model_info, ): - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model="test-model", usage=usage, custom_llm_provider="test-provider" ) @@ -1814,7 +1531,7 @@ def test_string_cost_values(): assert round(completion_cost, 12) == round(expected_completion_cost, 12) -def test_generic_cost_per_token_overlapping_cached_and_image_tokens(): +def test_cost_per_text_overlapping_cached_and_image_tokens(): """Some providers report cached_tokens and image_tokens as overlapping subsets of prompt_tokens. Billing each in full charged the overlap twice, once at the cache rate and again at the input rate.""" @@ -1837,14 +1554,14 @@ def test_generic_cost_per_token_overlapping_cached_and_image_tokens(): prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=None, cached_tokens=90, image_tokens=80), ) - prompt_cost, completion_cost = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="openai") + prompt_cost, completion_cost = cost_per_text(model=model, usage=usage, custom_llm_provider="openai") # 90 cached at 1e-7, the remaining 10 uncached tokens once at 1e-6 assert prompt_cost == pytest.approx(90 * 1e-7 + 10 * 1e-6) assert completion_cost == pytest.approx(10 * 2e-6) -def test_generic_cost_per_token_warm_prefix_cache_spanning_text_and_image_tokens(): +def test_cost_per_text_warm_prefix_cache_spanning_text_and_image_tokens(): """xAI reports text_tokens + image_tokens = prompt_tokens with cached_tokens overlapping both, so a warm prefix cache covering the whole image exceeds the text-only count. Observed live on grok-4.6 (issue #37281): the image tokens were billed a second time at @@ -1869,7 +1586,7 @@ def test_generic_cost_per_token_warm_prefix_cache_spanning_text_and_image_tokens prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=1319, cached_tokens=2432, image_tokens=1142), ) - prompt_cost, completion_cost = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="openai") + prompt_cost, completion_cost = cost_per_text(model=model, usage=usage, custom_llm_provider="openai") # 2432 cached at the cache-read rate, the 29 uncached tokens once at the input rate assert prompt_cost == pytest.approx(2432 * 5e-7 + 29 * 2e-6) @@ -1953,7 +1670,7 @@ def test_string_cost_values_edge_cases(): "litellm.litellm_core_utils.llm_cost_calc.utils.get_model_info", return_value=mock_model_info, ): - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model="test-model", usage=usage, custom_llm_provider="test-provider" ) @@ -1983,7 +1700,7 @@ def test_string_cost_values_with_threshold(): "litellm.litellm_core_utils.llm_cost_calc.utils.get_model_info", return_value=mock_model_info, ): - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model="test-model", usage=usage, custom_llm_provider="test-provider" ) @@ -2103,7 +1820,7 @@ def test_calculate_cache_writing_cost_bills_writes_of_an_unknown_lifetime_at_the (300_000, "priority", 6e-6, 12e-6), ], ) -def test_generic_cost_per_token_prices_12h_cache_writes_at_prompt_threshold( +def test_cost_per_text_prices_12h_cache_writes_at_prompt_threshold( prompt_tokens: int, service_tier: str, write_rate: float, twelve_hour_rate: float ) -> None: model_info: Final[ModelInfo] = { @@ -2134,7 +1851,7 @@ def test_generic_cost_per_token_prices_12h_cache_writes_at_prompt_threshold( ), ) - result: Final = generic_cost_per_token( + result: Final = cost_per_text( model="test-12h-cache-write", usage=usage, custom_llm_provider="openai", @@ -2149,7 +1866,7 @@ def test_generic_cost_per_token_prices_12h_cache_writes_at_prompt_threshold( ) -def test_generic_cost_per_token_falls_back_to_default_for_missing_12h_rate() -> None: +def test_cost_per_text_falls_back_to_default_for_missing_12h_rate() -> None: model_info: Final[ModelInfo] = { "input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6, @@ -2165,7 +1882,7 @@ def test_generic_cost_per_token_falls_back_to_default_for_missing_12h_rate() -> ), ) - result: Final = generic_cost_per_token( + result: Final = cost_per_text( model="test-12h-cache-write-fallback", usage=usage, custom_llm_provider="openai", @@ -2176,7 +1893,7 @@ def test_generic_cost_per_token_falls_back_to_default_for_missing_12h_rate() -> @pytest.mark.parametrize(("prompt_tokens", "twelve_hour_rate"), [(10_000, 7e-6), (300_000, 14e-6)]) -def test_generic_cost_per_token_prices_ultrafast_12h_cache_writes_at_the_ultrafast_rate( +def test_cost_per_text_prices_ultrafast_12h_cache_writes_at_the_ultrafast_rate( prompt_tokens: int, twelve_hour_rate: float ) -> None: model_info: Final[ModelInfo] = { @@ -2202,7 +1919,7 @@ def test_generic_cost_per_token_prices_ultrafast_12h_cache_writes_at_the_ultrafa ), ) - prompt_cost, _ = generic_cost_per_token( + prompt_cost, _ = cost_per_text( model="test-ultrafast-12h-cache-write", usage=usage, custom_llm_provider="openai", @@ -2238,7 +1955,7 @@ def test_cache_writing_cost_with_zero_creation_tokens_and_ephemeral_details(): ), ) - prompt_cost, _ = generic_cost_per_token( + prompt_cost, _ = cost_per_text( model="litellm-test-ephemeral-cache-writes", usage=usage, custom_llm_provider="anthropic", @@ -2285,7 +2002,7 @@ def test_service_tier_ultrafast_pricing(): "cache_read_input_token_cost_ultrafast": 5e-06, } - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model="gpt-5.6-sol", usage=usage, custom_llm_provider="openai", @@ -2308,13 +2025,13 @@ def test_service_tier_ultrafast_fallback_pricing(_local_model_cost_map): usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) - std_prompt_cost, std_completion_cost = generic_cost_per_token( + std_prompt_cost, std_completion_cost = cost_per_text( model="gpt-5.6-sol", usage=usage, custom_llm_provider="openai", service_tier=None, ) - ultrafast_prompt_cost, ultrafast_completion_cost = generic_cost_per_token( + ultrafast_prompt_cost, ultrafast_completion_cost = cost_per_text( model="gpt-5.6-sol", usage=usage, custom_llm_provider="openai", @@ -2365,8 +2082,7 @@ def test_gemini_image_generation_cost_with_zero_text_tokens(_local_model_cost_ma ), ) - model_cost_map = litellm.model_cost[model] - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model, usage=usage, custom_llm_provider=custom_llm_provider, @@ -2378,8 +2094,9 @@ def test_gemini_image_generation_cost_with_zero_text_tokens(_local_model_cost_ma # - reasoning_tokens: 225 * output_cost_per_token # Total completion should include both image + reasoning costs. - output_cost_per_image_token = model_cost_map.get("output_cost_per_image_token", 0) - output_cost_per_token = model_cost_map.get("output_cost_per_token", 0) + model_info = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider) + output_cost_per_image_token = model_info.get("output_cost_per_image_token") or 0.0 + output_cost_per_token = model_info.get("output_cost_per_token") or 0.0 expected_image_cost = 1120 * output_cost_per_image_token expected_reasoning_cost = 225 * output_cost_per_token # reasoning uses base token cost @@ -2465,7 +2182,7 @@ def test_query_count_is_free_without_a_per_query_price(_local_model_cost_map): prompt_tokens_details=PromptTokensDetailsWrapper(query_count=1), ) - prompt_cost, _ = generic_cost_per_token(model="text-embedding-3-small", usage=usage, custom_llm_provider="openai") + prompt_cost, _ = cost_per_text(model="text-embedding-3-small", usage=usage, custom_llm_provider="openai") assert prompt_cost == 0.0 @@ -2485,12 +2202,12 @@ def test_data_residency_applies_uplift(data_residency, model, _local_model_cost_ usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) - base = generic_cost_per_token( + base = cost_per_text( model=model, usage=usage, custom_llm_provider="openai", ) - regional = generic_cost_per_token( + regional = cost_per_text( model=model, usage=usage, custom_llm_provider="openai", @@ -2513,8 +2230,8 @@ def test_data_residency_no_uplift_for_pre_march_2026_models(model, _local_model_ usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) - base = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="openai") - regional = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="openai", data_residency="eu") + base = cost_per_text(model=model, usage=usage, custom_llm_provider="openai") + regional = cost_per_text(model=model, usage=usage, custom_llm_provider="openai", data_residency="eu") assert base == regional, f"{model} should not have a regional uplift, but cost changed with data_residency" @@ -2526,12 +2243,12 @@ def test_data_residency_no_uplift_for_unmarked_model(_local_model_cost_map): usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) - base = generic_cost_per_token( + base = cost_per_text( model="gpt-3.5-turbo", usage=usage, custom_llm_provider="openai", ) - with_residency = generic_cost_per_token( + with_residency = cost_per_text( model="gpt-3.5-turbo", usage=usage, custom_llm_provider="openai", @@ -2547,12 +2264,12 @@ def test_data_residency_none_no_uplift(_local_model_cost_map): usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) - base = generic_cost_per_token( + base = cost_per_text( model="gpt-5.4", usage=usage, custom_llm_provider="openai", ) - explicit_none = generic_cost_per_token( + explicit_none = cost_per_text( model="gpt-5.4", usage=usage, custom_llm_provider="openai", @@ -2568,13 +2285,13 @@ def test_data_residency_composes_with_service_tier(_local_model_cost_map): usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) - priority_base = generic_cost_per_token( + priority_base = cost_per_text( model="gpt-5.4", usage=usage, custom_llm_provider="openai", service_tier="priority", ) - priority_eu = generic_cost_per_token( + priority_eu = cost_per_text( model="gpt-5.4", usage=usage, custom_llm_provider="openai", @@ -2599,8 +2316,8 @@ def test_vertex_regional_location_applies_uplift(vertex_location, model, _local_ usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) - base = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="vertex_ai") - regional = generic_cost_per_token( + base = cost_per_text(model=model, usage=usage, custom_llm_provider="vertex_ai") + regional = cost_per_text( model=model, usage=usage, custom_llm_provider="vertex_ai", @@ -2624,8 +2341,8 @@ def test_vertex_global_or_absent_location_no_uplift(vertex_location, _local_mode usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) - base = generic_cost_per_token(model="claude-haiku-4-5@20251001", usage=usage, custom_llm_provider="vertex_ai") - located = generic_cost_per_token( + base = cost_per_text(model="claude-haiku-4-5@20251001", usage=usage, custom_llm_provider="vertex_ai") + located = cost_per_text( model="claude-haiku-4-5@20251001", usage=usage, custom_llm_provider="vertex_ai", @@ -2669,89 +2386,6 @@ def test_get_cost_per_unit_falls_back_from_service_tier_key_to_base(): assert get_cost_per_unit(model_info_direct, "input_cost_per_token_priority") == 5e-6 -@pytest.mark.parametrize( - "field", ("output_cost_per_token", "cache_read_input_token_cost", "cache_creation_input_token_cost") -) -def test_flat_input_price_keeps_other_token_thresholds(field: str) -> None: - from typing import Final, cast - from litellm.litellm_core_utils.llm_cost_calc.utils import _get_token_base_cost - - info: Final = cast( - ModelInfo, - { - "input_cost_per_token": 0.001, - "output_cost_per_token": 0.002, - "cache_read_input_token_cost": 0.003, - "cache_creation_input_token_cost": 0.004, - f"{field}_above_300k_tokens": 0.005, - }, - ) - rates: Final = _get_token_base_cost( - model_info=info, - usage=Usage(prompt_tokens=300_001, completion_tokens=3, total_tokens=300_004), - ) - assert rates.input_rate == 0.001 - assert rates.output_rate == (0.005 if field == "output_cost_per_token" else 0.002) - assert rates.cache_creation_rate == (0.005 if field == "cache_creation_input_token_cost" else 0.004) - assert rates.cache_read_rate == (0.005 if field == "cache_read_input_token_cost" else 0.003) - - -@pytest.mark.parametrize("prompt_tokens", (200_000, 200_001, 512_000, 512_001)) -@pytest.mark.parametrize("higher_rate", (0.0, 0.009)) -def test_mixed_token_thresholds_preserve_each_crossed_rate(prompt_tokens: int, higher_rate: float) -> None: - from typing import Final, cast - from litellm.litellm_core_utils.llm_cost_calc.utils import _get_token_base_cost - - info: Final = cast( - ModelInfo, - { - "input_cost_per_token": 0.001, - "output_cost_per_token": 0.002, - "cache_read_input_token_cost": 0.003, - "cache_creation_input_token_cost": 0.004, - "input_cost_per_token_above_200k_tokens": 0.005, - "output_cost_per_token_above_200k_tokens": 0.006, - "cache_creation_input_token_cost_above_200k_tokens": 0.007, - "cache_read_input_token_cost_above_512k_tokens": higher_rate, - "output_cost_per_token_above_512k_tokens": higher_rate, - }, - ) - rates: Final = _get_token_base_cost( - model_info=info, - usage=Usage(prompt_tokens=prompt_tokens, completion_tokens=3, total_tokens=prompt_tokens + 3), - ) - assert rates.input_rate == (0.005 if prompt_tokens > 200_000 else 0.001) - assert rates.output_rate == ( - higher_rate if prompt_tokens > 512_000 else 0.006 if prompt_tokens > 200_000 else 0.002 - ) - assert rates.cache_creation_rate == (0.007 if prompt_tokens > 200_000 else 0.004) - assert rates.cache_read_rate == (higher_rate if prompt_tokens > 512_000 else 0.003) - - -def test_threshold_keys_exclude_service_tier_variants(): - from typing import cast - - from litellm.litellm_core_utils.llm_cost_calc.utils import _get_token_base_cost - from litellm.types.utils import ModelInfo, Usage - - # The service-tier-suffixed above-threshold key must be excluded from - # threshold detection. The _priority variant has a higher threshold (300k), - # so if it were not excluded it would sort first and drive a 9e-6 rate for - # this non-tier request. With the exclusion only the standard 200k key - # applies, giving 3e-6. - model_info = cast( - ModelInfo, - { - "input_cost_per_token": 1e-6, - "input_cost_per_token_above_200k_tokens": 3e-6, - "input_cost_per_token_above_300k_tokens_priority": 9e-6, - "output_cost_per_token": 2e-6, - }, - ) - usage = Usage(prompt_tokens=350_000, completion_tokens=1_000, total_tokens=351_000) - assert _get_token_base_cost(model_info=model_info, usage=usage).input_rate == 3e-6 - - @pytest.mark.parametrize( "model,custom_llm_provider,reasoning_tokens,cached_tokens", [ @@ -2769,7 +2403,7 @@ def test_token_type_cost_breakdown_is_provider_agnostic( """ Reasoning and cache-read costs must be surfaced for every provider that reports those tokens, regardless of which cost calculator the provider routes through - (Perplexity, Cerebras, Dashscope bypass generic_cost_per_token entirely). + (Perplexity, Cerebras, Dashscope bypass cost_per_text entirely). Cache tokens always land in prompt_tokens_details.cached_tokens, so reading from there - not the top-level cache_read_input_tokens attribute the old breakdown code @@ -2839,7 +2473,7 @@ def test_token_type_cost_breakdown_reads_cache_write_tokens(_local_model_cost_ma assert breakdown.cache_creation_cost == pytest.approx(300 * model_info["cache_creation_input_token_cost"]) -def test_generic_cost_per_token_openai_cache_write_tokens_gpt_5_6(_local_model_cost_map): +def test_cost_per_text_openai_cache_write_tokens_gpt_5_6(_local_model_cost_map): """ Regression: OpenAI gpt-5.6 reports cache-write tokens under prompt_tokens_details.cache_write_tokens (not the Anthropic cache_creation_tokens @@ -2859,7 +2493,7 @@ def test_generic_cost_per_token_openai_cache_write_tokens_gpt_5_6(_local_model_c assert usage.prompt_tokens_details.cache_write_tokens == 800 assert usage.prompt_tokens_details.cache_creation_tokens == 800 - prompt_cost, _ = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="openai") + prompt_cost, _ = cost_per_text(model=model, usage=usage, custom_llm_provider="openai") info = litellm.get_model_info(model=model, custom_llm_provider="openai") expected_prompt = (1000 - 800) * info["input_cost_per_token"] + 800 * info["cache_creation_input_token_cost"] @@ -2868,7 +2502,7 @@ def test_generic_cost_per_token_openai_cache_write_tokens_gpt_5_6(_local_model_c assert prompt_cost > 1000 * info["input_cost_per_token"] -def test_generic_cost_per_token_backs_out_cache_write_tokens_from_text_tokens(_local_model_cost_map): +def test_cost_per_text_backs_out_cache_write_tokens_from_text_tokens(_local_model_cost_map): """ Regression for #34801: when a provider reports text_tokens covering the whole prompt alongside cache-write tokens (and no cache reads), the cache-write tokens @@ -2883,7 +2517,7 @@ def test_generic_cost_per_token_backs_out_cache_write_tokens_from_text_tokens(_l prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=0, cache_write_tokens=800, text_tokens=1000), ) - prompt_cost, _ = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="openai") + prompt_cost, _ = cost_per_text(model=model, usage=usage, custom_llm_provider="openai") info = litellm.get_model_info(model=model, custom_llm_provider="openai") expected_prompt = 200 * info["input_cost_per_token"] + 800 * info["cache_creation_input_token_cost"] @@ -2894,7 +2528,7 @@ def test_token_type_cost_breakdown_reconciles_with_generic_total(_local_model_co """ Both-ways check: the reasoning subset must sum with the remaining (text) output cost to exactly the completion total, and the cache-read subset with the remaining - input cost to exactly the prompt total, as computed by generic_cost_per_token. + input cost to exactly the prompt total, as computed by cost_per_text. A mismatch here would mean the breakdown misrepresents what was actually billed. """ @@ -2908,7 +2542,7 @@ def test_token_type_cost_breakdown_reconciles_with_generic_total(_local_model_co prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=300, text_tokens=700), ) - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model, usage=usage, custom_llm_provider=custom_llm_provider ) breakdown = get_token_type_cost_breakdown(model=model, custom_llm_provider=custom_llm_provider, usage=usage) @@ -3049,12 +2683,12 @@ def test_a_pinned_billing_time_prices_the_totals_and_the_reported_rates_at_one_m usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) with pinned_billing_time(datetime(2026, 1, 1, 2, 30, tzinfo=timezone.utc)): - off_peak_prompt_cost, off_peak_completion_cost = generic_cost_per_token( + off_peak_prompt_cost, off_peak_completion_cost = cost_per_text( model="off-peak-model", usage=usage, custom_llm_provider="openai" ) off_peak_rates = get_billed_token_rates(model="off-peak-model", custom_llm_provider="openai", usage=usage) with pinned_billing_time(datetime(2026, 1, 1, 12, 30, tzinfo=timezone.utc)): - peak_prompt_cost, peak_completion_cost = generic_cost_per_token( + peak_prompt_cost, peak_completion_cost = cost_per_text( model="off-peak-model", usage=usage, custom_llm_provider="openai" ) peak_rates = get_billed_token_rates(model="off-peak-model", custom_llm_provider="openai", usage=usage) @@ -3219,14 +2853,17 @@ def test_token_type_cost_breakdown_applies_regional_uplift(_local_model_cost_map assert eu.cache_read_cost == pytest.approx(base.cache_read_cost * uplift) # The uplifted breakdown must still reconcile with the uplifted totals. - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model, usage=usage, custom_llm_provider=custom_llm_provider, data_residency="eu", ) - text_output_cost = 300 * model_info["output_cost_per_token"] * uplift - text_input_cost = 600 * model_info["input_cost_per_token"] * uplift + output_rate = model_info["output_cost_per_token"] + input_rate = model_info["input_cost_per_token"] + assert output_rate is not None and input_rate is not None + text_output_cost = 300 * output_rate * uplift + text_input_cost = 600 * input_rate * uplift assert text_output_cost + eu.reasoning_cost == pytest.approx(completion_cost) assert text_input_cost + eu.cache_read_cost == pytest.approx(prompt_cost) @@ -3264,7 +2901,7 @@ def test_token_type_cost_breakdown_applies_vertex_regional_uplift(_local_model_c assert regional.cache_read_cost == pytest.approx(base.cache_read_cost * uplift) # The uplifted breakdown must still reconcile with the uplifted totals. - prompt_cost, _completion_cost = generic_cost_per_token( + prompt_cost, _completion_cost = cost_per_text( model=model, usage=usage, custom_llm_provider=custom_llm_provider, @@ -3420,9 +3057,9 @@ def test_fast_service_tier_is_case_insensitive(_local_model_cost_map): usage = Usage(prompt_tokens=1_000, completion_tokens=500) - assert generic_cost_per_token( + assert cost_per_text( model="gpt-5.6-sol", usage=usage, custom_llm_provider="openai", service_tier="FAST" - ) == generic_cost_per_token(model="gpt-5.6-sol", usage=usage, custom_llm_provider="openai", service_tier="fast") + ) == cost_per_text(model="gpt-5.6-sol", usage=usage, custom_llm_provider="openai", service_tier="fast") def test_priority_reasoning_tokens_bill_at_the_priority_output_rate(_local_model_cost_map): @@ -3444,13 +3081,13 @@ def test_priority_reasoning_tokens_bill_at_the_priority_output_rate(_local_model assert priority_output_rate is not None assert priority_output_rate != standard_reasoning_rate - standard = generic_cost_per_token( + standard = cost_per_text( model="gemini-3.5-flash", usage=usage, custom_llm_provider="gemini", service_tier=None ) - priority = generic_cost_per_token( + priority = cost_per_text( model="gemini-3.5-flash", usage=usage, custom_llm_provider="gemini", service_tier="priority" ) - fast = generic_cost_per_token( + fast = cost_per_text( model="gemini-3.5-flash", usage=usage, custom_llm_provider="gemini", service_tier="fast" ) @@ -3476,7 +3113,7 @@ def test_explicit_tier_reasoning_key_wins_over_the_tier_output_rate(): completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=600), ) - _, completion_cost = generic_cost_per_token( + _, completion_cost = cost_per_text( model="synthetic-model", usage=usage, custom_llm_provider="openai", @@ -3506,7 +3143,7 @@ def test_null_tier_reasoning_key_falls_back_to_the_tier_output_rate(): completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=600), ) - _, completion_cost = generic_cost_per_token( + _, completion_cost = cost_per_text( model="synthetic-model", usage=usage, custom_llm_provider="openai", @@ -3531,7 +3168,7 @@ def test_tier_request_without_tier_pricing_keeps_the_standard_reasoning_rate(): completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=600), ) - _, completion_cost = generic_cost_per_token( + _, completion_cost = cost_per_text( model="synthetic-model", usage=usage, custom_llm_provider="openai", @@ -3661,7 +3298,7 @@ def test_route_image_generation_cost_falls_back_to_requested_size(monkeypatch, r assert cost == expected_cost -def test_generic_cost_per_token_bills_reasoning_nested_in_text_tokens_once(_local_model_cost_map: None) -> None: +def test_cost_per_text_bills_reasoning_nested_in_text_tokens_once(_local_model_cost_map: None) -> None: """ Realtime usage (OpenAI and Azure) reports output_tokens == text_tokens + audio_tokens with reasoning_tokens already counted inside text_tokens, so reasoning must not be billed on top. @@ -3678,7 +3315,7 @@ def test_generic_cost_per_token_bills_reasoning_nested_in_text_tokens_once(_loca completion_tokens_details=CompletionTokensDetailsWrapper(text_tokens=29, audio_tokens=0, reasoning_tokens=19), ) - prompt_cost, completion_cost = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="openai") + prompt_cost, completion_cost = cost_per_text(model=model, usage=usage, custom_llm_provider="openai") breakdown = get_token_type_cost_breakdown(model=model, custom_llm_provider="openai", usage=usage) info = litellm.get_model_info(model=model, custom_llm_provider="openai") @@ -3691,7 +3328,7 @@ def test_generic_cost_per_token_bills_reasoning_nested_in_text_tokens_once(_loca ) -def test_generic_cost_per_token_keeps_billing_reasoning_reported_beside_text_tokens( +def test_cost_per_text_keeps_billing_reasoning_reported_beside_text_tokens( _local_model_cost_map: None, ) -> None: """Providers whose text_tokens exclude reasoning (text + reasoning == completion) stay billed in full.""" @@ -3704,13 +3341,13 @@ def test_generic_cost_per_token_keeps_billing_reasoning_reported_beside_text_tok completion_tokens_details=CompletionTokensDetailsWrapper(text_tokens=25, audio_tokens=0, reasoning_tokens=19), ) - _, completion_cost = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="openai") + _, completion_cost = cost_per_text(model=model, usage=usage, custom_llm_provider="openai") info = litellm.get_model_info(model=model, custom_llm_provider="openai") assert completion_cost == pytest.approx(44 * info["output_cost_per_token"]) -def test_generic_cost_per_token_strips_only_the_reasoning_share_when_text_over_reports( +def test_cost_per_text_strips_only_the_reasoning_share_when_text_over_reports( _local_model_cost_map: None, ) -> None: """Text over-reported past the reasoning share keeps its extra tokens billed; only the nested reasoning is netted out.""" @@ -3723,7 +3360,7 @@ def test_generic_cost_per_token_strips_only_the_reasoning_share_when_text_over_r completion_tokens_details=CompletionTokensDetailsWrapper(text_tokens=100, audio_tokens=70, reasoning_tokens=10), ) - _, completion_cost = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="openai") + _, completion_cost = cost_per_text(model=model, usage=usage, custom_llm_provider="openai") breakdown = get_token_type_cost_breakdown(model=model, custom_llm_provider="openai", usage=usage) info = litellm.get_model_info(model=model, custom_llm_provider="openai") @@ -3733,7 +3370,7 @@ def test_generic_cost_per_token_strips_only_the_reasoning_share_when_text_over_r ) -def test_generic_cost_per_token_bills_nested_reasoning_once_beside_audio_output(_local_model_cost_map: None) -> None: +def test_cost_per_text_bills_nested_reasoning_once_beside_audio_output(_local_model_cost_map: None) -> None: """Audio-output realtime usage nests reasoning inside text_tokens next to audio_tokens; text is billed net of it.""" model = "gpt-realtime-2.1-mini" @@ -3744,7 +3381,7 @@ def test_generic_cost_per_token_bills_nested_reasoning_once_beside_audio_output( completion_tokens_details=CompletionTokensDetailsWrapper(text_tokens=30, audio_tokens=70, reasoning_tokens=20), ) - _, completion_cost = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="openai") + _, completion_cost = cost_per_text(model=model, usage=usage, custom_llm_provider="openai") breakdown = get_token_type_cost_breakdown(model=model, custom_llm_provider="openai", usage=usage) info = litellm.get_model_info(model=model, custom_llm_provider="openai") @@ -3772,7 +3409,7 @@ def test_cached_audio_tokens_fall_back_to_cache_read_input_token_cost() -> None: ), ) - prompt_cost, _ = generic_cost_per_token( + prompt_cost, _ = cost_per_text( model="some-realtime-model", usage=usage, custom_llm_provider="openai", @@ -3782,7 +3419,7 @@ def test_cached_audio_tokens_fall_back_to_cache_read_input_token_cost() -> None: assert prompt_cost == pytest.approx(expected) -def test_generic_cost_per_token_bills_cache_creation_at_the_input_rate_without_a_write_price(): +def test_cost_per_text_bills_cache_creation_at_the_input_rate_without_a_write_price(): """Azure and OpenAI publish no cache-write price and bill cache writes as ordinary input. A deployment priced with only input, output, and cache-read rates must bill the creation tokens the provider reports at the input rate, never at 0. The numbers are a cold 7,336-token @@ -3799,7 +3436,7 @@ def test_generic_cost_per_token_bills_cache_creation_at_the_input_rate_without_a prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=0, cache_creation_tokens=7333), ) - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model="custom-priced-deployment", usage=usage, custom_llm_provider="azure", model_info=model_info ) @@ -3807,75 +3444,6 @@ def test_generic_cost_per_token_bills_cache_creation_at_the_input_rate_without_a assert completion_cost == pytest.approx(23 * 1.25e-6) -@pytest.mark.parametrize( - ("cache_rates", "current_time", "expected_creation", "expected_creation_1h"), - ( - pytest.param({}, None, 2e-7, 2e-7, id="no-write-price-uses-the-input-rate"), - pytest.param( - {"cache_creation_input_token_cost": 2.5e-7}, None, 2.5e-7, 2.5e-7, id="no-1h-price-uses-the-write-price" - ), - pytest.param({"cache_creation_input_token_cost": 0.0}, None, 0.0, 0.0, id="explicit-zero-stays-zero"), - pytest.param( - {"off_peak_pricing": {"hours_utc": "00:00-23:59", "input_cost_per_token": 1e-7}}, - datetime(2026, 9, 14, 12, tzinfo=timezone.utc), - 1e-7, - 1e-7, - id="no-write-price-uses-the-off-peak-input-rate", - ), - pytest.param( - { - "off_peak_pricing": { - "hours_utc": "00:00-23:59", - "input_cost_per_token": 1e-7, - "cache_creation_input_token_cost": 3e-7, - } - }, - datetime(2026, 9, 14, 12, tzinfo=timezone.utc), - 3e-7, - 3e-7, - id="no-1h-price-uses-the-off-peak-write-price", - ), - pytest.param( - { - "tiered_pricing": [ - { - "range": [0, 128000], - "input_cost_per_token": 2e-7, - "output_cost_per_token": 1.25e-6, - "cache_creation_input_token_cost": 2.5e-7, - } - ], - "off_peak_pricing": {"hours_utc": "00:00-23:59", "cache_creation_input_token_cost": 3e-7}, - }, - datetime(2026, 9, 14, 12, tzinfo=timezone.utc), - 3e-7, - 3e-7, - id="tiered-no-1h-price-uses-the-off-peak-write-price", - ), - ), -) -def test_get_token_base_cost_resolves_missing_cache_write_rates_like_the_tiered_path( - cache_rates: Mapping[str, float | Mapping[str, float | str]], - current_time: datetime | None, - expected_creation: float, - expected_creation_1h: float, -) -> None: - model_info = {"input_cost_per_token": 2e-7, "output_cost_per_token": 1.25e-6, **cache_rates} - usage = Usage(prompt_tokens=10, completion_tokens=1, total_tokens=11) - - rates = _get_token_base_cost(model_info, usage, current_time=current_time) - - assert rates.cache_creation_rate == pytest.approx(expected_creation) - assert calculate_cache_writing_cost( - cache_creation_tokens=3, - cache_creation_token_details=CacheCreationTokenDetails( - ephemeral_1h_input_tokens=1, ephemeral_12h_input_tokens=2 - ), - cache_creation_cost=rates.cache_creation_rate, - cache_creation_cost_by_ttl=rates.cache_creation_rates_by_ttl, - ) == pytest.approx(3 * expected_creation_1h) - - def _image_response(num_images: int = 1, usage: ImageUsage | None = None) -> ImageResponse: return ImageResponse( data=[ImageObject(url="https://example.com/img.png") for _ in range(num_images)], @@ -4211,7 +3779,7 @@ def test_azure_gpt_5_6_alias_matches_sol_pricing(_local_model_cost_map, region_p ("vertex_ai/claude-haiku-5-5", "vertex_ai", 100_001, 5e-07, 5e-08, 2.5e-06), ], ) -def test_generic_cost_per_token_claude_haiku_5_5_prompt_length_tiers( +def test_cost_per_text_claude_haiku_5_5_prompt_length_tiers( _local_model_cost_map: None, model: str, custom_llm_provider: str, @@ -4230,7 +3798,7 @@ def test_generic_cost_per_token_claude_haiku_5_5_prompt_length_tiers( prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=cached_tokens), ) - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model, usage=usage, custom_llm_provider=custom_llm_provider, @@ -4256,7 +3824,7 @@ def test_vertex_regional_endpoint_uplift_scales_claude_haiku_5_5_over_100k_rates prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=cached_tokens), ) - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model="vertex_ai/claude-haiku-5-5", usage=usage, custom_llm_provider="vertex_ai", @@ -4394,7 +3962,7 @@ def test_token_breakdown_uses_supplied_pricing_without_catalog_lookup(rate: floa prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=5), cache_creation_input_tokens=5, ) - prompt_cost, output_cost = generic_cost_per_token( + prompt_cost, output_cost = cost_per_text( model="supplied-pricing-test", usage=usage, custom_llm_provider="bedrock", diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_openai_cache_write_cost.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_openai_cache_write_cost.py index 6c333a83d14..8dd57d4aa1f 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_openai_cache_write_cost.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_openai_cache_write_cost.py @@ -13,7 +13,7 @@ discarded by ``get_model_info``. import pytest import litellm -from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text from litellm.responses.utils import ResponseAPILoggingUtils from litellm.types.utils import Usage @@ -46,7 +46,7 @@ def test_openai_cache_write_tokens_billed_at_the_cache_creation_rate(local_model fresh_tokens = prompt_tokens - cache_write_tokens completion_tokens = 5 - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=MODEL, usage=_openai_chat_usage(prompt_tokens, cache_write_tokens, completion_tokens), custom_llm_provider="openai", @@ -77,12 +77,12 @@ def test_responses_api_cache_write_costs_the_same_as_chat(local_model_cost_map): ) assert responses_usage.prompt_tokens_details.cache_write_tokens == cache_write_tokens - responses_prompt_cost, responses_completion_cost = generic_cost_per_token( + responses_prompt_cost, responses_completion_cost = cost_per_text( model=MODEL, usage=responses_usage, custom_llm_provider="openai", ) - chat_prompt_cost, chat_completion_cost = generic_cost_per_token( + chat_prompt_cost, chat_completion_cost = cost_per_text( model=MODEL, usage=_openai_chat_usage(prompt_tokens, cache_write_tokens, completion_tokens), custom_llm_provider="openai", @@ -126,7 +126,7 @@ def test_tiered_cache_creation_rates_are_registered_and_billed( }[rate_key] input_rate = litellm.model_cost[MODEL][input_rate_key] - prompt_cost, _ = generic_cost_per_token( + prompt_cost, _ = cost_per_text( model=MODEL, usage=_openai_chat_usage(prompt_tokens, cache_write_tokens, completion_tokens=100), custom_llm_provider="openai", diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index ef002a73e52..3047d447c30 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -1,7 +1,7 @@ import copy import json from itertools import chain -from typing import Final +from typing import Final, cast from unittest.mock import MagicMock, patch import httpx @@ -39,7 +39,7 @@ from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation im ) from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES, ANTHROPIC_TOOL_SEARCH_TOOL_TYPES from litellm.types.llms.openai import ChatCompletionToolCallFunctionChunk -from litellm.types.utils import ChatCompletionToolCallChunk, ModelResponse, ServerToolUse, Usage +from litellm.types.utils import ChatCompletionToolCallChunk, ModelInfo, ModelResponse, ServerToolUse, Usage from tests.unit.llms.anthropic.anthropic_docs_refusal import ANTHROPIC_DOCS_REFUSAL_MESSAGE @@ -4468,86 +4468,37 @@ def test_fast_mode_usage_calculation(): assert usage.speed == "fast" -def test_fast_mode_cost_calculation(): - """ - Test that fast mode applies the 'fast' multiplier from provider_specific_entry - on top of the base model cost (1.1x for claude-opus-4-6). - """ +_FAST_MODE_ROW: Final = cast( + ModelInfo, + { + "litellm_provider": "anthropic", + "input_cost_per_token": 5e-06, + "output_cost_per_token": 2.5e-05, + "provider_specific_entry": {"fast": 1.1, "us": 1.1}, + }, +) + +@pytest.mark.parametrize( + ("speed", "inference_geo", "multiplier"), + [ + pytest.param("fast", None, 1.1, id="fast"), + pytest.param("fast", "us", 1.1 * 1.1, id="fast-and-us"), + pytest.param(None, "global", 1.0, id="global-geo-is-no-uplift"), + ], +) +def test_fast_mode_and_inference_geo_multiply_the_base_cost( + speed: str | None, inference_geo: str | None, multiplier: float +) -> None: from litellm.llms.anthropic.cost_calculation import cost_per_token from litellm.types.utils import Usage - base_prompt = 0.005 - base_completion = 0.025 + usage = Usage(prompt_tokens=1000, completion_tokens=1000, speed=speed, inference_geo=inference_geo) - with ( - patch("litellm.llms.anthropic.cost_calculation.generic_cost_per_token") as mock_cost, - patch("litellm.get_model_info") as mock_info, - ): - mock_cost.return_value = (base_prompt, base_completion) - mock_info.return_value = {"provider_specific_entry": {"fast": 1.1, "us": 1.1}} + prompt_cost, completion_cost = cost_per_token(model="claude-opus-4-6", usage=usage, model_info=_FAST_MODE_ROW) - usage_fast = Usage( - prompt_tokens=1000, - completion_tokens=1000, - speed="fast", - ) - - prompt_cost, completion_cost = cost_per_token( - model="claude-opus-4-6", - usage=usage_fast, - ) - - # generic_cost_per_token called with the plain base model name - mock_cost.assert_called_once() - assert mock_cost.call_args[1]["model"] == "claude-opus-4-6" - assert mock_cost.call_args[1]["custom_llm_provider"] == "anthropic" - - # 1.1x multiplier applied - assert abs(prompt_cost - base_prompt * 1.1) < 1e-10 - assert abs(completion_cost - base_completion * 1.1) < 1e-10 - - -def test_fast_mode_with_inference_geo(): - """ - Test that fast mode + inference_geo both apply their multipliers from - provider_specific_entry (1.1 * 1.1 = 1.21x for claude-opus-4-6). - """ - - from litellm.llms.anthropic.cost_calculation import cost_per_token - from litellm.types.utils import Usage - - base_prompt = 0.005 - base_completion = 0.025 - - with ( - patch("litellm.llms.anthropic.cost_calculation.generic_cost_per_token") as mock_cost, - patch("litellm.get_model_info") as mock_info, - ): - mock_cost.return_value = (base_prompt, base_completion) - mock_info.return_value = {"provider_specific_entry": {"fast": 1.1, "us": 1.1}} - - usage = Usage( - prompt_tokens=1000, - completion_tokens=1000, - speed="fast", - inference_geo="us", - ) - - prompt_cost, completion_cost = cost_per_token( - model="claude-opus-4-6", - usage=usage, - ) - - # generic_cost_per_token called with the plain base model name - mock_cost.assert_called_once() - assert mock_cost.call_args[1]["model"] == "claude-opus-4-6" - assert mock_cost.call_args[1]["custom_llm_provider"] == "anthropic" - - # 1.1 (fast) * 1.1 (us) = 1.21x multiplier applied - expected_multiplier = 1.1 * 1.1 - assert abs(prompt_cost - base_prompt * expected_multiplier) < 1e-10 - assert abs(completion_cost - base_completion * expected_multiplier) < 1e-10 + assert prompt_cost == pytest.approx(1000 * 5e-06 * multiplier) + assert completion_cost == pytest.approx(1000 * 2.5e-05 * multiplier) def test_calculate_usage_captures_service_tier(): diff --git a/tests/unit/llms/dashscope/test_dashscope_cost_calculator.py b/tests/unit/llms/dashscope/test_dashscope_cost_calculator.py index 551caab7153..7f4379368d1 100644 --- a/tests/unit/llms/dashscope/test_dashscope_cost_calculator.py +++ b/tests/unit/llms/dashscope/test_dashscope_cost_calculator.py @@ -19,7 +19,7 @@ import litellm from litellm.litellm_core_utils.llm_cost_calc.utils import ( resolve_explicit_cache_read_cost_rate, calculate_prompt_caching_savings, - generic_cost_per_token, + cost_per_text, get_token_type_cost_breakdown, ) from litellm.llms.dashscope.cost_calculator import ( @@ -475,7 +475,7 @@ class TestDashscopeCostCalculator: prompt_tokens_details={"cached_tokens": 600, "cache_type": "ephemeral"}, ) - prompt_cost, _ = generic_cost_per_token( + prompt_cost, _ = cost_per_text( model="custom-qwen", usage=usage, custom_llm_provider="openai", diff --git a/tests/unit/llms/fireworks_ai/test_fireworks_ai_cache_pricing.py b/tests/unit/llms/fireworks_ai/test_fireworks_ai_cache_pricing.py deleted file mode 100644 index c21943cfe75..00000000000 --- a/tests/unit/llms/fireworks_ai/test_fireworks_ai_cache_pricing.py +++ /dev/null @@ -1,65 +0,0 @@ -from copy import deepcopy - -import pytest - -from litellm.constants import FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO -from litellm.llms.fireworks_ai.cache_pricing import with_default_cache_read_rate -from litellm.types.utils import ModelInfo - - -def test_explicit_cache_read_rate_and_missing_input_rate_keep_the_entry_untouched() -> None: - explicit_info: ModelInfo = {"input_cost_per_token": 2e-6, "cache_read_input_token_cost": 1e-6} - no_input_rate_info: ModelInfo = {"output_cost_per_token": 3e-6} - - assert with_default_cache_read_rate(explicit_info) is explicit_info - assert with_default_cache_read_rate(no_input_rate_info) is no_input_rate_info - - -def test_missing_cache_read_rate_is_derived_for_standard_and_off_peak_without_mutating_the_entry() -> None: - model_info: ModelInfo = { - "input_cost_per_token": 2e-6, - "off_peak_pricing": { - "hours_utc": "14:00-00:00", - "input_cost_per_token": 1e-6, - "output_cost_per_token": 3e-6, - }, - } - original: ModelInfo = deepcopy(model_info) - - derived = with_default_cache_read_rate(model_info) - - assert model_info == original - assert derived is not model_info - assert derived["cache_read_input_token_cost"] == pytest.approx(2e-6 * FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO) - assert derived["off_peak_pricing"] == { - "hours_utc": "14:00-00:00", - "input_cost_per_token": 1e-6, - "output_cost_per_token": 3e-6, - "cache_read_input_token_cost": 1e-6 * FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO, - } - - -def test_off_peak_window_without_its_own_input_rate_reuses_the_standard_derived_rate() -> None: - model_info: ModelInfo = { - "input_cost_per_token": 2e-6, - "off_peak_pricing": {"hours_utc": "14:00-00:00", "output_cost_per_token": 3e-6}, - } - - derived = with_default_cache_read_rate(model_info) - - assert derived["off_peak_pricing"]["cache_read_input_token_cost"] == derived["cache_read_input_token_cost"] - - -def test_string_rates_from_config_are_coerced_before_the_discount_is_applied() -> None: - model_info: ModelInfo = { - "input_cost_per_token": "2e-6", - "off_peak_pricing": { - "hours_utc": "14:00-00:00", - "input_cost_per_token": "1e-6", - }, - } - - derived = with_default_cache_read_rate(model_info) - - assert derived["cache_read_input_token_cost"] == pytest.approx(1e-6) - assert derived["off_peak_pricing"]["cache_read_input_token_cost"] == pytest.approx(5e-7) diff --git a/tests/unit/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py b/tests/unit/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py index c3aec979aa6..aa5990ae0b3 100644 --- a/tests/unit/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py +++ b/tests/unit/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py @@ -9,7 +9,7 @@ import pytest import litellm from litellm.litellm_core_utils.llm_cost_calc.utils import ( calculate_prompt_caching_savings, - generic_cost_per_token, + cost_per_text, get_token_type_cost_breakdown, ) from litellm.llms.fireworks_ai.cost_calculator import cost_per_token @@ -213,7 +213,7 @@ def test_fireworks_cache_read_rates_match_breakdown_and_caching_savings(): assert math.isclose(savings, 300 * INPUT_COST * 0.5, rel_tol=1e-10) -def test_generic_cost_per_token_applies_fireworks_cache_read_default_with_or_without_model_info(): +def test_cost_per_text_applies_fireworks_cache_read_default_with_or_without_model_info(): model = "accounts/fireworks/models/generic-cache-read-test" litellm.model_cost = { # test-quality-ok: the restore_model_cost fixture returns litellm.model_cost to the original object after each test **litellm.model_cost, @@ -227,12 +227,12 @@ def test_generic_cost_per_token_applies_fireworks_cache_read_default_with_or_wit usage = _usage(prompt_tokens=1000, cached_tokens=300, completion_tokens=200) expected_prompt_cost = (700 * INPUT_COST) + (300 * INPUT_COST * 0.5) - implicit_model_info_cost, _ = generic_cost_per_token( + implicit_model_info_cost, _ = cost_per_text( model=model, usage=usage, custom_llm_provider="fireworks_ai", ) - explicit_model_info_cost, _ = generic_cost_per_token( + explicit_model_info_cost, _ = cost_per_text( model=model, usage=usage, custom_llm_provider="fireworks_ai", @@ -392,6 +392,6 @@ def test_a_routed_xai_model_keeps_xais_inclusive_token_threshold(): routed_prompt_cost, routed_completion_cost = cost_per_token(model=f"fireworks_ai/{key}", usage=usage) - assert (routed_prompt_cost, routed_completion_cost) == generic_cost_per_token( + assert (routed_prompt_cost, routed_completion_cost) == cost_per_text( model=key, usage=usage, custom_llm_provider="xai" ) diff --git a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 0f5c21d08c7..833f12fac63 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -19,7 +19,7 @@ from litellm._logging import verbose_logger from litellm.cost_calculator import batch_cost_calculator from litellm.litellm_core_utils.exception_mapping_utils import exception_type from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap -from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token, parse_prompt_tokens_details +from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text, parse_prompt_tokens_details from litellm.llms.anthropic.messages.native import handler as anthropic_messages_handler from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.gemini.chat.transformation import GoogleAIStudioGeminiConfig @@ -1620,7 +1620,7 @@ def test_gemini_raw_usage_metadata_prices_cached_audio_at_its_standard_rate() -> + 5 * model_info["cache_read_input_audio_token_cost"] ) - prompt_cost, _ = generic_cost_per_token( + prompt_cost, _ = cost_per_text( model="gemini-audio-standard-test-model", usage=usage, custom_llm_provider="gemini", @@ -1640,13 +1640,13 @@ def test_gemini_usage_totals_are_subtracted_once_and_audio_cache_rate_matches_au cached_usage: Final = _gemini_usage_with_cached_audio(5) uncached_usage: Final = _gemini_usage_with_cached_audio(0) cached_details: Final = parse_prompt_tokens_details(cached_usage) - cached_prompt_cost, _ = generic_cost_per_token( + cached_prompt_cost, _ = cost_per_text( model="gemini-audio-standard-test-model", usage=cached_usage, custom_llm_provider="gemini", model_info=model_info, ) - uncached_prompt_cost, _ = generic_cost_per_token( + uncached_prompt_cost, _ = cost_per_text( model="gemini-audio-standard-test-model", usage=uncached_usage, custom_llm_provider="gemini", @@ -6299,7 +6299,7 @@ def test_vertex_ai_usage_metadata_with_document_tokens_cached(): "input_cost_per_token": 2e-6, "cache_read_input_token_cost": 5e-7, } - prompt_cost, _ = generic_cost_per_token( + prompt_cost, _ = cost_per_text( model="gemini-document-cache-test-model", usage=result, custom_llm_provider="gemini", diff --git a/tests/unit/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py b/tests/unit/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py index d7d921ab35c..efa13e5ccad 100644 --- a/tests/unit/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py +++ b/tests/unit/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py @@ -18,7 +18,7 @@ import respx import litellm from litellm.exceptions import BadRequestError -from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text from litellm.llms.vertex_ai.gemini_embeddings.batch_embed_content_transformation import ( _build_part_for_input, file_reference_name, @@ -348,7 +348,7 @@ class TestProcessEmbedContentResponseUsage: assert result.usage.total_tokens == 258 assert result.usage.prompt_tokens_details.image_tokens == 258 - prompt_cost, _ = generic_cost_per_token( + prompt_cost, _ = cost_per_text( model=self.MODEL, usage=result.usage, custom_llm_provider="vertex_ai", @@ -373,7 +373,7 @@ class TestProcessEmbedContentResponseUsage: assert result.usage.prompt_tokens == 12 assert result.usage.prompt_tokens_details.text_tokens == 12 - prompt_cost, _ = generic_cost_per_token( + prompt_cost, _ = cost_per_text( model=self.MODEL, usage=result.usage, custom_llm_provider="vertex_ai", diff --git a/tests/unit/proxy/management_endpoints/test_ptu_model_settings.py b/tests/unit/proxy/management_endpoints/test_ptu_model_settings.py index efb84ee8f3a..b01cd0b08ed 100644 --- a/tests/unit/proxy/management_endpoints/test_ptu_model_settings.py +++ b/tests/unit/proxy/management_endpoints/test_ptu_model_settings.py @@ -11,7 +11,7 @@ from unittest.mock import patch as patch_ctx import pytest from fastapi import HTTPException -from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text from litellm.litellm_core_utils.ptu_pricing import ptu_terms from litellm.llms.gemini.cost_calculator import cost_per_web_search_request from litellm.proxy._types import ( @@ -1089,7 +1089,7 @@ class TestPtuDeploymentsAreNotBilledPerToken: registered = router.get_deployment_model_info(model_id="dep-ptu", model_name="dashscope/qwen-flash") assert registered is not None assert registered["tiered_pricing"] == [] - assert generic_cost_per_token( + assert cost_per_text( model="dashscope/qwen-flash", usage=Usage(prompt_tokens=1000, completion_tokens=100, total_tokens=1100), custom_llm_provider="dashscope", diff --git a/tests/unit/proxy/pass_through_endpoints/test_vertex_ai_live_passthrough.py b/tests/unit/proxy/pass_through_endpoints/test_vertex_ai_live_passthrough.py index 9c372380749..dbbc4801d24 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_vertex_ai_live_passthrough.py +++ b/tests/unit/proxy/pass_through_endpoints/test_vertex_ai_live_passthrough.py @@ -574,7 +574,7 @@ class TestVertexAILivePassthroughLoggingHandler: def test_reporting_tool_use_tokens_does_not_move_the_bill(self, handler, mock_logging_obj): """Deliberate boundary: these tokens are reported here, and priced nowhere. - generic_cost_per_token reads the input bill out of prompt_tokens_details, and falls + cost_per_text reads the input bill out of prompt_tokens_details, and falls back to prompt_tokens only when the details carry no text or a cache hit overlaps them, so adding tool-use tokens to prompt_tokens is worth nothing on an ordinary Live turn and over-charges against the cache-overlap correction when it is not. Pricing them belongs diff --git a/tests/unit/proxy/spend_tracking/test_savings.py b/tests/unit/proxy/spend_tracking/test_savings.py index f9566fd1ba0..1adcd6b0fa9 100644 --- a/tests/unit/proxy/spend_tracking/test_savings.py +++ b/tests/unit/proxy/spend_tracking/test_savings.py @@ -3,7 +3,7 @@ from typing import Final, Literal import pytest import litellm -from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text from litellm.llms.anthropic.cost_calculation import cost_per_token as anthropic_cost_per_token from litellm.proxy.spend_tracking.savings import ( _baseline_usage, @@ -108,7 +108,7 @@ def _cached_usage_object() -> dict: def _cost_on(model: str, usage_object: dict) -> float: - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model, usage=Usage(**usage_object), custom_llm_provider="anthropic" ) return prompt_cost + completion_cost @@ -159,7 +159,7 @@ def _net_caching_savings_against_biller(usage_object: dict, model: str = "claude Bills the request as it happened, then bills the same token total with nothing cached, and returns the difference. Deriving the expectation from - ``generic_cost_per_token`` rather than restating the formula is what makes these + ``cost_per_text`` rather than restating the formula is what makes these tests able to fail: a wrong formula in savings.py cannot also be wrong here. """ prompt_tokens = usage_object["prompt_tokens"] @@ -245,7 +245,7 @@ def test_caching_savings_agree_with_biller_on_the_request_pricing_basis( ) costs: Final = tuple( sum( - generic_cost_per_token( + cost_per_text( model=model, usage=arm, custom_llm_provider=provider, @@ -486,8 +486,8 @@ def test_supplied_baseline_usage_is_priced_independently( baseline: str, selected: str, actual: Usage, modeled: Usage, loses_money: bool, ) -> None: result: Final = _savings(baseline, selected, actual, modeled) - expected: Final = sum(generic_cost_per_token(model=baseline, usage=modeled, custom_llm_provider="anthropic")) - sum( - generic_cost_per_token(model=selected, usage=actual, custom_llm_provider="anthropic") + expected: Final = sum(cost_per_text(model=baseline, usage=modeled, custom_llm_provider="anthropic")) - sum( + cost_per_text(model=selected, usage=actual, custom_llm_provider="anthropic") ) assert result == pytest.approx(expected) assert result is not None and (result < 0) is loses_money diff --git a/tests/unit/responses/test_responses_utils.py b/tests/unit/responses/test_responses_utils.py index 0863abd1f1e..37d876495bf 100644 --- a/tests/unit/responses/test_responses_utils.py +++ b/tests/unit/responses/test_responses_utils.py @@ -8,7 +8,7 @@ import pytest import litellm from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig -from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text from litellm.responses.litellm_completion_transformation.transformation import LiteLLMCompletionResponsesConfig from litellm.responses.utils import ResponseAPILoggingUtils, ResponsesAPIRequestUtils from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIOptionalRequestParams, ResponsesAPIResponse @@ -440,7 +440,7 @@ class TestResponseAPILoggingUtils: assert usage.prompt_tokens_details.cache_creation_token_details is not None assert usage.prompt_tokens_details.cache_creation_token_details.ephemeral_12h_input_tokens == 2_000 assert usage.prompt_tokens_details.cache_creation_token_details.ephemeral_30m_input_tokens == 3_000 - cost: Final = generic_cost_per_token( + cost: Final = cost_per_text( model="test-12h-cache-write", usage=usage, custom_llm_provider="openai", diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 7ad78c956d4..6ebfacdf539 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -1927,7 +1927,7 @@ def test_gemini_25_explicit_caching_cost_direct_usage(): This test reproduces the issue from #11156 where cached tokens should receive a 75% discount. """ - from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token + from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text from litellm.types.utils import ( CompletionTokensDetailsWrapper, PromptTokensDetailsWrapper, @@ -1953,7 +1953,7 @@ def test_gemini_25_explicit_caching_cost_direct_usage(): ), ) - input_cost, output_cost = generic_cost_per_token( + input_cost, output_cost = cost_per_text( model="gemini/gemini-2.5-pro", usage=usage, custom_llm_provider="gemini", @@ -1987,13 +1987,13 @@ def test_gemini_25_explicit_caching_cost_direct_usage(): def test_azure_ai_cache_cost_calculation(_local_model_cost_map): """ - Test that azure_ai provider correctly calculates cache costs using generic_cost_per_token. + Test that azure_ai provider correctly calculates cache costs using cost_per_text. This verifies that azure_ai models with custom cache pricing in model_info will have their cache_creation_input_token_cost and cache_read_input_token_cost applied correctly. """ - from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token + from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text from litellm.types.utils import PromptTokensDetailsWrapper, Usage # Register a custom azure_ai model with cache pricing @@ -2023,7 +2023,7 @@ def test_azure_ai_cache_cost_calculation(_local_model_cost_map): cache_creation_input_tokens=100, # 100 cache creation tokens ) - input_cost, output_cost = generic_cost_per_token( + input_cost, output_cost = cost_per_text( model=test_model_id, usage=usage, custom_llm_provider="azure_ai", @@ -2887,7 +2887,7 @@ def test_completion_cost_service_tier_for_anthropic(_local_model_cost_map): Regression for LIT-3771: the Anthropic cost route dropped ``service_tier``, so priority requests (whose tier is reported on the response usage) were always billed at the standard rate. The tier is captured by the - transformation and must flow through to ``generic_cost_per_token``. + transformation and must flow through to ``cost_per_text``. """ from litellm import completion_cost from litellm.llms.anthropic.chat.transformation import AnthropicConfig diff --git a/tests/unit/test_openai_service_tier_long_context_pricing.py b/tests/unit/test_openai_service_tier_long_context_pricing.py index 9777af1af70..e315aa330f9 100644 --- a/tests/unit/test_openai_service_tier_long_context_pricing.py +++ b/tests/unit/test_openai_service_tier_long_context_pricing.py @@ -6,7 +6,7 @@ from typing import Final import pytest import litellm -from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text from litellm.types.utils import PromptTokensDetailsWrapper, Usage REPO_ROOT = Path(__file__).parents[2] @@ -151,7 +151,7 @@ def test_tiered_long_context_cost_uses_catalog_rates( completion_tokens=COMPLETION_TOKENS, total_tokens=LONG_CONTEXT_PROMPT_TOKENS + COMPLETION_TOKENS, ) - prompt_cost, completion_cost = generic_cost_per_token( + prompt_cost, completion_cost = cost_per_text( model=model, usage=usage, custom_llm_provider="openai", @@ -169,13 +169,13 @@ def test_gpt_6_astra_ultrafast_long_context_costs_and_controls() -> None: total_tokens=301_000, prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=100, cache_creation_tokens=200), ) - ultrafast_prompt_cost, ultrafast_completion_cost = generic_cost_per_token( + ultrafast_prompt_cost, ultrafast_completion_cost = cost_per_text( model="gpt-6-astra", usage=ultrafast_usage, custom_llm_provider="openai", service_tier="ultrafast", ) - standard_prompt_cost, standard_completion_cost = generic_cost_per_token( + standard_prompt_cost, standard_completion_cost = cost_per_text( model="gpt-6-astra", usage=ultrafast_usage, custom_llm_provider="openai", @@ -186,7 +186,7 @@ def test_gpt_6_astra_ultrafast_long_context_costs_and_controls() -> None: total_tokens=272_000, prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=100, cache_creation_tokens=200), ) - below_threshold_prompt_cost, below_threshold_completion_cost = generic_cost_per_token( + below_threshold_prompt_cost, below_threshold_completion_cost = cost_per_text( model="gpt-6-astra", usage=below_threshold_usage, custom_llm_provider="openai", diff --git a/tests/unit/test_private_usage_aliases.py b/tests/unit/test_private_usage_aliases.py index d4d5e6478d8..92ae90190e0 100644 --- a/tests/unit/test_private_usage_aliases.py +++ b/tests/unit/test_private_usage_aliases.py @@ -1480,7 +1480,6 @@ ALIAS_CASES: Final = ( "get_service_tier_cost_key", False, ), - ("litellm.litellm_core_utils.llm_cost_calc.utils", "", "_is_above_128k", "is_above_128k", False), ( "litellm.litellm_core_utils.llm_request_utils", "", diff --git a/tests/unit/test_register_model_custom_pricing.py b/tests/unit/test_register_model_custom_pricing.py index f125ec14bed..b63d76905a5 100644 --- a/tests/unit/test_register_model_custom_pricing.py +++ b/tests/unit/test_register_model_custom_pricing.py @@ -121,7 +121,7 @@ def test_register_model_inherits_builtin_cache_pricing_for_unmapped_key(monkeypa Regression for the cache-pricing dropout under partial overrides. """ - from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token + from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text from litellm.types.utils import PromptTokensDetailsWrapper, Usage original_model_cost = litellm.model_cost @@ -168,7 +168,7 @@ def test_register_model_inherits_builtin_cache_pricing_for_unmapped_key(monkeypa cache_creation_input_tokens=200, ) - input_cost, output_cost = generic_cost_per_token( + input_cost, output_cost = cost_per_text( model=registered_key, usage=usage, custom_llm_provider="bedrock", diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 914589be55d..86d8a7a372d 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -5069,17 +5069,15 @@ class _EntryReadDuringARegistration(dict[str, object]): return super().__contains__(key) -@pytest.mark.parametrize("walk", ["fal_ai", "ollama", "gpt-5"]) +@pytest.mark.parametrize("walk", ["ollama", "gpt-5"]) def test_request_path_cost_map_walks_survive_a_registration_mid_walk( monkeypatch: pytest.MonkeyPatch, walk: str ) -> None: - from litellm.llms.fal_ai.cost_calculator import _keyed_rows from litellm.llms.ollama.common_utils import OllamaModelInfo from litellm.llms.openai.chat.gpt_reasoning_transformation import _catalogue_declares_default_effort from litellm.utils import _invalidate_model_cost_lowercase_map walks: Final[Mapping[str, Callable[[], object]]] = { - "fal_ai": lambda: _keyed_rows("lit-5853-model", "standard"), "ollama": lambda: OllamaModelInfo._is_static_ollama_model("ollama/lit-5853-model"), "gpt-5": _catalogue_declares_default_effort, }