mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
refactor(cost): price images, search and TTS with the Rust cost core [10/12]
cost::calculate gains image, pixel, query tier, web search, Maps grounding, citation and feature arms, with Gemini's search defaults as fallback lines and the Anthropic speed and geo multipliers as uplifts. The generic_cost_per_token shim is gone, and search, Lyria, per-image and web search pricing go through Prices.price Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
2f65871b42
commit
6a486c3b43
65 changed files with 1416 additions and 1767 deletions
|
|
@ -5,9 +5,11 @@ use crate::price::{
|
|||
Meter, OffPeakRates, OutputUnit, Price, Prices, ServiceTier, TierScope, TokenKind, Uplift,
|
||||
When,
|
||||
};
|
||||
use crate::price::{Resolution, TextUnit, WallClock};
|
||||
use crate::price::{Resolution, SearchDefaults, TextUnit, WallClock, WebSearchUnit};
|
||||
use crate::priced::{Line, Outcome, Priced, Quantity, Side};
|
||||
use crate::usage::{DurationMeter, Usage};
|
||||
use std::num::NonZeroU64;
|
||||
|
||||
use crate::usage::{DurationMeter, ImageOut, Usage};
|
||||
|
||||
const TEXT_INPUT: Meter = Meter::Input(InputUnit::Token(TokenKind::Text));
|
||||
const TEXT_OUTPUT: Meter = Meter::Output(OutputUnit::Token(TokenKind::Text));
|
||||
|
|
@ -35,6 +37,8 @@ pub fn calculate(
|
|||
let billed: Vec<Billed> = token_meters(&rates, usage)
|
||||
.chain(character_meters(&rates, usage))
|
||||
.chain(second_meters(&rates, usage))
|
||||
.chain(image_meters(&rates, usage))
|
||||
.chain(search_meters(&rates, usage))
|
||||
.chain([(
|
||||
Meter::Input(InputUnit::TextMessage),
|
||||
rates.banded(&Meter::Input(InputUnit::TextMessage)),
|
||||
|
|
@ -55,7 +59,6 @@ pub fn calculate(
|
|||
let missing: Vec<Meter> = billed
|
||||
.into_iter()
|
||||
.filter_map(|(meter, rate, ..)| rate.is_none().then_some(meter))
|
||||
.chain(without_an_arm(usage))
|
||||
.collect();
|
||||
if lines.is_empty() && !missing.is_empty() {
|
||||
return Outcome::Unpriced { missing };
|
||||
|
|
@ -437,6 +440,38 @@ impl<'a> Rates<'a> {
|
|||
)
|
||||
}
|
||||
|
||||
/// `search_provider_cost_per_query`'s tier walk. The tier whose range holds `max_results`, past the last range the
|
||||
/// highest tier, without tiers the flat query price
|
||||
fn query_rate(&self, max_results: u64) -> (Meter, Option<Rate>) {
|
||||
let tiers: Vec<(Option<NonZeroU64>, UnitPrice)> = self
|
||||
.prices
|
||||
.iter()
|
||||
.filter_map(|(price, rate)| match price.meter {
|
||||
Meter::Input(InputUnit::Query { over_results })
|
||||
if price.channel == Channel::Online =>
|
||||
{
|
||||
Some((over_results, rate))
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
let floor = |over: Option<NonZeroU64>| over.map_or(0, NonZeroU64::get);
|
||||
let held = tiers
|
||||
.iter()
|
||||
.filter(|(over, _)| max_results > floor(*over))
|
||||
.max_by_key(|(over, _)| floor(*over))
|
||||
.or_else(|| tiers.iter().max_by_key(|(over, _)| floor(*over)));
|
||||
match held {
|
||||
Some((over, rate)) => (
|
||||
Meter::Input(InputUnit::Query {
|
||||
over_results: *over,
|
||||
}),
|
||||
Some(Rate::own(*rate)),
|
||||
),
|
||||
None => (Meter::Input(InputUnit::Query { over_results: None }), None),
|
||||
}
|
||||
}
|
||||
|
||||
/// The first of `meters` with a price, borrowed when it is not the first
|
||||
fn first_of(&self, meters: &[Meter]) -> Option<Rate> {
|
||||
let (own, rest) = meters.split_first()?;
|
||||
|
|
@ -556,47 +591,142 @@ impl<'a> Rates<'a> {
|
|||
}
|
||||
}
|
||||
|
||||
/// Meters with usage that no arm prices yet, the per-unit meters 10 adds
|
||||
fn without_an_arm(usage: &Usage) -> impl Iterator<Item = Meter> + '_ {
|
||||
let counted = [
|
||||
(usage.images.input, Meter::Input(InputUnit::Image)),
|
||||
(usage.images.input_pixels, Meter::Input(InputUnit::Pixel)),
|
||||
(
|
||||
usage.images.output.len() as u64,
|
||||
Meter::Output(OutputUnit::Image(None)),
|
||||
),
|
||||
(
|
||||
usage.search.queries,
|
||||
Meter::Input(InputUnit::Query { over_results: None }),
|
||||
),
|
||||
(
|
||||
usage.search.maps_grounding,
|
||||
Meter::Feature(Feature::GoogleMapsGroundingQuery),
|
||||
),
|
||||
(
|
||||
usage.search.browser_opens,
|
||||
Meter::Feature(Feature::BrowserOpen),
|
||||
),
|
||||
(
|
||||
usage.search.additional_results,
|
||||
Meter::Feature(Feature::AdditionalResult),
|
||||
),
|
||||
(
|
||||
usage.tokens.citation,
|
||||
Meter::Feature(Feature::CitationToken),
|
||||
),
|
||||
];
|
||||
let web_search = usage
|
||||
.search
|
||||
.web_search
|
||||
.filter(|search| search.requests > 0)
|
||||
.map(|search| Meter::WebSearch(search.context));
|
||||
counted
|
||||
.into_iter()
|
||||
.filter_map(|(count, meter)| (count > 0).then_some(meter))
|
||||
.chain(web_search)
|
||||
const MEGAPIXEL: u64 = 1_048_576;
|
||||
|
||||
/// main's `default_image_cost_calculator` and fal's image rules. A requested size with its own price bills that
|
||||
/// price, then the first of the input per-image price, the output per-image price and the input per-pixel price
|
||||
/// times the image's pixels, then fal's output per-pixel price per started megapixel
|
||||
fn image_meters(rates: &Rates<'_>, usage: &Usage) -> Vec<Billed> {
|
||||
let per_image = |image: &ImageOut| -> Billed {
|
||||
let pixels = u64::from(image.width.unwrap_or(0)) * u64::from(image.height.unwrap_or(0));
|
||||
let sized = image.size.clone().and_then(|size| {
|
||||
let meter = Meter::Output(OutputUnit::Image(Some(size)));
|
||||
Some((meter.clone(), rates.banded(&meter)?))
|
||||
});
|
||||
let candidates = [
|
||||
(Meter::Input(InputUnit::Image), 1),
|
||||
(Meter::Output(OutputUnit::Image(None)), 1),
|
||||
(Meter::Input(InputUnit::Pixel), pixels),
|
||||
(
|
||||
Meter::Output(OutputUnit::Pixel),
|
||||
MEGAPIXEL * pixels.div_ceil(MEGAPIXEL),
|
||||
),
|
||||
];
|
||||
let flat = || {
|
||||
candidates
|
||||
.into_iter()
|
||||
.filter(|(_, quantity)| *quantity > 0)
|
||||
.find_map(|(meter, quantity)| {
|
||||
Some((meter.clone(), rates.banded(&meter)?, quantity))
|
||||
})
|
||||
};
|
||||
match sized.map(|(meter, rate)| (meter, rate, 1)).or_else(flat) {
|
||||
Some((meter, rate, quantity)) => (
|
||||
meter,
|
||||
Some(rate),
|
||||
Quantity::Count(quantity),
|
||||
Side::Completion,
|
||||
),
|
||||
None => (
|
||||
Meter::Output(OutputUnit::Image(None)),
|
||||
None,
|
||||
Quantity::Count(1),
|
||||
Side::Completion,
|
||||
),
|
||||
}
|
||||
};
|
||||
let input_image = Meter::Input(InputUnit::Image);
|
||||
let input_pixel = Meter::Input(InputUnit::Pixel);
|
||||
usage
|
||||
.images
|
||||
.output
|
||||
.iter()
|
||||
.map(per_image)
|
||||
.chain([
|
||||
(
|
||||
input_image.clone(),
|
||||
rates.banded(&input_image),
|
||||
Quantity::Count(usage.images.input),
|
||||
Side::Prompt,
|
||||
),
|
||||
(
|
||||
input_pixel.clone(),
|
||||
rates.banded(&input_pixel),
|
||||
Quantity::Count(usage.images.input_pixels),
|
||||
Side::Prompt,
|
||||
),
|
||||
])
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Search query tiers, web search, Maps grounding, citation tokens, Groq browser opens and Parallel AI's extra
|
||||
/// results. Web search and Maps bill once per prompt unless the row bills them per query
|
||||
fn search_meters(rates: &Rates<'_>, usage: &Usage) -> Vec<Billed> {
|
||||
let search = &usage.search;
|
||||
let per_prompt = rates.prices.billing().web_search == WebSearchUnit::PerPrompt;
|
||||
let billed_requests = |requests: u64| match per_prompt {
|
||||
true => requests.min(1),
|
||||
false => requests,
|
||||
};
|
||||
let google = rates.prices.billing().search_defaults == SearchDefaults::Google;
|
||||
let default = |price: f64| google.then(|| UnitPrice::new(price)).flatten();
|
||||
let web_search = search.web_search.map(|web_search| {
|
||||
let meter = Meter::WebSearch(web_search.context);
|
||||
let rate = rates.banded(&meter).or_else(|| {
|
||||
default(GOOGLE_WEB_SEARCH_DEFAULT).map(|price| Rate::own(price).borrowed_from(&meter))
|
||||
});
|
||||
(
|
||||
meter,
|
||||
rate,
|
||||
Quantity::Count(billed_requests(web_search.requests)),
|
||||
Side::Prompt,
|
||||
)
|
||||
});
|
||||
let maps = Meter::Feature(Feature::GoogleMapsGroundingQuery);
|
||||
let maps_default = match per_prompt {
|
||||
true => GOOGLE_MAPS_DEFAULT_PER_PROMPT,
|
||||
false => GOOGLE_MAPS_DEFAULT_PER_QUERY,
|
||||
};
|
||||
let maps_line = (
|
||||
maps.clone(),
|
||||
rates
|
||||
.banded(&maps)
|
||||
.or_else(|| default(maps_default).map(|price| Rate::own(price).borrowed_from(&maps))),
|
||||
Quantity::Count(billed_requests(search.maps_grounding)),
|
||||
Side::Prompt,
|
||||
);
|
||||
let feature = |feature: Feature, count: u64| {
|
||||
let meter = Meter::Feature(feature);
|
||||
(
|
||||
meter.clone(),
|
||||
rates.banded(&meter),
|
||||
Quantity::Count(count),
|
||||
Side::Prompt,
|
||||
)
|
||||
};
|
||||
let query = (search.queries > 0).then(|| {
|
||||
let (meter, rate) = rates.query_rate(search.max_results.unwrap_or(DEFAULT_MAX_RESULTS));
|
||||
(meter, rate, Quantity::Count(search.queries), Side::Prompt)
|
||||
});
|
||||
query
|
||||
.into_iter()
|
||||
.chain(web_search)
|
||||
.chain([
|
||||
maps_line,
|
||||
feature(Feature::CitationToken, usage.tokens.citation),
|
||||
feature(Feature::BrowserOpen, search.browser_opens),
|
||||
feature(Feature::AdditionalResult, search.additional_results),
|
||||
])
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Gemini's defaults for a Google row with no web search or Maps grounding price
|
||||
const GOOGLE_WEB_SEARCH_DEFAULT: f64 = 0.035;
|
||||
const GOOGLE_MAPS_DEFAULT_PER_QUERY: f64 = 0.014;
|
||||
const GOOGLE_MAPS_DEFAULT_PER_PROMPT: f64 = 0.025;
|
||||
/// The results a search returns when the request names no `max_results`, main's default
|
||||
const DEFAULT_MAX_RESULTS: u64 = 10;
|
||||
|
||||
fn uplifts(adjustments: &Adjustments, call: &CallContext) -> Vec<Multiplier> {
|
||||
let residency = call
|
||||
.residency
|
||||
|
|
@ -605,7 +735,19 @@ fn uplifts(adjustments: &Adjustments, call: &CallContext) -> Vec<Multiplier> {
|
|||
.non_global_endpoint
|
||||
.then(|| adjustments.get(&Uplift::NonGlobalEndpoint))
|
||||
.flatten();
|
||||
residency.into_iter().chain(endpoint).collect()
|
||||
let speed = call
|
||||
.speed
|
||||
.and_then(|speed| adjustments.get(&Uplift::Speed(speed)));
|
||||
let geo = call
|
||||
.inference_geo
|
||||
.clone()
|
||||
.and_then(|geo| adjustments.get(&Uplift::InferenceGeo(geo)));
|
||||
residency
|
||||
.into_iter()
|
||||
.chain(endpoint)
|
||||
.chain(speed)
|
||||
.chain(geo)
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
@ -616,7 +758,7 @@ mod tests {
|
|||
use rstest::rstest;
|
||||
|
||||
use super::*;
|
||||
use crate::price::{Billing, Residency};
|
||||
use crate::price::{Billing, Residency, SearchContextSize};
|
||||
|
||||
fn price(
|
||||
meter: &Meter,
|
||||
|
|
@ -1152,4 +1294,194 @@ mod tests {
|
|||
let expected = if prompt > 32_000 { rate } else { 0.1e-6 };
|
||||
close(total(&prices, &usage, &call()), 100.0 * expected);
|
||||
}
|
||||
|
||||
fn images(images: Vec<ImageOut>) -> Usage {
|
||||
Usage {
|
||||
images: crate::usage::Images {
|
||||
output: images,
|
||||
..crate::usage::Images::default()
|
||||
},
|
||||
..Usage::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn image(size: Option<&str>, width: u32, height: u32) -> ImageOut {
|
||||
ImageOut {
|
||||
size: size.and_then(crate::price::ImageSize::new),
|
||||
width: Some(width),
|
||||
height: Some(height),
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::input_price_wins(Some(0.04), Some(0.08), None, 0.04)]
|
||||
#[case::output_price(None, Some(0.08), None, 0.08)]
|
||||
#[case::input_pixels(None, None, Some(1e-8), 1e-8 * 1024.0 * 1024.0)]
|
||||
fn an_image_bills_the_first_per_image_price_in_mains_order(
|
||||
#[case] input: Option<f64>,
|
||||
#[case] output: Option<f64>,
|
||||
#[case] per_pixel: Option<f64>,
|
||||
#[case] expected: f64,
|
||||
) {
|
||||
let prices = row(input
|
||||
.map(|rate| standard(&Meter::Input(InputUnit::Image), rate))
|
||||
.into_iter()
|
||||
.chain(output.map(|rate| standard(&Meter::Output(OutputUnit::Image(None)), rate)))
|
||||
.chain(per_pixel.map(|rate| standard(&Meter::Input(InputUnit::Pixel), rate))));
|
||||
close(
|
||||
total(
|
||||
&prices,
|
||||
&images(vec![image(None, 1024, 1024), image(None, 1024, 1024)]),
|
||||
&call(),
|
||||
),
|
||||
2.0 * expected,
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn a_requested_size_with_its_own_price_bills_that_price() {
|
||||
let size = crate::price::ImageSize::new("1K").unwrap();
|
||||
let prices = row([
|
||||
standard(&Meter::Output(OutputUnit::Image(None)), 0.08),
|
||||
standard(&Meter::Output(OutputUnit::Image(Some(size))), 0.03),
|
||||
]);
|
||||
close(
|
||||
total(
|
||||
&prices,
|
||||
&images(vec![image(Some("1K"), 1024, 1024)]),
|
||||
&call(),
|
||||
),
|
||||
0.03,
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn output_pixels_bill_per_started_megapixel() {
|
||||
let prices = row([standard(&Meter::Output(OutputUnit::Pixel), 1e-8)]);
|
||||
let outcome = priced(calculate(
|
||||
&prices,
|
||||
&Adjustments::default(),
|
||||
&images(vec![image(None, 1500, 1000)]),
|
||||
&call(),
|
||||
));
|
||||
assert_eq!(outcome.lines[0].quantity, Quantity::Count(2 * MEGAPIXEL));
|
||||
}
|
||||
|
||||
fn queries(count: u64, max_results: Option<u64>) -> Usage {
|
||||
Usage {
|
||||
search: crate::usage::Search {
|
||||
queries: count,
|
||||
max_results,
|
||||
..crate::usage::Search::default()
|
||||
},
|
||||
..Usage::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::first_tier(Some(10), 0.005)]
|
||||
#[case::second_tier(Some(50), 0.025)]
|
||||
#[case::past_the_last_range(Some(500), 0.025)]
|
||||
#[case::default_max_results(None, 0.005)]
|
||||
fn a_query_bills_the_tier_its_max_results_falls_in(
|
||||
#[case] max_results: Option<u64>,
|
||||
#[case] rate: f64,
|
||||
) {
|
||||
let query = |over: u64| {
|
||||
Meter::Input(InputUnit::Query {
|
||||
over_results: NonZeroU64::new(over),
|
||||
})
|
||||
};
|
||||
let prices = row([standard(&query(0), 0.005), standard(&query(25), 0.025)]);
|
||||
close(
|
||||
total(&prices, &queries(2, max_results), &call()),
|
||||
2.0 * rate,
|
||||
);
|
||||
}
|
||||
|
||||
fn web_searches(requests: u64) -> Usage {
|
||||
Usage {
|
||||
search: crate::usage::Search {
|
||||
web_search: Some(crate::usage::WebSearch {
|
||||
requests,
|
||||
context: SearchContextSize::Medium,
|
||||
}),
|
||||
..crate::usage::Search::default()
|
||||
},
|
||||
..Usage::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn billed(rates: impl IntoIterator<Item = (Price, UnitPrice)>, billing: Billing) -> Prices {
|
||||
Prices::new(rates.into_iter().collect(), None, billing)
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::per_prompt(WebSearchUnit::PerPrompt, 0.01)]
|
||||
#[case::per_query(WebSearchUnit::PerQuery, 0.03)]
|
||||
fn web_search_bills_once_per_prompt_unless_the_row_bills_per_query(
|
||||
#[case] web_search: WebSearchUnit,
|
||||
#[case] expected: f64,
|
||||
) {
|
||||
let prices = billed(
|
||||
[standard(&Meter::WebSearch(SearchContextSize::Medium), 0.01)],
|
||||
Billing {
|
||||
web_search,
|
||||
..Billing::default()
|
||||
},
|
||||
);
|
||||
close(total(&prices, &web_searches(3), &call()), expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::google_row(SearchDefaults::Google, Some(0.035))]
|
||||
#[case::other_row(SearchDefaults::None, None)]
|
||||
fn a_google_row_without_a_web_search_price_bills_geminis_default(
|
||||
#[case] search_defaults: SearchDefaults,
|
||||
#[case] expected: Option<f64>,
|
||||
) {
|
||||
let prices = billed(
|
||||
[standard(&TEXT_INPUT, 1e-6)],
|
||||
Billing {
|
||||
search_defaults,
|
||||
web_search: WebSearchUnit::PerPrompt,
|
||||
..Billing::default()
|
||||
},
|
||||
);
|
||||
let outcome = calculate(&prices, &Adjustments::default(), &web_searches(3), &call());
|
||||
let web_search_total = match outcome {
|
||||
Outcome::Priced(priced) => Some(priced.total.get()),
|
||||
Outcome::Unpriced { .. } => None,
|
||||
};
|
||||
assert_eq!(web_search_total, expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::fast(Some(crate::call::Speed::Fast), None, 2.0)]
|
||||
#[case::geo(None, Some("US"), 1.1)]
|
||||
#[case::neither(None, None, 1.0)]
|
||||
fn speed_and_geo_uplifts_multiply_every_line(
|
||||
#[case] speed: Option<crate::call::Speed>,
|
||||
#[case] geo: Option<&str>,
|
||||
#[case] factor: f64,
|
||||
) {
|
||||
let prices = row([standard(&TEXT_INPUT, 1e-6), standard(&TEXT_OUTPUT, 2e-6)]);
|
||||
let adjustments = Adjustments::new([
|
||||
(
|
||||
Uplift::Speed(crate::call::Speed::Fast),
|
||||
Multiplier::new(2.0).unwrap(),
|
||||
),
|
||||
(
|
||||
Uplift::InferenceGeo(crate::call::InferenceGeo::new("us")),
|
||||
Multiplier::new(1.1).unwrap(),
|
||||
),
|
||||
]);
|
||||
let call = CallContext {
|
||||
speed,
|
||||
inference_geo: geo.map(crate::call::InferenceGeo::new),
|
||||
..call()
|
||||
};
|
||||
let outcome = priced(calculate(&prices, &adjustments, &usage(100, 0, 10), &call));
|
||||
close(outcome.total.get(), (100.0 * 1e-6 + 10.0 * 2e-6) * factor);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -65,8 +65,11 @@ pub struct Billing {
|
|||
pub cache_read_fallback: CacheReadFallback,
|
||||
/// Whether a row with no token price is priced by wall clock, from its `mode`
|
||||
pub wall_clock: WallClock,
|
||||
/// From `web_search_billing_unit`, whether web search and Maps grounding bill per query or once per prompt
|
||||
/// From `web_search_billing_unit`, whether web search and Maps grounding bill per query or once per prompt. A
|
||||
/// Google row without one bills per prompt, Gemini's default, and every other row per query
|
||||
pub web_search: WebSearchUnit,
|
||||
/// Prices a Google row bills web search and Maps grounding at when it sets none, Gemini's defaults on main
|
||||
pub search_defaults: SearchDefaults,
|
||||
/// Whether `tiered_pricing` bills each meter in its own tier or the whole call at the tier the input picks.
|
||||
/// A prompt-token tier table bills the whole call, main's `_get_tiered_base_costs`
|
||||
pub tier_scope: TierScope,
|
||||
|
|
@ -105,11 +108,18 @@ pub enum WallClock {
|
|||
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub enum WebSearchUnit {
|
||||
PerQuery,
|
||||
#[default]
|
||||
PerQuery,
|
||||
PerPrompt,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub enum SearchDefaults {
|
||||
#[default]
|
||||
None,
|
||||
Google,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub enum TierScope {
|
||||
#[default]
|
||||
|
|
|
|||
|
|
@ -2,11 +2,13 @@ use std::collections::BTreeMap;
|
|||
use std::collections::btree_map::Entry;
|
||||
use std::num::NonZeroU64;
|
||||
|
||||
use litellm_cost::money::UnitPrice;
|
||||
use litellm_cost::call::{InferenceGeo, Speed};
|
||||
use litellm_cost::money::{Multiplier, UnitPrice};
|
||||
use litellm_cost::price::{
|
||||
Adjustments, Band, BandEdge, Billing, CacheReadFallback, CacheReadMode, CacheTtl,
|
||||
GuardrailUnit, InputUnit, Meter, OffPeak, OffPeakRates, OutputUnit, Price, Prices, Residency,
|
||||
SearchContextSize, TextUnit, TierScope, TokenKind, Uplift, WallClock, WebSearchUnit, When,
|
||||
SearchContextSize, SearchDefaults, TextUnit, TierScope, TokenKind, Uplift, WallClock,
|
||||
WebSearchUnit, When,
|
||||
};
|
||||
use litellm_cost::schedule::{Schedule, Window};
|
||||
|
||||
|
|
@ -92,6 +94,10 @@ impl TryFrom<&EntryWire> for Prices {
|
|||
|
||||
fn billing(wire: &EntryWire, rates: &BTreeMap<Price, UnitPrice>) -> Billing {
|
||||
let provider = wire.attributes.litellm_provider.as_deref();
|
||||
let google = provider.is_some_and(|provider| {
|
||||
provider == "gemini"
|
||||
|| (provider.starts_with("vertex_ai") && !provider.contains("anthropic"))
|
||||
});
|
||||
let prices_meter = |priced: fn(&Meter) -> bool| rates.keys().any(|price| priced(&price.meter));
|
||||
let prices_tokens = prices_meter(|meter| {
|
||||
matches!(
|
||||
|
|
@ -137,9 +143,14 @@ fn billing(wire: &EntryWire, rates: &BTreeMap<Price, UnitPrice>) -> Billing {
|
|||
Some(Mode::AudioTranscription) => WallClock::AudioFile,
|
||||
Some(_) => WallClock::Never,
|
||||
},
|
||||
web_search: match wire.attributes.web_search_billing_unit {
|
||||
Some(WebSearchBillingUnit::PerQuery) => WebSearchUnit::PerQuery,
|
||||
Some(WebSearchBillingUnit::PerPrompt) | None => WebSearchUnit::PerPrompt,
|
||||
web_search: match (wire.attributes.web_search_billing_unit, google) {
|
||||
(Some(WebSearchBillingUnit::PerQuery), _) | (None, false) => WebSearchUnit::PerQuery,
|
||||
(Some(WebSearchBillingUnit::PerPrompt), _) | (None, true) => WebSearchUnit::PerPrompt,
|
||||
},
|
||||
search_defaults: if google {
|
||||
SearchDefaults::Google
|
||||
} else {
|
||||
SearchDefaults::None
|
||||
},
|
||||
tier_scope: if wire
|
||||
.tier_rows
|
||||
|
|
@ -155,7 +166,21 @@ fn billing(wire: &EntryWire, rates: &BTreeMap<Price, UnitPrice>) -> Billing {
|
|||
}
|
||||
|
||||
impl From<&EntryWire> for Adjustments {
|
||||
/// `provider_specific_entry.fast` is Anthropic's speed uplift and every other number in it a geo uplift, as
|
||||
/// main's `get_provider_specific_speed_multiplier` and `get_provider_specific_geo_multiplier` read it
|
||||
fn from(wire: &EntryWire) -> Self {
|
||||
let provider_specific = wire
|
||||
.attributes
|
||||
.provider_specific_entry
|
||||
.iter()
|
||||
.flatten()
|
||||
.filter_map(|(key, value)| {
|
||||
let uplift = match key.as_str() {
|
||||
"fast" => Uplift::Speed(Speed::Fast),
|
||||
geo => Uplift::InferenceGeo(InferenceGeo::new(geo)),
|
||||
};
|
||||
Some((uplift, Multiplier::new(value.as_f64()?)?))
|
||||
});
|
||||
Adjustments::new(
|
||||
[
|
||||
(Uplift::NonGlobalEndpoint, wire.regional_endpoint_uplift),
|
||||
|
|
@ -163,7 +188,8 @@ impl From<&EntryWire> for Adjustments {
|
|||
(Uplift::Residency(Residency::Us), wire.residency_uplift_us),
|
||||
]
|
||||
.into_iter()
|
||||
.filter_map(|(uplift, factor)| Some((uplift, factor?))),
|
||||
.filter_map(|(uplift, factor)| Some((uplift, factor?)))
|
||||
.chain(provider_specific),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
|
@ -929,11 +955,16 @@ mod tests {
|
|||
#[rstest]
|
||||
#[case::tokens_win_over_characters(
|
||||
json!({"litellm_provider": "vertex_ai-language-models", "input_cost_per_character": 1e-6, "input_cost_per_token": 1e-6}),
|
||||
Billing::default()
|
||||
Billing { web_search: WebSearchUnit::PerPrompt, search_defaults: SearchDefaults::Google, ..Billing::default() }
|
||||
)]
|
||||
#[case::characters_only(
|
||||
json!({"litellm_provider": "vertex_ai-language-models", "input_cost_per_character": 1e-6}),
|
||||
Billing { text_unit: TextUnit::Character, ..Billing::default() }
|
||||
Billing {
|
||||
text_unit: TextUnit::Character,
|
||||
web_search: WebSearchUnit::PerPrompt,
|
||||
search_defaults: SearchDefaults::Google,
|
||||
..Billing::default()
|
||||
}
|
||||
)]
|
||||
#[case::xai_band_is_inclusive(
|
||||
json!({"litellm_provider": "xai", "input_cost_per_token": 1e-6}),
|
||||
|
|
@ -961,7 +992,15 @@ mod tests {
|
|||
)]
|
||||
#[case::gemini_per_query_search(
|
||||
json!({"litellm_provider": "gemini", "web_search_billing_unit": "per_query"}),
|
||||
Billing { web_search: WebSearchUnit::PerQuery, ..Billing::default() }
|
||||
Billing { web_search: WebSearchUnit::PerQuery, search_defaults: SearchDefaults::Google, ..Billing::default() }
|
||||
)]
|
||||
#[case::vertex_rows_take_googles_search_defaults(
|
||||
json!({"litellm_provider": "vertex_ai-language-models"}),
|
||||
Billing { web_search: WebSearchUnit::PerPrompt, search_defaults: SearchDefaults::Google, ..Billing::default() }
|
||||
)]
|
||||
#[case::vertex_claude_rows_bill_search_like_anthropic(
|
||||
json!({"litellm_provider": "vertex_ai-anthropic_models"}),
|
||||
Billing::default()
|
||||
)]
|
||||
#[case::token_tiers_bill_the_whole_call(
|
||||
json!({"litellm_provider": "openai", "tiered_pricing": [{"range": [0, 32000], "input_cost_per_token": 1e-6}]}),
|
||||
|
|
@ -977,4 +1016,26 @@ mod tests {
|
|||
) {
|
||||
assert_eq!(prices(entry).unwrap().billing(), expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn provider_specific_entry_lowers_to_speed_and_geo_uplifts() {
|
||||
let wire = EntryWire::read(
|
||||
json!({"litellm_provider": "anthropic", "provider_specific_entry": {"fast": 6.0, "us": 1.1}})
|
||||
.as_object()
|
||||
.unwrap(),
|
||||
);
|
||||
let adjustments = Adjustments::from(&wire);
|
||||
assert_eq!(
|
||||
adjustments
|
||||
.get(&Uplift::Speed(Speed::Fast))
|
||||
.map(Multiplier::get),
|
||||
Some(6.0)
|
||||
);
|
||||
assert_eq!(
|
||||
adjustments
|
||||
.get(&Uplift::InferenceGeo(InferenceGeo::new("US")))
|
||||
.map(Multiplier::get),
|
||||
Some(1.1)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,3 +16,161 @@ pub struct ImageKey {
|
|||
pub height: u32,
|
||||
pub model: String,
|
||||
}
|
||||
|
||||
impl ImageCandidates {
|
||||
/// main's `default_image_cost_calculator` key spellings, in its order
|
||||
pub fn spellings(&self) -> Vec<String> {
|
||||
let (first, second) = self.size.unwrap_or((1024, 1024));
|
||||
let size = format!("{first}-x-{second}");
|
||||
let provider_prefix = self
|
||||
.provider
|
||||
.as_deref()
|
||||
.map(|provider| format!("{provider}/"));
|
||||
let without_provider = provider_prefix
|
||||
.as_deref()
|
||||
.and_then(|prefix| self.model.strip_prefix(prefix))
|
||||
.map(|rest| rest.replace(provider_prefix.as_deref().unwrap_or_default(), ""));
|
||||
let base = match (&self.provider, &without_provider) {
|
||||
(Some(provider), Some(rest)) => format!("{provider}/{size}/{rest}"),
|
||||
_ => format!("{size}/{}", self.model),
|
||||
};
|
||||
let quality = self
|
||||
.quality
|
||||
.as_deref()
|
||||
.filter(|quality| !quality.is_empty());
|
||||
let with_quality = |name: &str| {
|
||||
quality.map_or_else(|| name.to_owned(), |quality| format!("{quality}/{name}"))
|
||||
};
|
||||
let provider_first = quality
|
||||
.zip(self.provider.as_deref())
|
||||
.map(|(quality, provider)| {
|
||||
format!(
|
||||
"{provider}/{quality}/{size}/{}",
|
||||
without_provider.as_deref().unwrap_or(&self.model)
|
||||
)
|
||||
});
|
||||
let last_segment = self.model.rsplit('/').next().unwrap_or(&self.model);
|
||||
let model_without_provider = format!("{size}/{last_segment}");
|
||||
[
|
||||
Some(with_quality(&base)),
|
||||
provider_first,
|
||||
Some(base.clone()),
|
||||
Some(format!("high/{base}")),
|
||||
Some(with_quality(&model_without_provider)),
|
||||
Some(model_without_provider.clone()),
|
||||
Some(self.model.clone()),
|
||||
without_provider,
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
impl ImageKey {
|
||||
/// A `{provider}/{quality}/{W}-x-{H}/{model}` key, `None` for any other shape
|
||||
pub fn parse(key: &str, provider: &str) -> Option<Self> {
|
||||
let rest = key.strip_prefix(provider)?.strip_prefix('/')?;
|
||||
let (quality, rest) = rest.split_once('/')?;
|
||||
let (size, model) = rest.split_once('/')?;
|
||||
let (width, height) = size.split_once("-x-")?;
|
||||
let width: u32 = width.parse().ok().filter(|width| *width > 0)?;
|
||||
let height: u32 = height.parse().ok().filter(|height| *height > 0)?;
|
||||
Some(Self {
|
||||
quality: Some(quality.to_owned()),
|
||||
width,
|
||||
height,
|
||||
model: model.to_owned(),
|
||||
})
|
||||
}
|
||||
|
||||
fn pixels(&self) -> u64 {
|
||||
u64::from(self.width) * u64::from(self.height)
|
||||
}
|
||||
}
|
||||
|
||||
/// fal's nearest size match. Of the `{provider}/{quality}/{W}-x-{H}/{model}` keys the row filter keeps, the one whose
|
||||
/// pixel count is closest to the target's, the smaller one on a tie
|
||||
pub fn nearest_image_key<'a>(
|
||||
keys: impl IntoIterator<Item = &'a str>,
|
||||
provider: &str,
|
||||
model: &str,
|
||||
quality: &str,
|
||||
target: (u32, u32),
|
||||
) -> Option<&'a str> {
|
||||
let target_pixels = u64::from(target.0) * u64::from(target.1);
|
||||
keys.into_iter()
|
||||
.filter_map(|key| Some((key, ImageKey::parse(key, provider)?)))
|
||||
.filter(|(_, parsed)| parsed.model == model && parsed.quality.as_deref() == Some(quality))
|
||||
.min_by_key(|(_, parsed)| (parsed.pixels().abs_diff(target_pixels), parsed.pixels()))
|
||||
.map(|(key, _)| key)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[rstest]
|
||||
fn the_spellings_follow_mains_order() {
|
||||
let candidates = ImageCandidates {
|
||||
model: "azure/dall-e-3".into(),
|
||||
provider: Some("azure".into()),
|
||||
size: Some((1024, 1792)),
|
||||
quality: Some("hd".into()),
|
||||
};
|
||||
assert_eq!(
|
||||
candidates.spellings(),
|
||||
[
|
||||
"hd/azure/1024-x-1792/dall-e-3",
|
||||
"azure/hd/1024-x-1792/dall-e-3",
|
||||
"azure/1024-x-1792/dall-e-3",
|
||||
"high/azure/1024-x-1792/dall-e-3",
|
||||
"hd/1024-x-1792/dall-e-3",
|
||||
"1024-x-1792/dall-e-3",
|
||||
"azure/dall-e-3",
|
||||
"dall-e-3",
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn without_a_provider_prefix_or_quality_the_spellings_are_the_bare_ones() {
|
||||
let candidates = ImageCandidates {
|
||||
model: "dall-e-2".into(),
|
||||
provider: Some("openai".into()),
|
||||
size: None,
|
||||
quality: None,
|
||||
};
|
||||
assert_eq!(
|
||||
candidates.spellings(),
|
||||
[
|
||||
"1024-x-1024/dall-e-2",
|
||||
"1024-x-1024/dall-e-2",
|
||||
"high/1024-x-1024/dall-e-2",
|
||||
"1024-x-1024/dall-e-2",
|
||||
"1024-x-1024/dall-e-2",
|
||||
"dall-e-2",
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::exact((1024, 1024), "fal_ai/high/1024-x-1024/flux")]
|
||||
#[case::nearest((1100, 1100), "fal_ai/high/1024-x-1024/flux")]
|
||||
#[case::tie_takes_the_smaller((768, 768), "fal_ai/high/512-x-512/flux")]
|
||||
fn the_nearest_size_wins(#[case] target: (u32, u32), #[case] expected: &str) {
|
||||
let keys = [
|
||||
"fal_ai/high/512-x-512/flux",
|
||||
"fal_ai/high/1024-x-1024/flux",
|
||||
"fal_ai/low/1024-x-1024/flux",
|
||||
"fal_ai/high/1024-x-1024/other",
|
||||
"fal_ai/flux",
|
||||
];
|
||||
assert_eq!(
|
||||
nearest_image_key(keys, "fal_ai", "flux", "high", target),
|
||||
Some(expected)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,12 +4,17 @@ use std::sync::Arc;
|
|||
use std::time::Duration;
|
||||
|
||||
use litellm_cost::call::CallContext as CostCallContext;
|
||||
use litellm_cost::call::{InferenceGeo, Speed};
|
||||
use litellm_cost::price::{
|
||||
Adjustments, CacheReadMode, CacheTtl, Channel, InputUnit, Meter, OutputUnit,
|
||||
Prices as CostPrices, Residency, Resolution, ServiceTier, TextUnit, TokenKind,
|
||||
Adjustments, CacheReadMode, CacheTtl, Channel, Feature, ImageSize, InputUnit, Meter,
|
||||
OutputUnit, Prices as CostPrices, Residency, Resolution, SearchContextSize, ServiceTier,
|
||||
TextUnit, TokenKind,
|
||||
};
|
||||
use litellm_cost::priced::{Outcome, Quantity, Side};
|
||||
use litellm_cost::usage::{Characters, DurationMeter, Usage as CostUsage};
|
||||
use litellm_cost::usage::{
|
||||
Characters, DurationMeter, ImageOut as CostImageOut, Images, Search, Usage as CostUsage,
|
||||
WebSearch,
|
||||
};
|
||||
use litellm_host_python::from_py;
|
||||
use litellm_model_catalog::rejects_row;
|
||||
use litellm_model_catalog::wire::{EntryWire, Fields, FlatKey};
|
||||
|
|
@ -190,7 +195,9 @@ impl Usage {
|
|||
cache_read_explicit = 0, cache_write_5m = 0, cache_write_1h = 0, cache_write_12h = 0, output_audio = 0,
|
||||
output_image = 0, output_video = 0, reasoning = 0, prompt_characters = 0, completion_characters = 0,
|
||||
wall_clock = None, input_audio = None, live_voice = None, output_video_duration = None,
|
||||
video_resolution = None, text_messages = 0,
|
||||
video_resolution = None, text_messages = 0, images_in = 0, images_out = Vec::new(), pixels_in = 0,
|
||||
search_queries = 0, max_results = None, web_search_requests = 0, web_search_context = "medium",
|
||||
maps_grounding_requests = 0, citation_tokens = 0, browser_opens = 0, additional_results = 0,
|
||||
))]
|
||||
#[expect(
|
||||
clippy::too_many_arguments,
|
||||
|
|
@ -221,7 +228,18 @@ impl Usage {
|
|||
output_video_duration: Option<Duration>,
|
||||
video_resolution: Option<&str>,
|
||||
text_messages: u64,
|
||||
) -> Self {
|
||||
images_in: u64,
|
||||
images_out: Vec<PyRef<'_, ImageOut>>,
|
||||
pixels_in: u64,
|
||||
search_queries: u64,
|
||||
max_results: Option<u64>,
|
||||
web_search_requests: u64,
|
||||
web_search_context: &str,
|
||||
maps_grounding_requests: u64,
|
||||
citation_tokens: u64,
|
||||
browser_opens: u64,
|
||||
additional_results: u64,
|
||||
) -> PyResult<Self> {
|
||||
let mut usage = CostUsage {
|
||||
prompt_tokens,
|
||||
..CostUsage::default()
|
||||
|
|
@ -276,7 +294,34 @@ impl Usage {
|
|||
.filter_map(|(meter, duration)| Some((meter, duration?)))
|
||||
.collect();
|
||||
usage.text_messages = text_messages;
|
||||
Self(usage)
|
||||
usage.tokens.citation = citation_tokens;
|
||||
usage.images = Images {
|
||||
input: images_in,
|
||||
input_pixels: pixels_in,
|
||||
output: images_out.iter().map(|image| image.0.clone()).collect(),
|
||||
};
|
||||
let context = match web_search_context {
|
||||
"low" => SearchContextSize::Low,
|
||||
"medium" => SearchContextSize::Medium,
|
||||
"high" => SearchContextSize::High,
|
||||
other => {
|
||||
return Err(PyValueError::new_err(format!(
|
||||
"unknown search context size {other:?}"
|
||||
)));
|
||||
}
|
||||
};
|
||||
usage.search = Search {
|
||||
queries: search_queries,
|
||||
max_results,
|
||||
web_search: (web_search_requests > 0).then_some(WebSearch {
|
||||
requests: web_search_requests,
|
||||
context,
|
||||
}),
|
||||
maps_grounding: maps_grounding_requests,
|
||||
browser_opens,
|
||||
additional_results,
|
||||
};
|
||||
Ok(Self(usage))
|
||||
}
|
||||
|
||||
/// The provider-reported prompt size, which picks the band
|
||||
|
|
@ -286,10 +331,34 @@ impl Usage {
|
|||
}
|
||||
}
|
||||
|
||||
/// One returned image. `size` picks a size-keyed price, `width` and `height` the per-pixel ones
|
||||
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
|
||||
pub(crate) struct ImageOut(CostImageOut);
|
||||
|
||||
#[pymethods]
|
||||
impl ImageOut {
|
||||
#[new]
|
||||
#[pyo3(signature = (*, size = None, width = None, height = None))]
|
||||
fn new(size: Option<&str>, width: Option<u32>, height: Option<u32>) -> Self {
|
||||
Self(CostImageOut {
|
||||
size: size.and_then(ImageSize::new),
|
||||
width,
|
||||
height,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl CallContext {
|
||||
#[new]
|
||||
#[pyo3(signature = (*, service_tier, channel, data_residency, vertex_location, at, custom_pricing))]
|
||||
#[pyo3(signature = (
|
||||
*, service_tier, channel, data_residency, vertex_location, at, custom_pricing, speed = None,
|
||||
inference_geo = None,
|
||||
))]
|
||||
#[expect(
|
||||
clippy::too_many_arguments,
|
||||
reason = "one keyword argument per call fact, the Python signature"
|
||||
)]
|
||||
fn new(
|
||||
service_tier: Option<&str>,
|
||||
channel: &str,
|
||||
|
|
@ -297,6 +366,8 @@ impl CallContext {
|
|||
vertex_location: Option<&str>,
|
||||
at: &Bound<'_, PyAny>,
|
||||
custom_pricing: Option<&Bound<'_, PyDict>>,
|
||||
speed: Option<&str>,
|
||||
inference_geo: Option<&str>,
|
||||
) -> PyResult<Self> {
|
||||
let channel = match channel {
|
||||
"online" => Channel::Online,
|
||||
|
|
@ -328,8 +399,12 @@ impl CallContext {
|
|||
residency: data_residency.and_then(residency),
|
||||
non_global_endpoint: vertex_location
|
||||
.is_some_and(|location| !location.eq_ignore_ascii_case("global")),
|
||||
speed: None,
|
||||
inference_geo: None,
|
||||
speed: (speed == Some("fast")).then_some(Speed::Fast),
|
||||
inference_geo: inference_geo
|
||||
.filter(|geo| {
|
||||
!matches!(geo.to_lowercase().as_str(), "global" | "not_available")
|
||||
})
|
||||
.map(InferenceGeo::new),
|
||||
at: timestamp(at)?,
|
||||
overlay,
|
||||
},
|
||||
|
|
@ -518,6 +593,28 @@ fn meter_name(meter: &Meter) -> String {
|
|||
Meter::Input(InputUnit::AudioSecond) => "input_cost_per_audio_per_second".to_owned(),
|
||||
Meter::Output(OutputUnit::VideoSecond) => "output_cost_per_video_per_second".to_owned(),
|
||||
Meter::Input(InputUnit::TextMessage) => "input_cost_per_text_message".to_owned(),
|
||||
Meter::Input(InputUnit::Image) => "input_cost_per_image".to_owned(),
|
||||
Meter::Output(OutputUnit::Image(None)) => "output_cost_per_image".to_owned(),
|
||||
Meter::Output(OutputUnit::Image(Some(size))) => {
|
||||
format!("output_cost_per_image_{}", size.as_str())
|
||||
}
|
||||
Meter::Input(InputUnit::Pixel) => "input_cost_per_pixel".to_owned(),
|
||||
Meter::Output(OutputUnit::Pixel) => "output_cost_per_pixel".to_owned(),
|
||||
Meter::Input(InputUnit::Query { .. }) => "input_cost_per_query".to_owned(),
|
||||
Meter::WebSearch(context) => format!(
|
||||
"search_context_cost_per_query.search_context_size_{}",
|
||||
match context {
|
||||
SearchContextSize::Low => "low",
|
||||
SearchContextSize::Medium => "medium",
|
||||
SearchContextSize::High => "high",
|
||||
}
|
||||
),
|
||||
Meter::Feature(Feature::GoogleMapsGroundingQuery) => {
|
||||
"google_maps_grounding_cost_per_query".to_owned()
|
||||
}
|
||||
Meter::Feature(Feature::CitationToken) => "citation_cost_per_token".to_owned(),
|
||||
Meter::Feature(Feature::BrowserOpen) => "browser_open_cost_per_request".to_owned(),
|
||||
Meter::Feature(Feature::AdditionalResult) => "cost_per_additional_result".to_owned(),
|
||||
other => format!("{other:?}"),
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ mod _native {
|
|||
};
|
||||
|
||||
#[pymodule_export]
|
||||
use crate::cost::{CallContext, Line, Priced, PricesHandle, Unpriced, Usage};
|
||||
use crate::cost::{CallContext, ImageOut, Line, Priced, PricesHandle, Unpriced, Usage};
|
||||
#[cfg(feature = "panic-test")]
|
||||
#[pymodule_export]
|
||||
use crate::diagnostics::_panic_for_test;
|
||||
|
|
|
|||
|
|
@ -245,6 +245,69 @@ impl CatalogHandle {
|
|||
})
|
||||
}
|
||||
|
||||
/// The row for an image call, through main's size and quality key spellings
|
||||
#[pyo3(signature = (model, provider = None, *, size = None, quality = None))]
|
||||
fn resolve_image(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
model: &str,
|
||||
provider: Option<&str>,
|
||||
size: Option<(u32, u32)>,
|
||||
quality: Option<&str>,
|
||||
) -> PyResult<Py<PyAny>> {
|
||||
let dict = self.bound_dict(py)?;
|
||||
let keys = resolve::LiveKeys {
|
||||
dict: &dict,
|
||||
index: &self.index,
|
||||
};
|
||||
let candidates = litellm_model_catalog::ImageCandidates {
|
||||
model: model.to_owned(),
|
||||
provider: provider.map(str::to_owned),
|
||||
size,
|
||||
quality: quality.map(str::to_owned),
|
||||
};
|
||||
resolve::resolve_image(py, &keys, &candidates)
|
||||
}
|
||||
|
||||
/// fal's nearest size row, `{provider}/{quality}/{W}-x-{H}/{model}` with an `output_cost_per_image` price, from the
|
||||
/// provider's keys in the index
|
||||
#[pyo3(signature = (model, provider, *, quality, size))]
|
||||
fn nearest_image_key(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
model: &str,
|
||||
provider: &str,
|
||||
quality: &str,
|
||||
size: (u32, u32),
|
||||
) -> PyResult<Option<String>> {
|
||||
let dict = self.bound_dict(py)?;
|
||||
let keys = resolve::LiveKeys {
|
||||
dict: &dict,
|
||||
index: &self.index,
|
||||
};
|
||||
let priced: Vec<String> = keys
|
||||
.provider_keys(provider)
|
||||
.into_iter()
|
||||
.filter(|key| {
|
||||
keys.fields_of(key)
|
||||
.and_then(|fields| {
|
||||
fields
|
||||
.get("output_cost_per_image")
|
||||
.map(serde_json::Value::is_number)
|
||||
})
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.collect();
|
||||
Ok(litellm_model_catalog::nearest_image_key(
|
||||
priced.iter().map(String::as_str),
|
||||
provider,
|
||||
model,
|
||||
quality,
|
||||
size,
|
||||
)
|
||||
.map(str::to_owned))
|
||||
}
|
||||
|
||||
/// The keys whose row names this `litellm_provider`, from the index
|
||||
fn provider_keys(&self, py: Python<'_>, provider: &str) -> PyResult<Vec<String>> {
|
||||
let dict = self.bound_dict(py)?;
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ use std::sync::{Arc, Mutex, PoisonError};
|
|||
use litellm_host_python::{from_py, to_py};
|
||||
use litellm_model_catalog::wire::Fields;
|
||||
use litellm_model_catalog::{
|
||||
Candidates, Catalog, Keys, Resolution, RowRef, model_info, rejects_row,
|
||||
Candidates, Catalog, ImageCandidates, Keys, Resolution, RowRef, model_info, rejects_row,
|
||||
};
|
||||
use pyo3::prelude::*;
|
||||
use pyo3::types::{PyDict, PyTuple};
|
||||
|
|
@ -167,34 +167,50 @@ pub(crate) fn resolve(
|
|||
Resolution::Missing { tried } => {
|
||||
Ok(Missing { tried }.into_pyobject(py)?.into_any().unbind())
|
||||
}
|
||||
Resolution::Found { key, .. } => {
|
||||
let (fields, converted) = keys.fields(&key).unwrap_or_default();
|
||||
let errors: Vec<String> = if converted {
|
||||
Catalog::row_errors(&key, &fields)
|
||||
.iter()
|
||||
.filter(|error| rejects_row(&error.error))
|
||||
.map(ToString::to_string)
|
||||
.collect()
|
||||
} else {
|
||||
vec![format!("{key} must contain JSON-compatible values")]
|
||||
};
|
||||
let fields = Arc::new(fields);
|
||||
if errors.is_empty() {
|
||||
Ok(Resolved { key, fields }
|
||||
.into_pyobject(py)?
|
||||
.into_any()
|
||||
.unbind())
|
||||
} else {
|
||||
Ok(Invalid {
|
||||
key,
|
||||
errors,
|
||||
fields,
|
||||
}
|
||||
.into_pyobject(py)?
|
||||
.into_any()
|
||||
.unbind())
|
||||
}
|
||||
Resolution::Found { key, .. } => found(py, keys, key),
|
||||
}
|
||||
}
|
||||
|
||||
/// The row at `key` as `Resolved`, or `Invalid` when it fails validation
|
||||
pub(crate) fn found(py: Python<'_>, keys: &LiveKeys<'_, '_>, key: String) -> PyResult<Py<PyAny>> {
|
||||
let (fields, converted) = keys.fields(&key).unwrap_or_default();
|
||||
let errors: Vec<String> = if converted {
|
||||
Catalog::row_errors(&key, &fields)
|
||||
.iter()
|
||||
.filter(|error| rejects_row(&error.error))
|
||||
.map(ToString::to_string)
|
||||
.collect()
|
||||
} else {
|
||||
vec![format!("{key} must contain JSON-compatible values")]
|
||||
};
|
||||
let fields = Arc::new(fields);
|
||||
if errors.is_empty() {
|
||||
Ok(Resolved { key, fields }
|
||||
.into_pyobject(py)?
|
||||
.into_any()
|
||||
.unbind())
|
||||
} else {
|
||||
Ok(Invalid {
|
||||
key,
|
||||
errors,
|
||||
fields,
|
||||
}
|
||||
.into_pyobject(py)?
|
||||
.into_any()
|
||||
.unbind())
|
||||
}
|
||||
}
|
||||
|
||||
/// main's `default_image_cost_calculator` lookup, the first of its eight key spellings in the live dict
|
||||
pub(crate) fn resolve_image(
|
||||
py: Python<'_>,
|
||||
keys: &LiveKeys<'_, '_>,
|
||||
candidates: &ImageCandidates,
|
||||
) -> PyResult<Py<PyAny>> {
|
||||
let tried = candidates.spellings();
|
||||
match tried.iter().find(|key| keys.contains(key)) {
|
||||
Some(key) => found(py, keys, key.clone()),
|
||||
None => Ok(Missing { tried }.into_pyobject(py)?.into_any().unbind()),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ from contextvars import ContextVar
|
|||
from typing import Final, Literal
|
||||
|
||||
from litellm._catalog import Invalid, report_invalid
|
||||
from litellm.rust_bridge._native import CallContext, Line, Priced, Prices, Unpriced, Usage
|
||||
from litellm.rust_bridge._native import CallContext, ImageOut, Line, Priced, Prices, Unpriced, Usage
|
||||
|
||||
Channel = Literal["online", "batch", "decisions"]
|
||||
|
||||
|
|
@ -45,6 +45,7 @@ def as_main_cost(outcome: Priced | Unpriced | Invalid) -> tuple[float, float]:
|
|||
__all__ = [
|
||||
"CallContext",
|
||||
"Channel",
|
||||
"ImageOut",
|
||||
"Line",
|
||||
"Priced",
|
||||
"Prices",
|
||||
|
|
|
|||
|
|
@ -693,7 +693,6 @@ FIREWORKS_AI_16_B: Final = get_env_int("FIREWORKS_AI_16_B", 16)
|
|||
FIREWORKS_AI_80_B: Final = get_env_int("FIREWORKS_AI_80_B", 80)
|
||||
# https://docs.fireworks.ai/guides/prompt-caching (accessed 2026-09-19): serverless cached prompt tokens
|
||||
# default to a 50% discount off the input rate
|
||||
FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO: Final = 0.5
|
||||
#### Logging callback constants ####
|
||||
REDACTED_BY_LITELM_STRING: Final = "REDACTED_BY_LITELM"
|
||||
MAX_LANGFUSE_INITIALIZED_CLIENTS: Final = get_env_int("MAX_LANGFUSE_INITIALIZED_CLIENTS", 50)
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ from typing_extensions import ReadOnly, TypedDict, assert_never
|
|||
import litellm
|
||||
import litellm._logging
|
||||
from litellm import verbose_logger
|
||||
from litellm._catalog import CATALOG, Missing
|
||||
from litellm._cost import Priced, as_main_cost, call_custom_pricing, custom_pricing_for_call
|
||||
from litellm._cost import Usage as TokenUsage
|
||||
from litellm.constants import (
|
||||
|
|
@ -43,13 +44,13 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import (
|
|||
calculate_cache_writing_cost,
|
||||
calculate_cost_component,
|
||||
cost_per_text,
|
||||
generic_cost_per_token,
|
||||
get_batch_cost_rates,
|
||||
get_billable_input_tokens,
|
||||
get_regional_uplift_multiplier,
|
||||
get_service_tier_cost_key,
|
||||
get_token_type_cost_breakdown,
|
||||
get_vertex_regional_endpoint_uplift,
|
||||
image_cost,
|
||||
parse_prompt_tokens_details,
|
||||
price_usage,
|
||||
pricing_row,
|
||||
|
|
@ -111,7 +112,6 @@ from litellm.types.extraction import ExtractionResponse, ExtractionUsage
|
|||
from litellm.types.llms.base import CachedTokensDetails, LiteLLMBaseModel
|
||||
from litellm.types.llms.openai import (
|
||||
HttpxBinaryResponseContent,
|
||||
ImageGenerationRequestQuality,
|
||||
OpenAIModerationResponse,
|
||||
OpenAIRealtimeStreamList,
|
||||
OpenAIRealtimeStreamResponseBaseObject,
|
||||
|
|
@ -436,7 +436,7 @@ def cost_per_token(
|
|||
|
||||
if custom_cost_per_token is not None:
|
||||
with custom_pricing_for_call(dict(custom_cost_per_token)):
|
||||
return generic_cost_per_token(
|
||||
return cost_per_text(
|
||||
model=model,
|
||||
usage=_flat_custom_pricing_usage(
|
||||
usage_object, prompt_tokens, completion_tokens, cache_read_input_tokens, cache_creation_input_tokens
|
||||
|
|
@ -514,7 +514,7 @@ def cost_per_token(
|
|||
litellm.model_cost.get(decisions_model_info["key"], {}) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # model cost map is dynamically loaded
|
||||
)
|
||||
if decisions_overrides:
|
||||
return generic_cost_per_token(
|
||||
return cost_per_text(
|
||||
model=model,
|
||||
usage=usage_block,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
|
|
@ -580,7 +580,7 @@ def cost_per_token(
|
|||
)
|
||||
elif call_type == "atranscription" or call_type == "transcription":
|
||||
if _transcription_usage_has_token_details(usage_block):
|
||||
return generic_cost_per_token(
|
||||
return cost_per_text(
|
||||
model=model_without_prefix,
|
||||
usage=usage_block,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
|
|
@ -613,7 +613,7 @@ def cost_per_token(
|
|||
) is not None:
|
||||
return per_second_cost
|
||||
elif call_custom_pricing() is not None and _cost_map_model_info(model, custom_llm_provider) is None:
|
||||
return generic_cost_per_token(
|
||||
return cost_per_text(
|
||||
model=model,
|
||||
usage=usage_block,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
|
|
@ -646,7 +646,7 @@ def cost_per_token(
|
|||
elif custom_llm_provider == "bedrock":
|
||||
resolved_bedrock_info: Final = bedrock_model_info or _bedrock_pricing_info(model, None, "bedrock", False)
|
||||
if resolved_bedrock_info is not None:
|
||||
return generic_cost_per_token(
|
||||
return cost_per_text(
|
||||
model=model,
|
||||
usage=usage_block,
|
||||
custom_llm_provider="bedrock",
|
||||
|
|
@ -699,7 +699,7 @@ def cost_per_token(
|
|||
service_tier=service_tier,
|
||||
)
|
||||
else:
|
||||
return generic_cost_per_token(
|
||||
return cost_per_text(
|
||||
model=model,
|
||||
usage=usage_block,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
|
|
@ -2259,15 +2259,15 @@ def _video_input_image_cost(
|
|||
) -> float:
|
||||
match usage:
|
||||
case {"billable_input_image_count": int(billable_input_images)} if billable_input_images > 0:
|
||||
deployment_rate: Final = model_info.get("input_cost_per_image") if model_info is not None else None
|
||||
image_rate: Final = (
|
||||
deployment_rate
|
||||
if deployment_rate is not None
|
||||
else litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider).get(
|
||||
"input_cost_per_image"
|
||||
)
|
||||
row: Final = (
|
||||
model_info
|
||||
if model_info is not None and model_info.get("input_cost_per_image") is not None
|
||||
else pricing_row(model, custom_llm_provider, None)
|
||||
)
|
||||
return billable_input_images * image_rate if isinstance(image_rate, (int, float)) else 0.0
|
||||
prompt_cost, completion_cost = as_main_cost(
|
||||
price_usage(row, TokenUsage(prompt_tokens=0, text=0, output_text=0, images_in=billable_input_images))
|
||||
)
|
||||
return prompt_cost + completion_cost
|
||||
case _:
|
||||
return 0.0
|
||||
|
||||
|
|
@ -2682,57 +2682,19 @@ def default_image_cost_calculator(
|
|||
# Parse dimensions
|
||||
height, width = map(int, size_str.split("-x-"))
|
||||
|
||||
# Build model names for cost lookup
|
||||
base_model_name = f"{size_str}/{model}"
|
||||
model_name_without_custom_llm_provider: str | None = None
|
||||
if custom_llm_provider and model.startswith(f"{custom_llm_provider}/"):
|
||||
model_name_without_custom_llm_provider = model.replace(f"{custom_llm_provider}/", "")
|
||||
base_model_name = f"{custom_llm_provider}/{size_str}/{model_name_without_custom_llm_provider}"
|
||||
model_name_with_quality: Final = f"{quality}/{base_model_name}" if quality else base_model_name
|
||||
provider_first_model_name_with_quality: Final = (
|
||||
f"{custom_llm_provider}/{quality}/{size_str}/{model_name_without_custom_llm_provider or model}"
|
||||
if quality and custom_llm_provider
|
||||
else None
|
||||
)
|
||||
|
||||
# gpt-image-1 models use low, medium, high quality. If user did not specify quality, use medium fot gpt-image-1 model family
|
||||
model_name_with_v2_quality: Final = f"{ImageGenerationRequestQuality.HIGH.value}/{base_model_name}"
|
||||
|
||||
verbose_logger.debug("Looking up cost for models: %s, %s", model_name_with_quality, base_model_name)
|
||||
|
||||
model_without_provider: Final = f"{size_str}/{model.split('/')[-1]}"
|
||||
model_with_quality_without_provider = f"{quality}/{model_without_provider}" if quality else model_without_provider
|
||||
|
||||
models_to_check: Final = (
|
||||
model_name_with_quality,
|
||||
provider_first_model_name_with_quality,
|
||||
base_model_name,
|
||||
model_name_with_v2_quality,
|
||||
model_with_quality_without_provider,
|
||||
model_without_provider,
|
||||
model,
|
||||
model_name_without_custom_llm_provider,
|
||||
)
|
||||
matched_model: Final = next(
|
||||
(_model for _model in models_to_check if _model is not None and _model in litellm.model_cost), None
|
||||
)
|
||||
if matched_model is None and model_info is None:
|
||||
resolution: Final = CATALOG.resolve_image(model, custom_llm_provider, size=(height, width), quality=quality or None)
|
||||
models_to_check: Final = tuple(resolution.tried) if isinstance(resolution, Missing) else (resolution.key,)
|
||||
if isinstance(resolution, Missing) and model_info is None:
|
||||
raise Exception(f"Model not found in cost map. Tried checking {models_to_check}")
|
||||
|
||||
shared_cost_info: Final = litellm.model_cost[matched_model] if matched_model is not None else None
|
||||
shared_cost_info: Final = None if isinstance(resolution, Missing) else litellm.model_cost[resolution.key]
|
||||
price_tables: Final = tuple(table for table in (model_info, shared_cost_info) if table is not None)
|
||||
image_count: Final = n if n is not None else 1
|
||||
unit_counts: Final = (
|
||||
("input_cost_per_image", image_count),
|
||||
("output_cost_per_image", image_count),
|
||||
("input_cost_per_pixel", height * width * image_count),
|
||||
)
|
||||
cost: Final = next(
|
||||
(
|
||||
price * units
|
||||
table_cost
|
||||
for price_table in price_tables
|
||||
for cost_key, units in unit_counts
|
||||
if isinstance(price := price_table.get(cost_key), (int, float))
|
||||
if (table_cost := image_cost(price_table, image_count, width=int(width), height=int(height))) is not None
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
|
@ -3382,7 +3344,7 @@ def _candidate_realtime_token_costs(
|
|||
data_residency: str | None,
|
||||
) -> tuple[float, float] | None:
|
||||
try:
|
||||
return generic_cost_per_token(
|
||||
return cost_per_text(
|
||||
model=model_name,
|
||||
usage=combined_usage_object,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
|
|
|
|||
|
|
@ -281,21 +281,33 @@ class StandardBuiltInToolCostTracking:
|
|||
tool_usage: Final = _responses_tool_usage(response_object)
|
||||
if tool_usage is None or tool_usage.image_gen is None or tool_usage.image_gen.total_tokens <= 0:
|
||||
return None
|
||||
from litellm._cost import Usage as TokenUsage
|
||||
from litellm._cost import as_main_cost
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import price_usage, pricing_row
|
||||
|
||||
try:
|
||||
model_info: Final = litellm.get_model_info(model=tool_model, custom_llm_provider=custom_llm_provider)
|
||||
row: Final = pricing_row(tool_model, custom_llm_provider, None)
|
||||
except Exception as e: # noqa: BLE001 # get_model_info raises bare Exception for unmapped models; fall back
|
||||
verbose_logger.debug("Could not resolve pricing for image tool model %s: %s", tool_model, e)
|
||||
return None
|
||||
image_gen: Final = tool_usage.image_gen
|
||||
input_details: Final = image_gen.input_tokens_details
|
||||
output_details: Final = image_gen.output_tokens_details
|
||||
token_cost: Final = (
|
||||
(input_details.text_tokens if input_details else 0) * (model_info.get("input_cost_per_token") or 0)
|
||||
+ (input_details.image_tokens if input_details else 0) * (model_info.get("input_cost_per_image_token") or 0)
|
||||
+ (output_details.image_tokens if output_details else 0)
|
||||
* (model_info.get("output_cost_per_image_token") or 0)
|
||||
+ (output_details.text_tokens if output_details else 0) * (model_info.get("output_cost_per_token") or 0)
|
||||
text_in: Final = input_details.text_tokens if input_details else 0
|
||||
image_in: Final = input_details.image_tokens if input_details else 0
|
||||
prompt_cost, completion_cost = as_main_cost(
|
||||
price_usage(
|
||||
row,
|
||||
TokenUsage(
|
||||
prompt_tokens=text_in + image_in,
|
||||
text=text_in,
|
||||
image=image_in,
|
||||
output_text=output_details.text_tokens if output_details else 0,
|
||||
output_image=output_details.image_tokens if output_details else 0,
|
||||
),
|
||||
)
|
||||
)
|
||||
token_cost: Final = prompt_cost + completion_cost
|
||||
return token_cost if token_cost > 0 else None
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -13,7 +13,16 @@ from typing_extensions import ReadOnly
|
|||
|
||||
import litellm
|
||||
from litellm._catalog import Invalid
|
||||
from litellm._cost import CallContext, Channel, Priced, Prices, Unpriced, as_main_cost, call_custom_pricing
|
||||
from litellm._cost import (
|
||||
CallContext,
|
||||
Channel,
|
||||
ImageOut,
|
||||
Priced,
|
||||
Prices,
|
||||
Unpriced,
|
||||
as_main_cost,
|
||||
call_custom_pricing,
|
||||
)
|
||||
from litellm._cost import Usage as TokenUsage
|
||||
from litellm._internal_context import current_billing_time
|
||||
from litellm._logging import verbose_logger
|
||||
|
|
@ -21,7 +30,6 @@ from litellm.litellm_core_utils.llm_cost_calc.tiered_pricing import (
|
|||
select_tier_for_input,
|
||||
tier_rate,
|
||||
)
|
||||
from litellm.llms.fireworks_ai.cache_pricing import with_default_cache_read_rate
|
||||
from litellm.types.utils import (
|
||||
CacheCreationTokenDetails,
|
||||
CallTypes,
|
||||
|
|
@ -109,19 +117,10 @@ _BATCH_RATE_PREFIXES: Final = (
|
|||
)
|
||||
_BATCH_TIER_PREFIXES: Final = (*_BATCH_RATE_PREFIXES, *_TTL_PRICED_CACHE_WRITE_RATE_KEYS.values())
|
||||
_BATCH_TIER_KEY: Final = re.compile(rf"^({'|'.join(_BATCH_TIER_PREFIXES)})_above_(\d+k?)_tokens{_BATCH_KEY_SUFFIX}$")
|
||||
_NON_STANDARD_THRESHOLD_SUFFIXES: Final = (*_SERVICE_TIER_SUFFIXES, _BATCH_KEY_SUFFIX)
|
||||
|
||||
|
||||
def _uses_inclusive_token_thresholds(custom_llm_provider: str | None) -> bool:
|
||||
return custom_llm_provider in _INCLUSIVE_THRESHOLD_PROVIDERS
|
||||
|
||||
|
||||
def apply_provider_cache_read_default(model_info: ModelInfo, custom_llm_provider: str | None) -> ModelInfo:
|
||||
if custom_llm_provider == "fireworks_ai":
|
||||
return with_default_cache_read_rate(model_info)
|
||||
return model_info
|
||||
|
||||
|
||||
def _get_token_detail_value(details: object, key: str) -> int | None:
|
||||
if isinstance(details, dict):
|
||||
value = details.get(key)
|
||||
|
|
@ -173,15 +172,6 @@ def get_web_search_requests_from_usage(usage: Usage) -> int | None:
|
|||
return get_web_search_requests(getattr(usage, "server_tool_use", None))
|
||||
|
||||
|
||||
def is_above_128k(tokens: float) -> bool:
|
||||
if tokens > 128000:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
_is_above_128k = is_above_128k
|
||||
|
||||
|
||||
def get_billable_input_tokens(usage: Usage) -> int:
|
||||
"""
|
||||
Returns the number of billable input tokens.
|
||||
|
|
@ -219,10 +209,6 @@ def _parse_token_threshold(threshold: str) -> float:
|
|||
return float(threshold.replace("k", "")) * (1000 if "k" in threshold else 1)
|
||||
|
||||
|
||||
def _parse_above_token_threshold(key: str) -> float:
|
||||
return _parse_token_threshold(key.rsplit("_above_", 1)[1].split("_tokens")[0])
|
||||
|
||||
|
||||
def _prompt_exceeds_threshold(prompt_tokens: int, threshold: float, inclusive: bool) -> bool:
|
||||
return prompt_tokens > threshold or (inclusive and prompt_tokens == threshold)
|
||||
|
||||
|
|
@ -324,46 +310,6 @@ def _select_priced_tier(model_info: ModelInfo, usage: Usage) -> Mapping[str, obj
|
|||
return tier
|
||||
|
||||
|
||||
def _get_tiered_reasoning_rate(model_info: ModelInfo, usage: Usage) -> float | None:
|
||||
tier: Final = _select_priced_tier(model_info=model_info, usage=usage)
|
||||
if tier is None:
|
||||
return None
|
||||
if "output_cost_per_reasoning_token" not in tier and "output_cost_per_token" not in tier:
|
||||
return None
|
||||
return tier_rate(tier, "output_cost_per_reasoning_token", "output_cost_per_token")
|
||||
|
||||
|
||||
def _get_tiered_base_costs(model_info: ModelInfo, usage: Usage) -> tuple[TokenRates, Mapping[str, object]] | None:
|
||||
"""
|
||||
Resolve the base rates from a model's ``tiered_pricing`` table, if it has one.
|
||||
|
||||
Tiered pricing is all-or-nothing: one tier is picked from the request's input tokens
|
||||
and every token of the request is billed at that tier's rate. Rates the tier does not
|
||||
declare fall back to the tier's input rate, so a request never mixes tiers.
|
||||
|
||||
An output rate is the exception: a tier table that spells out only input rates would
|
||||
otherwise serve every completion for free, so the model's own output rate stands in.
|
||||
"""
|
||||
tier: Final = _select_priced_tier(model_info=model_info, usage=usage)
|
||||
if tier is None:
|
||||
return None
|
||||
|
||||
completion_cost: Final = (
|
||||
tier_rate(tier, "output_cost_per_token")
|
||||
if "output_cost_per_token" in tier
|
||||
else get_cost_per_unit(model_info, "output_cost_per_token") or 0.0
|
||||
)
|
||||
cache_creation_rate: Final = tier_rate(tier, "cache_creation_input_token_cost", "input_cost_per_token")
|
||||
rates: Final = TokenRates(
|
||||
input_rate=tier_rate(tier, "input_cost_per_token"),
|
||||
output_rate=completion_cost,
|
||||
cache_read_rate=tier_rate(tier, "cache_read_input_token_cost", "input_cost_per_token"),
|
||||
cache_creation_rate=cache_creation_rate,
|
||||
reasoning_rate=None,
|
||||
)
|
||||
return rates, tier
|
||||
|
||||
|
||||
def _is_within_off_peak_window(off_peak_hours_utc: str | Sequence[str], current_time: datetime | None = None) -> bool:
|
||||
"""Return True if current_time (UTC, defaulting to now) falls inside any off-peak window.
|
||||
|
||||
|
|
@ -601,215 +547,6 @@ def apply_off_peak_pricing(model_info: ModelInfo, current_time: datetime | None,
|
|||
)
|
||||
|
||||
|
||||
def _apply_off_peak_to_base_costs(
|
||||
model_info: ModelInfo,
|
||||
current_time: datetime | None,
|
||||
base_rates: TokenRates,
|
||||
tier: Mapping[str, object],
|
||||
) -> TokenRates:
|
||||
"""Apply off-peak rates to the base costs resolved from a pricing tier. off_peak_pricing has no
|
||||
field for the long-TTL cache-creation rates, so a tier that declares one keeps it, and a tier
|
||||
that does not bills that TTL at the off-peak-adjusted write rate. Reasoning is left to
|
||||
_resolve_billed_reasoning_rate.
|
||||
"""
|
||||
rates: Final = apply_off_peak_pricing(model_info, current_time, base_rates)
|
||||
return replace(
|
||||
rates,
|
||||
cache_creation_rates_by_ttl=MappingProxyType(
|
||||
{
|
||||
ttl: rates.cache_creation_rate if tier.get(key) is None else tier_rate(tier, key)
|
||||
for ttl, key in _TTL_PRICED_CACHE_WRITE_RATE_KEYS.items()
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _first_present_key(model_info: ModelInfo, preferred_key: str, fallback_key: str) -> str:
|
||||
return preferred_key if model_info.get(preferred_key) is not None else fallback_key
|
||||
|
||||
|
||||
def _get_token_base_cost(
|
||||
model_info: ModelInfo,
|
||||
usage: Usage,
|
||||
service_tier: str | None = None,
|
||||
current_time: datetime | None = None,
|
||||
*,
|
||||
threshold_is_inclusive: bool = False,
|
||||
) -> TokenRates:
|
||||
"""
|
||||
Return prompt cost, completion cost, and cache costs for a given model and usage.
|
||||
|
||||
If input_tokens > threshold and `input_cost_per_token_above_[x]k_tokens` or `input_cost_per_token_above_[x]_tokens` is set,
|
||||
then we use the corresponding threshold cost for all token types.
|
||||
|
||||
`threshold_is_inclusive` switches that comparison to >=, for providers such as xAI
|
||||
that bill the higher tier once the prompt reaches the threshold.
|
||||
|
||||
An absent cache-creation or cache-read rate always resolves to the resolved input
|
||||
rate, the way the tiered table and custom deployment pricing already do, since a
|
||||
provider that publishes no cache price bills cached tokens as ordinary input. An
|
||||
absent long-TTL write rate resolves to the cache-creation rate, off-peak included. An
|
||||
explicit 0.0 stays a real price for all of them.
|
||||
"""
|
||||
tiered_base_costs: Final = _get_tiered_base_costs(model_info=model_info, usage=usage)
|
||||
if tiered_base_costs is not None:
|
||||
return _apply_off_peak_to_base_costs(model_info, current_time, *tiered_base_costs)
|
||||
|
||||
# Get service tier aware cost keys
|
||||
input_cost_key: Final = get_service_tier_cost_key("input_cost_per_token", service_tier)
|
||||
output_cost_key: Final = get_service_tier_cost_key("output_cost_per_token", service_tier)
|
||||
cache_creation_cost_key: Final = get_service_tier_cost_key("cache_creation_input_token_cost", service_tier)
|
||||
cache_read_cost_key: Final = get_service_tier_cost_key("cache_read_input_token_cost", service_tier)
|
||||
|
||||
prompt_base_cost = cast( # cast-ok: model pricing data is external
|
||||
float, get_cost_per_unit(model_info, input_cost_key)
|
||||
)
|
||||
completion_base_cost = cast( # cast-ok: model pricing data is external
|
||||
float, get_cost_per_unit(model_info, output_cost_key)
|
||||
)
|
||||
|
||||
# For image generation models that don't have output_cost_per_token,
|
||||
# use output_cost_per_image_token as the base cost (all output tokens are image tokens)
|
||||
if completion_base_cost == 0.0:
|
||||
output_image_cost: Final = get_cost_per_unit(model_info, "output_cost_per_image_token", None)
|
||||
if output_image_cost is not None:
|
||||
completion_base_cost = output_image_cost
|
||||
cache_creation_cost = get_cost_per_unit(model_info, cache_creation_cost_key, default_value=None)
|
||||
cache_creation_keys_by_ttl: Mapping[str, str] = MappingProxyType(
|
||||
{ttl: get_service_tier_cost_key(key, service_tier) for ttl, key in _TTL_PRICED_CACHE_WRITE_RATE_KEYS.items()}
|
||||
)
|
||||
cache_read_cost = get_cost_per_unit(model_info, cache_read_cost_key, default_value=None)
|
||||
|
||||
## CHECK IF ABOVE THRESHOLD
|
||||
# Optimization: collect threshold keys first to avoid sorting all model_info keys.
|
||||
# Standard thresholds and thresholds suffixed for this request's service tier both count.
|
||||
tier_key_suffix: Final = get_service_tier_cost_key("", service_tier)
|
||||
threshold_prefixes: Final = tuple(f"{prefix}_above_" for prefix in _BATCH_RATE_PREFIXES)
|
||||
threshold_keys: Final = [
|
||||
k
|
||||
for k in model_info
|
||||
if k.startswith(threshold_prefixes)
|
||||
and "_tokens" in k
|
||||
and (
|
||||
not k.endswith(_NON_STANDARD_THRESHOLD_SUFFIXES) or (tier_key_suffix != "" and k.endswith(tier_key_suffix))
|
||||
)
|
||||
]
|
||||
|
||||
# Only sort the threshold keys (typically 1-2 keys instead of 66+)
|
||||
threshold: float | None = None
|
||||
for key in sorted(threshold_keys, key=_parse_above_token_threshold):
|
||||
value = model_info.get(key)
|
||||
if value is not None:
|
||||
try:
|
||||
# Handle both formats: _above_128k_tokens and _above_128_tokens
|
||||
threshold_str = key.rsplit("_above_", 1)[1].split("_tokens")[0]
|
||||
threshold = _parse_above_token_threshold(key)
|
||||
if _prompt_exceeds_threshold(usage.prompt_tokens, threshold, threshold_is_inclusive):
|
||||
# Prefer a service_tier-specific above-threshold key when available,
|
||||
# e.g. input_cost_per_token_priority_above_200k_tokens for Gemini
|
||||
# ON_DEMAND_PRIORITY. Falls back to the standard key automatically
|
||||
# via _get_cost_per_unit's service_tier fallback logic.
|
||||
tiered_input_key = (
|
||||
get_service_tier_cost_key(
|
||||
f"input_cost_per_token_above_{threshold_str}_tokens",
|
||||
service_tier,
|
||||
)
|
||||
if service_tier
|
||||
else f"input_cost_per_token_above_{threshold_str}_tokens"
|
||||
)
|
||||
prompt_base_cost = cast(
|
||||
float,
|
||||
get_cost_per_unit(model_info, tiered_input_key, prompt_base_cost),
|
||||
)
|
||||
tiered_output_key = (
|
||||
get_service_tier_cost_key(
|
||||
f"output_cost_per_token_above_{threshold_str}_tokens",
|
||||
service_tier,
|
||||
)
|
||||
if service_tier
|
||||
else f"output_cost_per_token_above_{threshold_str}_tokens"
|
||||
)
|
||||
completion_base_cost = cast(
|
||||
float,
|
||||
get_cost_per_unit(
|
||||
model_info,
|
||||
tiered_output_key,
|
||||
completion_base_cost,
|
||||
),
|
||||
)
|
||||
|
||||
# Apply tiered pricing to cache costs
|
||||
cache_creation_tiered_key = (
|
||||
get_service_tier_cost_key(
|
||||
f"cache_creation_input_token_cost_above_{threshold_str}_tokens",
|
||||
service_tier,
|
||||
)
|
||||
if service_tier
|
||||
else f"cache_creation_input_token_cost_above_{threshold_str}_tokens"
|
||||
)
|
||||
cache_read_tiered_key = (
|
||||
get_service_tier_cost_key(
|
||||
f"cache_read_input_token_cost_above_{threshold_str}_tokens",
|
||||
service_tier,
|
||||
)
|
||||
if service_tier
|
||||
else f"cache_read_input_token_cost_above_{threshold_str}_tokens"
|
||||
)
|
||||
|
||||
cache_creation_cost = get_cost_per_unit(model_info, cache_creation_tiered_key, cache_creation_cost)
|
||||
|
||||
cache_creation_keys_by_ttl = MappingProxyType(
|
||||
{
|
||||
ttl: _first_present_key(
|
||||
model_info,
|
||||
get_service_tier_cost_key(
|
||||
f"{_TTL_PRICED_CACHE_WRITE_RATE_KEYS[ttl]}_above_{threshold_str}_tokens",
|
||||
service_tier,
|
||||
),
|
||||
key,
|
||||
)
|
||||
for ttl, key in cache_creation_keys_by_ttl.items()
|
||||
}
|
||||
)
|
||||
cache_read_cost = get_cost_per_unit(model_info, cache_read_tiered_key, cache_read_cost)
|
||||
except (IndexError, ValueError):
|
||||
continue
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
input_rate_for_missing_cache_rates: Final = _off_peak_rate(
|
||||
_open_off_peak_block(model_info, current_time) or MappingProxyType({}),
|
||||
"input_cost_per_token",
|
||||
prompt_base_cost,
|
||||
)
|
||||
resolved_cache_read_cost: Final = input_rate_for_missing_cache_rates if cache_read_cost is None else cache_read_cost
|
||||
resolved_cache_creation_cost: Final = (
|
||||
input_rate_for_missing_cache_rates if cache_creation_cost is None else cache_creation_cost
|
||||
)
|
||||
base_rates: Final = apply_off_peak_pricing(
|
||||
model_info,
|
||||
current_time,
|
||||
TokenRates(
|
||||
input_rate=prompt_base_cost,
|
||||
output_rate=completion_base_cost,
|
||||
cache_read_rate=resolved_cache_read_cost,
|
||||
cache_creation_rate=resolved_cache_creation_cost,
|
||||
reasoning_rate=None,
|
||||
),
|
||||
)
|
||||
return replace(
|
||||
base_rates,
|
||||
cache_creation_rates_by_ttl=MappingProxyType(
|
||||
{
|
||||
ttl: base_rates.cache_creation_rate
|
||||
if (rate := get_cost_per_unit(model_info, key, default_value=None)) is None
|
||||
else rate
|
||||
for ttl, key in cache_creation_keys_by_ttl.items()
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def calculate_cost_component(model_info: Mapping[str, object], cost_key: str, usage_value: float | None) -> float:
|
||||
"""
|
||||
Generic cost calculator for any usage component
|
||||
|
|
@ -1164,111 +901,6 @@ def get_vertex_regional_endpoint_uplift(model_info: Mapping[str, object], vertex
|
|||
return 1.0
|
||||
|
||||
|
||||
def get_provider_specific_geo_multiplier(model_info: ModelInfo, usage: Usage) -> float:
|
||||
"""
|
||||
Resolve the provider-specific regional pricing multiplier for the geo the
|
||||
request was served from (``usage.inference_geo``), e.g. Anthropic's ``us: 1.1``
|
||||
stored under ``provider_specific_entry``. The regional surcharge applies to
|
||||
every token type, so per-type cost breakdowns must scale by it too.
|
||||
|
||||
Returns 1.0 when the request was served globally or the model carries no
|
||||
multiplier for the geo.
|
||||
"""
|
||||
inference_geo: Final = getattr(usage, "inference_geo", None)
|
||||
if not isinstance(inference_geo, str) or inference_geo.lower() in ("global", "not_available"):
|
||||
return 1.0
|
||||
provider_specific_entry: Final[dict[str, float]] = model_info.get("provider_specific_entry") or {}
|
||||
return float(provider_specific_entry.get(inference_geo.lower(), 1.0))
|
||||
|
||||
|
||||
def get_provider_specific_speed_multiplier(model_info: ModelInfo, usage: Usage) -> float:
|
||||
provider_specific_entry: Final = model_info.get("provider_specific_entry")
|
||||
if getattr(usage, "speed", None) != "fast" or not provider_specific_entry:
|
||||
return 1.0
|
||||
return float(provider_specific_entry.get("fast", 1.0))
|
||||
|
||||
|
||||
def _resolve_reasoning_token_cost(
|
||||
model_info: ModelInfo,
|
||||
service_tier: str | None,
|
||||
completion_base_cost: float,
|
||||
) -> float:
|
||||
tier_reasoning_key: Final = get_service_tier_cost_key("output_cost_per_reasoning_token", service_tier)
|
||||
if model_info.get(tier_reasoning_key) is not None:
|
||||
tier_reasoning_cost: Final = get_cost_per_unit(model_info, tier_reasoning_key, None)
|
||||
if tier_reasoning_cost is not None:
|
||||
return tier_reasoning_cost
|
||||
tier_output_key: Final = get_service_tier_cost_key("output_cost_per_token", service_tier)
|
||||
if tier_output_key != "output_cost_per_token" and model_info.get(tier_output_key) is not None:
|
||||
return completion_base_cost
|
||||
standard_reasoning_cost: Final = get_cost_per_unit(model_info, "output_cost_per_reasoning_token", None)
|
||||
return standard_reasoning_cost if standard_reasoning_cost is not None else completion_base_cost
|
||||
|
||||
|
||||
def _resolve_billed_reasoning_rate(
|
||||
model_info: ModelInfo,
|
||||
usage: Usage,
|
||||
service_tier: str | None,
|
||||
completion_base_cost: float,
|
||||
current_time: datetime | None,
|
||||
) -> float:
|
||||
off_peak: Final = _open_off_peak_block(model_info, current_time)
|
||||
off_peak_reasoning_rate: Final = (
|
||||
None if off_peak is None else _parse_off_peak_rate(off_peak.get("output_cost_per_reasoning_token"))
|
||||
)
|
||||
if off_peak_reasoning_rate is not None:
|
||||
return off_peak_reasoning_rate
|
||||
tiered_reasoning_rate: Final = _get_tiered_reasoning_rate(model_info=model_info, usage=usage)
|
||||
if tiered_reasoning_rate is not None:
|
||||
return tiered_reasoning_rate
|
||||
return _resolve_reasoning_token_cost(
|
||||
model_info=model_info,
|
||||
service_tier=service_tier,
|
||||
completion_base_cost=completion_base_cost,
|
||||
)
|
||||
|
||||
|
||||
def generic_cost_per_token(
|
||||
model: str,
|
||||
usage: Usage,
|
||||
custom_llm_provider: str,
|
||||
service_tier: str | None = None,
|
||||
data_residency: str | None = None,
|
||||
model_info: ModelInfo | None = None,
|
||||
vertex_location: str | None = None,
|
||||
current_time: datetime | None = None,
|
||||
) -> tuple[float, float]:
|
||||
"""
|
||||
Calculates the cost per token for a given model, prompt tokens, and completion tokens.
|
||||
|
||||
Handles context caching as well.
|
||||
|
||||
Input:
|
||||
- model: str, the model name without provider prefix
|
||||
- usage: LiteLLM Usage block, containing anthropic caching information
|
||||
- data_residency: optional OpenAI data-residency region (e.g. "eu", "us"),
|
||||
used to apply the per-model regional-processing uplift multiplier.
|
||||
- vertex_location: optional Vertex AI location the request was served from
|
||||
(e.g. "us-east5", "global"), used to apply the per-model
|
||||
regional-endpoint uplift multiplier when non-global.
|
||||
- current_time: the moment the request is billed at, for off_peak_pricing; defaults to now, UTC
|
||||
|
||||
Returns:
|
||||
Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd
|
||||
"""
|
||||
|
||||
return cost_per_text(
|
||||
model,
|
||||
usage,
|
||||
custom_llm_provider,
|
||||
service_tier=service_tier,
|
||||
data_residency=data_residency,
|
||||
model_info=model_info,
|
||||
vertex_location=vertex_location,
|
||||
current_time=current_time,
|
||||
)
|
||||
|
||||
|
||||
def cost_per_text(
|
||||
model: str,
|
||||
usage: Usage,
|
||||
|
|
@ -1293,6 +925,8 @@ def cost_per_text(
|
|||
data_residency=data_residency,
|
||||
vertex_location=vertex_location,
|
||||
current_time=current_time,
|
||||
speed=_usage_str(usage, "speed"),
|
||||
inference_geo=_usage_str(usage, "inference_geo"),
|
||||
key=model,
|
||||
)
|
||||
)
|
||||
|
|
@ -1311,6 +945,8 @@ def price_usage(
|
|||
vertex_location: str | None = None,
|
||||
current_time: datetime | None = None,
|
||||
channel: Channel = "online",
|
||||
speed: str | None = None,
|
||||
inference_geo: str | None = None,
|
||||
key: str = "model_info",
|
||||
) -> Priced | Unpriced | Invalid:
|
||||
"""Prices ``usage`` over ``row`` with the call's custom pricing over it, the one place a cost is computed"""
|
||||
|
|
@ -1329,10 +965,40 @@ def price_usage(
|
|||
vertex_location=vertex_location,
|
||||
at=current_time if current_time is not None else current_billing_time(),
|
||||
custom_pricing=None if custom_pricing is None else dict(custom_pricing),
|
||||
speed=speed,
|
||||
inference_geo=inference_geo,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def image_cost(
|
||||
row: Mapping[str, object], images: int, width: int | None = None, height: int | None = None
|
||||
) -> float | None:
|
||||
"""``images`` images priced per image or per pixel over ``row``, None when it prices neither"""
|
||||
priced: Final = price_usage(
|
||||
row,
|
||||
TokenUsage(
|
||||
prompt_tokens=0,
|
||||
text=0,
|
||||
output_text=0,
|
||||
images_out=[ImageOut(width=width, height=height)] * max(images, 0),
|
||||
),
|
||||
)
|
||||
return priced.total if isinstance(priced, Priced) and priced.lines else None
|
||||
|
||||
|
||||
def search_cost(model_info: Mapping[str, object], usage: TokenUsage) -> float:
|
||||
"""The search, grounding and tool meters of ``usage`` priced over ``model_info``"""
|
||||
prompt_cost, completion_cost = as_main_cost(price_usage(model_info, usage))
|
||||
return prompt_cost + completion_cost
|
||||
|
||||
|
||||
def _usage_str(usage: Usage, name: str) -> str | None:
|
||||
"""Anthropic's ``speed`` and ``inference_geo``, which pick the row's speed and geo uplifts"""
|
||||
value: Final = getattr(usage, name, None)
|
||||
return value if isinstance(value, str) else None
|
||||
|
||||
|
||||
def pricing_row(model: str, custom_llm_provider: str | None, model_info: ModelInfo | None) -> Mapping[str, object]:
|
||||
"""The row main priced, as written rather than `get_model_info`'s copy, which writes `0` for a missing token
|
||||
price. A call with custom pricing prices an unmapped model from its overlay alone"""
|
||||
|
|
@ -1356,8 +1022,13 @@ def bills_characters(row: Mapping[str, object]) -> bool:
|
|||
|
||||
|
||||
def _without_tiers(row: Mapping[str, object]) -> dict[str, object]:
|
||||
"""Main picks no tier for an empty prompt and bills the row's flat prices"""
|
||||
return {name: value for name, value in row.items() if name != "tiered_pricing"}
|
||||
"""Main picks no prompt-token tier for an empty prompt and bills the row's flat prices. A search row's result
|
||||
tiers stay"""
|
||||
tiers: Final = row.get("tiered_pricing")
|
||||
token_tiers: Final = isinstance(tiers, list) and any(
|
||||
isinstance(tier, Mapping) and "range" in tier for tier in tiers
|
||||
)
|
||||
return {name: value for name, value in row.items() if not (token_tiers and name == "tiered_pricing")}
|
||||
|
||||
|
||||
def _billable_prompt_details(usage: Usage) -> PromptTokensDetailsResult:
|
||||
|
|
@ -1426,9 +1097,8 @@ def _token_usage(
|
|||
) -> TokenUsage:
|
||||
"""The meters main billed. A prompt billed per image or second of media does not bill the same tokens again, and
|
||||
a row that bills characters bills no text tokens (decision 32)"""
|
||||
cache_type: Final = getattr(usage.prompt_tokens_details, "cache_type", None)
|
||||
cached_text: Final = details["cache_hit_tokens"] - details["cache_hit_audio_tokens"]
|
||||
explicit_cache: Final = cache_type == "ephemeral"
|
||||
explicit_cache: Final = _reads_explicit_cache(usage)
|
||||
cache_write_5m, cache_write_1h, cache_write_12h = _cache_write_tokens(details)
|
||||
completion: Final = _completion_token_counts(usage)
|
||||
return TokenUsage(
|
||||
|
|
@ -1571,6 +1241,10 @@ def _custom_pricing_rates(custom_cost_per_token: CostPerToken) -> BilledTokenRat
|
|||
)
|
||||
|
||||
|
||||
def _reads_explicit_cache(usage: Usage) -> bool:
|
||||
return getattr(usage.prompt_tokens_details, "cache_type", None) == "ephemeral"
|
||||
|
||||
|
||||
def _cost_map_billed_rates(
|
||||
model_info: ModelInfo,
|
||||
usage: Usage,
|
||||
|
|
@ -1579,54 +1253,55 @@ def _cost_map_billed_rates(
|
|||
data_residency: str | None,
|
||||
vertex_location: str | None,
|
||||
current_time: datetime | None,
|
||||
*,
|
||||
provider_multipliers: bool = True,
|
||||
) -> BilledTokenRates:
|
||||
billing_time: Final = current_time if current_time is not None else current_billing_time()
|
||||
base_costs: Final = _get_token_base_cost(
|
||||
model_info=model_info,
|
||||
usage=usage,
|
||||
service_tier=service_tier,
|
||||
current_time=billing_time,
|
||||
threshold_is_inclusive=_uses_inclusive_token_thresholds(custom_llm_provider),
|
||||
)
|
||||
provider_cache_read_cost_rate: Final = resolve_explicit_cache_read_cost_rate(
|
||||
model_info=model_info,
|
||||
usage=usage,
|
||||
current_time=billing_time,
|
||||
service_tier=service_tier,
|
||||
)
|
||||
effective_cache_read_cost_rate: Final = (
|
||||
base_costs.cache_read_rate if provider_cache_read_cost_rate is None else provider_cache_read_cost_rate
|
||||
)
|
||||
|
||||
reasoning_rate: Final = _resolve_billed_reasoning_rate(
|
||||
model_info=model_info,
|
||||
usage=usage,
|
||||
service_tier=service_tier,
|
||||
completion_base_cost=base_costs.output_rate,
|
||||
current_time=billing_time,
|
||||
)
|
||||
audio_cache_read_rate: Final = get_cost_per_unit(
|
||||
"""The rate of every token meter this call would bill, read off one priced probe: one unit of each meter at the
|
||||
call's prompt size, so the band, tier, off-peak window and uplifts are the ones the call itself billed at"""
|
||||
explicit_cache: Final = _reads_explicit_cache(usage)
|
||||
priced: Final = price_usage(
|
||||
model_info,
|
||||
get_service_tier_cost_key("cache_read_input_audio_token_cost", service_tier),
|
||||
None,
|
||||
)
|
||||
multiplier: Final = (
|
||||
get_regional_uplift_multiplier(model_info, data_residency)
|
||||
* get_vertex_regional_endpoint_uplift(model_info, vertex_location)
|
||||
* get_provider_specific_geo_multiplier(model_info=model_info, usage=usage)
|
||||
* get_provider_specific_speed_multiplier(model_info=model_info, usage=usage)
|
||||
)
|
||||
return BilledTokenRates(
|
||||
input_cost_per_token=base_costs.input_rate,
|
||||
output_cost_per_token=base_costs.output_rate,
|
||||
cache_read_input_token_cost=effective_cache_read_cost_rate,
|
||||
cache_read_input_audio_token_cost=(
|
||||
audio_cache_read_rate if audio_cache_read_rate is not None else effective_cache_read_cost_rate
|
||||
TokenUsage(
|
||||
prompt_tokens=usage.prompt_tokens,
|
||||
text=1,
|
||||
output_text=1,
|
||||
cache_read=0 if explicit_cache else 1,
|
||||
cache_read_explicit=1 if explicit_cache else 0,
|
||||
cache_read_audio=1,
|
||||
cache_write_5m=1,
|
||||
cache_write_1h=1,
|
||||
cache_write_12h=1,
|
||||
reasoning=1,
|
||||
),
|
||||
cache_creation_input_token_cost=base_costs.cache_creation_rate,
|
||||
cache_creation_input_token_cost_by_ttl=base_costs.cache_creation_rates_by_ttl,
|
||||
output_cost_per_reasoning_token=reasoning_rate,
|
||||
).scaled(multiplier)
|
||||
service_tier=service_tier,
|
||||
data_residency=data_residency,
|
||||
vertex_location=vertex_location,
|
||||
current_time=current_time,
|
||||
speed=_usage_str(usage, "speed") if provider_multipliers else None,
|
||||
inference_geo=_usage_str(usage, "inference_geo") if provider_multipliers else None,
|
||||
key=str(model_info.get("key") or custom_llm_provider or "model_info"),
|
||||
)
|
||||
rates: Final = MappingProxyType(
|
||||
{line.meter: line.amount for line in priced.lines} if isinstance(priced, Priced) else {}
|
||||
)
|
||||
cache_read: Final = rates.get(
|
||||
"explicit_cache_read_input_token_cost" if explicit_cache else "cache_read_input_token_cost", 0.0
|
||||
)
|
||||
cache_write: Final = rates.get("cache_creation_input_token_cost", 0.0)
|
||||
return BilledTokenRates(
|
||||
input_cost_per_token=rates.get("input_cost_per_token", 0.0),
|
||||
output_cost_per_token=rates.get("output_cost_per_token", 0.0),
|
||||
cache_read_input_token_cost=cache_read,
|
||||
cache_read_input_audio_token_cost=rates.get("cache_read_input_audio_token_cost", cache_read),
|
||||
cache_creation_input_token_cost=cache_write,
|
||||
cache_creation_input_token_cost_by_ttl=MappingProxyType(
|
||||
{
|
||||
"1hr": rates.get("cache_creation_input_token_cost_above_1hr", cache_write),
|
||||
"12hr": rates.get("cache_creation_input_token_cost_above_12hr", cache_write),
|
||||
}
|
||||
),
|
||||
output_cost_per_reasoning_token=rates.get("output_cost_per_reasoning_token", 0.0),
|
||||
)
|
||||
|
||||
|
||||
def get_billed_token_rates(
|
||||
|
|
@ -1645,16 +1320,11 @@ def get_billed_token_rates(
|
|||
if custom_cost_per_token is not None:
|
||||
return _custom_pricing_rates(custom_cost_per_token)
|
||||
try:
|
||||
resolved_model_info: Final = apply_provider_cache_read_default(
|
||||
model_info
|
||||
if model_info is not None
|
||||
else get_model_info(model=model, custom_llm_provider=custom_llm_provider),
|
||||
custom_llm_provider,
|
||||
)
|
||||
resolved_model_info: Final = pricing_row(model, custom_llm_provider, model_info)
|
||||
except Exception: # noqa: BLE001 # get_model_info raises a bare Exception for an unmapped model: no rates
|
||||
return None
|
||||
return _cost_map_billed_rates(
|
||||
model_info=resolved_model_info,
|
||||
model_info=cast(ModelInfo, resolved_model_info), # cast-ok: the row as written, priced by its own meters
|
||||
usage=usage,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
service_tier=service_tier,
|
||||
|
|
@ -1680,7 +1350,7 @@ def get_token_type_cost_breakdown(
|
|||
object and model pricing alone.
|
||||
|
||||
This works for every provider, including Perplexity/Cerebras/Dashscope whose
|
||||
cost calculators bypass ``generic_cost_per_token``, because cache tokens always
|
||||
cost calculators bypass ``cost_per_text``, because cache tokens always
|
||||
land on ``prompt_tokens_details`` (via the Usage constructor and provider
|
||||
transformations) and reasoning tokens on ``completion_tokens_details``. It reuses
|
||||
the same rate resolution as the total-cost path (``get_billed_token_rates``) so the
|
||||
|
|
@ -1743,25 +1413,18 @@ def calculate_prompt_caching_savings(
|
|||
``billed_at`` is the request's completion time, so off-peak windows resolve as the
|
||||
biller saw them rather than at the later spend write.
|
||||
"""
|
||||
model_info_with_cache_read_default: Final = apply_provider_cache_read_default(model_info, custom_llm_provider)
|
||||
base_costs: Final = _get_token_base_cost(
|
||||
model_info=model_info_with_cache_read_default,
|
||||
usage=usage,
|
||||
service_tier=service_tier,
|
||||
current_time=billed_at,
|
||||
threshold_is_inclusive=_uses_inclusive_token_thresholds(custom_llm_provider),
|
||||
)
|
||||
provider_cache_read_cost: Final = resolve_explicit_cache_read_cost_rate(
|
||||
rates: Final = _cost_map_billed_rates(
|
||||
model_info=model_info,
|
||||
usage=usage,
|
||||
current_time=billed_at,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
service_tier=service_tier,
|
||||
data_residency=data_residency,
|
||||
vertex_location=vertex_location,
|
||||
current_time=billed_at,
|
||||
provider_multipliers=False,
|
||||
)
|
||||
prompt_base_cost: Final = base_costs.input_rate
|
||||
effective_cache_read_cost: Final = (
|
||||
base_costs.cache_read_rate if provider_cache_read_cost is None else provider_cache_read_cost
|
||||
)
|
||||
write_rate: Final = base_costs.cache_creation_rate or prompt_base_cost
|
||||
input_rate: Final = rates.input_cost_per_token
|
||||
write_rate: Final = rates.cache_creation_input_token_cost or input_rate
|
||||
prompt_tokens_details: Final = parse_prompt_tokens_details(usage)
|
||||
cache_read_tokens: Final = max(prompt_tokens_details["cache_hit_tokens"], 0)
|
||||
cache_creation_tokens: Final = max(prompt_tokens_details["cache_creation_tokens"], 0)
|
||||
|
|
@ -1776,19 +1439,16 @@ def calculate_prompt_caching_savings(
|
|||
if details is not None
|
||||
else None
|
||||
)
|
||||
read_discount: Final = cache_read_tokens * max(prompt_base_cost - effective_cache_read_cost, 0.0)
|
||||
read_discount: Final = cache_read_tokens * max(input_rate - rates.cache_read_input_token_cost, 0.0)
|
||||
write_premium: Final = calculate_cache_writing_cost(
|
||||
cache_creation_tokens=cache_creation_tokens,
|
||||
cache_creation_token_details=cache_creation_details,
|
||||
cache_creation_cost=write_rate - prompt_base_cost,
|
||||
cache_creation_cost=write_rate - input_rate,
|
||||
cache_creation_cost_by_ttl=MappingProxyType(
|
||||
{ttl: rate - prompt_base_cost for ttl, rate in base_costs.cache_creation_rates_by_ttl.items()}
|
||||
{ttl: rate - input_rate for ttl, rate in rates.cache_creation_input_token_cost_by_ttl.items()}
|
||||
),
|
||||
)
|
||||
uplift: Final = get_regional_uplift_multiplier(model_info, data_residency) * get_vertex_regional_endpoint_uplift(
|
||||
model_info, vertex_location
|
||||
)
|
||||
return (read_discount - write_premium) * uplift
|
||||
return read_discount - write_premium
|
||||
|
||||
|
||||
def calculate_image_response_cost_from_usage(
|
||||
|
|
@ -1875,7 +1535,7 @@ def calculate_image_response_cost_from_usage(
|
|||
completion_tokens_details=completion_tokens_details,
|
||||
)
|
||||
|
||||
prompt_cost, completion_cost = generic_cost_per_token(
|
||||
prompt_cost, completion_cost = cost_per_text(
|
||||
model=model,
|
||||
usage=normalized_usage,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
from typing import Final
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import resolve_image_model_info
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import image_cost, resolve_image_model_info
|
||||
from litellm.types.utils import ImageResponse, ModelInfo
|
||||
|
||||
|
||||
|
|
@ -18,11 +18,10 @@ def cost_calculator(
|
|||
custom_llm_provider=litellm.LlmProviders.AIML.value,
|
||||
model_info=model_info,
|
||||
)
|
||||
output_cost_per_image: Final[float] = _model_info.get("output_cost_per_image") or 0.0
|
||||
num_images: int = 0
|
||||
if isinstance(image_response, ImageResponse):
|
||||
if image_response.data:
|
||||
num_images = len(image_response.data)
|
||||
return output_cost_per_image * num_images
|
||||
return image_cost(_model_info, num_images) or 0.0
|
||||
else:
|
||||
raise ValueError(f"image_response must be of type ImageResponse got type={type(image_response)}")
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ Helper util for handling amazon nova cost calculation
|
|||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.types.utils import Usage
|
||||
|
|
@ -16,4 +16,4 @@ def cost_per_token(model: str, usage: "Usage") -> tuple[float, float]:
|
|||
Calculates the cost per token for a given model, prompt tokens, and completion tokens.
|
||||
Follows the same logic as Anthropic's cost per token calculation.
|
||||
"""
|
||||
return generic_cost_per_token(model=model, usage=usage, custom_llm_provider="amazon_nova")
|
||||
return cost_per_text(model=model, usage=usage, custom_llm_provider="amazon_nova")
|
||||
|
|
|
|||
|
|
@ -8,16 +8,13 @@ from typing import TYPE_CHECKING, Final, Optional
|
|||
from pydantic import ValidationError
|
||||
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import (
|
||||
generic_cost_per_token,
|
||||
get_provider_specific_geo_multiplier,
|
||||
get_provider_specific_speed_multiplier,
|
||||
cost_per_text,
|
||||
get_web_search_requests_from_usage,
|
||||
)
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.types.utils import ModelInfo, Usage
|
||||
import litellm
|
||||
|
||||
|
||||
def cost_per_token(
|
||||
|
|
@ -36,7 +33,7 @@ def cost_per_token(
|
|||
Returns:
|
||||
Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd
|
||||
"""
|
||||
prompt_cost, completion_cost = generic_cost_per_token(
|
||||
return cost_per_text(
|
||||
model=model,
|
||||
usage=usage,
|
||||
custom_llm_provider="anthropic",
|
||||
|
|
@ -44,28 +41,6 @@ def cost_per_token(
|
|||
model_info=model_info,
|
||||
)
|
||||
|
||||
# Apply provider_specific_entry multipliers for geo/speed routing
|
||||
try:
|
||||
effective_info: Final = (
|
||||
model_info
|
||||
if model_info is not None
|
||||
else litellm.get_model_info(model=model, custom_llm_provider="anthropic")
|
||||
)
|
||||
geo_multiplier: Final = get_provider_specific_geo_multiplier(model_info=effective_info, usage=usage)
|
||||
speed_multiplier: Final = get_provider_specific_speed_multiplier(model_info=effective_info, usage=usage)
|
||||
|
||||
if speed_multiplier != 1.0:
|
||||
prompt_cost *= speed_multiplier
|
||||
completion_cost *= speed_multiplier
|
||||
|
||||
if geo_multiplier != 1.0:
|
||||
prompt_cost *= geo_multiplier
|
||||
completion_cost *= geo_multiplier
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return prompt_cost, completion_cost
|
||||
|
||||
|
||||
class _AnthropicServerToolUseProbe(LiteLLMBaseModel):
|
||||
web_search_requests: int | None = None
|
||||
|
|
@ -99,29 +74,15 @@ def get_cost_for_anthropic_web_search(
|
|||
model_info: Optional["ModelInfo"] = None,
|
||||
usage: Optional["Usage"] = None,
|
||||
) -> float:
|
||||
"""
|
||||
Get the cost of using a web search tool for Anthropic.
|
||||
"""
|
||||
from litellm.types.utils import SearchContextCostPerQuery
|
||||
"""``server_tool_use.web_search_requests`` at the row's medium search context price"""
|
||||
from litellm._cost import Usage as TokenUsage
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import search_cost
|
||||
|
||||
## Check if web search requests are in the usage object
|
||||
if model_info is None:
|
||||
return 0.0
|
||||
|
||||
if usage is None:
|
||||
if model_info is None or usage is None:
|
||||
return 0.0
|
||||
web_search_requests: Final = get_web_search_requests_from_usage(usage)
|
||||
if web_search_requests is None:
|
||||
return 0.0
|
||||
|
||||
## Get the cost per web search request
|
||||
search_context_pricing: Final[SearchContextCostPerQuery] = (
|
||||
model_info.get("search_context_cost_per_query") or SearchContextCostPerQuery()
|
||||
return search_cost(
|
||||
model_info, TokenUsage(prompt_tokens=0, text=0, output_text=0, web_search_requests=web_search_requests)
|
||||
)
|
||||
cost_per_web_search_request: Final = search_context_pricing.get("search_context_size_medium", 0.0)
|
||||
if cost_per_web_search_request is None or cost_per_web_search_request == 0.0:
|
||||
return 0.0
|
||||
|
||||
## Calculate the total cost
|
||||
total_cost: Final = cost_per_web_search_request * web_search_requests
|
||||
return total_cost
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ Helper util for handling azure openai-specific cost calculation
|
|||
- e.g.: prompt caching, audio tokens
|
||||
"""
|
||||
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
|
||||
|
|
@ -25,7 +25,7 @@ def cost_per_token(
|
|||
"""
|
||||
## Use generic cost calculator for all other cases
|
||||
## This properly handles: text tokens, audio tokens, cached tokens, reasoning tokens, etc.
|
||||
return generic_cost_per_token(
|
||||
return cost_per_text(
|
||||
model=model,
|
||||
usage=usage,
|
||||
custom_llm_provider="azure",
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ Handles Azure AI Foundry Model Router flat cost and other Azure AI specific pric
|
|||
from typing import Final
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text
|
||||
from litellm.types.utils import Usage
|
||||
from litellm.utils import get_model_info
|
||||
|
||||
|
|
@ -65,9 +65,7 @@ def calculate_azure_model_router_flat_cost(model: str, prompt_tokens: int) -> fl
|
|||
|
||||
def _response_model_cost(model: str, usage: Usage, service_tier: str | None) -> tuple[float, float]:
|
||||
try:
|
||||
return generic_cost_per_token(
|
||||
model=model, usage=usage, custom_llm_provider="azure_ai", service_tier=service_tier
|
||||
)
|
||||
return cost_per_text(model=model, usage=usage, custom_llm_provider="azure_ai", service_tier=service_tier)
|
||||
except Exception as e:
|
||||
if not is_azure_model_router(model):
|
||||
raise
|
||||
|
|
|
|||
|
|
@ -4,22 +4,23 @@ from typing import Final
|
|||
import litellm
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import (
|
||||
calculate_image_response_cost_from_usage,
|
||||
get_cost_per_unit,
|
||||
image_cost,
|
||||
resolve_image_model_info,
|
||||
)
|
||||
from litellm.types.utils import ImageResponse, ModelInfo
|
||||
|
||||
|
||||
def _input_cost_per_pixel(resolved: ModelInfo) -> float:
|
||||
deployment_price: Final = get_cost_per_unit(resolved, "input_cost_per_pixel", default_value=None)
|
||||
if deployment_price is not None:
|
||||
return deployment_price
|
||||
model_cost_key: Final = resolved.get("key")
|
||||
shared_entry: Final = litellm.model_cost.get(model_cost_key) if model_cost_key is not None else None
|
||||
if shared_entry is None:
|
||||
return 0.0
|
||||
shared_price: Final = shared_entry.get("input_cost_per_pixel")
|
||||
return float(shared_price) if isinstance(shared_price, (int, float)) else 0.0
|
||||
def _requested_dimensions(optional_params: Mapping[str, object] | None, size: object) -> tuple[int, int] | None:
|
||||
width: Final = optional_params.get("width") if optional_params else None
|
||||
height: Final = optional_params.get("height") if optional_params else None
|
||||
if type(width) is int and type(height) is int and width > 0 and height > 0:
|
||||
return width, height
|
||||
if not isinstance(size, str):
|
||||
return None
|
||||
parts: Final = size.replace("-x-", "x").split("x")
|
||||
if len(parts) != 2 or not all(part.isdigit() for part in parts):
|
||||
return None
|
||||
return int(parts[0]), int(parts[1])
|
||||
|
||||
|
||||
def cost_calculator(
|
||||
|
|
@ -50,27 +51,10 @@ def cost_calculator(
|
|||
return token_based_cost
|
||||
|
||||
num_images: Final = n if n is not None else len(image_response.data or ())
|
||||
output_cost_per_image: Final[float] = _model_info.get("output_cost_per_image") or 0.0
|
||||
if output_cost_per_image:
|
||||
return output_cost_per_image * num_images
|
||||
|
||||
if _input_cost_per_pixel(_model_info):
|
||||
from litellm.cost_calculator import default_image_cost_calculator
|
||||
|
||||
width: Final = optional_params.get("width") if optional_params else None
|
||||
height: Final = optional_params.get("height") if optional_params else None
|
||||
pixel_size: Final = (
|
||||
f"{width}x{height}"
|
||||
if type(width) is int and type(height) is int and width > 0 and height > 0
|
||||
else size or image_response.size
|
||||
)
|
||||
return default_image_cost_calculator(
|
||||
model=_model_info.get("key", model),
|
||||
custom_llm_provider=litellm.LlmProviders.AZURE_AI.value,
|
||||
size=pixel_size,
|
||||
n=num_images,
|
||||
model_info=model_info,
|
||||
)
|
||||
return 0.0
|
||||
width, height = _requested_dimensions(optional_params, size or image_response.size) or (1024, 1024)
|
||||
key: Final = _model_info.get("key")
|
||||
catalog_row: Final = litellm.model_cost.get(key) if isinstance(key, str) else None
|
||||
row: Final = {**_model_info, **(catalog_row or {}), **(model_info or {})}
|
||||
return image_cost(row, num_images, width=width, height=height) or 0.0
|
||||
|
||||
raise ValueError(f"image_response must be of type ImageResponse got type={type(image_response)}")
|
||||
|
|
|
|||
|
|
@ -3,13 +3,9 @@ Helper util for handling bedrock-specific cost calculation
|
|||
- e.g.: prompt caching
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING, Final
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import (
|
||||
generic_cost_per_token,
|
||||
get_provider_specific_geo_multiplier,
|
||||
)
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.types.utils import Usage
|
||||
|
|
@ -21,13 +17,4 @@ def cost_per_token(model: str, usage: "Usage", service_tier: str | None = None)
|
|||
|
||||
Follows the same logic as Anthropic's cost per token calculation.
|
||||
"""
|
||||
model_info: Final = litellm.get_model_info(model=model, custom_llm_provider="bedrock")
|
||||
prompt_cost, completion_cost = generic_cost_per_token(
|
||||
model=model,
|
||||
usage=usage,
|
||||
custom_llm_provider="bedrock",
|
||||
service_tier=service_tier,
|
||||
model_info=model_info,
|
||||
)
|
||||
geo_multiplier: Final = get_provider_specific_geo_multiplier(model_info=model_info, usage=usage)
|
||||
return prompt_cost * geo_multiplier, completion_cost * geo_multiplier
|
||||
return cost_per_text(model=model, usage=usage, custom_llm_provider="bedrock", service_tier=service_tier)
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ from typing import Any, Final
|
|||
|
||||
from openai.types.image import Image
|
||||
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import image_cost
|
||||
from litellm.llms.bedrock.common_utils import get_cached_model_info
|
||||
from litellm.types.llms.bedrock import (
|
||||
AmazonNovaCanvasColorGuidedGenerationParams,
|
||||
|
|
@ -206,9 +207,7 @@ class AmazonNovaCanvasConfig:
|
|||
model=model,
|
||||
custom_llm_provider="bedrock",
|
||||
)
|
||||
|
||||
output_cost_per_image: Final[float] = model_info.get("output_cost_per_image") or 0.0
|
||||
num_images: int = 0
|
||||
if image_response.data:
|
||||
num_images = len(image_response.data)
|
||||
return output_cost_per_image * num_images
|
||||
return image_cost(model_info, num_images) or 0.0
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ from typing import Final
|
|||
|
||||
from openai.types.image import Image
|
||||
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import image_cost
|
||||
from litellm.llms.bedrock.common_utils import get_cached_model_info
|
||||
from litellm.types.utils import ImageResponse
|
||||
|
||||
|
|
@ -156,9 +157,7 @@ class AmazonStabilityConfig:
|
|||
model=model,
|
||||
custom_llm_provider="bedrock",
|
||||
)
|
||||
|
||||
output_cost_per_image: Final[float] = model_info.get("output_cost_per_image") or 0.0
|
||||
num_images: int = 0
|
||||
if image_response.data:
|
||||
num_images = len(image_response.data)
|
||||
return output_cost_per_image * num_images
|
||||
return image_cost(model_info, num_images) or 0.0
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ from typing import Final
|
|||
|
||||
from openai.types.image import Image
|
||||
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import image_cost
|
||||
from litellm.llms.bedrock.common_utils import BedrockError, get_cached_model_info
|
||||
from litellm.types.llms.bedrock import (
|
||||
AmazonStability3TextToImageRequest,
|
||||
|
|
@ -126,9 +127,7 @@ class AmazonStability3Config:
|
|||
model=model,
|
||||
custom_llm_provider="bedrock",
|
||||
)
|
||||
|
||||
output_cost_per_image: Final[float] = model_info.get("output_cost_per_image") or 0.0
|
||||
num_images: int = 0
|
||||
if image_response.data:
|
||||
num_images = len(image_response.data)
|
||||
return output_cost_per_image * num_images
|
||||
return image_cost(model_info, num_images) or 0.0
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
from litellm.litellm_core_utils.llm_cost_calc.utils import image_cost
|
||||
|
||||
"""
|
||||
Transformation logic for Amazon Titan Image Generation.
|
||||
"""
|
||||
|
|
@ -152,8 +154,7 @@ class AmazonTitanImageGenerationConfig:
|
|||
optional_params: dict | None = None,
|
||||
) -> float:
|
||||
model_info: Final = get_model_info(model=model)
|
||||
output_cost_per_image: Final = model_info.get("output_cost_per_image") or 0.0
|
||||
if not image_response.data:
|
||||
return 0.0
|
||||
num_images: Final = len(image_response.data)
|
||||
return output_cost_per_image * num_images
|
||||
return image_cost(model_info, num_images) or 0.0
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
from typing import Final
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import resolve_image_model_info
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import image_cost, resolve_image_model_info
|
||||
from litellm.types.utils import ImageResponse, ModelInfo
|
||||
|
||||
|
||||
|
|
@ -18,11 +18,10 @@ def cost_calculator(
|
|||
custom_llm_provider=litellm.LlmProviders.COMETAPI.value,
|
||||
model_info=model_info,
|
||||
)
|
||||
output_cost_per_image: Final[float] = _model_info.get("output_cost_per_image") or 0.0
|
||||
num_images: int = 0
|
||||
if isinstance(image_response, ImageResponse):
|
||||
if image_response.data:
|
||||
num_images = len(image_response.data)
|
||||
return output_cost_per_image * num_images
|
||||
return image_cost(_model_info, num_images) or 0.0
|
||||
else:
|
||||
raise ValueError(f"image_response must be of type ImageResponse got type={type(image_response)}")
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ Helper util for handling databricks-specific cost calculation
|
|||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
_LEGACY_ENDPOINT_NAMES: Final = MappingProxyType(
|
||||
|
|
@ -41,7 +41,7 @@ def cost_per_token(model: str, usage: Usage, service_tier: str | None = None) ->
|
|||
Returns:
|
||||
Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd
|
||||
"""
|
||||
return generic_cost_per_token(
|
||||
return cost_per_text(
|
||||
model=_registry_key(model),
|
||||
usage=usage,
|
||||
custom_llm_provider="databricks",
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ Cost calculator for DeepSeek Chat models.
|
|||
Handles prompt caching scenario.
|
||||
"""
|
||||
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
|
||||
|
|
@ -14,4 +14,4 @@ def cost_per_token(model: str, usage: Usage) -> tuple[float, float]:
|
|||
|
||||
Follows the same logic as Anthropic's cost per token calculation.
|
||||
"""
|
||||
return generic_cost_per_token(model=model, usage=usage, custom_llm_provider="deepseek")
|
||||
return cost_per_text(model=model, usage=usage, custom_llm_provider="deepseek")
|
||||
|
|
|
|||
|
|
@ -7,7 +7,8 @@ from typing import Final
|
|||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import deployment_pricing, resolve_image_model_info
|
||||
from litellm._catalog import CATALOG
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import deployment_pricing, image_cost, resolve_image_model_info
|
||||
from litellm.types.utils import ImageObject, ImageResponse, ModelInfo
|
||||
|
||||
FAL_KEYED_PRICING_DEFAULT_QUALITY: Final[str] = "high"
|
||||
|
|
@ -82,38 +83,20 @@ def _parse_keyed_dimensions(size: str | None) -> tuple[int, int] | None:
|
|||
return (width, height) if width > 0 and height > 0 else None
|
||||
|
||||
|
||||
def _keyed_rows(model: str, quality: str) -> tuple[tuple[int, int, float], ...]:
|
||||
prefix: Final = f"fal_ai/{quality}/"
|
||||
suffix: Final = f"/{model}"
|
||||
return tuple(
|
||||
(width, height, float(raw_cost))
|
||||
for key in (*litellm.model_cost,)
|
||||
if isinstance(key, str) and key.startswith(prefix) and key.endswith(suffix)
|
||||
for size in (key[len(prefix) : -len(suffix)],)
|
||||
for dimensions in (_parse_keyed_dimensions(size),)
|
||||
if dimensions is not None
|
||||
for entry in (_entry(key),)
|
||||
if entry is not None
|
||||
for raw_cost in (entry.get("output_cost_per_image"),)
|
||||
if isinstance(raw_cost, (int, float))
|
||||
for width, height in (dimensions,)
|
||||
)
|
||||
|
||||
|
||||
def _keyed_cost_per_image(
|
||||
model: str,
|
||||
image: object,
|
||||
optional_params: Mapping[str, object],
|
||||
) -> float | None:
|
||||
quality: Final = _keyed_quality(optional_params)
|
||||
rows: Final = _keyed_rows(model, quality)
|
||||
if not rows:
|
||||
return None
|
||||
target_dimensions: Final = (
|
||||
"""The per-image price of the size-keyed row nearest the image's size"""
|
||||
target: Final = (
|
||||
_image_dimensions(image) or _parse_keyed_dimensions(_keyed_size(optional_params)) or _DEFAULT_KEYED_DIMENSIONS
|
||||
)
|
||||
target_pixels: Final = target_dimensions[0] * target_dimensions[1]
|
||||
return min(rows, key=lambda row: (abs(row[0] * row[1] - target_pixels), row[0] * row[1]))[2]
|
||||
key: Final = CATALOG.nearest_image_key(
|
||||
model, litellm.LlmProviders.FAL_AI.value, quality=_keyed_quality(optional_params), size=target
|
||||
)
|
||||
entry: Final = _entry(key) if key is not None else None
|
||||
return image_cost(entry, 1) if entry is not None else None
|
||||
|
||||
|
||||
def _flat_cost_per_image(
|
||||
|
|
|
|||
|
|
@ -1,42 +0,0 @@
|
|||
from typing import (
|
||||
Final,
|
||||
cast, # noqa: TID251 # the derived entry is a dict copy of a ReadOnly TypedDict; no cast-free way to retype it
|
||||
)
|
||||
|
||||
from litellm.constants import FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO
|
||||
from litellm.types.utils import ModelInfo
|
||||
|
||||
|
||||
def _as_rate(value: object) -> float | None:
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float, str)):
|
||||
return None
|
||||
try:
|
||||
return float(value)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def with_default_cache_read_rate(model_info: ModelInfo) -> ModelInfo:
|
||||
input_rate: Final = _as_rate(model_info.get("input_cost_per_token"))
|
||||
if model_info.get("cache_read_input_token_cost") is not None or input_rate is None:
|
||||
return model_info
|
||||
cache_read_rate: Final = input_rate * FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO
|
||||
off_peak: Final = model_info.get("off_peak_pricing")
|
||||
if off_peak is None or "cache_read_input_token_cost" in off_peak:
|
||||
return cast(ModelInfo, {**model_info, "cache_read_input_token_cost": cache_read_rate})
|
||||
off_peak_input_rate: Final = _as_rate(off_peak.get("input_cost_per_token"))
|
||||
return cast(
|
||||
ModelInfo,
|
||||
{
|
||||
**model_info,
|
||||
"cache_read_input_token_cost": cache_read_rate,
|
||||
"off_peak_pricing": {
|
||||
**off_peak,
|
||||
"cache_read_input_token_cost": (
|
||||
off_peak_input_rate * FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO
|
||||
if off_peak_input_rate is not None
|
||||
else cache_read_rate
|
||||
),
|
||||
},
|
||||
},
|
||||
)
|
||||
|
|
@ -11,7 +11,7 @@ from litellm.constants import (
|
|||
FIREWORKS_AI_56_B_MOE,
|
||||
FIREWORKS_AI_176_B_MOE,
|
||||
)
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text
|
||||
from litellm.types.utils import ModelInfo, Usage
|
||||
from litellm.utils import get_model_info
|
||||
|
||||
|
|
@ -85,7 +85,7 @@ def cost_per_token(model: str, usage: Usage, current_time: datetime | None = Non
|
|||
Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd
|
||||
"""
|
||||
model_info: Final = _resolve_model_info(model)
|
||||
return generic_cost_per_token(
|
||||
return cost_per_text(
|
||||
model=model,
|
||||
usage=usage,
|
||||
custom_llm_provider=model_info["litellm_provider"],
|
||||
|
|
|
|||
|
|
@ -16,9 +16,9 @@ def cost_per_token(model: str, usage: "Usage", service_tier: str | None = None)
|
|||
|
||||
Follows the same logic as Anthropic's cost per token calculation.
|
||||
"""
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text
|
||||
|
||||
return generic_cost_per_token(
|
||||
return cost_per_text(
|
||||
model=model,
|
||||
usage=usage,
|
||||
custom_llm_provider="gemini",
|
||||
|
|
@ -26,52 +26,36 @@ def cost_per_token(model: str, usage: "Usage", service_tier: str | None = None)
|
|||
)
|
||||
|
||||
|
||||
def cost_per_web_search_request(usage: "Usage", model_info: "ModelInfo") -> float:
|
||||
"""
|
||||
Calculates the cost of web search (grounding with Google Search).
|
||||
|
||||
Billing mode is determined by ``web_search_billing_unit`` in model_info:
|
||||
- ``"per_query"``: charged per individual search query (Gemini 3.x).
|
||||
- ``"per_prompt"`` (default): charged per grounded prompt (Gemini 2.x),
|
||||
regardless of how many queries were executed internally.
|
||||
|
||||
Reads the per-request cost from ``search_context_cost_per_query`` in
|
||||
``model_info`` when available, falling back to $0.035 for models not
|
||||
yet updated in the pricing JSON.
|
||||
"""
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import (
|
||||
get_web_search_requests_from_usage,
|
||||
)
|
||||
def web_search_requests(usage: "Usage | None") -> int:
|
||||
"""Grounded searches, Gemini's ``prompt_tokens_details.web_search_requests`` else ``server_tool_use``"""
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import get_web_search_requests_from_usage
|
||||
from litellm.types.utils import PromptTokensDetailsWrapper
|
||||
|
||||
_DEFAULT_COST: Final = 35e-3
|
||||
search_costs: Final = model_info.get("search_context_cost_per_query") or {}
|
||||
_cost: Final = search_costs.get("search_context_size_medium", _DEFAULT_COST)
|
||||
|
||||
requests_from_prompt_details: Final = (
|
||||
usage.prompt_tokens_details.web_search_requests
|
||||
if (
|
||||
usage is not None
|
||||
and usage.prompt_tokens_details is not None
|
||||
and isinstance(usage.prompt_tokens_details, PromptTokensDetailsWrapper)
|
||||
and hasattr(usage.prompt_tokens_details, "web_search_requests")
|
||||
and usage.prompt_tokens_details.web_search_requests is not None
|
||||
)
|
||||
details: Final = usage.prompt_tokens_details if usage is not None else None
|
||||
from_details: Final = (
|
||||
details.web_search_requests
|
||||
if isinstance(details, PromptTokensDetailsWrapper) and getattr(details, "web_search_requests", None)
|
||||
else None
|
||||
)
|
||||
requests_from_server_tool_use: Final = get_web_search_requests_from_usage(usage)
|
||||
number_of_web_search_requests: Final = requests_from_prompt_details or requests_from_server_tool_use or 0
|
||||
return from_details or (get_web_search_requests_from_usage(usage) if usage is not None else None) or 0
|
||||
|
||||
billing_mode: Final = model_info.get("web_search_billing_unit") or "per_prompt"
|
||||
billable_requests: Final = (
|
||||
1 if (number_of_web_search_requests > 0 and billing_mode == "per_prompt") else number_of_web_search_requests
|
||||
|
||||
def cost_per_web_search_request(usage: "Usage", model_info: "ModelInfo") -> float:
|
||||
"""Grounding with Google Search, per query or once per prompt as the row's ``web_search_billing_unit`` says"""
|
||||
from litellm._cost import Usage as TokenUsage
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import search_cost
|
||||
|
||||
return search_cost(
|
||||
_as_google_row(model_info),
|
||||
TokenUsage(prompt_tokens=0, text=0, output_text=0, web_search_requests=web_search_requests(usage)),
|
||||
)
|
||||
|
||||
return _cost * billable_requests
|
||||
|
||||
|
||||
GOOGLE_MAPS_GROUNDING_DEFAULT_COST_PER_QUERY: Final = 14e-3
|
||||
GOOGLE_MAPS_GROUNDING_DEFAULT_COST_PER_PROMPT: Final = 25e-3
|
||||
def _as_google_row(model_info: "ModelInfo") -> dict[str, object]:
|
||||
"""The row with a Google provider, so its missing search and Maps prices take Gemini's defaults"""
|
||||
provider: Final = model_info.get("litellm_provider")
|
||||
google: Final = isinstance(provider, str) and (provider == "gemini" or provider.startswith("vertex_ai"))
|
||||
return {**model_info, "litellm_provider": provider if google else "gemini"}
|
||||
|
||||
|
||||
def google_maps_grounding_requests(usage: "Usage | None") -> int | None:
|
||||
|
|
@ -84,26 +68,16 @@ def google_maps_grounding_requests(usage: "Usage | None") -> int | None:
|
|||
|
||||
|
||||
def cost_per_google_maps_grounding_request(usage: "Usage", model_info: "ModelInfo") -> float:
|
||||
"""
|
||||
Calculates the cost of Grounding with Google Maps.
|
||||
"""Grounding with Google Maps, under the same billing unit as Google Search grounding"""
|
||||
from litellm._cost import Usage as TokenUsage
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import search_cost
|
||||
|
||||
Billing follows ``web_search_billing_unit`` in model_info the same way Google Search grounding
|
||||
does: ``"per_query"`` (Gemini 3.x) multiplies the executed Maps queries, ``"per_prompt"``
|
||||
(default, Gemini 2.x) charges one flat fee per grounded prompt.
|
||||
|
||||
The rate comes from ``google_maps_grounding_cost_per_query`` in ``model_info``, falling back
|
||||
to Google's list price for that billing unit when the pricing JSON has no entry yet.
|
||||
"""
|
||||
requests: Final = google_maps_grounding_requests(usage)
|
||||
if not requests or requests <= 0:
|
||||
return 0.0
|
||||
billing_mode: Final = model_info.get("web_search_billing_unit") or "per_prompt"
|
||||
default_cost: Final = (
|
||||
GOOGLE_MAPS_GROUNDING_DEFAULT_COST_PER_QUERY
|
||||
if billing_mode == "per_query"
|
||||
else GOOGLE_MAPS_GROUNDING_DEFAULT_COST_PER_PROMPT
|
||||
return search_cost(
|
||||
_as_google_row(model_info),
|
||||
TokenUsage(
|
||||
prompt_tokens=0,
|
||||
text=0,
|
||||
output_text=0,
|
||||
maps_grounding_requests=max(google_maps_grounding_requests(usage) or 0, 0),
|
||||
),
|
||||
)
|
||||
configured_cost: Final = model_info.get("google_maps_grounding_cost_per_query")
|
||||
cost: Final = default_cost if configured_cost is None else configured_cost
|
||||
billed_requests: Final = requests if billing_mode == "per_query" else 1
|
||||
return cost * billed_requests
|
||||
|
|
|
|||
|
|
@ -16,15 +16,22 @@ if TYPE_CHECKING:
|
|||
|
||||
|
||||
def cost_per_web_search_request(usage: Usage, model_info: "ModelInfo") -> float:
|
||||
search_costs: Final = model_info.get("search_context_cost_per_query")
|
||||
cost_per_search: Final = search_costs.get("search_context_size_medium", 0.0) if search_costs else 0.0
|
||||
from litellm._cost import Usage as TokenUsage
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import search_cost
|
||||
|
||||
try:
|
||||
server_tool_use: Final = usage.server_tool_use
|
||||
except AttributeError:
|
||||
return 0.0
|
||||
if server_tool_use is None:
|
||||
return 0.0
|
||||
searches: Final = server_tool_use.web_search_requests or 0
|
||||
opens: Final = server_tool_use.browser_open_requests or 0
|
||||
cost_per_open: Final = model_info.get("browser_open_cost_per_request") or 0.0
|
||||
return searches * cost_per_search + opens * cost_per_open
|
||||
return search_cost(
|
||||
model_info,
|
||||
TokenUsage(
|
||||
prompt_tokens=0,
|
||||
text=0,
|
||||
output_text=0,
|
||||
web_search_requests=server_tool_use.web_search_requests or 0,
|
||||
browser_opens=server_tool_use.browser_open_requests or 0,
|
||||
),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ Helper util for handling openai-specific cost calculation
|
|||
|
||||
from typing import Literal
|
||||
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text
|
||||
from litellm.types.utils import CallTypes, Usage
|
||||
|
||||
|
||||
|
|
@ -35,7 +35,7 @@ def cost_per_token(
|
|||
Returns:
|
||||
Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd
|
||||
"""
|
||||
return generic_cost_per_token(
|
||||
return cost_per_text(
|
||||
model=model,
|
||||
usage=usage,
|
||||
custom_llm_provider="openai",
|
||||
|
|
|
|||
|
|
@ -9,8 +9,8 @@ from typing import Final
|
|||
from litellm import verbose_logger
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import (
|
||||
calculate_image_response_cost_from_usage,
|
||||
cost_per_text,
|
||||
flat_image_cost,
|
||||
generic_cost_per_token,
|
||||
resolve_image_model_info,
|
||||
)
|
||||
from litellm.types.utils import ImageResponse, ModelInfo, Usage
|
||||
|
|
@ -35,9 +35,9 @@ def cost_calculator(
|
|||
verbose_logger.debug("No usage data available for %s, cannot calculate token-based cost", model)
|
||||
return flat_image_cost(price_table, image_response)
|
||||
|
||||
# A chat Usage with an explicit output breakdown: cost via generic_cost_per_token.
|
||||
# A chat Usage with an explicit output breakdown: cost via cost_per_text.
|
||||
if isinstance(usage, Usage) and usage.completion_tokens_details is not None:
|
||||
prompt_cost, completion_cost = generic_cost_per_token(
|
||||
prompt_cost, completion_cost = cost_per_text(
|
||||
model=model, usage=usage, custom_llm_provider=provider, model_info=price_table
|
||||
)
|
||||
return prompt_cost + completion_cost
|
||||
|
|
@ -54,9 +54,9 @@ def cost_calculator(
|
|||
return token_based_cost
|
||||
|
||||
# Fallback: a Usage with no output breakdown that the image helper can't read —
|
||||
# cost via generic_cost_per_token (text rate) instead of returning 0.0.
|
||||
# cost via cost_per_text (text rate) instead of returning 0.0.
|
||||
if isinstance(usage, Usage):
|
||||
prompt_cost, completion_cost = generic_cost_per_token(
|
||||
prompt_cost, completion_cost = cost_per_text(
|
||||
model=model, usage=usage, custom_llm_provider=provider, model_info=price_table
|
||||
)
|
||||
return prompt_cost + completion_cost
|
||||
|
|
|
|||
|
|
@ -4,7 +4,9 @@ from typing import Final
|
|||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from litellm.utils import get_model_info
|
||||
from litellm._cost import Usage as TokenUsage
|
||||
from litellm._cost import as_main_cost
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import price_usage, pricing_row
|
||||
|
||||
PARALLEL_AI_DEFAULT_RESULTS: Final = 10
|
||||
PARALLEL_AI_USAGE_PARAM: Final = "_parallel_ai_usage"
|
||||
|
|
@ -60,16 +62,6 @@ def _effective_max_results(optional_params: Mapping[str, object]) -> int:
|
|||
return max_results if max_results is not None else PARALLEL_AI_DEFAULT_RESULTS
|
||||
|
||||
|
||||
def _request_cost(mode: str) -> tuple[float, float]:
|
||||
"""The per-request price and the price of each result past the included ones"""
|
||||
pricing_model: Final = PARALLEL_AI_PRICING_MODEL_BY_MODE.get(mode, PARALLEL_AI_STANDARD_SEARCH_MODEL)
|
||||
model_info: Final = get_model_info(model=pricing_model, custom_llm_provider="parallel_ai")
|
||||
return (
|
||||
float(model_info.get("input_cost_per_query") or 0.0),
|
||||
float(model_info.get("cost_per_additional_result") or 0.0),
|
||||
)
|
||||
|
||||
|
||||
def _additional_results(
|
||||
optional_params: Mapping[str, object],
|
||||
usage: Sequence[Mapping[str, object]] | None,
|
||||
|
|
@ -86,8 +78,19 @@ def parallel_ai_search_cost(
|
|||
optional_params: Mapping[str, object],
|
||||
usage: Sequence[Mapping[str, object]] | None,
|
||||
) -> float:
|
||||
request_cost, additional_result_cost = _request_cost(_effective_mode(optional_params))
|
||||
mode: Final = _effective_mode(optional_params)
|
||||
pricing_model: Final = PARALLEL_AI_PRICING_MODEL_BY_MODE.get(mode, PARALLEL_AI_STANDARD_SEARCH_MODEL)
|
||||
request_count_from_usage: Final = _usage_count(usage, "sku_search") if usage is not None else None
|
||||
request_count: Final = request_count_from_usage if request_count_from_usage is not None else 1
|
||||
additional_results: Final = _additional_results(optional_params, usage)
|
||||
return request_count * request_cost + additional_results * additional_result_cost
|
||||
prompt_cost, completion_cost = as_main_cost(
|
||||
price_usage(
|
||||
pricing_row(pricing_model, "parallel_ai", None),
|
||||
TokenUsage(
|
||||
prompt_tokens=0,
|
||||
text=0,
|
||||
output_text=0,
|
||||
search_queries=request_count_from_usage if request_count_from_usage is not None else 1,
|
||||
additional_results=_additional_results(optional_params, usage),
|
||||
),
|
||||
)
|
||||
)
|
||||
return prompt_cost + completion_cost
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
from typing import Final
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import resolve_image_model_info
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import image_cost, resolve_image_model_info
|
||||
from litellm.types.utils import ImageResponse, ModelInfo
|
||||
|
||||
|
||||
|
|
@ -18,11 +18,10 @@ def cost_calculator(
|
|||
custom_llm_provider=litellm.LlmProviders.RECRAFT.value,
|
||||
model_info=model_info,
|
||||
)
|
||||
output_cost_per_image: Final[float] = _model_info.get("output_cost_per_image") or 0.0
|
||||
num_images: int = 0
|
||||
if isinstance(image_response, ImageResponse):
|
||||
if image_response.data:
|
||||
num_images = len(image_response.data)
|
||||
return output_cost_per_image * num_images
|
||||
return image_cost(_model_info, num_images) or 0.0
|
||||
else:
|
||||
raise ValueError(f"image_response must be of type ImageResponse got type={type(image_response)}")
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
from typing import Final
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import resolve_image_model_info
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import image_cost, resolve_image_model_info
|
||||
from litellm.types.utils import ImageResponse, ModelInfo
|
||||
|
||||
|
||||
|
|
@ -21,11 +21,10 @@ def cost_calculator(
|
|||
custom_llm_provider=litellm.LlmProviders.RUNWAYML.value,
|
||||
model_info=model_info,
|
||||
)
|
||||
output_cost_per_image: Final[float] = _model_info.get("output_cost_per_image") or 0.0
|
||||
num_images: int = 0
|
||||
if isinstance(image_response, ImageResponse):
|
||||
if image_response.data:
|
||||
num_images = len(image_response.data)
|
||||
return output_cost_per_image * num_images
|
||||
return image_cost(_model_info, num_images) or 0.0
|
||||
else:
|
||||
raise ValueError(f"image_response must be of type ImageResponse, got type={type(image_response)}")
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
|
||||
def cost_per_token(model: str, usage: Usage) -> tuple[float, float]:
|
||||
return generic_cost_per_token(model=model, usage=usage, custom_llm_provider="tencent")
|
||||
return cost_per_text(model=model, usage=usage, custom_llm_provider="tencent")
|
||||
|
|
|
|||
|
|
@ -80,18 +80,30 @@ def _vertex_ai_lyria_model_key(model: str) -> str:
|
|||
return model if model.startswith("vertex_ai/") else f"vertex_ai/{model}"
|
||||
|
||||
|
||||
def _vertex_ai_lyria_generation_cost(model_info: VertexAILyriaModelInfo | None) -> float | None:
|
||||
return None if model_info is None else model_info.get("output_cost_per_image")
|
||||
|
||||
|
||||
def get_vertex_ai_lyria_model_info(model: str) -> VertexAILyriaModelInfo | None:
|
||||
model_key: Final = _vertex_ai_lyria_model_key(model)
|
||||
return _validate_vertex_ai_lyria_model_info(litellm.model_cost.get(model_key))
|
||||
|
||||
|
||||
def get_vertex_ai_lyria_generation_cost(model: str) -> float | None:
|
||||
def get_vertex_ai_lyria_generation_cost(model: str, generations: int = 1) -> float | None:
|
||||
"""``generations`` Lyria generations at the row's per-generation price, None when the model is not a priced Lyria
|
||||
row"""
|
||||
from litellm._cost import ImageOut, Priced
|
||||
from litellm._cost import Usage as TokenUsage
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import price_usage
|
||||
|
||||
model_key: Final = _vertex_ai_lyria_model_key(model)
|
||||
return _vertex_ai_lyria_generation_cost(_validate_vertex_ai_lyria_model_info(litellm.model_cost.get(model_key)))
|
||||
row: Final = litellm.model_cost.get(model_key)
|
||||
if row is None or _validate_vertex_ai_lyria_model_info(row) is None:
|
||||
return None
|
||||
priced: Final = price_usage(
|
||||
row,
|
||||
TokenUsage(prompt_tokens=0, text=0, output_text=0, images_out=[ImageOut()] * max(generations, 0)),
|
||||
key=model_key,
|
||||
)
|
||||
if not isinstance(priced, Priced) or (generations > 0 and not priced.lines):
|
||||
return None
|
||||
return priced.total
|
||||
|
||||
|
||||
class VertexAIError(BaseLLMException):
|
||||
|
|
|
|||
|
|
@ -1,14 +1,8 @@
|
|||
# What is this?
|
||||
## Cost calculation for Google AI Studio / Vertex AI models
|
||||
from typing import Final
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import (
|
||||
generic_cost_per_token,
|
||||
get_vertex_regional_endpoint_uplift,
|
||||
is_above_128k,
|
||||
)
|
||||
from litellm.types.utils import ModelInfo, Usage
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
"""
|
||||
Gemini pricing covers:
|
||||
|
|
@ -19,32 +13,6 @@ Gemini pricing covers:
|
|||
"""
|
||||
|
||||
|
||||
def _handle_128k_pricing(
|
||||
model_info: ModelInfo,
|
||||
usage: Usage,
|
||||
) -> tuple[float, float]:
|
||||
## CALCULATE INPUT COST
|
||||
input_cost_per_token_above_128k_tokens: Final = model_info.get("input_cost_per_token_above_128k_tokens")
|
||||
output_cost_per_token_above_128k_tokens = model_info.get("output_cost_per_token_above_128k_tokens")
|
||||
|
||||
prompt_tokens: Final = usage.prompt_tokens
|
||||
completion_tokens: Final = usage.completion_tokens
|
||||
|
||||
if is_above_128k(tokens=prompt_tokens) and input_cost_per_token_above_128k_tokens is not None:
|
||||
prompt_cost = prompt_tokens * input_cost_per_token_above_128k_tokens
|
||||
else:
|
||||
prompt_cost = prompt_tokens * (model_info["input_cost_per_token"] or 0.0)
|
||||
|
||||
## CALCULATE OUTPUT COST
|
||||
output_cost_per_token_above_128k_tokens = model_info.get("output_cost_per_token_above_128k_tokens")
|
||||
if is_above_128k(tokens=completion_tokens) and output_cost_per_token_above_128k_tokens is not None:
|
||||
completion_cost = completion_tokens * output_cost_per_token_above_128k_tokens
|
||||
else:
|
||||
completion_cost = completion_tokens * (model_info["output_cost_per_token"] or 0.0)
|
||||
|
||||
return prompt_cost, completion_cost
|
||||
|
||||
|
||||
def cost_per_token(
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
|
|
@ -67,26 +35,9 @@ def cost_per_token(
|
|||
|
||||
Returns:
|
||||
Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd
|
||||
|
||||
Raises:
|
||||
Exception if model requires >128k pricing, but model cost not mapped
|
||||
"""
|
||||
|
||||
## GET MODEL INFO
|
||||
model_info: Final = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)
|
||||
|
||||
## HANDLE 128k+ PRICING
|
||||
input_cost_per_token_above_128k_tokens: Final = model_info.get("input_cost_per_token_above_128k_tokens")
|
||||
output_cost_per_token_above_128k_tokens: Final = model_info.get("output_cost_per_token_above_128k_tokens")
|
||||
if input_cost_per_token_above_128k_tokens is not None or output_cost_per_token_above_128k_tokens is not None:
|
||||
prompt_cost_128k, completion_cost_128k = _handle_128k_pricing(
|
||||
model_info=model_info,
|
||||
usage=usage,
|
||||
)
|
||||
vertex_uplift: Final = get_vertex_regional_endpoint_uplift(model_info, vertex_location)
|
||||
return prompt_cost_128k * vertex_uplift, completion_cost_128k * vertex_uplift
|
||||
|
||||
return generic_cost_per_token(
|
||||
return cost_per_text(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
usage=usage,
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ Vertex AI Image Edit Cost Calculator
|
|||
from typing import Final
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import image_cost
|
||||
from litellm.types.utils import ImageResponse
|
||||
|
||||
|
||||
|
|
@ -23,10 +24,8 @@ def cost_calculator(
|
|||
custom_llm_provider="vertex_ai",
|
||||
)
|
||||
|
||||
output_cost_per_image: Final[float] = model_info.get("output_cost_per_image") or 0.0
|
||||
|
||||
if not isinstance(image_response, ImageResponse):
|
||||
raise ValueError(f"image_response must be of type ImageResponse got type={type(image_response)}")
|
||||
|
||||
num_images: Final = len(image_response.data or [])
|
||||
return output_cost_per_image * num_images
|
||||
return image_cost(model_info, num_images) or 0.0
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import math
|
|||
from collections.abc import Mapping
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text
|
||||
from litellm.types.utils import PromptTokensDetailsWrapper, Usage
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -53,7 +53,7 @@ def _split_reported_cost(model: str, usage: Usage, reported_cost: float) -> tupl
|
|||
"""xAI reports one total, so attribute it to input and output in the ratio our own token math gives."""
|
||||
try:
|
||||
priced_input, priced_output = _token_math_cost_per_token(model, usage)
|
||||
except Exception: # noqa: BLE001 # generic_cost_per_token raises bare Exception for unmapped models
|
||||
except Exception: # noqa: BLE001 # cost_per_text raises bare Exception for unmapped models
|
||||
return 0.0, reported_cost
|
||||
priced_total: Final = priced_input + priced_output
|
||||
if priced_total <= 0:
|
||||
|
|
@ -105,7 +105,7 @@ def _token_math_cost_per_token(model: str, usage: Usage) -> tuple[float, float]:
|
|||
completion_tokens_details=None,
|
||||
)
|
||||
|
||||
prompt_cost, completion_cost = generic_cost_per_token(model=model, usage=modified_usage, custom_llm_provider="xai")
|
||||
prompt_cost, completion_cost = cost_per_text(model=model, usage=modified_usage, custom_llm_provider="xai")
|
||||
|
||||
return prompt_cost, completion_cost
|
||||
|
||||
|
|
|
|||
|
|
@ -442,7 +442,7 @@ class VertexPassthroughLoggingHandler:
|
|||
prediction_count: Final = VertexPassthroughLoggingHandler._get_audio_prediction_count(
|
||||
json_response=json_response
|
||||
)
|
||||
response_cost: Final = (get_vertex_ai_lyria_generation_cost(model=model) or 0.0) * prediction_count
|
||||
response_cost: Final = get_vertex_ai_lyria_generation_cost(model=model, generations=prediction_count) or 0.0
|
||||
|
||||
logging_obj.model = model # rebind-ok: passthrough attribution records the resolved Vertex model
|
||||
logging_obj.model_call_details[ # rebind-ok: passthrough attribution enriches callback metadata
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY
|
|||
from litellm.litellm_core_utils.get_llm_provider_logic import resolve_model_and_provider_for_metadata
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import (
|
||||
calculate_prompt_caching_savings,
|
||||
generic_cost_per_token,
|
||||
cost_per_text,
|
||||
get_cost_per_unit,
|
||||
)
|
||||
from litellm.types.integrations.anthropic_cache_control_hook import (
|
||||
|
|
@ -195,7 +195,7 @@ def _pricing_basis(cost_breakdown: Mapping[str, object] | None) -> PricingBasis:
|
|||
they price at standard rates, which is what they already did.
|
||||
|
||||
These values survive a JSON round trip on the way here, so none is guaranteed to be
|
||||
a string. `generic_cost_per_token` calls `.lower()` on them without a type check, and
|
||||
a string. `cost_per_text` calls `.lower()` on them without a type check, and
|
||||
the resulting `AttributeError` would be swallowed into a silent zero by the caller's
|
||||
`except`, so anything that is not a string is dropped here instead.
|
||||
"""
|
||||
|
|
@ -215,7 +215,7 @@ def _recorded_token_cost(cost_breakdown: Mapping[str, object] | None) -> float |
|
|||
"""What the biller charged for this request's tokens, or ``None`` when unrecorded.
|
||||
|
||||
``input_cost`` already carries the cache buckets, so it and ``output_cost`` sum to
|
||||
exactly what `generic_cost_per_token` returns for the same request; the separate
|
||||
exactly what `cost_per_text` returns for the same request; the separate
|
||||
``cache_read_cost`` and ``cache_creation_cost`` entries decompose that sum rather than
|
||||
adding to it, and including them would charge those tokens twice.
|
||||
|
||||
|
|
@ -251,7 +251,7 @@ def _cost_of_usage(
|
|||
model_info=model_info,
|
||||
)
|
||||
else:
|
||||
prompt_cost, completion_cost = generic_cost_per_token(
|
||||
prompt_cost, completion_cost = cost_per_text(
|
||||
model=model.model,
|
||||
usage=usage,
|
||||
custom_llm_provider=model.provider,
|
||||
|
|
|
|||
|
|
@ -161,7 +161,7 @@ from litellm.litellm_core_utils.core_helpers import (
|
|||
from litellm.litellm_core_utils.duration_parser import get_budget_window_start
|
||||
from litellm.litellm_core_utils.env_utils import get_env_float, get_env_int
|
||||
from litellm.litellm_core_utils.litellm_logging import BilledRoundsDispatch, Logging, deployment_pricing_model_info
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
|
||||
from litellm.litellm_core_utils.served_output_texts import (
|
||||
|
|
@ -1289,12 +1289,12 @@ def _deployment_priced_input_cost(
|
|||
model: str, custom_llm_provider: str | None, prompt_tokens: int, deployment_pricing: ModelInfo
|
||||
) -> float:
|
||||
"""Input price for the estimated prompt at the deployment's registered rates, read through
|
||||
``deployment_pricing_model_info`` and priced by ``generic_cost_per_token`` the way the success
|
||||
``deployment_pricing_model_info`` and priced by ``cost_per_text`` the way the success
|
||||
path prices them, so a long-context tier the deployment declares (``input_cost_per_token_above_200k_tokens``)
|
||||
is honoured instead of flattened to the standard rate. An entry it cannot price keeps the row at $0."""
|
||||
usage: Final = Usage(prompt_tokens=prompt_tokens, completion_tokens=0, total_tokens=prompt_tokens)
|
||||
try:
|
||||
prompt_cost, _ = generic_cost_per_token(
|
||||
prompt_cost, _ = cost_per_text(
|
||||
model=model,
|
||||
usage=usage,
|
||||
custom_llm_provider=custom_llm_provider or str(deployment_pricing.get("litellm_provider") or ""),
|
||||
|
|
|
|||
|
|
@ -110,14 +110,14 @@ def _priced(router: "Router", candidate: Baseline) -> tuple[float, Baseline] | N
|
|||
dimension to that engine. A candidate that prices to nothing there cannot stand in
|
||||
for what the traffic would have cost.
|
||||
"""
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text
|
||||
|
||||
provider, _, model_name = candidate.model.partition("/")
|
||||
try:
|
||||
info: Final = router.cached_deployment_model_info(candidate.deployment_id or "", candidate.model)
|
||||
if info is None:
|
||||
return None
|
||||
prompt_cost, completion_cost = generic_cost_per_token(
|
||||
prompt_cost, completion_cost = cost_per_text(
|
||||
model=model_name or candidate.model,
|
||||
usage=_REFERENCE_REQUEST,
|
||||
custom_llm_provider=provider,
|
||||
|
|
|
|||
|
|
@ -156,8 +156,23 @@ class Usage:
|
|||
output_video_duration: timedelta | None = None,
|
||||
video_resolution: str | None = None,
|
||||
text_messages: int = 0,
|
||||
images_in: int = 0,
|
||||
images_out: Sequence[ImageOut] = (),
|
||||
pixels_in: int = 0,
|
||||
search_queries: int = 0,
|
||||
max_results: int | None = None,
|
||||
web_search_requests: int = 0,
|
||||
web_search_context: Literal["low", "medium", "high"] = "medium",
|
||||
maps_grounding_requests: int = 0,
|
||||
citation_tokens: int = 0,
|
||||
browser_opens: int = 0,
|
||||
additional_results: int = 0,
|
||||
) -> None: ...
|
||||
|
||||
@final
|
||||
class ImageOut:
|
||||
def __init__(self, *, size: str | None = None, width: int | None = None, height: int | None = None) -> None: ...
|
||||
|
||||
|
||||
@final
|
||||
class CallContext:
|
||||
|
|
@ -170,6 +185,8 @@ class CallContext:
|
|||
vertex_location: str | None,
|
||||
at: datetime,
|
||||
custom_pricing: Mapping[str, object] | None,
|
||||
speed: str | None = None,
|
||||
inference_geo: str | None = None,
|
||||
) -> None: ...
|
||||
|
||||
@final
|
||||
|
|
@ -208,6 +225,15 @@ class Catalog:
|
|||
def bound_to(module: ModuleType, attr: str) -> Catalog: ...
|
||||
def bundled(self) -> Fetched: ...
|
||||
def fetch(self, source: Remote | File | Bundled, *, timeout: float, attempts: int) -> Fetched | Unavailable: ...
|
||||
def resolve_image(
|
||||
self,
|
||||
model: str,
|
||||
provider: str | None = None,
|
||||
*,
|
||||
size: tuple[int, int] | None = None,
|
||||
quality: str | None = None,
|
||||
) -> Resolved | Missing | Invalid: ...
|
||||
def nearest_image_key(self, model: str, provider: str, *, quality: str, size: tuple[int, int]) -> str | None: ...
|
||||
def reload(self, source: Remote | File | Bundled) -> Installed | Unavailable: ...
|
||||
def provenance(self) -> Provenance | None: ...
|
||||
def resolve(self, model: str, provider: str | None = None) -> Resolved | Missing | Invalid: ...
|
||||
|
|
|
|||
|
|
@ -9,8 +9,10 @@ from typing import Final
|
|||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm._cost import Usage as TokenUsage
|
||||
from litellm._cost import as_main_cost
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
from litellm.utils import get_model_info
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import price_usage, pricing_row
|
||||
|
||||
PROVIDER_USAGE_ADAPTER: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] = TypeAdapter(
|
||||
tuple[Mapping[str, object], ...]
|
||||
|
|
@ -93,28 +95,16 @@ def search_provider_cost_per_query(
|
|||
)
|
||||
return (input_cost, 0.0)
|
||||
|
||||
model_info: Final = get_model_info(
|
||||
model=_pricing_model(model, custom_llm_provider, optional_params),
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
max_results: Final = (optional_params or {}).get("max_results")
|
||||
return as_main_cost(
|
||||
price_usage(
|
||||
pricing_row(_pricing_model(model, custom_llm_provider, optional_params), custom_llm_provider, None),
|
||||
TokenUsage(
|
||||
prompt_tokens=0,
|
||||
text=0,
|
||||
output_text=0,
|
||||
search_queries=number_of_queries,
|
||||
max_results=max_results if isinstance(max_results, int) and not isinstance(max_results, bool) else None,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
# Check for tiered pricing (e.g., Exa AI based on max_results)
|
||||
tiered_pricing: Final = model_info.get("tiered_pricing")
|
||||
if tiered_pricing and isinstance(tiered_pricing, list):
|
||||
max_results: Final = (optional_params or {}).get("max_results", 10) # default 10 results
|
||||
cost_per_query = 0.0
|
||||
|
||||
for tier in tiered_pricing:
|
||||
range_min, range_max = tier["max_results_range"]
|
||||
if range_min <= max_results <= range_max:
|
||||
cost_per_query = tier["input_cost_per_query"]
|
||||
break
|
||||
else:
|
||||
# Fallback to highest tier if out of range
|
||||
cost_per_query = tiered_pricing[-1]["input_cost_per_query"]
|
||||
else:
|
||||
# Simple flat rate
|
||||
cost_per_query = float(model_info.get("input_cost_per_query") or 0.0)
|
||||
|
||||
total_cost: Final = number_of_queries * cost_per_query
|
||||
return (total_cost, 0.0) # (input_cost, output_cost)
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -13,7 +13,7 @@ discarded by ``get_model_info``.
|
|||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text
|
||||
from litellm.responses.utils import ResponseAPILoggingUtils
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
|
|
@ -46,7 +46,7 @@ def test_openai_cache_write_tokens_billed_at_the_cache_creation_rate(local_model
|
|||
fresh_tokens = prompt_tokens - cache_write_tokens
|
||||
completion_tokens = 5
|
||||
|
||||
prompt_cost, completion_cost = generic_cost_per_token(
|
||||
prompt_cost, completion_cost = cost_per_text(
|
||||
model=MODEL,
|
||||
usage=_openai_chat_usage(prompt_tokens, cache_write_tokens, completion_tokens),
|
||||
custom_llm_provider="openai",
|
||||
|
|
@ -77,12 +77,12 @@ def test_responses_api_cache_write_costs_the_same_as_chat(local_model_cost_map):
|
|||
)
|
||||
assert responses_usage.prompt_tokens_details.cache_write_tokens == cache_write_tokens
|
||||
|
||||
responses_prompt_cost, responses_completion_cost = generic_cost_per_token(
|
||||
responses_prompt_cost, responses_completion_cost = cost_per_text(
|
||||
model=MODEL,
|
||||
usage=responses_usage,
|
||||
custom_llm_provider="openai",
|
||||
)
|
||||
chat_prompt_cost, chat_completion_cost = generic_cost_per_token(
|
||||
chat_prompt_cost, chat_completion_cost = cost_per_text(
|
||||
model=MODEL,
|
||||
usage=_openai_chat_usage(prompt_tokens, cache_write_tokens, completion_tokens),
|
||||
custom_llm_provider="openai",
|
||||
|
|
@ -126,7 +126,7 @@ def test_tiered_cache_creation_rates_are_registered_and_billed(
|
|||
}[rate_key]
|
||||
input_rate = litellm.model_cost[MODEL][input_rate_key]
|
||||
|
||||
prompt_cost, _ = generic_cost_per_token(
|
||||
prompt_cost, _ = cost_per_text(
|
||||
model=MODEL,
|
||||
usage=_openai_chat_usage(prompt_tokens, cache_write_tokens, completion_tokens=100),
|
||||
custom_llm_provider="openai",
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import copy
|
||||
import json
|
||||
from itertools import chain
|
||||
from typing import Final
|
||||
from typing import Final, cast
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import httpx
|
||||
|
|
@ -39,7 +39,7 @@ from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation im
|
|||
)
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES, ANTHROPIC_TOOL_SEARCH_TOOL_TYPES
|
||||
from litellm.types.llms.openai import ChatCompletionToolCallFunctionChunk
|
||||
from litellm.types.utils import ChatCompletionToolCallChunk, ModelResponse, ServerToolUse, Usage
|
||||
from litellm.types.utils import ChatCompletionToolCallChunk, ModelInfo, ModelResponse, ServerToolUse, Usage
|
||||
from tests.unit.llms.anthropic.anthropic_docs_refusal import ANTHROPIC_DOCS_REFUSAL_MESSAGE
|
||||
|
||||
|
||||
|
|
@ -4468,86 +4468,37 @@ def test_fast_mode_usage_calculation():
|
|||
assert usage.speed == "fast"
|
||||
|
||||
|
||||
def test_fast_mode_cost_calculation():
|
||||
"""
|
||||
Test that fast mode applies the 'fast' multiplier from provider_specific_entry
|
||||
on top of the base model cost (1.1x for claude-opus-4-6).
|
||||
"""
|
||||
_FAST_MODE_ROW: Final = cast(
|
||||
ModelInfo,
|
||||
{
|
||||
"litellm_provider": "anthropic",
|
||||
"input_cost_per_token": 5e-06,
|
||||
"output_cost_per_token": 2.5e-05,
|
||||
"provider_specific_entry": {"fast": 1.1, "us": 1.1},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("speed", "inference_geo", "multiplier"),
|
||||
[
|
||||
pytest.param("fast", None, 1.1, id="fast"),
|
||||
pytest.param("fast", "us", 1.1 * 1.1, id="fast-and-us"),
|
||||
pytest.param(None, "global", 1.0, id="global-geo-is-no-uplift"),
|
||||
],
|
||||
)
|
||||
def test_fast_mode_and_inference_geo_multiply_the_base_cost(
|
||||
speed: str | None, inference_geo: str | None, multiplier: float
|
||||
) -> None:
|
||||
from litellm.llms.anthropic.cost_calculation import cost_per_token
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
base_prompt = 0.005
|
||||
base_completion = 0.025
|
||||
usage = Usage(prompt_tokens=1000, completion_tokens=1000, speed=speed, inference_geo=inference_geo)
|
||||
|
||||
with (
|
||||
patch("litellm.llms.anthropic.cost_calculation.generic_cost_per_token") as mock_cost,
|
||||
patch("litellm.get_model_info") as mock_info,
|
||||
):
|
||||
mock_cost.return_value = (base_prompt, base_completion)
|
||||
mock_info.return_value = {"provider_specific_entry": {"fast": 1.1, "us": 1.1}}
|
||||
prompt_cost, completion_cost = cost_per_token(model="claude-opus-4-6", usage=usage, model_info=_FAST_MODE_ROW)
|
||||
|
||||
usage_fast = Usage(
|
||||
prompt_tokens=1000,
|
||||
completion_tokens=1000,
|
||||
speed="fast",
|
||||
)
|
||||
|
||||
prompt_cost, completion_cost = cost_per_token(
|
||||
model="claude-opus-4-6",
|
||||
usage=usage_fast,
|
||||
)
|
||||
|
||||
# generic_cost_per_token called with the plain base model name
|
||||
mock_cost.assert_called_once()
|
||||
assert mock_cost.call_args[1]["model"] == "claude-opus-4-6"
|
||||
assert mock_cost.call_args[1]["custom_llm_provider"] == "anthropic"
|
||||
|
||||
# 1.1x multiplier applied
|
||||
assert abs(prompt_cost - base_prompt * 1.1) < 1e-10
|
||||
assert abs(completion_cost - base_completion * 1.1) < 1e-10
|
||||
|
||||
|
||||
def test_fast_mode_with_inference_geo():
|
||||
"""
|
||||
Test that fast mode + inference_geo both apply their multipliers from
|
||||
provider_specific_entry (1.1 * 1.1 = 1.21x for claude-opus-4-6).
|
||||
"""
|
||||
|
||||
from litellm.llms.anthropic.cost_calculation import cost_per_token
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
base_prompt = 0.005
|
||||
base_completion = 0.025
|
||||
|
||||
with (
|
||||
patch("litellm.llms.anthropic.cost_calculation.generic_cost_per_token") as mock_cost,
|
||||
patch("litellm.get_model_info") as mock_info,
|
||||
):
|
||||
mock_cost.return_value = (base_prompt, base_completion)
|
||||
mock_info.return_value = {"provider_specific_entry": {"fast": 1.1, "us": 1.1}}
|
||||
|
||||
usage = Usage(
|
||||
prompt_tokens=1000,
|
||||
completion_tokens=1000,
|
||||
speed="fast",
|
||||
inference_geo="us",
|
||||
)
|
||||
|
||||
prompt_cost, completion_cost = cost_per_token(
|
||||
model="claude-opus-4-6",
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
# generic_cost_per_token called with the plain base model name
|
||||
mock_cost.assert_called_once()
|
||||
assert mock_cost.call_args[1]["model"] == "claude-opus-4-6"
|
||||
assert mock_cost.call_args[1]["custom_llm_provider"] == "anthropic"
|
||||
|
||||
# 1.1 (fast) * 1.1 (us) = 1.21x multiplier applied
|
||||
expected_multiplier = 1.1 * 1.1
|
||||
assert abs(prompt_cost - base_prompt * expected_multiplier) < 1e-10
|
||||
assert abs(completion_cost - base_completion * expected_multiplier) < 1e-10
|
||||
assert prompt_cost == pytest.approx(1000 * 5e-06 * multiplier)
|
||||
assert completion_cost == pytest.approx(1000 * 2.5e-05 * multiplier)
|
||||
|
||||
|
||||
def test_calculate_usage_captures_service_tier():
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ import litellm
|
|||
from litellm.litellm_core_utils.llm_cost_calc.utils import (
|
||||
resolve_explicit_cache_read_cost_rate,
|
||||
calculate_prompt_caching_savings,
|
||||
generic_cost_per_token,
|
||||
cost_per_text,
|
||||
get_token_type_cost_breakdown,
|
||||
)
|
||||
from litellm.llms.dashscope.cost_calculator import (
|
||||
|
|
@ -475,7 +475,7 @@ class TestDashscopeCostCalculator:
|
|||
prompt_tokens_details={"cached_tokens": 600, "cache_type": "ephemeral"},
|
||||
)
|
||||
|
||||
prompt_cost, _ = generic_cost_per_token(
|
||||
prompt_cost, _ = cost_per_text(
|
||||
model="custom-qwen",
|
||||
usage=usage,
|
||||
custom_llm_provider="openai",
|
||||
|
|
|
|||
|
|
@ -1,65 +0,0 @@
|
|||
from copy import deepcopy
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.constants import FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO
|
||||
from litellm.llms.fireworks_ai.cache_pricing import with_default_cache_read_rate
|
||||
from litellm.types.utils import ModelInfo
|
||||
|
||||
|
||||
def test_explicit_cache_read_rate_and_missing_input_rate_keep_the_entry_untouched() -> None:
|
||||
explicit_info: ModelInfo = {"input_cost_per_token": 2e-6, "cache_read_input_token_cost": 1e-6}
|
||||
no_input_rate_info: ModelInfo = {"output_cost_per_token": 3e-6}
|
||||
|
||||
assert with_default_cache_read_rate(explicit_info) is explicit_info
|
||||
assert with_default_cache_read_rate(no_input_rate_info) is no_input_rate_info
|
||||
|
||||
|
||||
def test_missing_cache_read_rate_is_derived_for_standard_and_off_peak_without_mutating_the_entry() -> None:
|
||||
model_info: ModelInfo = {
|
||||
"input_cost_per_token": 2e-6,
|
||||
"off_peak_pricing": {
|
||||
"hours_utc": "14:00-00:00",
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 3e-6,
|
||||
},
|
||||
}
|
||||
original: ModelInfo = deepcopy(model_info)
|
||||
|
||||
derived = with_default_cache_read_rate(model_info)
|
||||
|
||||
assert model_info == original
|
||||
assert derived is not model_info
|
||||
assert derived["cache_read_input_token_cost"] == pytest.approx(2e-6 * FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO)
|
||||
assert derived["off_peak_pricing"] == {
|
||||
"hours_utc": "14:00-00:00",
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 3e-6,
|
||||
"cache_read_input_token_cost": 1e-6 * FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO,
|
||||
}
|
||||
|
||||
|
||||
def test_off_peak_window_without_its_own_input_rate_reuses_the_standard_derived_rate() -> None:
|
||||
model_info: ModelInfo = {
|
||||
"input_cost_per_token": 2e-6,
|
||||
"off_peak_pricing": {"hours_utc": "14:00-00:00", "output_cost_per_token": 3e-6},
|
||||
}
|
||||
|
||||
derived = with_default_cache_read_rate(model_info)
|
||||
|
||||
assert derived["off_peak_pricing"]["cache_read_input_token_cost"] == derived["cache_read_input_token_cost"]
|
||||
|
||||
|
||||
def test_string_rates_from_config_are_coerced_before_the_discount_is_applied() -> None:
|
||||
model_info: ModelInfo = {
|
||||
"input_cost_per_token": "2e-6",
|
||||
"off_peak_pricing": {
|
||||
"hours_utc": "14:00-00:00",
|
||||
"input_cost_per_token": "1e-6",
|
||||
},
|
||||
}
|
||||
|
||||
derived = with_default_cache_read_rate(model_info)
|
||||
|
||||
assert derived["cache_read_input_token_cost"] == pytest.approx(1e-6)
|
||||
assert derived["off_peak_pricing"]["cache_read_input_token_cost"] == pytest.approx(5e-7)
|
||||
|
|
@ -9,7 +9,7 @@ import pytest
|
|||
import litellm
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import (
|
||||
calculate_prompt_caching_savings,
|
||||
generic_cost_per_token,
|
||||
cost_per_text,
|
||||
get_token_type_cost_breakdown,
|
||||
)
|
||||
from litellm.llms.fireworks_ai.cost_calculator import cost_per_token
|
||||
|
|
@ -213,7 +213,7 @@ def test_fireworks_cache_read_rates_match_breakdown_and_caching_savings():
|
|||
assert math.isclose(savings, 300 * INPUT_COST * 0.5, rel_tol=1e-10)
|
||||
|
||||
|
||||
def test_generic_cost_per_token_applies_fireworks_cache_read_default_with_or_without_model_info():
|
||||
def test_cost_per_text_applies_fireworks_cache_read_default_with_or_without_model_info():
|
||||
model = "accounts/fireworks/models/generic-cache-read-test"
|
||||
litellm.model_cost = { # test-quality-ok: the restore_model_cost fixture returns litellm.model_cost to the original object after each test
|
||||
**litellm.model_cost,
|
||||
|
|
@ -227,12 +227,12 @@ def test_generic_cost_per_token_applies_fireworks_cache_read_default_with_or_wit
|
|||
usage = _usage(prompt_tokens=1000, cached_tokens=300, completion_tokens=200)
|
||||
expected_prompt_cost = (700 * INPUT_COST) + (300 * INPUT_COST * 0.5)
|
||||
|
||||
implicit_model_info_cost, _ = generic_cost_per_token(
|
||||
implicit_model_info_cost, _ = cost_per_text(
|
||||
model=model,
|
||||
usage=usage,
|
||||
custom_llm_provider="fireworks_ai",
|
||||
)
|
||||
explicit_model_info_cost, _ = generic_cost_per_token(
|
||||
explicit_model_info_cost, _ = cost_per_text(
|
||||
model=model,
|
||||
usage=usage,
|
||||
custom_llm_provider="fireworks_ai",
|
||||
|
|
@ -392,6 +392,6 @@ def test_a_routed_xai_model_keeps_xais_inclusive_token_threshold():
|
|||
|
||||
routed_prompt_cost, routed_completion_cost = cost_per_token(model=f"fireworks_ai/{key}", usage=usage)
|
||||
|
||||
assert (routed_prompt_cost, routed_completion_cost) == generic_cost_per_token(
|
||||
assert (routed_prompt_cost, routed_completion_cost) == cost_per_text(
|
||||
model=key, usage=usage, custom_llm_provider="xai"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ from litellm._logging import verbose_logger
|
|||
from litellm.cost_calculator import batch_cost_calculator
|
||||
from litellm.litellm_core_utils.exception_mapping_utils import exception_type
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token, parse_prompt_tokens_details
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text, parse_prompt_tokens_details
|
||||
from litellm.llms.anthropic.messages.native import handler as anthropic_messages_handler
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.llms.gemini.chat.transformation import GoogleAIStudioGeminiConfig
|
||||
|
|
@ -1620,7 +1620,7 @@ def test_gemini_raw_usage_metadata_prices_cached_audio_at_its_standard_rate() ->
|
|||
+ 5 * model_info["cache_read_input_audio_token_cost"]
|
||||
)
|
||||
|
||||
prompt_cost, _ = generic_cost_per_token(
|
||||
prompt_cost, _ = cost_per_text(
|
||||
model="gemini-audio-standard-test-model",
|
||||
usage=usage,
|
||||
custom_llm_provider="gemini",
|
||||
|
|
@ -1640,13 +1640,13 @@ def test_gemini_usage_totals_are_subtracted_once_and_audio_cache_rate_matches_au
|
|||
cached_usage: Final = _gemini_usage_with_cached_audio(5)
|
||||
uncached_usage: Final = _gemini_usage_with_cached_audio(0)
|
||||
cached_details: Final = parse_prompt_tokens_details(cached_usage)
|
||||
cached_prompt_cost, _ = generic_cost_per_token(
|
||||
cached_prompt_cost, _ = cost_per_text(
|
||||
model="gemini-audio-standard-test-model",
|
||||
usage=cached_usage,
|
||||
custom_llm_provider="gemini",
|
||||
model_info=model_info,
|
||||
)
|
||||
uncached_prompt_cost, _ = generic_cost_per_token(
|
||||
uncached_prompt_cost, _ = cost_per_text(
|
||||
model="gemini-audio-standard-test-model",
|
||||
usage=uncached_usage,
|
||||
custom_llm_provider="gemini",
|
||||
|
|
@ -6299,7 +6299,7 @@ def test_vertex_ai_usage_metadata_with_document_tokens_cached():
|
|||
"input_cost_per_token": 2e-6,
|
||||
"cache_read_input_token_cost": 5e-7,
|
||||
}
|
||||
prompt_cost, _ = generic_cost_per_token(
|
||||
prompt_cost, _ = cost_per_text(
|
||||
model="gemini-document-cache-test-model",
|
||||
usage=result,
|
||||
custom_llm_provider="gemini",
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ import respx
|
|||
|
||||
import litellm
|
||||
from litellm.exceptions import BadRequestError
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text
|
||||
from litellm.llms.vertex_ai.gemini_embeddings.batch_embed_content_transformation import (
|
||||
_build_part_for_input,
|
||||
file_reference_name,
|
||||
|
|
@ -348,7 +348,7 @@ class TestProcessEmbedContentResponseUsage:
|
|||
assert result.usage.total_tokens == 258
|
||||
assert result.usage.prompt_tokens_details.image_tokens == 258
|
||||
|
||||
prompt_cost, _ = generic_cost_per_token(
|
||||
prompt_cost, _ = cost_per_text(
|
||||
model=self.MODEL,
|
||||
usage=result.usage,
|
||||
custom_llm_provider="vertex_ai",
|
||||
|
|
@ -373,7 +373,7 @@ class TestProcessEmbedContentResponseUsage:
|
|||
assert result.usage.prompt_tokens == 12
|
||||
assert result.usage.prompt_tokens_details.text_tokens == 12
|
||||
|
||||
prompt_cost, _ = generic_cost_per_token(
|
||||
prompt_cost, _ = cost_per_text(
|
||||
model=self.MODEL,
|
||||
usage=result.usage,
|
||||
custom_llm_provider="vertex_ai",
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ from unittest.mock import patch as patch_ctx
|
|||
import pytest
|
||||
from fastapi import HTTPException
|
||||
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text
|
||||
from litellm.litellm_core_utils.ptu_pricing import ptu_terms
|
||||
from litellm.llms.gemini.cost_calculator import cost_per_web_search_request
|
||||
from litellm.proxy._types import (
|
||||
|
|
@ -1089,7 +1089,7 @@ class TestPtuDeploymentsAreNotBilledPerToken:
|
|||
registered = router.get_deployment_model_info(model_id="dep-ptu", model_name="dashscope/qwen-flash")
|
||||
assert registered is not None
|
||||
assert registered["tiered_pricing"] == []
|
||||
assert generic_cost_per_token(
|
||||
assert cost_per_text(
|
||||
model="dashscope/qwen-flash",
|
||||
usage=Usage(prompt_tokens=1000, completion_tokens=100, total_tokens=1100),
|
||||
custom_llm_provider="dashscope",
|
||||
|
|
|
|||
|
|
@ -574,7 +574,7 @@ class TestVertexAILivePassthroughLoggingHandler:
|
|||
def test_reporting_tool_use_tokens_does_not_move_the_bill(self, handler, mock_logging_obj):
|
||||
"""Deliberate boundary: these tokens are reported here, and priced nowhere.
|
||||
|
||||
generic_cost_per_token reads the input bill out of prompt_tokens_details, and falls
|
||||
cost_per_text reads the input bill out of prompt_tokens_details, and falls
|
||||
back to prompt_tokens only when the details carry no text or a cache hit overlaps them,
|
||||
so adding tool-use tokens to prompt_tokens is worth nothing on an ordinary Live turn and
|
||||
over-charges against the cache-overlap correction when it is not. Pricing them belongs
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ from typing import Final, Literal
|
|||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text
|
||||
from litellm.llms.anthropic.cost_calculation import cost_per_token as anthropic_cost_per_token
|
||||
from litellm.proxy.spend_tracking.savings import (
|
||||
_baseline_usage,
|
||||
|
|
@ -108,7 +108,7 @@ def _cached_usage_object() -> dict:
|
|||
|
||||
|
||||
def _cost_on(model: str, usage_object: dict) -> float:
|
||||
prompt_cost, completion_cost = generic_cost_per_token(
|
||||
prompt_cost, completion_cost = cost_per_text(
|
||||
model=model, usage=Usage(**usage_object), custom_llm_provider="anthropic"
|
||||
)
|
||||
return prompt_cost + completion_cost
|
||||
|
|
@ -159,7 +159,7 @@ def _net_caching_savings_against_biller(usage_object: dict, model: str = "claude
|
|||
|
||||
Bills the request as it happened, then bills the same token total with nothing
|
||||
cached, and returns the difference. Deriving the expectation from
|
||||
``generic_cost_per_token`` rather than restating the formula is what makes these
|
||||
``cost_per_text`` rather than restating the formula is what makes these
|
||||
tests able to fail: a wrong formula in savings.py cannot also be wrong here.
|
||||
"""
|
||||
prompt_tokens = usage_object["prompt_tokens"]
|
||||
|
|
@ -245,7 +245,7 @@ def test_caching_savings_agree_with_biller_on_the_request_pricing_basis(
|
|||
)
|
||||
costs: Final = tuple(
|
||||
sum(
|
||||
generic_cost_per_token(
|
||||
cost_per_text(
|
||||
model=model,
|
||||
usage=arm,
|
||||
custom_llm_provider=provider,
|
||||
|
|
@ -486,8 +486,8 @@ def test_supplied_baseline_usage_is_priced_independently(
|
|||
baseline: str, selected: str, actual: Usage, modeled: Usage, loses_money: bool,
|
||||
) -> None:
|
||||
result: Final = _savings(baseline, selected, actual, modeled)
|
||||
expected: Final = sum(generic_cost_per_token(model=baseline, usage=modeled, custom_llm_provider="anthropic")) - sum(
|
||||
generic_cost_per_token(model=selected, usage=actual, custom_llm_provider="anthropic")
|
||||
expected: Final = sum(cost_per_text(model=baseline, usage=modeled, custom_llm_provider="anthropic")) - sum(
|
||||
cost_per_text(model=selected, usage=actual, custom_llm_provider="anthropic")
|
||||
)
|
||||
assert result == pytest.approx(expected)
|
||||
assert result is not None and (result < 0) is loses_money
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import pytest
|
|||
|
||||
import litellm
|
||||
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text
|
||||
from litellm.responses.litellm_completion_transformation.transformation import LiteLLMCompletionResponsesConfig
|
||||
from litellm.responses.utils import ResponseAPILoggingUtils, ResponsesAPIRequestUtils
|
||||
from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIOptionalRequestParams, ResponsesAPIResponse
|
||||
|
|
@ -440,7 +440,7 @@ class TestResponseAPILoggingUtils:
|
|||
assert usage.prompt_tokens_details.cache_creation_token_details is not None
|
||||
assert usage.prompt_tokens_details.cache_creation_token_details.ephemeral_12h_input_tokens == 2_000
|
||||
assert usage.prompt_tokens_details.cache_creation_token_details.ephemeral_30m_input_tokens == 3_000
|
||||
cost: Final = generic_cost_per_token(
|
||||
cost: Final = cost_per_text(
|
||||
model="test-12h-cache-write",
|
||||
usage=usage,
|
||||
custom_llm_provider="openai",
|
||||
|
|
|
|||
|
|
@ -1927,7 +1927,7 @@ def test_gemini_25_explicit_caching_cost_direct_usage():
|
|||
This test reproduces the issue from #11156 where cached tokens should receive
|
||||
a 75% discount.
|
||||
"""
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text
|
||||
from litellm.types.utils import (
|
||||
CompletionTokensDetailsWrapper,
|
||||
PromptTokensDetailsWrapper,
|
||||
|
|
@ -1953,7 +1953,7 @@ def test_gemini_25_explicit_caching_cost_direct_usage():
|
|||
),
|
||||
)
|
||||
|
||||
input_cost, output_cost = generic_cost_per_token(
|
||||
input_cost, output_cost = cost_per_text(
|
||||
model="gemini/gemini-2.5-pro",
|
||||
usage=usage,
|
||||
custom_llm_provider="gemini",
|
||||
|
|
@ -1987,13 +1987,13 @@ def test_gemini_25_explicit_caching_cost_direct_usage():
|
|||
|
||||
def test_azure_ai_cache_cost_calculation(_local_model_cost_map):
|
||||
"""
|
||||
Test that azure_ai provider correctly calculates cache costs using generic_cost_per_token.
|
||||
Test that azure_ai provider correctly calculates cache costs using cost_per_text.
|
||||
|
||||
This verifies that azure_ai models with custom cache pricing in model_info
|
||||
will have their cache_creation_input_token_cost and cache_read_input_token_cost
|
||||
applied correctly.
|
||||
"""
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text
|
||||
from litellm.types.utils import PromptTokensDetailsWrapper, Usage
|
||||
|
||||
# Register a custom azure_ai model with cache pricing
|
||||
|
|
@ -2023,7 +2023,7 @@ def test_azure_ai_cache_cost_calculation(_local_model_cost_map):
|
|||
cache_creation_input_tokens=100, # 100 cache creation tokens
|
||||
)
|
||||
|
||||
input_cost, output_cost = generic_cost_per_token(
|
||||
input_cost, output_cost = cost_per_text(
|
||||
model=test_model_id,
|
||||
usage=usage,
|
||||
custom_llm_provider="azure_ai",
|
||||
|
|
@ -2887,7 +2887,7 @@ def test_completion_cost_service_tier_for_anthropic(_local_model_cost_map):
|
|||
Regression for LIT-3771: the Anthropic cost route dropped ``service_tier``,
|
||||
so priority requests (whose tier is reported on the response usage) were
|
||||
always billed at the standard rate. The tier is captured by the
|
||||
transformation and must flow through to ``generic_cost_per_token``.
|
||||
transformation and must flow through to ``cost_per_text``.
|
||||
"""
|
||||
from litellm import completion_cost
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ from typing import Final
|
|||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text
|
||||
from litellm.types.utils import PromptTokensDetailsWrapper, Usage
|
||||
|
||||
REPO_ROOT = Path(__file__).parents[2]
|
||||
|
|
@ -151,7 +151,7 @@ def test_tiered_long_context_cost_uses_catalog_rates(
|
|||
completion_tokens=COMPLETION_TOKENS,
|
||||
total_tokens=LONG_CONTEXT_PROMPT_TOKENS + COMPLETION_TOKENS,
|
||||
)
|
||||
prompt_cost, completion_cost = generic_cost_per_token(
|
||||
prompt_cost, completion_cost = cost_per_text(
|
||||
model=model,
|
||||
usage=usage,
|
||||
custom_llm_provider="openai",
|
||||
|
|
@ -169,13 +169,13 @@ def test_gpt_6_astra_ultrafast_long_context_costs_and_controls() -> None:
|
|||
total_tokens=301_000,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=100, cache_creation_tokens=200),
|
||||
)
|
||||
ultrafast_prompt_cost, ultrafast_completion_cost = generic_cost_per_token(
|
||||
ultrafast_prompt_cost, ultrafast_completion_cost = cost_per_text(
|
||||
model="gpt-6-astra",
|
||||
usage=ultrafast_usage,
|
||||
custom_llm_provider="openai",
|
||||
service_tier="ultrafast",
|
||||
)
|
||||
standard_prompt_cost, standard_completion_cost = generic_cost_per_token(
|
||||
standard_prompt_cost, standard_completion_cost = cost_per_text(
|
||||
model="gpt-6-astra",
|
||||
usage=ultrafast_usage,
|
||||
custom_llm_provider="openai",
|
||||
|
|
@ -186,7 +186,7 @@ def test_gpt_6_astra_ultrafast_long_context_costs_and_controls() -> None:
|
|||
total_tokens=272_000,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=100, cache_creation_tokens=200),
|
||||
)
|
||||
below_threshold_prompt_cost, below_threshold_completion_cost = generic_cost_per_token(
|
||||
below_threshold_prompt_cost, below_threshold_completion_cost = cost_per_text(
|
||||
model="gpt-6-astra",
|
||||
usage=below_threshold_usage,
|
||||
custom_llm_provider="openai",
|
||||
|
|
|
|||
|
|
@ -1480,7 +1480,6 @@ ALIAS_CASES: Final = (
|
|||
"get_service_tier_cost_key",
|
||||
False,
|
||||
),
|
||||
("litellm.litellm_core_utils.llm_cost_calc.utils", "", "_is_above_128k", "is_above_128k", False),
|
||||
(
|
||||
"litellm.litellm_core_utils.llm_request_utils",
|
||||
"",
|
||||
|
|
|
|||
|
|
@ -121,7 +121,7 @@ def test_register_model_inherits_builtin_cache_pricing_for_unmapped_key(monkeypa
|
|||
|
||||
Regression for the cache-pricing dropout under partial overrides.
|
||||
"""
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import cost_per_text
|
||||
from litellm.types.utils import PromptTokensDetailsWrapper, Usage
|
||||
|
||||
original_model_cost = litellm.model_cost
|
||||
|
|
@ -168,7 +168,7 @@ def test_register_model_inherits_builtin_cache_pricing_for_unmapped_key(monkeypa
|
|||
cache_creation_input_tokens=200,
|
||||
)
|
||||
|
||||
input_cost, output_cost = generic_cost_per_token(
|
||||
input_cost, output_cost = cost_per_text(
|
||||
model=registered_key,
|
||||
usage=usage,
|
||||
custom_llm_provider="bedrock",
|
||||
|
|
|
|||
|
|
@ -5069,17 +5069,15 @@ class _EntryReadDuringARegistration(dict[str, object]):
|
|||
return super().__contains__(key)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("walk", ["fal_ai", "ollama", "gpt-5"])
|
||||
@pytest.mark.parametrize("walk", ["ollama", "gpt-5"])
|
||||
def test_request_path_cost_map_walks_survive_a_registration_mid_walk(
|
||||
monkeypatch: pytest.MonkeyPatch, walk: str
|
||||
) -> None:
|
||||
from litellm.llms.fal_ai.cost_calculator import _keyed_rows
|
||||
from litellm.llms.ollama.common_utils import OllamaModelInfo
|
||||
from litellm.llms.openai.chat.gpt_reasoning_transformation import _catalogue_declares_default_effort
|
||||
from litellm.utils import _invalidate_model_cost_lowercase_map
|
||||
|
||||
walks: Final[Mapping[str, Callable[[], object]]] = {
|
||||
"fal_ai": lambda: _keyed_rows("lit-5853-model", "standard"),
|
||||
"ollama": lambda: OllamaModelInfo._is_static_ollama_model("ollama/lit-5853-model"),
|
||||
"gpt-5": _catalogue_declares_default_effort,
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue