scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-29 06:33:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 200

Merge branch 'main' into litellm_mcp_admin_team_key_scope

Browse source
This commit is contained in:
mateo-berri 2026-10-28 20:56:30 -07:00
commit 4de4961587
164 changed files with 12366 additions and 3229 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/test-rust.yml vendored
View file

@ -4,6 +4,7 @@ on:
push:
paths:
- "litellm-rust/**"
- "model_prices_and_context_window.json"
- "litellm/rust_bridge/**"
- "scripts/generate_trace_types.py"
- "scripts/trace_codegen/**"
@ -38,6 +39,7 @@ on:
- "litellm_**"
paths:
- "litellm-rust/**"
- "model_prices_and_context_window.json"
- "litellm/rust_bridge/**"
- "scripts/generate_trace_types.py"
- "scripts/trace_codegen/**"

2
.github/workflows/test-unit.yml vendored
View file

@ -42,7 +42,7 @@ jobs:
id: rust-bridge-key
if: steps.changes.outputs.decision != 'skip'
env:
RUST_BRIDGE_CACHE_KEY: ${{ runner.os }}-rust-bridge-editable-${{ hashFiles('litellm-rust/Cargo.lock', 'litellm-rust/Cargo.toml', 'litellm-rust/crates/**', 'pyproject.toml') }}
RUST_BRIDGE_CACHE_KEY: ${{ runner.os }}-rust-bridge-editable-${{ hashFiles('litellm-rust/Cargo.lock', 'litellm-rust/Cargo.toml', 'litellm-rust/crates/**', 'model_prices_and_context_window.json', 'pyproject.toml') }}
run: echo "key=$RUST_BRIDGE_CACHE_KEY" >> "$GITHUB_OUTPUT"
- name: Set up Python

10
enterprise/enterprise_hooks/google_text_moderation.py
View file

@ -15,7 +15,10 @@ import litellm
from litellm._logging import verbose_proxy_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.guardrails._content_utils import iter_message_text
from litellm.proxy.guardrails._content_utils import (
is_non_conversational_request,
iter_message_text,
)
from litellm.types.utils import CallTypesLiteral
_THRESHOLD_VALUE: Final = TypeAdapter(object)
@ -94,7 +97,7 @@ class ENTERPRISE_GoogleTextModeration(CustomLogger):
async def async_moderation_hook(
self,
data: dict,
data: dict[str, object],
user_api_key_dict: UserAPIKeyAuth,
call_type: CallTypesLiteral,
):
@ -102,6 +105,9 @@ class ENTERPRISE_GoogleTextModeration(CustomLogger):
- Calls Google's Text Moderation API
- Rejects request if it fails safety check
"""
if is_non_conversational_request(call_type, data):
verbose_proxy_logger.debug("Google text moderation: skipping non-conversational call type %s", call_type)
return
# Covers multimodal list content + Responses-API input.
text = "".join(iter_message_text(data))
if text:

3
enterprise/litellm_enterprise/enterprise_callbacks/callback_controls.py
View file

@ -33,9 +33,8 @@ class EnterpriseCallbackControls:
try:
disabled_callbacks = EnterpriseCallbackControls.get_disabled_callbacks(litellm_params, standard_callback_dynamic_params)
verbose_logger.debug(f"Dynamically disabled callbacks from {X_LITELLM_DISABLE_CALLBACKS}: {disabled_callbacks}")
verbose_logger.debug(f"Checking if {callback} is disabled via headers. Disable callbacks from headers: {disabled_callbacks}")
if disabled_callbacks is not None:
verbose_logger.debug(f"Checking if {callback} is in dynamically disabled callbacks: {disabled_callbacks}")
#########################################################
# premium user check
#########################################################

34
litellm-rust/Cargo.lock generated
View file

@ -3150,6 +3150,7 @@ dependencies = [
"defmt",
"jiff-core",
"jiff-static",
"jiff-tzdb",
"log",
"portable-atomic",
"portable-atomic-util",
@ -3178,6 +3179,12 @@ dependencies = [
"syn 2.0.119",
]
[[package]]
name = "jiff-tzdb"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa8377070c6bae868759445e5a77f66d84f0b72f3a054bfb00e6d038b8282da7"
[[package]]
name = "jni"
version = "0.22.4"
@ -3729,9 +3736,12 @@ dependencies = [
name = "litellm-cost"
version = "0.1.0"
dependencies = [
"criterion",
"proptest",
"jiff",
"rstest",
"schemars 1.2.2",
"serde",
"serde_json",
"strum",
"thiserror 2.0.19",
]
@ -4277,17 +4287,35 @@ name = "litellm-model-catalog"
version = "0.1.0"
dependencies = [
"indexmap 2.14.0",
"jiff",
"jsonschema",
"litellm-cost",
"litellm-error",
"litellm-llms-types",
"macro_rules_attribute",
"rstest",
"schemars 1.2.2",
"serde",
"serde_json",
"serde_with",
"strum",
"thiserror 2.0.19",
"time",
]
[[package]]
name = "litellm-model-catalog-refresh"
version = "0.1.0"
dependencies = [
"litellm-error",
"litellm-http",
"litellm-model-catalog",
"reqwest",
"rstest",
"thiserror 2.0.19",
"tokio",
"url",
]
[[package]]
name = "litellm-python-bridge"
version = "0.1.0"

4
litellm-rust/Cargo.toml
View file

@ -73,6 +73,9 @@ litellm-python-compat = { path = "crates/python-compat" }
litellm-circuit-breaker = { path = "crates/circuit-breaker" }
litellm-redis-client = { path = "crates/redis-client" }
litellm-redis-testing = { path = "crates/redis-testing" }
litellm-cost = { path = "crates/cost" }
litellm-model-catalog = { path = "crates/model-catalog" }
litellm-model-catalog-refresh = { path = "crates/model-catalog-refresh" }
askama = { version = "0.16.1", default-features = false, features = ["derive", "std"] }
tracing = "0.1"
@ -109,6 +112,7 @@ serde = { version = "1.0", features = ["derive"] }
serde_json = { version = "1.0", features = ["float_roundtrip"] }
serde_with = { version = "=3.21.0", default-features = false, features = ["std", "macros"] }
sha2 = "0.10"
sha1 = "0.10"
syn = { version = "2", default-features = false }
sqlx = { version = "0.9.0", default-features = false, features = ["json", "macros", "postgres", "runtime-tokio", "chrono", "tls-rustls-ring-native-roots"] }
chrono = { version = "0.4.45", default-features = false, features = ["std"] }

15
litellm-rust/crates/cost/Cargo.toml
View file

@ -5,14 +5,17 @@ edition.workspace = true
license.workspace = true
repository.workspace = true
[features]
serde = ["dep:serde"]
schema = ["serde", "dep:schemars"]
[dependencies]
jiff = { version = "0.2.37", default-features = false, features = ["std"] }
schemars = { workspace = true, optional = true }
serde = { workspace = true, optional = true }
thiserror.workspace = true
strum.workspace = true
[dev-dependencies]
criterion.workspace = true
proptest.workspace = true
rstest.workspace = true
[[bench]]
name = "calculate"
harness = false
serde_json.workspace = true

12
litellm-rust/crates/cost/README.md
View file

@ -1,13 +1,3 @@
# litellm-cost
This crate calculates text token charges from rates and usage supplied by its caller. It is standalone and has no Python bridge or proxy integration
Call `compile(&pricing)` once for an immutable plan, then `plan.calculate(&request)` for each supported request. `calculate(&pricing, &request)` compiles on each call. A successful result exposes pre-multiplier component costs, selected rates, the multiplier, and derived `input()`, `output()`, and `total()` values
The caller states whether `prompt_tokens` includes cache tokens. Threshold selection uses total input tokens for either convention and selects one rate for the whole request. Thresholds are sorted when compiled, and duplicate thresholds or tier overrides fail deterministically. `Fast` selects priority rates; unknown tiers use standard rates
`Rate::Missing`, `Rate::Null`, and `Rate::Value(0.0)` remain distinct. Missing cache rates fall back to the selected input rate, and an absent one-hour write rate falls back to the selected write rate. Missing input or output rates return typed errors, including for zero usage. Python's sparse-entry behavior remains outside this native contract
The supported off-peak shape is one non-wrapping UTC daily window. The caller supplies the applicable regional multiplier after provider-specific selection. Negative or non-finite rates, ambiguous rules, inconsistent cache counts, incomplete write splits, invalid windows and overflow return errors. Callers must decline unsupported inputs before native execution if their public contract accepts those shapes
This crate does not select models, read catalogs, fetch provider prices, normalize multimodal usage, process provider-reported costs, or calculate non-token charges. It does not change proxy behavior. The reference fixture was generated by `tests/generate_python_reference.py` against the Python implementation at the commit recorded in `tests/python_reference.tsv`, using synthetic rates and fixed usage
This crate holds the price data model that `litellm-model-catalog` lowers each catalog entry into, plus the usage, call context and priced outcome types that `calculate` will compute over. There is no `calculate` until the token pricing PR lands. `UnitPrice` is USD per unit and `Usd` is an amount, and only the operations that make sense between them exist, so a price is never added to an amount. `Prices::band` and `Prices::rate` resolve a token rate the way Python's `_get_token_base_cost` does

66
litellm-rust/crates/cost/benches/calculate.rs
View file

@ -1,66 +0,0 @@
use criterion::{Criterion, criterion_group, criterion_main};
use litellm_cost::{
Pricing, PromptConvention, Rate, Rates, Request, ServiceTier, ThresholdPolicy, ThresholdRates,
Usage, calculate, compile,
};
use std::hint::black_box;
fn bench(c: &mut Criterion) {
let pricing = Pricing {
standard: Rates {
input: Rate::Value(0.000002),
output: Rate::Value(0.000008),
cache_read: Rate::Value(0.0000005),
cache_write: Rate::Missing,
cache_write_1h: Rate::Missing,
},
tiers: &[],
thresholds: &[],
off_peak: None,
};
let request = Request {
usage: Usage {
prompt_tokens: 1000,
completion_tokens: 200,
cache_read_tokens: 250,
cache_write_tokens: 0,
cache_write_5m_tokens: None,
cache_write_1h_tokens: None,
prompt_convention: PromptConvention::IncludesCache,
},
service_tier: ServiceTier::Standard,
threshold_policy: ThresholdPolicy::Exclusive,
region_multiplier: None,
billed_at_utc_minute: None,
};
let plan = compile(&pricing).unwrap();
c.bench_function("native_compiled_calculation", |b| {
b.iter(|| black_box(plan.calculate(black_box(&request)).unwrap()))
});
c.bench_function("native_full_wrapper", |b| {
b.iter(|| black_box(calculate(black_box(&pricing), black_box(&request)).unwrap()))
});
c.bench_function("native_rate_compilation", |b| {
b.iter(|| black_box(compile(black_box(&pricing)).unwrap()))
});
let threshold = ThresholdRates {
above_prompt_tokens: 1000,
standard: Rates {
input: Rate::Value(0.000004),
output: Rate::Value(0.000016),
..Rates::EMPTY
},
tiers: &[],
};
let threshold_pricing = Pricing {
thresholds: &[threshold],
..pricing
};
let threshold_plan = compile(&threshold_pricing).unwrap();
c.bench_function("native_threshold_boundary", |b| {
b.iter(|| black_box(threshold_plan.calculate(black_box(&request)).unwrap()))
});
}
criterion_group!(benches, bench);
criterion_main!(benches);

40
litellm-rust/crates/cost/examples/charge.rs
View file

@ -1,40 +0,0 @@
use litellm_cost::{
Pricing, PromptConvention, Rate, Rates, Request, ServiceTier, ThresholdPolicy, Usage, compile,
};
fn main() {
let pricing = Pricing {
standard: Rates {
input: Rate::Value(2.0),
output: Rate::Value(4.0),
cache_read: Rate::Value(0.5),
cache_write: Rate::Value(3.0),
cache_write_1h: Rate::Missing,
},
tiers: &[],
thresholds: &[],
off_peak: None,
};
let request = Request {
usage: Usage {
prompt_tokens: 100,
completion_tokens: 20,
cache_read_tokens: 25,
cache_write_tokens: 10,
cache_write_5m_tokens: None,
cache_write_1h_tokens: None,
prompt_convention: PromptConvention::IncludesCache,
},
service_tier: ServiceTier::Standard,
threshold_policy: ThresholdPolicy::Exclusive,
region_multiplier: None,
billed_at_utc_minute: None,
};
let cost = compile(&pricing).unwrap().calculate(&request).unwrap();
println!(
"input={} output={} total={}",
cost.input(),
cost.output(),
cost.total()
);
}

42
litellm-rust/crates/cost/src/call.rs Normal file
View file

@ -0,0 +1,42 @@
use crate::price::{Channel, Prices, Residency, ServiceTier};
/// Facts about the call, never about the row or the provider
#[derive(Clone, Debug, PartialEq)]
pub struct CallContext {
/// Requested service tier, `"fast"` maps to `Priority` as on main
pub tier: ServiceTier,
/// Online, batch or decisions
pub channel: Channel,
/// OpenAI data residency, picks the residency uplift
pub residency: Option<Residency>,
/// Vertex call sent to a regional endpoint, picks the regional uplift
pub non_global_endpoint: bool,
/// Anthropic `speed: fast`, picks the speed uplift
pub speed: Option<Speed>,
/// Anthropic and Bedrock `inference_geo`, picks the geo uplift
pub inference_geo: Option<InferenceGeo>,
/// When the call was billed, for off-peak windows
pub at: jiff::Timestamp,
/// Per-call custom pricing, lowered like a row and applied field by field over it
pub overlay: Option<Prices>,
}
/// Anthropic `speed`
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum Speed {
Fast,
}
/// Anthropic and Bedrock `inference_geo`, lowercased as on main
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct InferenceGeo(Box<str>);
impl InferenceGeo {
pub fn new(geo: &str) -> Self {
Self(geo.to_lowercase().into())
}
pub fn as_str(&self) -> &str {
&self.0
}
}

47
litellm-rust/crates/cost/src/error.rs
View file

@ -1,27 +1,24 @@
#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)]
use thiserror::Error;
#[derive(Clone, Copy, Debug, PartialEq, Eq, Error, strum::VariantArray)]
pub enum Error {
#[error("model has no input rate")]
MissingInputRate,
#[error("model has no output rate")]
MissingOutputRate,
#[error("rate must be finite and nonnegative")]
InvalidRate,
#[error("region multiplier must be finite and positive")]
InvalidRegionMultiplier,
#[error("off-peak pricing needs a billing minute within the day")]
InvalidBillingTime,
#[error("off-peak window must start before it ends within the day")]
InvalidOffPeakWindow,
#[error("cached tokens exceed prompt tokens")]
CacheExceedsPrompt,
#[error("cache write details do not add up to the cache write tokens")]
InvalidCacheWriteDetails,
#[error("token count or cost overflows")]
TokenCountOverflow,
#[error("service tier is repeated or cannot carry its own rates")]
DuplicateTier,
#[error("prompt token threshold is repeated")]
DuplicateThreshold,
#[error("service tier is repeated or cannot carry its own rates within a threshold")]
DuplicateThresholdTier,
#[error("expected a UTC window HH:MM-HH:MM")]
InvalidUtcWindow,
#[error("expected an ISO weekday number 1-7 or an English day name")]
InvalidWeekday,
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use super::Error;
use crate::schedule::{UtcWindow, Weekday};
#[rstest]
#[case::window("not-a-window".parse::<UtcWindow>().unwrap_err(), Error::InvalidUtcWindow)]
#[case::weekday("not-a-weekday".parse::<Weekday>().unwrap_err(), Error::InvalidWeekday)]
fn schedule_parsers_share_the_crate_error(#[case] error: Error, #[case] expected: Error) {
assert_eq!(error, expected);
}
}

388
litellm-rust/crates/cost/src/lib.rs
View file

@ -1,383 +1,9 @@
pub mod call;
mod error;
pub mod money;
pub mod price;
pub mod priced;
pub mod schedule;
pub mod usage;
pub use error::Error;
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum Rate {
Missing,
Null,
Value(f64),
}
impl Rate {
fn value(self) -> Option<f64> {
match self {
Self::Value(value) => Some(value),
Self::Missing | Self::Null => None,
}
}
fn or(self, fallback: Self) -> Self {
if self.value().is_some() {
self
} else {
fallback
}
}
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct Rates {
pub input: Rate,
pub output: Rate,
pub cache_read: Rate,
pub cache_write: Rate,
pub cache_write_1h: Rate,
}
impl Rates {
pub const EMPTY: Self = Self {
input: Rate::Missing,
output: Rate::Missing,
cache_read: Rate::Missing,
cache_write: Rate::Missing,
cache_write_1h: Rate::Missing,
};
fn overlay(self, base: Self) -> Self {
Self {
input: self.input.or(base.input),
output: self.output.or(base.output),
cache_read: self.cache_read.or(base.cache_read),
cache_write: self.cache_write.or(base.cache_write),
cache_write_1h: self.cache_write_1h.or(base.cache_write_1h),
}
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum ServiceTier {
Standard,
Flex,
Priority,
Fast,
Ultrafast,
Unknown,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum ThresholdPolicy {
Exclusive,
Inclusive,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum PromptConvention {
IncludesCache,
ExcludesCache,
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct Usage {
pub prompt_tokens: u64,
pub completion_tokens: u64,
pub cache_read_tokens: u64,
pub cache_write_tokens: u64,
pub cache_write_5m_tokens: Option<u64>,
pub cache_write_1h_tokens: Option<u64>,
pub prompt_convention: PromptConvention,
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct TierRates {
pub tier: ServiceTier,
pub rates: Rates,
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct ThresholdRates<'a> {
pub above_prompt_tokens: u64,
pub standard: Rates,
pub tiers: &'a [TierRates],
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct OffPeakRates {
pub start_utc_minute: u16,
pub end_utc_minute: u16,
pub rates: Rates,
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct Pricing<'a> {
pub standard: Rates,
pub tiers: &'a [TierRates],
pub thresholds: &'a [ThresholdRates<'a>],
pub off_peak: Option<OffPeakRates>,
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct Request {
pub usage: Usage,
pub service_tier: ServiceTier,
pub threshold_policy: ThresholdPolicy,
pub region_multiplier: Option<f64>,
pub billed_at_utc_minute: Option<u16>,
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct Cost {
pub uncached_input: f64,
pub cache_read: f64,
pub cache_write_5m: f64,
pub cache_write_1h: f64,
pub output: f64,
pub multiplier: f64,
pub rates: EffectiveRates,
}
impl Cost {
pub fn input(self) -> f64 {
(self.uncached_input + self.cache_read + self.cache_write_5m + self.cache_write_1h)
* self.multiplier
}
pub fn output(self) -> f64 {
self.output * self.multiplier
}
pub fn total(self) -> f64 {
self.input() + self.output()
}
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct EffectiveRates {
pub input: f64,
pub output: f64,
pub cache_read: f64,
pub cache_write_5m: f64,
pub cache_write_1h: f64,
}
fn selected_tier(tier: ServiceTier) -> ServiceTier {
if tier == ServiceTier::Fast {
ServiceTier::Priority
} else {
tier
}
}
#[derive(Clone, Debug)]
struct CompiledThreshold {
above_prompt_tokens: u64,
standard: Rates,
tiers: Vec<TierRates>,
}
#[derive(Clone, Debug)]
pub struct PricingPlan {
standard: Rates,
tiers: Vec<TierRates>,
thresholds: Vec<CompiledThreshold>,
off_peak: Option<OffPeakRates>,
}
fn valid_rates(rates: Rates) -> bool {
[
rates.input,
rates.output,
rates.cache_read,
rates.cache_write,
rates.cache_write_1h,
]
.into_iter()
.all(|rate| {
rate.value()
.is_none_or(|value| value.is_finite() && value >= 0.0)
})
}
fn validate_tiers(tiers: &[TierRates], duplicate: Error) -> Result<(), Error> {
if tiers.iter().any(|entry| !valid_rates(entry.rates)) {
return Err(Error::InvalidRate);
}
if tiers.iter().enumerate().any(|(index, entry)| {
matches!(
entry.tier,
ServiceTier::Standard | ServiceTier::Unknown | ServiceTier::Fast
) || tiers[..index]
.iter()
.any(|previous| previous.tier == entry.tier)
}) {
return Err(duplicate);
}
Ok(())
}
pub fn compile(pricing: &Pricing<'_>) -> Result<PricingPlan, Error> {
if !valid_rates(pricing.standard) {
return Err(Error::InvalidRate);
}
validate_tiers(pricing.tiers, Error::DuplicateTier)?;
if let Some(window) = pricing.off_peak {
if window.start_utc_minute >= 1440
|| window.end_utc_minute > 1440
|| window.start_utc_minute >= window.end_utc_minute
{
return Err(Error::InvalidOffPeakWindow);
}
if !valid_rates(window.rates) {
return Err(Error::InvalidRate);
}
}
let mut thresholds: Vec<_> = pricing
.thresholds
.iter()
.map(|entry| {
if !valid_rates(entry.standard) {
return Err(Error::InvalidRate);
}
validate_tiers(entry.tiers, Error::DuplicateThresholdTier)?;
Ok(CompiledThreshold {
above_prompt_tokens: entry.above_prompt_tokens,
standard: entry.standard,
tiers: entry.tiers.to_vec(),
})
})
.collect::<Result<_, _>>()?;
thresholds.sort_unstable_by_key(|entry| entry.above_prompt_tokens);
if thresholds
.windows(2)
.any(|pair| pair[0].above_prompt_tokens == pair[1].above_prompt_tokens)
{
return Err(Error::DuplicateThreshold);
}
Ok(PricingPlan {
standard: pricing.standard,
tiers: pricing.tiers.to_vec(),
thresholds,
off_peak: pricing.off_peak,
})
}
impl PricingPlan {
fn resolve_rates(&self, request: &Request, threshold_tokens: u64) -> Result<Rates, Error> {
let tier = selected_tier(request.service_tier);
let base = self
.tiers
.iter()
.find(|entry| tier != ServiceTier::Standard && entry.tier == tier)
.map_or(self.standard, |entry| entry.rates.overlay(self.standard));
let threshold = self.thresholds.iter().rev().find(|entry| {
threshold_tokens > entry.above_prompt_tokens
|| (request.threshold_policy == ThresholdPolicy::Inclusive
&& threshold_tokens == entry.above_prompt_tokens)
});
let selected = threshold.map_or(base, |entry| {
let standard = entry.standard.overlay(base);
entry
.tiers
.iter()
.find(|specific| tier != ServiceTier::Standard && specific.tier == tier)
.map_or(standard, |specific| specific.rates.overlay(standard))
});
match self.off_peak {
None => Ok(selected),
Some(window) => {
if window.start_utc_minute >= 1440
|| window.end_utc_minute > 1440
|| window.start_utc_minute >= window.end_utc_minute
{
return Err(Error::InvalidOffPeakWindow);
}
let minute = request
.billed_at_utc_minute
.ok_or(Error::InvalidBillingTime)?;
if minute >= 1440 {
return Err(Error::InvalidBillingTime);
}
if (window.start_utc_minute..window.end_utc_minute).contains(&minute) {
Ok(window.rates.overlay(selected))
} else {
Ok(selected)
}
}
}
}
fn checked_rate(rate: Rate, missing: Error) -> Result<f64, Error> {
let value = rate.value().ok_or(missing)?;
if !value.is_finite() || value < 0.0 {
return Err(Error::InvalidRate);
}
Ok(value)
}
pub fn calculate(&self, request: &Request) -> Result<Cost, Error> {
let usage = request.usage;
let cached = usage
.cache_read_tokens
.checked_add(usage.cache_write_tokens)
.ok_or(Error::TokenCountOverflow)?;
let (regular, threshold_tokens) = match usage.prompt_convention {
PromptConvention::IncludesCache => (
usage
.prompt_tokens
.checked_sub(cached)
.ok_or(Error::CacheExceedsPrompt)?,
usage.prompt_tokens,
),
PromptConvention::ExcludesCache => (
usage.prompt_tokens,
usage
.prompt_tokens
.checked_add(cached)
.ok_or(Error::TokenCountOverflow)?,
),
};
let writes = match (usage.cache_write_5m_tokens, usage.cache_write_1h_tokens) {
(None, None) => (usage.cache_write_tokens, 0),
(Some(five), Some(one)) if five.checked_add(one) == Some(usage.cache_write_tokens) => {
(five, one)
}
_ => return Err(Error::InvalidCacheWriteDetails),
};
let rates = self.resolve_rates(request, threshold_tokens)?;
let input = Self::checked_rate(rates.input, Error::MissingInputRate)?;
let output = Self::checked_rate(rates.output, Error::MissingOutputRate)?;
let read = Self::checked_rate(rates.cache_read.or(rates.input), Error::MissingInputRate)?;
let write = Self::checked_rate(rates.cache_write.or(rates.input), Error::MissingInputRate)?;
let write_1h = Self::checked_rate(
rates.cache_write_1h.or(rates.cache_write).or(rates.input),
Error::MissingInputRate,
)?;
let multiplier = request.region_multiplier.unwrap_or(1.0);
if !multiplier.is_finite() || multiplier <= 0.0 {
return Err(Error::InvalidRegionMultiplier);
}
let cost = Cost {
uncached_input: regular as f64 * input,
cache_read: usage.cache_read_tokens as f64 * read,
cache_write_5m: writes.0 as f64 * write,
cache_write_1h: writes.1 as f64 * write_1h,
output: usage.completion_tokens as f64 * output,
multiplier,
rates: EffectiveRates {
input,
output,
cache_read: read,
cache_write_5m: write,
cache_write_1h: write_1h,
},
};
if !cost.total().is_finite() {
return Err(Error::TokenCountOverflow);
}
Ok(cost)
}
}
pub fn calculate(pricing: &Pricing<'_>, request: &Request) -> Result<Cost, Error> {
compile(pricing)?.calculate(request)
}
pub use error::*;

257
litellm-rust/crates/cost/src/money.rs Normal file
View file

@ -0,0 +1,257 @@
use std::iter::Sum;
use std::ops::{Add, Mul};
use crate::priced::Quantity;
#[derive(Clone, Copy, Debug, PartialEq, PartialOrd)]
pub struct UnitPrice(f64);
#[derive(Clone, Copy, Debug, Default, PartialEq, PartialOrd)]
pub struct Usd(f64);
#[derive(Clone, Copy, Debug, PartialEq, PartialOrd)]
pub struct Multiplier(f64);
impl UnitPrice {
pub fn new(usd_per_unit: f64) -> Option<Self> {
(usd_per_unit.is_finite() && usd_per_unit >= 0.0).then_some(Self(usd_per_unit))
}
pub fn get(self) -> f64 {
self.0
}
}
impl Usd {
pub const ZERO: Self = Self(0.0);
pub fn new(amount: f64) -> Option<Self> {
(amount.is_finite() && amount >= 0.0).then_some(Self(amount))
}
pub fn get(self) -> f64 {
self.0
}
}
impl Multiplier {
pub fn new(factor: f64) -> Option<Self> {
(factor.is_finite() && factor >= 1.0).then_some(Self(factor))
}
pub fn get(self) -> f64 {
self.0
}
}
impl Mul<Quantity> for UnitPrice {
type Output = Usd;
fn mul(self, quantity: Quantity) -> Usd {
match quantity {
Quantity::Count(units) => Usd(self.0 * units as f64),
Quantity::Duration(duration) => Usd(self.0 * duration.as_secs_f64()),
}
}
}
impl Mul<Multiplier> for UnitPrice {
type Output = UnitPrice;
fn mul(self, factor: Multiplier) -> UnitPrice {
UnitPrice(self.0 * factor.0)
}
}
impl Mul<Multiplier> for Usd {
type Output = Usd;
fn mul(self, factor: Multiplier) -> Usd {
Usd(self.0 * factor.0)
}
}
impl Add for Usd {
type Output = Usd;
fn add(self, other: Usd) -> Usd {
Usd(self.0 + other.0)
}
}
impl Sum for Usd {
fn sum<I: Iterator<Item = Usd>>(amounts: I) -> Usd {
amounts.fold(Usd::ZERO, Add::add)
}
}
#[cfg(feature = "serde")]
mod serde_impls {
use std::fmt;
use serde::de::{self, Deserialize, Deserializer, Visitor};
use super::{Multiplier, UnitPrice};
#[derive(Clone, Copy)]
struct Number(&'static str);
impl Visitor<'_> for Number {
type Value = f64;
fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str(self.0)
}
fn visit_i64<E: de::Error>(self, value: i64) -> Result<f64, E> {
Ok(value as f64)
}
fn visit_u64<E: de::Error>(self, value: u64) -> Result<f64, E> {
Ok(value as f64)
}
fn visit_f64<E: de::Error>(self, value: f64) -> Result<f64, E> {
Ok(value)
}
fn visit_str<E: de::Error>(self, value: &str) -> Result<f64, E> {
value
.trim()
.parse()
.map_err(|_| E::invalid_value(de::Unexpected::Str(value), &self))
}
}
fn checked<'de, D: Deserializer<'de>, T>(
deserializer: D,
expected: Number,
new: fn(f64) -> Option<T>,
) -> Result<T, D::Error> {
let number = deserializer.deserialize_any(expected)?;
new(number)
.ok_or_else(|| de::Error::invalid_value(de::Unexpected::Float(number), &expected))
}
impl<'de> Deserialize<'de> for UnitPrice {
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
checked(deserializer, Number("a non-negative USD amount"), Self::new)
}
}
impl<'de> Deserialize<'de> for Multiplier {
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
checked(
deserializer,
Number("a multiplier of at least 1"),
Self::new,
)
}
}
}
#[cfg(feature = "schema")]
mod schema {
use std::borrow::Cow;
use schemars::{JsonSchema, Schema, SchemaGenerator, json_schema};
use super::{Multiplier, UnitPrice};
impl JsonSchema for UnitPrice {
fn inline_schema() -> bool {
true
}
fn schema_name() -> Cow<'static, str> {
"UnitPrice".into()
}
fn json_schema(_: &mut SchemaGenerator) -> Schema {
json_schema!({"type": "number", "minimum": 0})
}
}
impl JsonSchema for Multiplier {
fn inline_schema() -> bool {
true
}
fn schema_name() -> Cow<'static, str> {
"Multiplier".into()
}
fn json_schema(_: &mut SchemaGenerator) -> Schema {
json_schema!({"type": "number", "minimum": 1})
}
}
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use super::*;
#[rstest]
#[case::price(0.5)]
#[case::free(0.0)]
fn unit_price_keeps_finite_non_negative_values(#[case] value: f64) {
assert_eq!(UnitPrice::new(value).map(UnitPrice::get), Some(value));
}
#[rstest]
#[case::negative(-1e-9)]
#[case::nan(f64::NAN)]
#[case::infinite(f64::INFINITY)]
fn unit_price_rejects_what_no_price_can_be(#[case] value: f64) {
assert_eq!(UnitPrice::new(value), None);
}
#[rstest]
#[case::uplift(1.1, true)]
#[case::neutral(1.0, true)]
#[case::discount(0.5, false)]
#[case::nan(f64::NAN, false)]
fn multiplier_is_never_a_discount(#[case] value: f64, #[case] accepted: bool) {
assert_eq!(Multiplier::new(value).is_some(), accepted);
}
#[rstest]
fn amounts_follow_units_times_price_then_uplift() {
let price = UnitPrice::new(0.25).unwrap();
let uplift = Multiplier::new(2.0).unwrap();
let total: Usd = [
price * Quantity::Count(4),
(price * uplift) * Quantity::Count(2),
]
.into_iter()
.sum();
assert_eq!(total.get(), 2.0);
assert_eq!((total * uplift).get(), 4.0);
}
#[rstest]
fn a_duration_is_billed_in_fractional_seconds() {
let per_second = UnitPrice::new(0.01).unwrap();
let amount = per_second * Quantity::Duration(std::time::Duration::from_millis(2_500));
assert!((amount.get() - 0.025).abs() < 1e-12);
}
#[cfg(feature = "serde")]
#[rstest]
#[case::number(serde_json::json!(1.5e-6), Some(1.5e-6))]
#[case::integer(serde_json::json!(2), Some(2.0))]
#[case::numeric_string(serde_json::json!(" 0.000002 "), Some(2e-6))]
#[case::negative(serde_json::json!(-1), None)]
#[case::word(serde_json::json!("free"), None)]
#[case::boolean(serde_json::json!(true), None)]
fn unit_price_reads_numbers_and_numeric_strings(
#[case] written: serde_json::Value,
#[case] expected: Option<f64>,
) {
use serde::Deserialize;
assert_eq!(
UnitPrice::deserialize(&written).ok().map(UnitPrice::get),
expected
);
}
}

505
litellm-rust/crates/cost/src/price.rs Normal file
View file

@ -0,0 +1,505 @@
mod meter;
use std::collections::BTreeMap;
use std::num::NonZeroU64;
pub use meter::*;
use crate::call::{InferenceGeo, Speed};
use crate::money::{Multiplier, UnitPrice};
use crate::schedule::Schedule;
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum ServiceTier {
#[default]
Standard,
Priority,
Flex,
Ultrafast,
Balanced,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum Band {
#[default]
Base,
OverPromptTokens(NonZeroU64),
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct When {
pub tier: ServiceTier,
pub band: Band,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum Channel {
#[default]
Online,
Batch,
Decisions,
}
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct Price {
pub meter: Meter,
pub channel: Channel,
pub when: When,
}
#[derive(Clone, Debug, Default, PartialEq)]
pub struct Prices {
rates: BTreeMap<Price, UnitPrice>,
off_peak: Option<OffPeak>,
billing: Billing,
}
/// Row attributes pricing reads, lowered by model-catalog from the row and its provider
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct Billing {
/// Unit the row bills text in. `Character` only when the row prices characters and not tokens
pub text_unit: TextUnit,
/// Whether a band's floor is in the band. xAI rows bill the band from the floor itself
pub band_edge: BandEdge,
/// Rate a cache read borrows when the row has no cache-read price. Fireworks rows borrow half the input rate
pub cache_read_fallback: CacheReadFallback,
/// Whether a row with no token price is priced by wall clock, from its `mode`
pub wall_clock: WallClock,
/// From `web_search_billing_unit`, whether web search and Maps grounding bill per query or once per prompt
pub web_search: WebSearchUnit,
/// Whether `tiered_pricing` bills each meter in its own tier or the whole call at the tier the input picks.
/// Dashscope rows bill the whole call (inferred, 10 checks no other row has that shape)
pub tier_scope: TierScope,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum TextUnit {
#[default]
Token,
Character,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum BandEdge {
#[default]
Exclusive,
Inclusive,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum CacheReadFallback {
#[default]
InputRate,
HalfInputRate,
}
/// `ResponseTime` for `chat`, `completion`, `embedding`, `responses` or no mode, `AudioFile` for
/// `audio_transcription`, which prices the input audio when it has one, `Never` for every other mode
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum WallClock {
Never,
#[default]
ResponseTime,
AudioFile,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum WebSearchUnit {
PerQuery,
#[default]
PerPrompt,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum TierScope {
#[default]
PerMeter,
WholeCall,
}
#[derive(Clone, Debug, PartialEq)]
pub struct OffPeak {
schedule: Schedule,
rates: OffPeakRates,
}
#[derive(Clone, Copy, Debug, Default, PartialEq)]
pub struct OffPeakRates {
pub input: Option<UnitPrice>,
pub output: Option<UnitPrice>,
pub cache_read: Option<UnitPrice>,
pub explicit_cache_read: Option<UnitPrice>,
pub cache_write: Option<UnitPrice>,
pub reasoning: Option<UnitPrice>,
}
#[derive(Clone, Debug, Default, PartialEq)]
pub struct Adjustments(BTreeMap<Uplift, Multiplier>);
/// Which call fact picks which multiplier in `Adjustments`, lowered by model-catalog from the row
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum Uplift {
/// OpenAI data residency
Residency(Residency),
/// Vertex non-global endpoint, from `regional_endpoint_uplift_multiplier`
NonGlobalEndpoint,
/// Anthropic `speed: fast`, from `provider_specific_entry.fast`
Speed(Speed),
/// Anthropic and Bedrock geo, from `provider_specific_entry.<geo>`, lowercased as on main
InferenceGeo(InferenceGeo),
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum Residency {
Eu,
Us,
}
impl Band {
pub fn over_prompt_tokens(tokens: u64) -> Self {
NonZeroU64::new(tokens).map_or(Self::Base, Self::OverPromptTokens)
}
}
impl Price {
pub fn online(meter: Meter, when: When) -> Self {
Self {
meter,
channel: Channel::Online,
when,
}
}
}
impl Prices {
pub fn new(
rates: BTreeMap<Price, UnitPrice>,
off_peak: Option<OffPeak>,
billing: Billing,
) -> Self {
Self {
rates,
off_peak,
billing,
}
}
pub fn get(&self, price: &Price) -> Option<UnitPrice> {
self.rates.get(price).copied()
}
pub fn iter(&self) -> impl Iterator<Item = (&Price, UnitPrice)> {
self.rates.iter().map(|(price, rate)| (price, *rate))
}
pub fn off_peak(&self) -> Option<&OffPeak> {
self.off_peak.as_ref()
}
pub fn billing(&self) -> Billing {
self.billing
}
/// The online prompt band a request falls in. Every token meter's `_above_` prices count,
/// and the highest band the prompt passes wins. `billing.band_edge` says whether the floor is in the band.
pub fn band(&self, tier: ServiceTier, prompt_tokens: u64) -> Band {
let inclusive = self.billing.band_edge == BandEdge::Inclusive;
self.rates
.keys()
.filter(|price| {
price.channel == Channel::Online
&& price.meter.sets_prompt_band()
&& (price.when.tier == ServiceTier::Standard || price.when.tier == tier)
})
.filter_map(|price| match price.when.band {
Band::OverPromptTokens(floor) => Some(floor.get()),
Band::Base => None,
})
.filter(|floor| prompt_tokens > *floor || (inclusive && prompt_tokens == *floor))
.max()
.map_or(Band::Base, Band::over_prompt_tokens)
}
/// The online rate for `meter`, tried at `(tier, band)`, `(standard, band)`, `(tier, base)`
/// and `(standard, base)` in that order. A missing cache price falls back to the input price
/// resolved the same way.
pub fn rate(&self, meter: &Meter, tier: ServiceTier, band: Band) -> Option<UnitPrice> {
[
When { tier, band },
When {
tier: ServiceTier::Standard,
band,
},
When {
tier,
band: Band::Base,
},
When::default(),
]
.into_iter()
.find_map(|when| self.get(&Price::online(meter.clone(), when)))
.or_else(|| self.rate(&meter.fallback()?, tier, band))
}
}
impl OffPeak {
pub fn new(schedule: Schedule, rates: OffPeakRates) -> Option<Self> {
(rates != OffPeakRates::default()).then_some(Self { schedule, rates })
}
pub fn schedule(&self) -> &Schedule {
&self.schedule
}
pub fn rates(&self) -> &OffPeakRates {
&self.rates
}
}
impl Adjustments {
pub fn new(uplifts: impl IntoIterator<Item = (Uplift, Multiplier)>) -> Self {
Self(uplifts.into_iter().collect())
}
pub fn get(&self, uplift: &Uplift) -> Option<Multiplier> {
self.0.get(uplift).copied()
}
}
#[cfg(test)]
mod tests {
use rstest::{fixture, rstest};
use super::*;
const INPUT: Meter = Meter::TEXT_INPUT;
const OUTPUT: Meter = Meter::Output(OutputUnit::Token(TokenKind::Text));
const CACHE_READ: Meter = Meter::CacheRead {
kind: TokenKind::Text,
mode: CacheReadMode::Implicit,
};
const CACHE_WRITE_HOUR: Meter = Meter::CacheWrite {
kind: TokenKind::Text,
ttl: CacheTtl::OneHour,
};
fn over(tokens: u64) -> Band {
Band::over_prompt_tokens(tokens)
}
fn prices(rates: &[(Meter, Channel, ServiceTier, Band, f64)]) -> Prices {
priced_with(rates, Billing::default())
}
fn priced_with(rates: &[(Meter, Channel, ServiceTier, Band, f64)], billing: Billing) -> Prices {
Prices::new(
rates
.iter()
.map(|(meter, channel, tier, band, rate)| {
(
Price {
meter: meter.clone(),
channel: *channel,
when: When {
tier: *tier,
band: *band,
},
},
UnitPrice::new(*rate).unwrap(),
)
})
.collect(),
None,
billing,
)
}
#[fixture]
fn priority_with_a_standard_band() -> Prices {
use Channel::Online;
use ServiceTier::{Priority, Standard};
prices(&[
(INPUT, Online, Standard, Band::Base, 1.0),
(INPUT, Online, Priority, Band::Base, 2.0),
(INPUT, Online, Standard, over(200_000), 10.0),
(OUTPUT, Online, Standard, Band::Base, 3.0),
(OUTPUT, Online, Priority, over(200_000), 30.0),
])
}
#[rstest]
#[case::below(1_000, false, Band::Base)]
#[case::at_floor_exclusive(200_000, false, Band::Base)]
#[case::at_floor_inclusive(200_000, true, over(200_000))]
#[case::above(200_001, false, over(200_000))]
fn the_band_starts_strictly_above_its_floor_unless_inclusive(
#[case] prompt_tokens: u64,
#[case] inclusive: bool,
#[case] expected: Band,
) {
let band_edge = if inclusive {
BandEdge::Inclusive
} else {
BandEdge::Exclusive
};
let prices = priced_with(
&[(
INPUT,
Channel::Online,
ServiceTier::Standard,
over(200_000),
10.0,
)],
Billing {
band_edge,
..Billing::default()
},
);
assert_eq!(prices.band(ServiceTier::Standard, prompt_tokens), expected);
}
#[rstest]
fn a_band_only_another_tier_prices_does_not_apply() {
let prices = prices(&[(
OUTPUT,
Channel::Online,
ServiceTier::Flex,
over(100_000),
1.0,
)]);
assert_eq!(prices.band(ServiceTier::Standard, 300_000), Band::Base);
assert_eq!(prices.band(ServiceTier::Flex, 300_000), over(100_000));
}
#[rstest]
fn batch_bands_never_pick_the_online_band() {
let prices = prices(&[(
INPUT,
Channel::Batch,
ServiceTier::Standard,
over(100_000),
1.0,
)]);
assert_eq!(prices.band(ServiceTier::Standard, 300_000), Band::Base);
}
#[rstest]
fn the_highest_band_passed_wins() {
let prices = prices(&[
(
INPUT,
Channel::Online,
ServiceTier::Standard,
over(100_000),
1.0,
),
(
OUTPUT,
Channel::Online,
ServiceTier::Standard,
over(200_000),
1.0,
),
]);
assert_eq!(prices.band(ServiceTier::Standard, 300_000), over(200_000));
}
#[rstest]
#[case::tier_base(INPUT, ServiceTier::Priority, Band::Base, Some(2.0))]
#[case::standard_band_beats_tier_base(INPUT, ServiceTier::Priority, over(200_000), Some(10.0))]
#[case::tier_band(OUTPUT, ServiceTier::Priority, over(200_000), Some(30.0))]
#[case::standard_base_when_nothing_else(OUTPUT, ServiceTier::Flex, over(200_000), Some(3.0))]
#[case::cache_read_falls_back_to_the_resolved_input(
CACHE_READ,
ServiceTier::Priority,
over(200_000),
Some(10.0)
)]
#[case::one_hour_write_falls_back_through_write_to_input(
CACHE_WRITE_HOUR,
ServiceTier::Priority,
Band::Base,
Some(2.0)
)]
#[case::twelve_hour_write_falls_back_through_write_to_input(
Meter::CacheWrite { kind: TokenKind::Text, ttl: CacheTtl::TwelveHours },
ServiceTier::Standard,
Band::Base,
Some(1.0)
)]
#[case::no_fallback_for_other_meters(
Meter::Input(InputUnit::Token(TokenKind::Audio)),
ServiceTier::Standard,
Band::Base,
None
)]
fn rate_walks_tier_then_band(
priority_with_a_standard_band: Prices,
#[case] meter: Meter,
#[case] tier: ServiceTier,
#[case] band: Band,
#[case] expected: Option<f64>,
) {
assert_eq!(
priority_with_a_standard_band
.rate(&meter, tier, band)
.map(UnitPrice::get),
expected
);
}
#[rstest]
fn an_explicit_cache_price_wins_over_the_input_fallback() {
let prices = prices(&[
(
INPUT,
Channel::Online,
ServiceTier::Standard,
Band::Base,
1.0,
),
(
CACHE_READ,
Channel::Online,
ServiceTier::Standard,
Band::Base,
0.0,
),
]);
assert_eq!(
prices
.rate(&CACHE_READ, ServiceTier::Standard, Band::Base)
.map(UnitPrice::get),
Some(0.0)
);
}
#[rstest]
fn off_peak_needs_a_price() {
let schedule = Schedule {
windows: Vec::new(),
weekday_timezone: None,
};
assert_eq!(
OffPeak::new(schedule.clone(), OffPeakRates::default()),
None
);
assert!(
OffPeak::new(
schedule,
OffPeakRates {
output: UnitPrice::new(1.0),
..OffPeakRates::default()
}
)
.is_some()
);
}
#[rstest]
#[case::zero(0, Band::Base)]
#[case::positive(272_000, Band::OverPromptTokens(NonZeroU64::new(272_000).unwrap()))]
fn a_zero_floor_is_the_base_band(#[case] tokens: u64, #[case] expected: Band) {
assert_eq!(Band::over_prompt_tokens(tokens), expected);
}
}

197
litellm-rust/crates/cost/src/price/meter.rs Normal file
View file

@ -0,0 +1,197 @@
use std::num::NonZeroU64;
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum Meter {
Input(InputUnit),
Output(OutputUnit),
CacheRead {
kind: TokenKind,
mode: CacheReadMode,
},
CacheWrite {
kind: TokenKind,
ttl: CacheTtl,
},
Feature(Feature),
WallClockSecond,
WebSearch(SearchContextSize),
Guardrail(GuardrailUnit),
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum TokenKind {
Text,
Audio,
Image,
Video,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum InputUnit {
Token(TokenKind),
Character,
Image,
Pixel,
Query { over_results: Option<NonZeroU64> },
Request,
Second,
TextMessage,
AudioSecond,
VideoSecond { over_seconds: Option<NonZeroU64> },
}
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum OutputUnit {
Token(TokenKind),
ReasoningToken,
Character,
Image(Option<ImageSize>),
Pixel,
Second(Option<Resolution>),
VideoSecond,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum CacheReadMode {
Implicit,
Explicit,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum CacheTtl {
Default,
OneHour,
TwelveHours,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum Feature {
AnnotationPage,
CitationToken,
CodeInterpreterSession,
GoogleMapsGroundingQuery,
OcrCredit,
OcrPage,
/// `browser_open_cost_per_request`, Groq `browser.open`
BrowserOpen,
/// `cost_per_additional_result`, Parallel AI results past the ones the query price includes
AdditionalResult,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum SearchContextSize {
Low,
Medium,
High,
}
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct GuardrailUnit(Box<str>);
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct ImageSize(Box<str>);
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct Resolution(Box<str>);
impl Meter {
pub(crate) const TEXT_INPUT: Self = Self::Input(InputUnit::Token(TokenKind::Text));
pub(crate) fn sets_prompt_band(&self) -> bool {
matches!(
self,
Self::Input(InputUnit::Token(TokenKind::Text))
| Self::Output(OutputUnit::Token(TokenKind::Text))
| Self::CacheRead {
kind: TokenKind::Text,
mode: CacheReadMode::Implicit,
}
| Self::CacheWrite {
kind: TokenKind::Text,
..
}
)
}
pub(crate) fn fallback(&self) -> Option<Self> {
match self {
Self::CacheRead {
kind: TokenKind::Text,
mode: CacheReadMode::Implicit,
}
| Self::CacheWrite {
kind: TokenKind::Text,
ttl: CacheTtl::Default,
} => Some(Self::TEXT_INPUT),
Self::CacheWrite {
kind: TokenKind::Text,
ttl: CacheTtl::OneHour | CacheTtl::TwelveHours,
} => Some(Self::CacheWrite {
kind: TokenKind::Text,
ttl: CacheTtl::Default,
}),
_ => None,
}
}
}
impl GuardrailUnit {
pub fn new(counter: &str) -> Self {
Self(counter.into())
}
pub fn as_str(&self) -> &str {
&self.0
}
}
impl ImageSize {
pub fn new(label: &str) -> Option<Self> {
let number = label.strip_suffix('K').unwrap_or(label);
let (whole, fraction) = number.split_once('.').unwrap_or((number, "0"));
(is_digits(whole) && is_digits(fraction)).then(|| Self(label.into()))
}
pub fn as_str(&self) -> &str {
&self.0
}
}
impl Resolution {
pub fn new(label: &str) -> Option<Self> {
let digits = label.strip_suffix(['p', 'k'])?;
is_digits(digits).then(|| Self(label.into()))
}
pub fn as_str(&self) -> &str {
&self.0
}
}
fn is_digits(value: &str) -> bool {
!value.is_empty() && value.bytes().all(|byte| byte.is_ascii_digit())
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use super::*;
#[rstest]
#[case::kilo("1K", true)]
#[case::half_kilo("0.5K", true)]
#[case::pixels("1024", true)]
#[case::word("big", false)]
#[case::bare_suffix("K", false)]
#[case::trailing_dot("1.K", false)]
fn image_size_labels_are_numbers_with_an_optional_k(#[case] label: &str, #[case] valid: bool) {
assert_eq!(ImageSize::new(label).is_some(), valid);
}
#[rstest]
#[case::progressive("720p", true)]
#[case::kilo("4k", true)]
#[case::bare("720", false)]
#[case::word("hd", false)]
fn resolution_labels_are_numbers_with_p_or_k(#[case] label: &str, #[case] valid: bool) {
assert_eq!(Resolution::new(label).is_some(), valid);
}
}

53
litellm-rust/crates/cost/src/priced.rs Normal file
View file

@ -0,0 +1,53 @@
use std::time::Duration;
use crate::money::{UnitPrice, Usd};
use crate::price::Meter;
#[derive(Clone, Debug, PartialEq)]
pub enum Outcome {
Priced(Priced),
/// No meter with usage had a price
Unpriced {
missing: Vec<Meter>,
},
}
#[derive(Clone, Debug, PartialEq)]
pub struct Priced {
/// One line per billed meter, fallbacks included
pub lines: Vec<Line>,
/// Meters with usage and no price and no fallback
pub missing: Vec<Meter>,
/// Sum of `lines`
pub total: Usd,
}
#[derive(Clone, Debug, PartialEq)]
pub struct Line {
/// The meter this line bills
pub meter: Meter,
/// How much of the meter the call used
pub quantity: Quantity,
/// Rate after off-peak and before uplifts
pub unit_price: UnitPrice,
/// `unit_price * quantity` times every uplift that applies
pub amount: Usd,
/// The meter whose rate was borrowed, when this meter had none
pub fallback_from: Option<Meter>,
/// Which half of main's `(prompt, completion)` pair the line belongs to
pub side: Side,
}
/// A billed amount of one meter
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Quantity {
Count(u64),
Duration(Duration),
}
/// Meters main bills as one total (video, Replicate, Amazon Transcribe) take a fixed side, chosen in 09
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Side {
Prompt,
Completion,
}

241
litellm-rust/crates/cost/src/schedule.rs Normal file
View file

@ -0,0 +1,241 @@
use std::str::FromStr;
use crate::Error;
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Schedule {
pub windows: Vec<Window>,
pub weekday_timezone: Option<String>,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Window {
pub hours: Vec<UtcWindow>,
pub weekdays: Option<Vec<Weekday>>,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct UtcWindow {
pub start_minute: u16,
pub end_minute: u16,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum Weekday {
Monday,
Tuesday,
Wednesday,
Thursday,
Friday,
Saturday,
Sunday,
}
impl FromStr for UtcWindow {
type Err = Error;
fn from_str(window: &str) -> Result<Self, Self::Err> {
let (start, end) = window.split_once('-').ok_or(Error::InvalidUtcWindow)?;
Ok(Self {
start_minute: minute_of_day(start).ok_or(Error::InvalidUtcWindow)?,
end_minute: minute_of_day(end).ok_or(Error::InvalidUtcWindow)?,
})
}
}
fn minute_of_day(clock: &str) -> Option<u16> {
let (hour, minute) = clock.split_once(':')?;
if hour.len() != 2 || minute.len() != 2 {
return None;
}
let hour: u16 = hour.parse().ok().filter(|hour| *hour < 24)?;
let minute: u16 = minute.parse().ok().filter(|minute| *minute < 60)?;
Some(hour * 60 + minute)
}
impl Weekday {
pub fn from_iso(number: u8) -> Option<Self> {
match number {
1 => Some(Self::Monday),
2 => Some(Self::Tuesday),
3 => Some(Self::Wednesday),
4 => Some(Self::Thursday),
5 => Some(Self::Friday),
6 => Some(Self::Saturday),
7 => Some(Self::Sunday),
_ => None,
}
}
}
impl FromStr for Weekday {
type Err = Error;
fn from_str(name: &str) -> Result<Self, Self::Err> {
match name.trim().to_ascii_lowercase().as_str() {
"mon" | "monday" => Ok(Self::Monday),
"tue" | "tues" | "tuesday" => Ok(Self::Tuesday),
"wed" | "wednesday" => Ok(Self::Wednesday),
"thu" | "thur" | "thurs" | "thursday" => Ok(Self::Thursday),
"fri" | "friday" => Ok(Self::Friday),
"sat" | "saturday" => Ok(Self::Saturday),
"sun" | "sunday" => Ok(Self::Sunday),
_ => Err(Error::InvalidWeekday),
}
}
}
#[cfg(feature = "serde")]
mod serde_impls {
use std::fmt;
use serde::de::{self, Deserialize, Deserializer, Visitor};
use super::{UtcWindow, Weekday};
impl<'de> Deserialize<'de> for UtcWindow {
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
let window = String::deserialize(deserializer)?;
window.parse().map_err(de::Error::custom)
}
}
struct WrittenWeekday;
impl Visitor<'_> for WrittenWeekday {
type Value = Weekday;
fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("an ISO weekday number 1-7 or an English day name")
}
fn visit_u64<E: de::Error>(self, number: u64) -> Result<Weekday, E> {
u8::try_from(number)
.ok()
.and_then(Weekday::from_iso)
.ok_or_else(|| E::invalid_value(de::Unexpected::Unsigned(number), &self))
}
fn visit_i64<E: de::Error>(self, number: i64) -> Result<Weekday, E> {
u64::try_from(number)
.map_err(|_| E::invalid_value(de::Unexpected::Signed(number), &self))
.and_then(|number| self.visit_u64(number))
}
fn visit_str<E: de::Error>(self, name: &str) -> Result<Weekday, E> {
name.parse()
.map_err(|_| E::invalid_value(de::Unexpected::Str(name), &self))
}
}
impl<'de> Deserialize<'de> for Weekday {
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
deserializer.deserialize_any(WrittenWeekday)
}
}
}
#[cfg(feature = "schema")]
mod schema {
use std::borrow::Cow;
use schemars::{JsonSchema, Schema, SchemaGenerator, json_schema};
use super::{UtcWindow, Weekday};
impl JsonSchema for UtcWindow {
fn inline_schema() -> bool {
true
}
fn schema_name() -> Cow<'static, str> {
"UtcWindow".into()
}
fn json_schema(_: &mut SchemaGenerator) -> Schema {
json_schema!({
"type": "string",
"pattern": r"^([01]\d|2[0-3]):[0-5]\d-([01]\d|2[0-3]):[0-5]\d$"
})
}
}
impl JsonSchema for Weekday {
fn inline_schema() -> bool {
true
}
fn schema_name() -> Cow<'static, str> {
"Weekday".into()
}
fn json_schema(_: &mut SchemaGenerator) -> Schema {
json_schema!({
"anyOf": [
{"type": "integer", "minimum": 1, "maximum": 7},
{
"type": "string",
"pattern": r"(?i)^\s*(mon|monday|tue|tues|tuesday|wed|wednesday|thu|thur|thurs|thursday|fri|friday|sat|saturday|sun|sunday)\s*$"
}
]
})
}
}
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use super::*;
#[rstest]
#[case::same_day("10:00-18:30", Some((600, 1110)))]
#[case::wraps_midnight("22:00-02:00", Some((1320, 120)))]
#[case::whole_day("00:00-00:00", Some((0, 0)))]
#[case::hour_out_of_range("25:00-01:00", None)]
#[case::minute_out_of_range("10:60-11:00", None)]
#[case::one_digit_hour("9:00-10:00", None)]
#[case::no_separator("10:00", None)]
fn utc_window_reads_hh_mm_ranges(#[case] text: &str, #[case] expected: Option<(u16, u16)>) {
assert_eq!(
text.parse::<UtcWindow>()
.ok()
.map(|window| (window.start_minute, window.end_minute)),
expected
);
}
#[rstest]
#[case::short("thurs", Some(Weekday::Thursday))]
#[case::upper("SATURDAY", Some(Weekday::Saturday))]
#[case::padded(" mon ", Some(Weekday::Monday))]
#[case::unknown("someday", None)]
#[case::empty("", None)]
fn weekday_names_match_python_after_trimming(
#[case] name: &str,
#[case] expected: Option<Weekday>,
) {
assert_eq!(name.parse().ok(), expected);
}
#[rstest]
#[case::monday(1, Some(Weekday::Monday))]
#[case::sunday(7, Some(Weekday::Sunday))]
#[case::zero(0, None)]
#[case::eight(8, None)]
fn weekday_numbers_are_iso(#[case] number: u8, #[case] expected: Option<Weekday>) {
assert_eq!(Weekday::from_iso(number), expected);
}
#[cfg(feature = "serde")]
#[rstest]
#[case::number(serde_json::json!(6), Some(Weekday::Saturday))]
#[case::name(serde_json::json!("Sun"), Some(Weekday::Sunday))]
#[case::negative(serde_json::json!(-1), None)]
#[case::boolean(serde_json::json!(true), None)]
fn weekday_reads_numbers_and_names(
#[case] written: serde_json::Value,
#[case] expected: Option<Weekday>,
) {
use serde::Deserialize;
assert_eq!(Weekday::deserialize(&written).ok(), expected);
}
}

104
litellm-rust/crates/cost/src/usage.rs Normal file
View file

@ -0,0 +1,104 @@
use std::collections::BTreeMap;
use std::time::Duration;
use crate::price::{CacheReadMode, CacheTtl, ImageSize, Resolution, SearchContextSize, TokenKind};
/// What one call consumed. Python fills it from the provider response, `calculate` never reads the catalog for it
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct Usage {
/// Prompt size as the provider reported it. Only picks `_above_Nk_tokens` bands, never billed.
/// Filled even when the call bills characters, `0` when the provider reports none
pub prompt_tokens: u64,
/// Billed token quantities by meter
pub tokens: Tokens,
/// Billed character quantities. Used only when the row's `billing.text_unit` is `Character`
pub characters: Characters,
/// Billed time per second meter, never a bare number of seconds or milliseconds
pub durations: BTreeMap<DurationMeter, Duration>,
/// Images sent and returned
pub images: Images,
/// Search queries, web searches and grounding requests
pub search: Search,
/// Realtime text messages, xAI bills them per message
pub text_messages: u64,
}
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct Tokens {
/// Uncached prompt tokens by modality
pub input: BTreeMap<TokenKind, u64>,
/// Prompt tokens served from cache, by modality and implicit or explicit cache
pub cache_read: BTreeMap<(TokenKind, CacheReadMode), u64>,
/// Prompt tokens written to cache, by modality and cache lifetime
pub cache_write: BTreeMap<(TokenKind, CacheTtl), u64>,
/// Completion tokens by modality, reasoning excluded
pub output: BTreeMap<TokenKind, u64>,
/// Reasoning tokens, billed at the reasoning rate or the text output rate
pub reasoning: u64,
/// Perplexity citation tokens, prompt side
pub citation: u64,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct Characters {
/// Characters of the formatted prompt
pub prompt: u64,
/// Characters of the response text
pub completion: u64,
}
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum DurationMeter {
/// Response time of the call, for rows priced by wall clock
WallClock,
/// Length of the input audio, a transcription file or a realtime transcription
InputAudio,
/// Realtime voice session time
LiveVoice,
/// Length of generated video, already multiplied by the video count. `resolution` picks `output_cost_per_second_<resolution>`
OutputVideo { resolution: Option<Resolution> },
}
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct Images {
/// Images sent in the request
pub input: u64,
/// Pixels sent in the request, for rows priced per input pixel
pub input_pixels: u64,
/// One entry per returned image, since price can depend on each image's size
pub output: Vec<ImageOut>,
}
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct ImageOut {
/// Size the request asked for, picks a size keyed price like fal's `output_cost_per_image_<size>`
pub size: Option<ImageSize>,
/// Pixel width the response reported, for per-pixel and nearest-size pricing
pub width: Option<u32>,
/// Pixel height the response reported, for per-pixel and nearest-size pricing
pub height: Option<u32>,
}
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct Search {
/// Search API queries, for `input_cost_per_query` rows
pub queries: u64,
/// Results the query asked for, picks the `max_results_range` tier
pub max_results: Option<u64>,
/// Web searches a model ran as a tool
pub web_search: Option<WebSearch>,
/// Google Maps grounding requests
pub maps_grounding: u64,
/// Groq `browser.open` calls
pub browser_opens: u64,
/// Results past the ones the query price includes. Python counts them, since the included 10 is a request default
pub additional_results: u64,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct WebSearch {
/// Web search requests the model made
pub requests: u64,
/// Context size the request asked for, picks `search_context_size_<size>`
pub context: SearchContextSize,
}

535
litellm-rust/crates/cost/tests/calculation.rs
View file

@ -1,535 +0,0 @@
use litellm_cost::{
Error, OffPeakRates, Pricing, PromptConvention, Rate, Rates, Request, ServiceTier,
ThresholdPolicy, ThresholdRates, TierRates, Usage, calculate, compile,
};
use rstest::rstest;
fn rates(input: Rate, output: Rate) -> Rates {
Rates {
input,
output,
..Rates::EMPTY
}
}
fn request() -> Request {
Request {
usage: Usage {
prompt_tokens: 100,
completion_tokens: 20,
cache_read_tokens: 25,
cache_write_tokens: 10,
cache_write_5m_tokens: None,
cache_write_1h_tokens: None,
prompt_convention: PromptConvention::IncludesCache,
},
service_tier: ServiceTier::Standard,
threshold_policy: ThresholdPolicy::Exclusive,
region_multiplier: None,
billed_at_utc_minute: None,
}
}
fn pricing(standard: Rates) -> Pricing<'static> {
Pricing {
standard,
tiers: &[],
thresholds: &[],
off_peak: None,
}
}
#[test]
fn breakdown_and_total_agree() {
let standard = Rates {
cache_read: Rate::Value(0.5),
cache_write: Rate::Value(3.0),
..rates(Rate::Value(2.0), Rate::Value(4.0))
};
let result = calculate(&pricing(standard), &request()).unwrap();
assert_eq!(result.uncached_input, 65.0 * 2.0);
assert_eq!(result.cache_read, 25.0 * 0.5);
assert_eq!(result.cache_write_5m, 10.0 * 3.0);
assert_eq!(result.output(), 20.0 * 4.0);
assert_eq!(result.total(), result.input() + result.output());
assert_eq!(result.rates.cache_read, 0.5);
}
#[rstest]
#[case::missing(Rate::Missing, Rate::Missing, 200.0)]
#[case::null(Rate::Null, Rate::Missing, 200.0)]
#[case::zero(Rate::Value(0.0), Rate::Value(0.0), 130.0)]
fn absent_null_and_zero_cache_rates_are_distinct(
#[case] cache_read: Rate,
#[case] cache_write: Rate,
#[case] expected_input: f64,
) {
let base = rates(Rate::Value(2.0), Rate::Value(4.0));
let standard = Rates {
cache_read,
cache_write,
..base
};
assert_eq!(
calculate(&pricing(standard), &request()).unwrap().input(),
expected_input
);
}
#[test]
fn equivalent_prompt_conventions_select_the_same_threshold() {
let threshold = ThresholdRates {
above_prompt_tokens: 90,
standard: rates(Rate::Value(5.0), Rate::Value(8.0)),
tiers: &[],
};
let specification = Pricing {
standard: rates(Rate::Value(2.0), Rate::Value(4.0)),
tiers: &[],
thresholds: &[threshold],
off_peak: None,
};
let included = request();
let excluded = Request {
usage: Usage {
prompt_tokens: 65,
prompt_convention: PromptConvention::ExcludesCache,
..included.usage
},
..included
};
let plan = compile(&specification).unwrap();
assert_eq!(plan.calculate(&included), plan.calculate(&excluded));
assert_eq!(plan.calculate(&included).unwrap().rates.input, 5.0);
}
#[test]
fn split_writes_and_invalid_accounting() {
let standard = Rates {
cache_read: Rate::Value(0.5),
cache_write: Rate::Value(3.0),
cache_write_1h: Rate::Value(5.0),
..rates(Rate::Value(2.0), Rate::Value(4.0))
};
let base = request();
let split = Request {
usage: Usage {
cache_write_5m_tokens: Some(4),
cache_write_1h_tokens: Some(6),
..base.usage
},
..base
};
let result = calculate(&pricing(standard), &split).unwrap();
assert_eq!(result.cache_write_5m, 12.0);
assert_eq!(result.cache_write_1h, 30.0);
let overlapping = Request {
usage: Usage {
prompt_tokens: 30,
..split.usage
},
..split
};
assert_eq!(
calculate(&pricing(standard), &overlapping),
Err(Error::CacheExceedsPrompt)
);
let incomplete = Request {
usage: Usage {
cache_write_1h_tokens: None,
..split.usage
},
..split
};
assert_eq!(
calculate(&pricing(standard), &incomplete),
Err(Error::InvalidCacheWriteDetails)
);
}
#[test]
fn threshold_tiers_and_boundaries() {
let priority = TierRates {
tier: ServiceTier::Priority,
rates: rates(Rate::Value(3.0), Rate::Missing),
};
let threshold = ThresholdRates {
above_prompt_tokens: 100,
standard: rates(Rate::Value(5.0), Rate::Value(8.0)),
tiers: &[
TierRates {
tier: ServiceTier::Priority,
rates: rates(Rate::Value(7.0), Rate::Missing),
},
TierRates {
tier: ServiceTier::Flex,
rates: rates(Rate::Value(6.0), Rate::Missing),
},
],
};
let specification = Pricing {
standard: rates(Rate::Value(2.0), Rate::Value(4.0)),
tiers: &[priority],
thresholds: &[threshold],
off_peak: None,
};
let base = request();
let no_cache = Request {
usage: Usage {
cache_read_tokens: 0,
cache_write_tokens: 0,
..base.usage
},
..base
};
let fast = Request {
service_tier: ServiceTier::Fast,
..no_cache
};
let inclusive = Request {
threshold_policy: ThresholdPolicy::Inclusive,
..fast
};
let flex = Request {
service_tier: ServiceTier::Flex,
..inclusive
};
assert_eq!(calculate(&specification, &no_cache).unwrap().input(), 200.0);
assert_eq!(calculate(&specification, &fast).unwrap().input(), 300.0);
assert_eq!(
calculate(&specification, &inclusive).unwrap().input(),
700.0
);
assert_eq!(calculate(&specification, &flex).unwrap().input(), 600.0);
}
#[rstest]
#[case::ultrafast_above_threshold(ServiceTier::Ultrafast, 300_000, 9_301_000.0, 37_000.0)]
#[case::ultrafast_at_threshold(ServiceTier::Ultrafast, 272_000, 544_500.0, 5_000.0)]
#[case::standard_above_threshold(ServiceTier::Standard, 300_000, 3_300_600.0, 13_000.0)]
#[case::priority_above_threshold(ServiceTier::Priority, 300_000, 5_701_000.0, 23_000.0)]
fn tiered_long_context_rates_are_selected_by_service_tier(
#[case] service_tier: ServiceTier,
#[case] prompt_tokens: u64,
#[case] expected_input: f64,
#[case] expected_output: f64,
) {
let standard = Rates {
cache_read: Rate::Value(3.0),
..rates(Rate::Value(1.0), Rate::Value(2.0))
};
let tiers = [
TierRates {
tier: ServiceTier::Priority,
rates: Rates {
cache_read: Rate::Value(5.0),
..rates(Rate::Value(3.0), Rate::Value(4.0))
},
},
TierRates {
tier: ServiceTier::Ultrafast,
rates: Rates {
cache_read: Rate::Value(7.0),
..rates(Rate::Value(2.0), Rate::Value(5.0))
},
},
];
let threshold_tiers = [
TierRates {
tier: ServiceTier::Priority,
rates: Rates {
cache_read: Rate::Value(29.0),
..rates(Rate::Value(19.0), Rate::Value(23.0))
},
},
TierRates {
tier: ServiceTier::Ultrafast,
rates: Rates {
cache_read: Rate::Value(41.0),
..rates(Rate::Value(31.0), Rate::Value(37.0))
},
},
];
let thresholds = [ThresholdRates {
above_prompt_tokens: 272_000,
standard: Rates {
cache_read: Rate::Value(17.0),
..rates(Rate::Value(11.0), Rate::Value(13.0))
},
tiers: &threshold_tiers,
}];
let pricing = Pricing {
standard,
tiers: &tiers,
thresholds: &thresholds,
off_peak: None,
};
let base = request();
let long_context_request = Request {
usage: Usage {
prompt_tokens,
completion_tokens: 1_000,
cache_read_tokens: 100,
cache_write_tokens: 0,
..base.usage
},
service_tier,
..base
};
let cost = calculate(&pricing, &long_context_request).unwrap();
assert_eq!(cost.input(), expected_input);
assert_eq!(cost.output(), expected_output);
}
#[test]
fn compile_rejects_ambiguous_rates() {
let duplicate = ThresholdRates {
above_prompt_tokens: 100,
standard: Rates::EMPTY,
tiers: &[],
};
let specification = Pricing {
standard: rates(Rate::Value(1.0), Rate::Value(1.0)),
tiers: &[],
thresholds: &[duplicate, duplicate],
off_peak: None,
};
assert_eq!(
compile(&specification).err(),
Some(Error::DuplicateThreshold)
);
let invalid = pricing(rates(Rate::Value(f64::NAN), Rate::Value(1.0)));
assert_eq!(compile(&invalid).err(), Some(Error::InvalidRate));
}
#[test]
fn off_peak_is_one_non_wrapping_utc_window() {
let specification = Pricing {
standard: rates(Rate::Value(2.0), Rate::Value(4.0)),
tiers: &[],
thresholds: &[],
off_peak: Some(OffPeakRates {
start_utc_minute: 60,
end_utc_minute: 120,
rates: rates(Rate::Value(1.0), Rate::Value(2.0)),
}),
};
let base = request();
let start = Request {
billed_at_utc_minute: Some(60),
..base
};
let end = Request {
billed_at_utc_minute: Some(120),
..base
};
assert_eq!(
calculate(&specification, &base),
Err(Error::InvalidBillingTime)
);
assert_eq!(calculate(&specification, &start).unwrap().input(), 100.0);
assert_eq!(calculate(&specification, &end).unwrap().input(), 200.0);
}
#[test]
fn missing_rates_and_free_rates_remain_distinct() {
let base = request();
let empty = Request {
usage: Usage {
prompt_tokens: 0,
completion_tokens: 0,
cache_read_tokens: 0,
cache_write_tokens: 0,
..base.usage
},
..base
};
assert_eq!(
calculate(&pricing(Rates::EMPTY), &empty),
Err(Error::MissingInputRate)
);
assert_eq!(
calculate(&pricing(rates(Rate::Value(0.0), Rate::Missing)), &empty),
Err(Error::MissingOutputRate)
);
assert_eq!(
calculate(&pricing(rates(Rate::Value(0.0), Rate::Value(0.0))), &empty)
.unwrap()
.total(),
0.0
);
}
#[test]
fn matches_executed_python_reference_cases() {
for row in include_str!("python_reference.tsv")
.lines()
.filter(|line| !line.starts_with('#'))
{
let fields: Vec<_> = row.split('\t').collect();
let count = |index: usize| fields[index].parse::<u64>().unwrap();
let number = |index: usize| fields[index].parse::<f64>().unwrap();
let optional_rate = |index: usize| {
if fields[index].is_empty() {
Rate::Missing
} else {
Rate::Value(number(index))
}
};
let threshold = ThresholdRates {
above_prompt_tokens: if fields[9].is_empty() { 0 } else { count(9) },
standard: rates(optional_rate(10), optional_rate(11)),
tiers: &[],
};
let thresholds = if fields[9].is_empty() {
&[][..]
} else {
std::slice::from_ref(&threshold)
};
let specification = Pricing {
standard: Rates {
cache_read: optional_rate(7),
cache_write: optional_rate(8),
..rates(Rate::Value(number(5)), Rate::Value(number(6)))
},
tiers: &[],
thresholds,
off_peak: None,
};
let base = request();
let input = Request {
usage: Usage {
prompt_tokens: count(1),
completion_tokens: count(2),
cache_read_tokens: count(3),
cache_write_tokens: count(4),
..base.usage
},
..base
};
let actual = calculate(&specification, &input).unwrap();
assert_eq!(actual.input(), number(12), "{}", fields[0]);
assert_eq!(actual.output(), number(13), "{}", fields[0]);
}
}
proptest::proptest! {
#[test]
fn equivalent_usage_conventions_and_breakdown_agree(
regular in 0_u64..1000,
read in 0_u64..1000,
write in 0_u64..1000,
output in 0_u64..1000,
) {
let threshold = ThresholdRates {
above_prompt_tokens: 1000,
standard: rates(Rate::Value(5.0), Rate::Value(8.0)),
tiers: &[],
};
let specification = Pricing {
standard: rates(Rate::Value(2.0), Rate::Value(4.0)),
tiers: &[],
thresholds: &[threshold],
off_peak: None,
};
let base = request();
let included = Request {
usage: Usage {
prompt_tokens: regular + read + write,
completion_tokens: output,
cache_read_tokens: read,
cache_write_tokens: write,
..base.usage
},
..base
};
let excluded = Request {
usage: Usage {
prompt_tokens: regular,
prompt_convention: PromptConvention::ExcludesCache,
..included.usage
},
..included
};
let plan = compile(&specification).unwrap();
let left = plan.calculate(&included).unwrap();
let right = plan.calculate(&excluded).unwrap();
proptest::prop_assert_eq!(left, right);
proptest::prop_assert_eq!(left.total(), left.input() + left.output());
}
}
#[test]
fn regional_multiplier_applies_after_input_components_are_summed() {
let standard = Rates {
cache_read: Rate::Value(0.5),
cache_write: Rate::Value(3.0),
..rates(Rate::Value(2.0), Rate::Value(4.0))
};
let base = request();
let regional = Request {
region_multiplier: Some(1.1),
..base
};
let result = calculate(&pricing(standard), &regional).unwrap();
assert_eq!(result.input(), (65.0 * 2.0 + 25.0 * 0.5 + 10.0 * 3.0) * 1.1);
assert_eq!(result.output(), 20.0 * 4.0 * 1.1);
let invalid = Request {
region_multiplier: Some(f64::NAN),
..base
};
assert_eq!(
calculate(&pricing(standard), &invalid),
Err(Error::InvalidRegionMultiplier)
);
}
#[test]
fn compilation_sorts_thresholds_and_rejects_duplicate_tiers() {
let high = ThresholdRates {
above_prompt_tokens: 200,
standard: rates(Rate::Value(7.0), Rate::Missing),
tiers: &[],
};
let low = ThresholdRates {
above_prompt_tokens: 100,
standard: rates(Rate::Value(5.0), Rate::Missing),
tiers: &[],
};
let specification = Pricing {
standard: rates(Rate::Value(2.0), Rate::Value(4.0)),
tiers: &[],
thresholds: &[high, low],
off_peak: None,
};
let base = request();
let above_both = Request {
usage: Usage {
prompt_tokens: 201,
cache_read_tokens: 0,
cache_write_tokens: 0,
..base.usage
},
..base
};
assert_eq!(
compile(&specification)
.unwrap()
.calculate(&above_both)
.unwrap()
.rates
.input,
7.0
);
let duplicate = TierRates {
tier: ServiceTier::Flex,
rates: Rates::EMPTY,
};
let invalid = Pricing {
tiers: &[duplicate, duplicate],
thresholds: &[],
..specification
};
assert_eq!(compile(&invalid).err(), Some(Error::DuplicateTier));
}

96
litellm-rust/crates/cost/tests/generate_python_reference.py
View file

@ -1,96 +0,0 @@
import subprocess
from dataclasses import dataclass
from pathlib import Path
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
from litellm.types.utils import Usage
@dataclass(frozen=True, slots=True)
class Case:
name: str
prompt: int
completion: int
cache_read: int
cache_write: int
input_rate: float
output_rate: float
cache_read_rate: float | None = None
cache_write_rate: float | None = None
threshold: int | None = None
threshold_input_rate: float | None = None
threshold_output_rate: float | None = None
CASES = (
Case("ordinary", 100, 20, 0, 0, 2.0, 4.0),
Case("cache_fallback", 100, 20, 25, 10, 2.0, 4.0),
Case("cache_specific", 100, 20, 25, 10, 2.0, 4.0, 0.5, 3.0),
Case("free_cache", 100, 20, 25, 10, 2.0, 4.0, 0.0, 0.0),
Case("threshold_below", 99, 20, 0, 0, 2.0, 4.0, threshold=100, threshold_input_rate=5.0, threshold_output_rate=8.0),
Case("threshold_at", 100, 20, 0, 0, 2.0, 4.0, threshold=100, threshold_input_rate=5.0, threshold_output_rate=8.0),
Case(
"threshold_above", 101, 20, 0, 0, 2.0, 4.0, threshold=100, threshold_input_rate=5.0, threshold_output_rate=8.0
),
Case(
"cache_threshold_above",
101,
20,
25,
10,
2.0,
4.0,
threshold=100,
threshold_input_rate=5.0,
threshold_output_rate=8.0,
),
)
def reference(case: Case) -> tuple[float, float]:
info = {"input_cost_per_token": case.input_rate, "output_cost_per_token": case.output_rate}
if case.cache_read_rate is not None:
info["cache_read_input_token_cost"] = case.cache_read_rate
if case.cache_write_rate is not None:
info["cache_creation_input_token_cost"] = case.cache_write_rate
if case.threshold is not None:
info[f"input_cost_per_token_above_{case.threshold}_tokens"] = case.threshold_input_rate
info[f"output_cost_per_token_above_{case.threshold}_tokens"] = case.threshold_output_rate
details = {"cached_tokens": case.cache_read, "cache_write_tokens": case.cache_write}
usage = Usage(prompt_tokens=case.prompt, completion_tokens=case.completion, prompt_tokens_details=details)
return generic_cost_per_token(
model="synthetic",
usage=usage,
custom_llm_provider="openai",
model_info=info,
)
def main() -> None:
revision = subprocess.check_output(("git", "rev-parse", "HEAD"), text=True).strip()
rows = ("# Python reference commit: " + revision,) + tuple(
"\t".join(
str(value) if value is not None else ""
for value in (
case.name,
case.prompt,
case.completion,
case.cache_read,
case.cache_write,
case.input_rate,
case.output_rate,
case.cache_read_rate,
case.cache_write_rate,
case.threshold,
case.threshold_input_rate,
case.threshold_output_rate,
*reference(case),
)
)
for case in CASES
)
Path(__file__).with_name("python_reference.tsv").write_text("\n".join(rows) + "\n")
if __name__ == "__main__":
main()

9
litellm-rust/crates/cost/tests/python_reference.tsv
View file

@ -1,9 +0,0 @@
# Python reference commit: dc4be2fd987c993aefcf16444e34f960c12c8627
ordinary 100 20 0 0 2.0 4.0 200.0 80.0
cache_fallback 100 20 25 10 2.0 4.0 200.0 80.0
cache_specific 100 20 25 10 2.0 4.0 0.5 3.0 172.5 80.0
free_cache 100 20 25 10 2.0 4.0 0.0 0.0 130.0 80.0
threshold_below 99 20 0 0 2.0 4.0 100 5.0 8.0 198.0 80.0
threshold_at 100 20 0 0 2.0 4.0 100 5.0 8.0 200.0 80.0
threshold_above 101 20 0 0 2.0 4.0 100 5.0 8.0 505.0 160.0
cache_threshold_above 101 20 25 10 2.0 4.0 100 5.0 8.0 505.0 160.0
Can't render this file because it has a wrong number of fields in line 2.

133
litellm-rust/crates/inference-ocr/tests/ocr/azure_document_intelligence.rs
View file

@ -12,6 +12,7 @@ use rstest::rstest;
use super::*;
const MODEL: &str = "azure_ai/doc-intelligence/prebuilt-read";
const LAYOUT_MODEL: &str = "azure_ai/doc-intelligence/prebuilt-layout";
fn read_request(base: &str, options: Value) -> LiteLLMOcrRequest {
ocr_request(MODEL, base, options)
@ -70,6 +71,7 @@ async fn pages_features_and_extra_options_map_to_the_analyze_call() {
#[case(json!({"pages": "1&&features=bad"}), Error::Pages(PagesFault::Range))]
#[case(json!({"features": "languages&pages=1"}), Error::Features)]
#[case(json!({"req_format": "azure"}), Error::RequestFormat)]
#[case(json!({"outputContentFormat": "html"}), Error::OutputContentFormat)]
#[tokio::test]
async fn invalid_pages_features_and_format_are_rejected_before_sending(
#[case] options: Value,
@ -130,6 +132,137 @@ async fn an_inline_document_is_sent_as_base64_and_only_page_text_is_kept(#[case]
assert_eq!(sent.json(), json!({"base64Source": "YWJj"}));
}
#[rstest]
#[case::lower_case("markdown")]
#[case::capitalized("Markdown")]
#[tokio::test]
async fn markdown_output_is_requested_on_the_analyze_call_and_split_into_pages(
#[case] output_content_format: &str,
) {
let upstream = upstream([json_response(json!({
"status": "succeeded",
"analyzeResult": {
"content": "# Tïtle\n\n<!-- PageBreak -->\n\n| a |",
"contentFormat": "markdown",
"stringIndexType": "unicodeCodePoint",
"pages": [
{"pageNumber": 1, "spans": [{"offset": 0, "length": 29}], "lines": [{"content": "Tïtle"}]},
{"pageNumber": 2, "spans": [{"offset": 29, "length": 5}], "lines": [{"content": "a"}]}
]
}
}))])
.await;
let response = perform(ocr_request(
LAYOUT_MODEL,
&upstream.uri(),
json!({"outputContentFormat": output_content_format, "pages": [0, 1]}),
))
.await
.unwrap();
let sent = only_request(&upstream).await;
assert_eq!(
sent.query("outputContentFormat").as_deref(),
Some("markdown")
);
assert_eq!(
sent.query("stringIndexType").as_deref(),
Some("unicodeCodePoint")
);
assert_eq!(sent.query("pages").as_deref(), Some("1,2"));
assert_eq!(sent.json(), json!({"base64Source": "YWJj"}));
assert_eq!(
response.content.as_deref(),
Some("# Tïtle\n\n<!-- PageBreak -->\n\n| a |")
);
let pages = response
.pages
.iter()
.map(|page| page.markdown.as_str())
.collect::<Vec<_>>();
assert_eq!(pages, ["# Tïtle\n\n<!-- PageBreak -->\n\n", "| a |"]);
assert_eq!(
response.extra_fields.get("stringIndexType"),
Some(&json!("unicodeCodePoint"))
);
assert_eq!(
response.usage_info.as_ref().unwrap().pages_processed,
Some(2)
);
}
#[rstest]
#[case::native_markdown(LAYOUT_MODEL, json!({"outputContentFormat": "markdown", "req_format": "native"}), Some("markdown"))]
#[case::markdown_on_read(MODEL, json!({"outputContentFormat": "markdown"}), Some("markdown"))]
#[case::text(LAYOUT_MODEL, json!({"outputContentFormat": "text"}), Some("text"))]
#[case::omitted(LAYOUT_MODEL, json!({}), None)]
#[tokio::test]
async fn only_layout_markdown_in_litellm_format_asks_for_code_point_offsets(
#[case] model: &str,
#[case] options: Value,
#[case] output_content_format: Option<&str>,
) {
let upstream = upstream([json_response(json!({
"status": "succeeded",
"analyzeResult": {
"contentFormat": "text",
"stringIndexType": "textElements",
"pages": [{"pageNumber": 1, "lines": [{"content": "hello"}]}]
}
}))])
.await;
let response = perform(ocr_request(model, &upstream.uri(), options))
.await
.unwrap();
let sent = only_request(&upstream).await;
assert_eq!(
sent.query("outputContentFormat").as_deref(),
output_content_format
);
assert_eq!(sent.query("stringIndexType"), None);
assert_eq!(sent.json(), json!({"base64Source": "YWJj"}));
assert_eq!(response.pages[0].markdown, "hello");
assert_eq!(response.extra_fields.get("stringIndexType"), None);
}
#[rstest]
#[case::snake_case(json!({"output_content_format": "markdown"}), Some("markdown"))]
#[case::padded(json!({"outputContentFormat": " markdown "}), Some("markdown"))]
#[case::empty(json!({"outputContentFormat": ""}), None)]
#[case::empty_camel_case_falls_back_to_snake_case(
json!({"outputContentFormat": "", "output_content_format": "markdown"}),
Some("markdown")
)]
#[case::dropped(json!({"outputContentFormat": "html", "drop_params": true}), None)]
#[tokio::test]
async fn the_requested_output_content_format_reaches_azure_in_its_query_spelling(
#[case] options: Value,
#[case] output_content_format: Option<&str>,
) {
let upstream = upstream([json_response(json!({
"status": "succeeded",
"analyzeResult": {"pages": [{"pageNumber": 1, "lines": [{"content": "hello"}]}]}
}))])
.await;
let response = perform(ocr_request(MODEL, &upstream.uri(), options))
.await
.unwrap();
let sent = only_request(&upstream).await;
assert_eq!(
sent.query("outputContentFormat").as_deref(),
output_content_format
);
assert_eq!(sent.query("output_content_format"), None);
assert_eq!(sent.query("drop_params"), None);
assert_eq!(sent.json(), json!({"base64Source": "YWJj"}));
assert_eq!(response.pages[0].markdown, "hello");
}
#[tokio::test]
async fn native_format_normalizes_pages_and_keeps_the_provider_response() {
let operation = json!({

23
litellm-rust/crates/llms-types/src/formats/audio_transcription/response.rs
View file

@ -1,14 +1,31 @@
use serde_json::Value;
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct AudioTranscriptionUsage {
pub input_tokens: u64,
pub output_tokens: u64,
pub total_tokens: u64,
}
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct AudioTranscriptionResponseData {
pub text: String,
pub usage: Option<AudioTranscriptionUsage>,
}
impl AudioTranscriptionResponseData {
pub fn into_json(self) -> Value {
serde_json::json!({
"text": self.text,
})
match self.usage {
None => serde_json::json!({"text": self.text}),
Some(usage) => serde_json::json!({
"text": self.text,
"usage": {
"type": "tokens",
"input_tokens": usage.input_tokens,
"output_tokens": usage.output_tokens,
"total_tokens": usage.total_tokens,
},
}),
}
}
}

426
litellm-rust/crates/llms/src/azure_ai/ocr/document_intelligence/transformation.rs
View file

@ -1,4 +1,7 @@
use std::{collections::BTreeSet, time::Duration};
use std::{
collections::{BTreeMap, BTreeSet},
time::Duration,
};
use base64::{Engine, engine::general_purpose::STANDARD};
use litellm_auth::{InputSource, Sourced};
@ -8,7 +11,7 @@ use litellm_llms_types::serde_compat::{FiniteF64, LaxI64};
use reqwest::Url;
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use serde_with::serde_as;
use serde_with::{DefaultOnError, serde_as};
use tokio::time::Instant;
use crate::base_llm::ocr::{
@ -17,9 +20,10 @@ use crate::base_llm::ocr::{
handler::{CallHooks, OcrClient, read_json_response},
settings::OcrSettings,
transformation::{
BaseOcrConfig, DecodedOcrResponse, OCR_INLINE_MAX_BYTES, OCR_POLL_RETRY_SECS,
OcrConnection, OcrCredentialInputs, OcrResponseContext, PreparedOcrRequest,
ResolvedOcrCredentials, decode_and_normalize_response, decode_response,
BaseOcrConfig, DROP_PARAMS_PARAM, DecodedOcrResponse, OCR_INLINE_MAX_BYTES,
OCR_POLL_RETRY_SECS, OcrConnection, OcrCredentialInputs, OcrResponseContext,
PreparedOcrRequest, ResolvedOcrCredentials, decode_and_normalize_response, decode_response,
drops_unsupported_params, response_format,
},
};
use litellm_llms_types::formats::ocr::{
@ -32,6 +36,30 @@ const AZURE_DI_DEFAULT_HEIGHT: f64 = 11.0;
const AZURE_DI_API_KEY_ENV: &str = "AZURE_DOCUMENT_INTELLIGENCE_API_KEY";
const AZURE_DI_ENDPOINT_ENV: &str = "AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT";
const OUTPUT_CONTENT_FORMAT_PARAM: &str = "outputContentFormat";
const OUTPUT_CONTENT_FORMAT_ALIAS: &str = "output_content_format";
const CODE_POINT_STRING_INDEX: &str = "unicodeCodePoint";
const STRING_INDEX_TYPE_FIELD: &str = "stringIndexType";
const MARKDOWN_MODEL_ID: &str = "prebuilt-layout";
#[derive(
Clone,
Copy,
Debug,
PartialEq,
strum::EnumString,
strum::IntoStaticStr,
strum::Display,
strum::VariantArray,
serde_with::SerializeDisplay,
)]
#[strum(ascii_case_insensitive)]
pub enum OutputContentFormat {
#[strum(serialize = "text")]
Text,
#[strum(serialize = "markdown")]
Markdown,
}
#[derive(Clone, Debug, PartialEq, Serialize)]
pub struct DocumentIntelligenceParams {
@ -39,6 +67,13 @@ pub struct DocumentIntelligenceParams {
pub pages: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub features: Option<String>,
#[serde(
rename = "outputContentFormat",
skip_serializing_if = "Option::is_none"
)]
pub output_content_format: Option<OutputContentFormat>,
#[serde(rename = "stringIndexType", skip_serializing_if = "Option::is_none")]
pub string_index_type: Option<&'static str>,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
@ -80,6 +115,10 @@ pub struct AzureDocumentIntelligenceOperation {
#[derive(Clone, Debug, Default, Deserialize)]
struct AzureDocumentIntelligenceAnalyzeResult {
pub content: Option<String>,
#[serde(rename = "contentFormat")]
pub content_format: Option<String>,
#[serde(rename = "stringIndexType")]
pub string_index_type: Option<String>,
#[serde(default)]
pub pages: Vec<AzureDocumentIntelligencePage>,
pub tables: Option<Vec<Map<String, Value>>>,
@ -100,6 +139,9 @@ struct AzureDocumentIntelligencePage {
pub unit: Option<String>,
#[serde(default)]
pub lines: Vec<AzureDocumentIntelligenceLine>,
#[serde(default)]
#[serde_as(deserialize_as = "DefaultOnError")]
pub spans: Option<Vec<AzureDocumentIntelligenceSpan>>,
}
#[derive(Clone, Debug, Deserialize)]
@ -107,6 +149,18 @@ struct AzureDocumentIntelligenceLine {
pub content: Option<String>,
}
#[derive(Clone, Debug, Deserialize)]
struct AzureDocumentIntelligenceSpan {
pub offset: usize,
pub length: usize,
}
impl AzureDocumentIntelligenceSpan {
fn end(&self) -> usize {
self.offset.saturating_add(self.length)
}
}
#[derive(Clone, Debug)]
pub struct AzureDocumentIntelligenceOcrConfig;
@ -116,7 +170,14 @@ impl BaseOcrConfig for AzureDocumentIntelligenceOcrConfig {
type Environment = Vec<(String, String)>;
fn get_supported_ocr_params(&self, _model: &str) -> &'static [&'static str] {
&["pages", "features", "req_format"]
&[
"pages",
"features",
OUTPUT_CONTENT_FORMAT_PARAM,
OUTPUT_CONTENT_FORMAT_ALIAS,
"req_format",
DROP_PARAMS_PARAM,
]
}
fn get_api_key_env_var(&self) -> Option<&'static str> {
@ -154,11 +215,17 @@ impl BaseOcrConfig for AzureDocumentIntelligenceOcrConfig {
fn map_ocr_params(
&self,
non_default_params: &CallArguments,
_model: &str,
model: &str,
) -> Result<DocumentIntelligenceParams, Error> {
let output_content_format = requested_output_content_format(non_default_params)?;
let slices_markdown_pages = output_content_format == Some(OutputContentFormat::Markdown)
&& response_format(non_default_params)? != OcrResponseFormat::Native
&& model_id(model).is_ok_and(|id| id == MARKDOWN_MODEL_ID);
Ok(DocumentIntelligenceParams {
pages: normalize_pages_param(non_default_params.get("pages"))?,
features: normalize_features_param(non_default_params.get("features"))?,
output_content_format,
string_index_type: slices_markdown_pages.then_some(CODE_POINT_STRING_INDEX),
})
}
@ -337,6 +404,36 @@ fn normalize_features_param(features: Option<&Value>) -> Result<Option<String>,
Ok(Some(normalized.join(",")))
}
fn requested_output_content_format(
params: &CallArguments,
) -> Result<Option<OutputContentFormat>, Error> {
let requested = [OUTPUT_CONTENT_FORMAT_PARAM, OUTPUT_CONTENT_FORMAT_ALIAS]
.into_iter()
.filter_map(|name| params.get(name))
.find(|format| !is_unset(format));
match requested.map(parse_output_content_format).transpose() {
Err(Error::OutputContentFormat) if drops_unsupported_params(params) => Ok(None),
parsed => parsed,
}
}
fn is_unset(format: &Value) -> bool {
match format {
Value::Null => true,
Value::String(format) => format.trim().is_empty(),
_ => false,
}
}
fn parse_output_content_format(format: &Value) -> Result<OutputContentFormat, Error> {
format
.as_str()
.ok_or(Error::OutputContentFormat)?
.trim()
.parse()
.map_err(|_| Error::OutputContentFormat)
}
fn build_request(document: OcrDocument) -> Result<DocumentIntelligenceRequest, Error> {
let source = document.source();
if source.is_empty() {
@ -367,10 +464,27 @@ fn transform_completed_response(
));
}
let result = response.analyze_result.unwrap_or_default();
let markdown_content = result
.content
.as_deref()
.filter(|_| {
result.content_format.as_deref() == Some(<&str>::from(OutputContentFormat::Markdown))
&& result.string_index_type.as_deref() == Some(CODE_POINT_STRING_INDEX)
&& result.pages.iter().all(|page| page.spans.is_some())
})
.map(|content| {
CodePointText::new(
content,
result
.pages
.iter()
.flat_map(|page| page.spans.iter().flatten()),
)
});
let pages = result
.pages
.into_iter()
.map(|page| transform_azure_page(page, dpi))
.map(|page| transform_azure_page(page, markdown_content.as_ref(), dpi))
.collect::<Result<Vec<_>, _>>()?;
let pages_processed = i64::try_from(pages.len()).map_err(|_| Error::NumericRange("pages"))?;
Ok(LiteLLMOcrResponse {
@ -381,11 +495,20 @@ fn transform_completed_response(
pages_processed: Some(pages_processed),
..Default::default()
}),
extra_fields: result
.string_index_type
.filter(|unit| unit == CODE_POINT_STRING_INDEX)
.map(|unit| Map::from_iter([(STRING_INDEX_TYPE_FIELD.to_owned(), Value::String(unit))]))
.unwrap_or_default(),
..LiteLLMOcrResponse::new(model, pages)
})
}
fn transform_azure_page(page: AzureDocumentIntelligencePage, dpi: i64) -> Result<OcrPage, Error> {
fn transform_azure_page(
page: AzureDocumentIntelligencePage,
markdown_content: Option<&CodePointText<'_>>,
dpi: i64,
) -> Result<OcrPage, Error> {
let index = page
.page_number
.unwrap_or(1)
@ -397,12 +520,20 @@ fn transform_azure_page(page: AzureDocumentIntelligencePage, dpi: i64) -> Result
page.unit.as_deref().unwrap_or("inch"),
dpi,
)?;
let markdown = page
.lines
.iter()
.map(|line| line.content.as_deref().unwrap_or_default())
.collect::<Vec<_>>()
.join("\n");
let markdown = match markdown_content {
Some(content) => page
.spans
.iter()
.flatten()
.map(|span| content.slice(span))
.collect::<String>(),
None => page
.lines
.iter()
.map(|line| line.content.as_deref().unwrap_or_default())
.collect::<Vec<_>>()
.join("\n"),
};
Ok(OcrPage {
index,
markdown,
@ -411,6 +542,43 @@ fn transform_azure_page(page: AzureDocumentIntelligencePage, dpi: i64) -> Result
})
}
struct CodePointText<'a> {
text: &'a str,
byte_offsets: BTreeMap<usize, usize>,
}
impl<'a> CodePointText<'a> {
fn new<'s>(
text: &'a str,
spans: impl Iterator<Item = &'s AzureDocumentIntelligenceSpan>,
) -> Self {
let span_bounds = spans
.flat_map(|span| [span.offset, span.end()])
.collect::<BTreeSet<_>>();
Self {
text,
byte_offsets: text
.char_indices()
.map(|(byte_offset, _)| byte_offset)
.enumerate()
.filter(|(code_point, _)| span_bounds.contains(code_point))
.collect(),
}
}
fn slice(&self, span: &AzureDocumentIntelligenceSpan) -> &'a str {
let byte_offset = |code_point: usize| {
self.byte_offsets
.get(&code_point)
.copied()
.unwrap_or(self.text.len())
};
self.text
.get(byte_offset(span.offset)..byte_offset(span.end()))
.unwrap_or_default()
}
}
fn convert_dimensions(
width: f64,
height: f64,
@ -578,6 +746,16 @@ impl AzureDocumentIntelligenceOcrConfig {
.features
.iter()
.map(|features| ("features", features.as_str())),
)
.chain(
params
.output_content_format
.map(|format| (OUTPUT_CONTENT_FORMAT_PARAM, format.into())),
)
.chain(
params
.string_index_type
.map(|index_type| ("stringIndexType", index_type)),
),
)
.into_string()
@ -697,8 +875,12 @@ mod tests {
}
fn map(value: Value) -> Result<DocumentIntelligenceParams, Error> {
map_for_model(value, "azure_ai/doc-intelligence/prebuilt-layout")
}
fn map_for_model(value: Value, model: &str) -> Result<DocumentIntelligenceParams, Error> {
let arguments = serde_json::from_value(value).unwrap();
AzureDocumentIntelligenceOcrConfig.map_ocr_params(&arguments, "model")
AzureDocumentIntelligenceOcrConfig.map_ocr_params(&arguments, model)
}
#[test]
@ -828,6 +1010,218 @@ mod tests {
assert_eq!(map(json!({"features": []})).unwrap().features, None);
}
#[rstest]
#[case::markdown(
json!({"outputContentFormat": "markdown"}),
Some(OutputContentFormat::Markdown),
Some("unicodeCodePoint")
)]
#[case::markdown_litellm_format(
json!({"outputContentFormat": "markdown", "req_format": "litellm"}),
Some(OutputContentFormat::Markdown),
Some("unicodeCodePoint")
)]
#[case::markdown_native_format(
json!({"outputContentFormat": "markdown", "req_format": "native"}),
Some(OutputContentFormat::Markdown),
None
)]
#[case::capitalized_markdown(
json!({"outputContentFormat": "Markdown"}),
Some(OutputContentFormat::Markdown),
Some("unicodeCodePoint")
)]
#[case::text(json!({"outputContentFormat": "text"}), Some(OutputContentFormat::Text), None)]
#[case::upper_case_text(json!({"outputContentFormat": "TEXT"}), Some(OutputContentFormat::Text), None)]
#[case::padded_markdown(
json!({"outputContentFormat": " markdown\n"}),
Some(OutputContentFormat::Markdown),
Some("unicodeCodePoint")
)]
#[case::snake_case_markdown(
json!({"output_content_format": "markdown"}),
Some(OutputContentFormat::Markdown),
Some("unicodeCodePoint")
)]
#[case::camel_case_wins_over_snake_case(
json!({"outputContentFormat": "text", "output_content_format": "markdown"}),
Some(OutputContentFormat::Text),
None
)]
#[case::empty_camel_case_falls_back_to_snake_case(
json!({"outputContentFormat": "", "output_content_format": "markdown"}),
Some(OutputContentFormat::Markdown),
Some("unicodeCodePoint")
)]
#[case::null_camel_case_falls_back_to_snake_case(
json!({"outputContentFormat": null, "output_content_format": "markdown"}),
Some(OutputContentFormat::Markdown),
Some("unicodeCodePoint")
)]
#[case::empty(json!({"outputContentFormat": ""}), None, None)]
#[case::blank(json!({"outputContentFormat": " "}), None, None)]
#[case::null(json!({"outputContentFormat": null}), None, None)]
#[case::omitted(json!({}), None, None)]
fn output_content_format_maps_to_query_fields(
#[case] options: Value,
#[case] format: Option<OutputContentFormat>,
#[case] string_index_type: Option<&str>,
) {
let mapped = map(options).unwrap();
assert_eq!(mapped.output_content_format, format);
assert_eq!(mapped.string_index_type, string_index_type);
}
#[rstest]
#[case::layout("azure_ai/doc-intelligence/prebuilt-layout", Some("unicodeCodePoint"))]
#[case::bare_layout("prebuilt-layout", Some("unicodeCodePoint"))]
#[case::read("azure_ai/doc-intelligence/prebuilt-read", None)]
#[case::invoice("azure_ai/doc-intelligence/prebuilt-invoice", None)]
#[case::custom("azure_ai/doc-intelligence/my-extraction-model", None)]
fn only_the_layout_model_asks_for_code_point_offsets(
#[case] model: &str,
#[case] string_index_type: Option<&str>,
) {
let mapped = map_for_model(json!({"outputContentFormat": "markdown"}), model).unwrap();
assert_eq!(
mapped.output_content_format,
Some(OutputContentFormat::Markdown)
);
assert_eq!(mapped.string_index_type, string_index_type);
}
#[rstest]
#[case::unknown(json!({"outputContentFormat": "html"}))]
#[case::number(json!({"outputContentFormat": 1}))]
#[case::boolean(json!({"outputContentFormat": true}))]
#[case::list(json!({"outputContentFormat": ["markdown"]}))]
#[case::snake_case_unknown(json!({"output_content_format": "html"}))]
#[case::drop_params_off(json!({"outputContentFormat": "html", "drop_params": false}))]
fn invalid_output_content_format_is_a_request_error(#[case] options: Value) {
let error = map(options).unwrap_err();
assert!(matches!(error, Error::OutputContentFormat), "{error:?}");
assert_eq!(error.http_status_code(), Some(400));
}
#[rstest]
#[case::unknown(json!({"outputContentFormat": "html", "drop_params": true, "pages": [0]}))]
#[case::number(json!({"outputContentFormat": 1, "drop_params": true, "pages": [0]}))]
#[case::snake_case_unknown(
json!({"output_content_format": "html", "drop_params": true, "pages": [0]})
)]
fn drop_params_drops_an_invalid_output_content_format(#[case] options: Value) {
let mapped = map(options).unwrap();
assert_eq!(serde_json::to_value(mapped).unwrap(), json!({"pages": "1"}));
}
#[rstest]
#[case::camel_case(json!({"outputContentFormat": "markdown", "drop_params": true}))]
#[case::snake_case(json!({"output_content_format": "markdown", "drop_params": true}))]
fn drop_params_keeps_a_valid_output_content_format(#[case] options: Value) {
let mapped = map(options).unwrap();
assert_eq!(
mapped.output_content_format,
Some(OutputContentFormat::Markdown)
);
}
fn markdown_pages(analyze_result: Value) -> Vec<String> {
let raw =
serde_json::to_vec(&json!({"status": "succeeded", "analyzeResult": analyze_result}))
.unwrap();
AzureDocumentIntelligenceOcrConfig
.transform_ocr_response("model", &raw, OcrResponseFormat::Litellm)
.unwrap()
.pages
.into_iter()
.map(|page| page.markdown)
.collect()
}
#[rstest]
fn markdown_pages_are_code_point_slices_of_the_content() {
let content = "# Ünïcode क्ष\n\n<!-- PageBreak -->\n\n## Två 🙂\n\n|a|\n\n> tail";
let second_page_start = "# Ünïcode क्ष\n\n<!-- PageBreak -->\n\n".chars().count();
let pages = markdown_pages(json!({
"content": content,
"contentFormat": "markdown",
"stringIndexType": "unicodeCodePoint",
"pages": [
{"pageNumber": 1, "spans": [{"offset": 0, "length": second_page_start}],
"lines": [{"content": "Ünïcode क्ष"}]},
{"pageNumber": 2, "spans": [
{"offset": second_page_start, "length": 8},
{"offset": second_page_start + 10, "length": 3}
], "lines": [{"content": "Två"}]}
]
}));
assert_eq!(
pages,
["# Ünïcode क्ष\n\n<!-- PageBreak -->\n\n", "## Två 🙂|a|"]
);
}
#[rstest]
fn spans_past_the_content_end_are_clamped() {
let pages = markdown_pages(json!({
"content": "ab",
"contentFormat": "markdown",
"stringIndexType": "unicodeCodePoint",
"pages": [{"spans": [{"offset": 1, "length": 9}, {"offset": 7, "length": 1}]}]
}));
assert_eq!(pages, ["b"]);
}
#[rstest]
#[case::text_content(json!("text"), json!("unicodeCodePoint"))]
#[case::grapheme_offsets(json!("markdown"), json!("textElements"))]
#[case::no_index_type(json!("markdown"), Value::Null)]
#[case::no_content_format(Value::Null, json!("unicodeCodePoint"))]
fn other_responses_keep_line_joined_pages(
#[case] content_format: Value,
#[case] string_index_type: Value,
) {
let pages = markdown_pages(json!({
"content": "# A\n\nB",
"contentFormat": content_format,
"stringIndexType": string_index_type,
"pages": [{"spans": [{"offset": 0, "length": 6}],
"lines": [{"content": "A"}, {"content": "B"}]}]
}));
assert_eq!(pages, ["A\nB"]);
}
#[rstest]
#[case::null_spans(json!("markdown"), json!(null))]
#[case::fractional_offset(json!("markdown"), json!([{"offset": 0.5, "length": 1}]))]
#[case::missing_length(json!("markdown"), json!([{"offset": 0}]))]
#[case::negative_offset(json!("markdown"), json!([{"offset": -1, "length": 1}]))]
#[case::text_content(json!("text"), json!([{"offset": "0", "length": 1}]))]
fn unusable_page_spans_keep_line_joined_pages(
#[case] content_format: Value,
#[case] broken_spans: Value,
) {
let pages = markdown_pages(json!({
"content": "# A\n\nB",
"contentFormat": content_format,
"stringIndexType": "unicodeCodePoint",
"pages": [
{"spans": [{"offset": 0, "length": 5}], "lines": [{"content": "A"}]},
{"spans": broken_spans, "lines": [{"content": "B"}]}
]
}));
assert_eq!(pages, ["A", "B"]);
}
#[tokio::test]
async fn request_endpoint_cannot_receive_environment_key() {
let connection = OcrConnection {

8
litellm-rust/crates/llms/src/base_llm/ocr/error.rs
View file

@ -59,6 +59,10 @@ pub enum Error {
Pages(PagesFault),
#[error("invalid OCR features")]
Features,
#[error(
"invalid OCR outputContentFormat, expected text or markdown. To drop unsupported params, set `litellm.drop_params = True`."
)]
OutputContentFormat,
#[error("OCR model cannot be a dot segment")]
DotModel,
#[error("OCR response exceeds the size limit of {limit} bytes")]
@ -200,6 +204,7 @@ impl Error {
| Self::InlineDocumentTooLarge
| Self::Pages(_)
| Self::Features
| Self::OutputContentFormat
| Self::DotModel
| Self::TooLarge { .. }
| Self::ResponseField { .. }
@ -242,6 +247,7 @@ impl Error {
| Self::InlineDocumentTooLarge
| Self::Pages(_)
| Self::Features
| Self::OutputContentFormat
| Self::DotModel
| Self::TooLarge { .. }
| Self::ResponseField { .. }
@ -291,6 +297,7 @@ impl Error {
| Self::InlineDocumentTooLarge
| Self::Pages(_)
| Self::Features
| Self::OutputContentFormat
| Self::DotModel
| Self::InvalidProvider(_)
| Self::InvalidModel { .. }
@ -345,6 +352,7 @@ impl Error {
| Self::InlineDocumentTooLarge
| Self::Pages(_)
| Self::Features
| Self::OutputContentFormat
| Self::DotModel
| Self::InvalidProvider(_)
| Self::InvalidModel { .. }

5
litellm-rust/crates/llms/src/base_llm/ocr/transformation.rs
View file

@ -21,6 +21,7 @@ pub const OCR_RESPONSE_MAX_BYTES: usize = 64 * 1024 * 1024;
pub const OCR_INLINE_MAX_BYTES: usize = 50 * 1024 * 1024;
pub const OCR_MAX_FETCH_REDIRECTS: usize = 10;
pub const OCR_POLL_RETRY_SECS: u64 = 2;
pub const DROP_PARAMS_PARAM: &str = "drop_params";
#[derive(Clone, Default)]
pub struct OcrCredentialInputs {
@ -185,6 +186,10 @@ pub fn response_format(optional_params: &CallArguments) -> Result<OcrResponseFor
.map(|format| format.unwrap_or_default())
}
pub fn drops_unsupported_params(optional_params: &CallArguments) -> bool {
optional_params.get(DROP_PARAMS_PARAM) == Some(&Value::Bool(true))
}
#[derive(Debug)]
pub struct DecodedOcrResponse<T> {
pub data: T,

83
litellm-rust/crates/llms/src/bedrock/audio_transcription/mod.rs
View file

@ -6,8 +6,8 @@ use litellm_auth_aws::{
};
use litellm_core_utils::core_helpers::json_type_name;
use litellm_llms_types::{
formats::audio_transcription::AudioTranscriptionResponseData,
providers::bedrock::ConverseResponse,
formats::audio_transcription::{AudioTranscriptionResponseData, AudioTranscriptionUsage},
providers::bedrock::{ConverseResponse, ConverseUsage},
};
use serde::Deserialize;
use serde_json::{Map, Value, json};
@ -101,10 +101,10 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
if let Some(prompt) = optional_string(&optional_params, "prompt") {
instruction.push_str(&format!(" Additional context: {prompt}"));
}
let mut inference_config = Map::from_iter([("maxTokens".to_string(), json!(4096))]);
if let Some(temperature) = optional_params.get("temperature") {
inference_config.insert("temperature".to_string(), temperature.clone());
}
let temperature = optional_params
.get("temperature")
.cloned()
.unwrap_or(json!(0));
Ok(AudioTranscriptionRequestData {
body: json!({
"messages": [{
@ -114,8 +114,7 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
{"text": instruction}
]
}],
"system": [{"text": "You are a transcription assistant."}],
"inferenceConfig": inference_config,
"inferenceConfig": {"maxTokens": 4096, "temperature": temperature},
}),
})
}
@ -125,6 +124,7 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
_model: &str,
response_json: Value,
) -> Result<AudioTranscriptionResponseData, Error> {
let reports_usage = response_json.get("usage").is_some();
let response: ConverseResponse =
serde_json::from_value(response_json).map_err(|error| {
Error::InvalidResponse(ErrorDetail::invalid("Converse response", error))
@ -137,6 +137,7 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
}
Ok(AudioTranscriptionResponseData {
text: response.content_text(),
usage: reports_usage.then(|| transcription_usage(&response.usage)),
})
}
@ -191,22 +192,41 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
}
}
fn transcription_usage(usage: &ConverseUsage) -> AudioTranscriptionUsage {
AudioTranscriptionUsage {
input_tokens: usage.input_tokens,
output_tokens: usage.output_tokens,
total_tokens: usage
.total_tokens
.unwrap_or(usage.input_tokens + usage.output_tokens),
}
}
#[cfg(test)]
mod tests {
use super::*;
use rstest::rstest;
fn no_env(_: &str) -> Option<String> {
None
}
#[test]
fn request_matches_python_shape() {
let params = Map::from_iter([
("language".to_string(), json!("en")),
("prompt".to_string(), json!("Speaker names")),
("temperature".to_string(), json!(0)),
("timestamp_granularities".to_string(), json!(["word"])),
]);
#[rstest]
#[case::caller_temperature(Some(json!(0.4)), json!(0.4))]
#[case::default_temperature(None, json!(0))]
fn request_sends_audio_and_instruction_without_system_block(
#[case] temperature: Option<Value>,
#[case] expected_temperature: Value,
) {
let params = Map::from_iter(
[
("language".to_string(), json!("en")),
("prompt".to_string(), json!("Speaker names")),
("timestamp_granularities".to_string(), json!(["word"])),
]
.into_iter()
.chain(temperature.map(|value| ("temperature".to_string(), value))),
);
let params = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG.map_transcription_params(&params);
let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG
.transform_audio_transcription_request(
@ -225,22 +245,39 @@ mod tests {
{"text": "Transcribe the audio. Respond with only the transcript. The audio language is en. Additional context: Speaker names"}
]
}],
"system": [{"text": "You are a transcription assistant."}],
"inferenceConfig": {"maxTokens": 4096, "temperature": 0}
"inferenceConfig": {"maxTokens": 4096, "temperature": expected_temperature}
})
);
}
#[test]
fn response_concatenates_content_blocks() {
#[rstest]
#[case::reported_total(
json!({"inputTokens": 390, "outputTokens": 60, "totalTokens": 450}),
json!({"text": "hello world", "usage": {"type": "tokens", "input_tokens": 390, "output_tokens": 60, "total_tokens": 450}})
)]
#[case::derived_total(
json!({"inputTokens": 1, "outputTokens": 2}),
json!({"text": "hello world", "usage": {"type": "tokens", "input_tokens": 1, "output_tokens": 2, "total_tokens": 3}})
)]
fn response_carries_text_and_converse_usage(#[case] usage: Value, #[case] expected: Value) {
let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG
.transform_audio_transcription_response(
"model",
json!({"output": {"message": {"content": [{"text": "hello "}, {"text": "world"}]}}, "usage": {"inputTokens": 1, "outputTokens": 2}}),
json!({"output": {"message": {"content": [{"text": "hello "}, {"text": "world"}]}}, "usage": usage}),
)
.expect("response");
assert_eq!(result.text, "hello world");
assert_eq!(result.into_json(), json!({"text": "hello world"}));
assert_eq!(result.into_json(), expected);
}
#[rstest]
fn response_without_usage_reports_none() {
let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG
.transform_audio_transcription_response(
"model",
json!({"output": {"message": {"content": [{"text": "hello"}]}}}),
)
.expect("response");
assert_eq!(result.into_json(), json!({"text": "hello"}));
}
#[test]

19
litellm-rust/crates/model-catalog-refresh/Cargo.toml Normal file
View file

@ -0,0 +1,19 @@
[package]
name = "litellm-model-catalog-refresh"
version = "0.1.0"
edition.workspace = true
license.workspace = true
repository.workspace = true
[dependencies]
litellm-http.workspace = true
litellm-model-catalog.workspace = true
thiserror.workspace = true
url.workspace = true
[dev-dependencies]
litellm-error.workspace = true
litellm-http = { workspace = true, features = ["test-support"] }
reqwest.workspace = true
rstest.workspace = true
tokio.workspace = true

105
litellm-rust/crates/model-catalog-refresh/src/error.rs Normal file
View file

@ -0,0 +1,105 @@
use std::io;
use thiserror::Error;
#[derive(Debug, Error)]
#[error(transparent)]
pub struct Error(Box<Repr>);
#[derive(Debug, Error)]
enum Repr {
#[error(transparent)]
Http(#[from] litellm_http::Error),
#[error("could not read the catalog")]
Read(#[source] io::Error),
#[error("invalid catalog URL")]
InvalidUrl(#[source] url::ParseError),
#[error("unsupported catalog URL scheme {scheme}")]
UnsupportedScheme { scheme: String },
}
impl Error {
pub fn unsupported_scheme(scheme: String) -> Self {
Self(Box::new(Repr::UnsupportedScheme { scheme }))
}
}
impl From<litellm_http::Error> for Error {
fn from(cause: litellm_http::Error) -> Self {
Self(Box::new(Repr::Http(cause)))
}
}
impl From<io::Error> for Error {
fn from(cause: io::Error) -> Self {
Self(Box::new(Repr::Read(cause)))
}
}
impl From<url::ParseError> for Error {
fn from(cause: url::ParseError) -> Self {
Self(Box::new(Repr::InvalidUrl(cause)))
}
}
#[cfg(test)]
mod tests {
use litellm_error::{Report, find};
use rstest::rstest;
use super::*;
use crate::{DoneReason, Reply, Step};
#[rstest]
fn exhausted_transport_keeps_its_typed_cause() {
let reply = Reply::Transport {
error: io::Error::from(io::ErrorKind::ConnectionReset).into(),
};
let Reply::Transport { error } = reply else {
panic!("transport failure")
};
let step = Step::Done {
reason: DoneReason::Exhausted(error),
};
let Step::Done {
reason: DoneReason::Exhausted(error),
} = step
else {
panic!("exhausted attempts")
};
let cause = find::<io::Error>(&error).expect("typed I/O cause");
assert_eq!(cause.kind(), io::ErrorKind::ConnectionReset);
assert_eq!(
Report::new(&error)
.to_string()
.matches(&cause.to_string())
.count(),
1
);
assert!(!error.to_string().contains(&cause.to_string()));
}
#[rstest]
fn invalid_source_keeps_its_typed_cause() {
let error = Error::from(url::Url::parse("http://[invalid").unwrap_err());
assert!(find::<url::ParseError>(&error).is_some());
}
#[rstest]
#[tokio::test]
async fn http_failures_keep_the_cause_without_the_url() {
let cause = litellm_http::Client::for_test()
.get("http://localhost:invalid/private?api_key=secret")
.send()
.await
.unwrap_err();
let error = Error::from(litellm_http::Error::from(cause));
assert!(find::<reqwest::Error>(&error).is_some());
let report = Report::new(&error).to_string();
assert!(!report.contains("secret"), "{report}");
assert!(!report.contains("private"), "{report}");
let debug = format!("{error:?}");
assert!(!debug.contains("secret"), "{debug}");
assert!(!debug.contains("private"), "{debug}");
}
}

9
litellm-rust/crates/model-catalog-refresh/src/lib.rs Normal file
View file

@ -0,0 +1,9 @@
mod error;
mod policy;
mod source;
mod step;
pub use error::Error;
pub use policy::*;
pub use source::*;
pub use step::*;

20
litellm-rust/crates/model-catalog-refresh/src/policy.rs Normal file
View file

@ -0,0 +1,20 @@
use std::time::Duration;
use litellm_model_catalog::IntegrityLimits;
/// Main's defaults, 3 attempts, 30 s max wait, 5 s timeout, backoff base 2.0, jitter 0.0 to 1.0 s
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct RetryPolicy {
pub max_attempts: u32,
pub max_wait: Duration,
pub timeout: Duration,
pub backoff_base_seconds: f64,
pub jitter_min_seconds: f64,
pub jitter_max_seconds: f64,
/// Main's 50 model floor and 0.5 shrink ratio
pub limits: IntegrityLimits,
}
/// Main's default `LITELLM_MODEL_COST_MAP_URL`
pub const DEFAULT_REGISTRY_URL: &str =
"https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json";

17
litellm-rust/crates/model-catalog-refresh/src/source.rs Normal file
View file

@ -0,0 +1,17 @@
use std::path::PathBuf;
use url::Url;
/// Where a fetched catalog comes from
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum Source {
Http(Url),
File(PathBuf),
S3(S3Object),
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct S3Object {
pub bucket: String,
pub key: String,
}

66
litellm-rust/crates/model-catalog-refresh/src/step.rs Normal file
View file

@ -0,0 +1,66 @@
use std::time::Duration;
use litellm_model_catalog::Catalog;
use crate::Error;
use crate::source::Source;
/// One attempt. Where to fetch from is the `Source`, so the request carries no URL
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct FetchRequest {
pub if_none_match: Option<String>,
pub timeout: Duration,
}
#[derive(Debug)]
pub enum Reply {
/// An HTTP, S3 or file answer. The body is read only on 200
Response {
status: u16,
etag: Option<String>,
retry_after: Option<Duration>,
body: Option<Vec<u8>>,
},
/// No answer reached us
Transport { error: Error },
}
/// What the fetch state machine asks its driver to do next
#[derive(Debug)]
pub enum Step {
Fetch(FetchRequest),
Sleep(Duration),
/// A parsed document to install, with its rejected row errors
Adopt {
catalog: Box<Catalog>,
installed: Installed,
errors: Vec<litellm_model_catalog::Error>,
},
/// 304, a final status, or retries exhausted
Done {
reason: DoneReason,
},
}
#[derive(Debug)]
pub enum DoneReason {
NotModified,
Status(u16),
Exhausted(Error),
}
/// What is installed now, the input to the next revalidation
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Installed {
pub origin: Origin,
pub etag: Option<String>,
/// `git_blob_id` of the installed bytes
pub revision: String,
pub model_count: usize,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum Origin {
Bundled,
Remote { source: Source },
}

28
litellm-rust/crates/model-catalog/AGENTS.md
View file

@ -1,5 +1,27 @@
## Validation
## Layout
For `model_prices_and_context_window.json` validation, we should eventually:
`wire/` reads one entry as written. `EntryWire::read` makes one pass over the entry's keys and puts every non-null key in exactly one place, which is an attribute, a flat price parsed by `FlatKey::parse`, a price block, a `supports_*` capability or `unknown`. A known key with a bad value becomes an `Error` that names the key, and the rest of the entry is still read
- Stop skipping this crate's tests
`lower.rs` is the only code that knows both the wire types and `litellm-cost`. It lowers an `EntryWire` into `Prices`, `Adjustments` and `Model` and checks the rules that span keys, such as one price per meter and condition, contiguous tiers, no suffix inside a block, and off-peak needing hours and prices. Lowering only goes one way. Nothing turns `Prices` back into JSON, since the raw fields are kept
`catalog.rs` reads the document, indexes aliases and runs the checks across entries. A bad entry keeps its raw fields and its errors are listed in `Loaded.errors`. Deciding what to do with them is the caller's job. `Catalog::written` gives back every top-level key and value as the document wrote them, in document order. `Catalog::served` gives back every model and alias as Python serves it, without `sample_spec` and `fallback_generalizations`
`Billing` on `Prices` holds the row attributes pricing reads (text unit, band edge, cache-read fallback, wall clock, web search unit, tier scope). `lower.rs` derives it from the row and its `litellm_provider`, so `litellm-cost` holds no provider list
`store/` is the catalog's in-memory store and the only place that indexes model names. Its rules are in [src/store/AGENTS.md](src/store/AGENTS.md)
`layers.rs`, `resolve/`, `model_info.rs` and `register.rs` hold types only. The functions over them land with their first caller
Catalog errors use one opaque `Error` with typed causes and JSON Pointer accessors (`path`, `key`, `field`). `Error::rejects` alone decides whether an error fails the document, rejects its row or neither, and `ErrorKind` is the stable name hosts report it by
## Prices
The price vocabulary lives in `litellm-cost`, and only this crate knows JSON key names. Every price is keyed by `Meter`, `Channel` and `When`. A variation only one meter has stays in the meter (cache TTL, image size, resolution, `_above_Ns_interval`, `max_results_range`). Service tier and prompt band make up `When`. Batch and decisions are a `Channel` because Python bills them with their own rules. Off-peak is an overlay on `Prices`, and regional uplifts are `Adjustments`, never prices
A new `N`, image size or resolution in an existing key shape parses with no Rust change. A new unit, token kind, feature or tier needs a variant in `litellm-cost` plus its spelling in `wire/flat_key.rs` and `FLAT_KEY_PATTERN` in `schema.rs`
`Prices::band` and `Prices::rate` are pinned to Python's `_get_token_base_cost` by `tests/price_lookup_reference.json`. After a billing change in Python, regenerate it from the repo root with `PYTHONPATH=. python litellm-rust/crates/model-catalog/tests/generate_price_lookup_reference.py`
## Tests
`BUNDLED_REGISTRY` embeds the root `model_prices_and_context_window.json`, so the tests over it fail a JSON change with a bad value or a key no type reads

10
litellm-rust/crates/model-catalog/Cargo.toml
View file

@ -6,15 +6,19 @@ license.workspace = true
repository.workspace = true
[features]
schema = ["dep:schemars", "litellm-llms-types/schema"]
schema = ["dep:schemars", "litellm-cost/schema"]
[dependencies]
litellm-llms-types.workspace = true
litellm-cost = { workspace = true, features = ["serde"] }
indexmap = { version = "2.14.0", features = ["serde"] }
jiff = { version = "0.2.37", default-features = false, features = ["std", "tzdb-bundle-always"] }
macro_rules_attribute.workspace = true
schemars = { workspace = true, optional = true }
serde.workspace = true
serde_json.workspace = true
serde_json = { workspace = true, features = ["preserve_order"] }
serde_with.workspace = true
strum.workspace = true
thiserror.workspace = true
time.workspace = true

3
litellm-rust/crates/model-catalog/src/bundled.rs Normal file
View file

@ -0,0 +1,3 @@
/// The registry document this build serves when no other one is loaded.
pub const BUNDLED_REGISTRY: &[u8] =
include_bytes!("../../../../model_prices_and_context_window.json");

67
litellm-rust/crates/model-catalog/src/capabilities.rs
View file

@ -1,67 +0,0 @@
use serde::{Deserialize, Serialize};
/// Primary API surface / task type of the model.
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(rename_all = "snake_case")]
pub enum Mode {
AudioSpeech,
AudioTranscription,
Chat,
Completion,
Decisions,
Embedding,
Evaluation,
Guardrail,
ImageEdit,
ImageGeneration,
Moderation,
Ocr,
Realtime,
Rerank,
Responses,
Search,
VectorStore,
VideoGeneration,
}
/// Gemini audio generation API the model is served through.
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(rename_all = "snake_case")]
pub enum VertexAiAudioApi {
LyriaPredict,
LyriaInteractions,
}
/// Audio container format the model can return.
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(rename_all = "snake_case")]
pub enum AudioFormat {
Mp3,
Wav,
}
/// Input modality the model accepts.
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(rename_all = "snake_case")]
pub enum InputModality {
Text,
Image,
Audio,
Video,
}
/// Output modality the model can produce.
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(rename_all = "snake_case")]
pub enum OutputModality {
Text,
Image,
Audio,
Video,
Code,
}

485
litellm-rust/crates/model-catalog/src/catalog.rs
View file

@ -1,54 +1,29 @@
use crate::error::Error;
use crate::fallback::{FallbackGeneralizations, FallbackRule};
use crate::model_info::ModelInfo;
use std::borrow::Cow;
use std::collections::HashSet;
use indexmap::IndexMap;
use serde::Deserialize;
use serde_json::{Map, Value};
use std::collections::HashMap;
use serde_json::Value;
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct Provenance {
pub source: Option<String>,
pub revision: Option<String>,
pub etag: Option<String>,
}
use crate::error::{Error, Loaded, Rejects, Repr};
use crate::fallback::{FallbackGeneralizations, FallbackRule};
use crate::model::Model;
use crate::store::{Rows, Store};
use crate::wire::{EntryWire, Fields};
const SAMPLE_SPEC: &str = "sample_spec";
const FALLBACK_GENERALIZATIONS: &str = "fallback_generalizations";
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct IntegrityLimits {
pub reference_model_count: usize,
pub min_model_count: usize,
pub min_reference_ratio: f64,
}
#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
pub enum AliasIssue {
#[error("model {model:?} has aliases that are not an array")]
InvalidList { model: String },
#[error("model {model:?} has an alias that is not a string")]
InvalidName { model: String },
#[error("alias {alias:?} of model {model:?} is already a model name")]
CanonicalCollision { model: String, alias: String },
#[error("alias {alias:?} of model {model:?} is already an alias of another model")]
AliasCollision { model: String, alias: String },
min_model_count: usize,
min_reference_ratio: f64,
}
#[derive(Clone, Debug)]
pub struct ModelEntry {
fields: Map<String, Value>,
info: ModelInfo,
}
impl ModelEntry {
pub fn field(&self, name: &str) -> Option<&Value> {
self.fields.get(name)
}
pub fn fields(&self) -> &Map<String, Value> {
&self.fields
}
/// The entry deserialized into the typed mirror of the catalog schema.
pub fn info(&self) -> &ModelInfo {
&self.info
}
fields: Fields,
model: Option<Model>,
}
#[derive(Clone, Copy, Debug)]
@ -58,142 +33,312 @@ pub struct ModelMatch<'a> {
pub entry: &'a ModelEntry,
}
#[derive(Debug)]
#[derive(Clone, Debug)]
struct Section {
index: usize,
key: String,
value: Value,
}
/// One fetched key kept off the served map because its row failed validation
#[derive(Clone, Debug)]
pub struct Rejection<'a> {
pub key: String,
pub errors: Vec<&'a Error>,
/// The bundled row for the key is served instead. `false` means the key is left out
pub replaced_by_bundled: bool,
}
#[derive(Clone, Copy, Debug, serde::Serialize)]
#[serde(untagged)]
pub enum Written<'a> {
Entry(&'a Fields),
Section(&'a Value),
}
#[derive(Clone, Debug)]
pub struct Catalog {
store: Store<CatalogRows>,
sections: Vec<Section>,
fallback_generalizations: Option<FallbackGeneralizations>,
}
/// Models and aliases, an alias served as its owner's row
#[derive(Clone, Debug)]
struct CatalogRows {
entries: IndexMap<String, ModelEntry>,
aliases: IndexMap<String, String>,
lowercase_keys: HashMap<String, String>,
sample_spec: Option<Value>,
fallback_generalizations: Option<FallbackGeneralizations>,
provenance: Provenance,
alias_issues: Vec<AliasIssue>,
}
impl CatalogRows {
fn owner<'a>(&'a self, key: &'a str) -> &'a str {
self.aliases.get(key).map_or(key, String::as_str)
}
fn fields(&self, key: &str) -> Option<&Fields> {
Some(self.entries.get(self.owner(key))?.fields())
}
}
/// Model names match ignoring case, and keys are grouped by their raw `litellm_provider`
impl Rows for CatalogRows {
type Row = Fields;
type Stamp = ();
fn fold(name: &str) -> String {
name.to_lowercase()
}
fn stamp(&self) -> Self::Stamp {}
fn contains(&self, key: &str) -> bool {
self.entries.contains_key(key) || self.aliases.contains_key(key)
}
fn row(&self, key: &str) -> Option<Cow<'_, Fields>> {
self.fields(key).map(Cow::Borrowed)
}
fn scan(&self, visit: &mut dyn FnMut(&str, Option<&str>)) {
let keys = self.entries.keys().chain(self.aliases.keys());
for key in keys {
let provider = self
.fields(key)
.and_then(|fields| fields.get("litellm_provider")?.as_str());
visit(key, provider);
}
}
}
impl IntegrityLimits {
pub fn new(min_model_count: usize, min_reference_ratio: f64) -> Option<Self> {
(0.0..=1.0).contains(&min_reference_ratio).then_some(Self {
min_model_count,
min_reference_ratio,
})
}
}
impl ModelEntry {
pub fn field(&self, name: &str) -> Option<&Value> {
self.fields.get(name)
}
pub fn fields(&self) -> &Fields {
&self.fields
}
pub fn model(&self) -> Option<&Model> {
self.model.as_ref()
}
}
impl Catalog {
pub fn parse(body: &[u8], provenance: Provenance) -> Result<Self, Error> {
let root: IndexMap<String, Value> = serde_json::from_slice(body)?;
if root.is_empty() {
return Err(Error::Empty);
}
pub fn parse(body: &[u8]) -> Result<Loaded<Self>, Error> {
let document = serde_json::from_slice::<Fields>(body)
.map_err(|error| Error::new("", Repr::Json(error)))?;
let mut entries = IndexMap::with_capacity(root.len());
let mut problems = Vec::new();
let mut sections = Vec::new();
let mut entries = IndexMap::with_capacity(document.len());
let mut alias_lists = Vec::new();
let mut alias_issues = Vec::new();
let mut sample_spec = None;
let mut fallback_generalizations = None;
for (name, value) in root {
match name.as_str() {
"sample_spec" => {
sample_spec = Some(value);
for (index, (name, value)) in document.into_iter().enumerate() {
let fields = match value {
Value::Object(fields) if !is_section(&name) => fields,
value => {
if !is_section(&name) {
problems.push(Error::from_key(&name, Repr::NotAnObject));
}
sections.push(Section {
index,
key: name,
value,
});
continue;
}
"fallback_generalizations" => {
fallback_generalizations =
Some(serde_json::from_value::<FallbackGeneralizations>(value)?);
continue;
}
_ => {}
}
let Value::Object(mut fields) = value else {
return Err(Error::EntryNotObject { model: name });
};
let info = ModelInfo::deserialize(&fields)?;
if let Some(aliases) = fields.remove("aliases") {
match aliases {
Value::Array(names) => alias_lists.push((name.clone(), names)),
_ => alias_issues.push(AliasIssue::InvalidList {
model: name.clone(),
}),
}
let (model, aliases, row_problems) = read_row(&fields);
problems.extend(
row_problems
.into_iter()
.map(|problem| problem.prepend(&name)),
);
if let Some(aliases) = aliases {
alias_lists.push((name.clone(), aliases));
}
entries.insert(name, ModelEntry { fields, info });
entries.insert(name, ModelEntry { fields, model });
}
if entries.is_empty() {
return Err(Error::new("", Repr::Empty));
}
let mut aliases = IndexMap::new();
let mut aliases: IndexMap<String, String> = IndexMap::new();
for (model, names) in alias_lists {
for name in names {
let Value::String(alias) = name else {
alias_issues.push(AliasIssue::InvalidName {
model: model.clone(),
});
continue;
};
for alias in names {
if entries.contains_key(&alias) {
alias_issues.push(AliasIssue::CanonicalCollision {
model: model.clone(),
alias,
});
} else if aliases.contains_key(&alias) {
alias_issues.push(AliasIssue::AliasCollision {
model: model.clone(),
alias,
});
problems.push(Error::from_key(&model, Repr::AliasShadowsModel { alias }));
} else if let Some(taken_by) = aliases.get(&alias) {
problems.push(Error::from_key(
&model,
Repr::AliasTaken {
alias,
taken_by: taken_by.clone(),
},
));
} else {
aliases.insert(alias, model.clone());
}
}
}
let lowercase_keys = entries
.keys()
.chain(aliases.keys())
.map(|key| (key.to_lowercase(), key.clone()))
.collect();
Ok(Self {
entries,
aliases,
lowercase_keys,
sample_spec,
fallback_generalizations,
provenance,
alias_issues,
let fallback_generalizations = sections
.iter()
.find(|section| section.key == FALLBACK_GENERALIZATIONS)
.and_then(|section| {
FallbackGeneralizations::deserialize(&section.value)
.map_err(|error| {
problems.push(Error::from_key(&section.key, Repr::Invalid(error)))
})
.ok()
});
Ok(Loaded {
value: Self {
store: Store::new(CatalogRows { entries, aliases }),
sections,
fallback_generalizations,
},
errors: problems,
})
}
pub fn validate(&self, limits: IntegrityLimits) -> Result<(), Error> {
if !limits.min_reference_ratio.is_finite()
|| !(0.0..=1.0).contains(&limits.min_reference_ratio)
pub fn row_errors(model: &str, fields: &Fields) -> Vec<Error> {
let (_, _, errors) = read_row(fields);
errors
.into_iter()
.map(|error| error.prepend(model))
.collect()
}
pub fn written(&self) -> impl Iterator<Item = (&str, Written<'_>)> {
let mut sections = self.sections.iter().peekable();
let mut entries = self.rows().entries.iter();
(0..self.rows().entries.len() + self.sections.len()).map_while(move |index| match sections
.next_if(|section| section.index == index)
{
return Err(Error::InvalidRatio);
}
let actual = self.entries.len();
Some(section) => Some((section.key.as_str(), Written::Section(&section.value))),
None => entries
.next()
.map(|(name, entry)| (name.as_str(), Written::Entry(&entry.fields))),
})
}
pub fn served(&self) -> impl Iterator<Item = (&str, &Fields)> {
let rows = self.rows();
rows.entries
.iter()
.map(|(name, entry)| (name.as_str(), entry.fields()))
.chain(
rows.aliases
.iter()
.map(|(alias, owner)| (alias.as_str(), rows.entries[owner].fields())),
)
}
pub fn check_size(&self, reference: usize, limits: IntegrityLimits) -> Result<(), Error> {
let actual = self.model_count();
if actual < limits.min_model_count {
return Err(Error::BelowMinimum {
actual,
minimum: limits.min_model_count,
});
return Err(Error::new(
"",
Repr::BelowMinimum {
actual,
minimum: limits.min_model_count,
},
));
}
if limits.reference_model_count > 0
&& (actual as f64) < (limits.reference_model_count as f64) * limits.min_reference_ratio
{
return Err(Error::Shrunk {
actual,
reference: limits.reference_model_count,
ratio: limits.min_reference_ratio,
});
if (actual as f64) < (reference as f64) * limits.min_reference_ratio {
return Err(Error::new(
"",
Repr::Shrunk {
actual,
reference,
ratio: limits.min_reference_ratio,
},
));
}
Ok(())
}
pub fn lookup(&self, key: &str) -> Option<ModelMatch<'_>> {
let matched_key = if self.entries.contains_key(key) || self.aliases.contains_key(key) {
key
} else {
self.lowercase_keys.get(&key.to_lowercase())?.as_str()
};
let canonical_key = self
.aliases
.get(matched_key)
.map(String::as_str)
.unwrap_or(matched_key);
let (canonical_key, entry) = self.entries.get_key_value(canonical_key)?;
let matched_key = self
pub fn without_rejected(self, errors: &[Error]) -> (Self, Vec<&Error>) {
let row_names: HashSet<String> = self
.rows()
.entries
.get_key_value(matched_key)
.keys()
.cloned()
.chain(
self.sections
.iter()
.filter(|section| !is_section(&section.key))
.map(|section| section.key.clone()),
)
.collect();
let rejected: HashSet<String> = errors
.iter()
.filter(|error| error.rejects() == Rejects::Row)
.filter_map(|error| crate::error::pointer_root(error.path()))
.filter(|root| row_names.contains(root))
.collect();
let removed_errors = errors
.iter()
.filter(|error| {
crate::error::pointer_root(error.path())
.is_some_and(|root| rejected.contains(&root))
})
.collect();
let removed_positions: HashSet<usize> = self
.written()
.enumerate()
.filter(|(_, (key, _))| rejected.contains(*key))
.map(|(index, _)| index)
.collect();
let mut catalog = self;
catalog.store.edit(|rows| {
rows.entries.retain(|name, _| !rejected.contains(name));
rows.aliases.retain(|_, owner| !rejected.contains(owner));
});
catalog
.sections
.retain(|section| is_section(&section.key) || !rejected.contains(&section.key));
for section in &mut catalog.sections {
let removed_before = removed_positions
.iter()
.filter(|index| **index < section.index)
.count();
section.index -= removed_before;
}
(catalog, removed_errors)
}
pub fn unrecognized(&self) -> impl Iterator<Item = (&str, &str)> {
self.rows().entries.iter().flat_map(|(name, entry)| {
entry
.model
.iter()
.flat_map(|model| &model.unrecognized)
.map(move |key| (name.as_str(), key.as_str()))
})
}
pub fn lookup(&self, key: &str) -> Option<ModelMatch<'_>> {
let rows = self.rows();
let found = self.store.find_key(key)?;
let (canonical_key, entry) = rows.entries.get_key_value(rows.owner(&found))?;
let matched_key = rows
.entries
.get_key_value(found.as_str())
.map(|(key, _)| key.as_str())
.or_else(|| {
self.aliases
.get_key_value(matched_key)
rows.aliases
.get_key_value(found.as_str())
.map(|(key, _)| key.as_str())
})?;
Some(ModelMatch {
@ -204,30 +349,58 @@ impl Catalog {
}
pub fn model_count(&self) -> usize {
self.entries.len()
self.rows().entries.len()
}
pub fn model_names(&self) -> impl Iterator<Item = &str> {
self.entries.keys().map(String::as_str)
}
pub fn alias_count(&self) -> usize {
self.aliases.len()
self.rows().entries.keys().map(String::as_str)
}
pub fn aliases(&self) -> &IndexMap<String, String> {
&self.aliases
&self.rows().aliases
}
pub fn alias_issues(&self) -> &[AliasIssue] {
&self.alias_issues
fn rows(&self) -> &CatalogRows {
self.store.rows()
}
pub fn sample_spec(&self) -> Option<&Value> {
self.sample_spec.as_ref()
}
pub fn fallback_generalizations(&self) -> Option<&FallbackGeneralizations> {
self.fallback_generalizations.as_ref()
self.sections
.iter()
.find(|section| section.key == SAMPLE_SPEC)
.map(|section| &section.value)
}
pub fn fallback_rules(&self) -> Option<&[FallbackRule]> {
Some(self.fallback_generalizations.as_ref()?.rules.as_slice())
}
pub fn provenance(&self) -> &Provenance {
&self.provenance
}
fn read_row(fields: &Fields) -> (Option<Model>, Option<Vec<String>>, Vec<Error>) {
let wire = EntryWire::read(fields);
let aliases = wire.attributes.aliases.clone();
let timezone_error = unknown_timezone(&wire);
match Model::try_from(wire) {
Ok(model) => (Some(model), aliases, timezone_error.into_iter().collect()),
Err(mut errors) => {
errors.extend(timezone_error);
(None, aliases, errors)
}
}
}
fn is_section(key: &str) -> bool {
key == SAMPLE_SPEC || key == FALLBACK_GENERALIZATIONS
}
fn unknown_timezone(wire: &EntryWire) -> Option<Error> {
let zone = wire.off_peak.as_ref()?.weekday_timezone.as_deref()?.trim();
(!zone.is_empty() && jiff::tz::TimeZone::get(zone).is_err()).then(|| {
Error::from_path(
"off_peak_pricing.weekday_timezone",
Repr::UnknownTimezone {
zone: zone.to_owned(),
},
)
})
}

391
litellm-rust/crates/model-catalog/src/error.rs
View file

@ -1,21 +1,392 @@
use std::num::ParseIntError;
use strum::{EnumDiscriminants, IntoStaticStr, VariantArray};
use thiserror::Error;
#[derive(Debug, Error)]
pub enum Error {
#[error("invalid catalog JSON")]
Json(#[from] serde_json::Error),
#[error("catalog is empty")]
#[error(transparent)]
pub struct Error(Box<Context>);
#[derive(Debug, Error)]
enum Context {
#[error(transparent)]
Bare(Repr),
#[error("{}", display_path(path))]
At {
path: String,
#[source]
cause: Repr,
},
}
#[derive(Debug, Error)]
pub(crate) enum DateCause {
#[error("invalid date component")]
Number(#[from] ParseIntError),
#[error("calendar component out of range")]
Range(#[from] time::error::ComponentRange),
}
#[derive(Debug, Error, EnumDiscriminants)]
#[strum_discriminants(name(ErrorKind), derive(IntoStaticStr, VariantArray), vis(pub))]
pub(crate) enum Repr {
#[error("invalid JSON")]
#[strum_discriminants(strum(serialize = "json"))]
Json(#[source] serde_json::Error),
#[error("has no models")]
#[strum_discriminants(strum(serialize = "empty"))]
Empty,
#[error("model {model:?} must be an object")]
EntryNotObject { model: String },
#[error("catalog has {actual} models, below minimum {minimum}")]
#[error("has {actual} models, below minimum {minimum}")]
#[strum_discriminants(strum(serialize = "below_minimum"))]
BelowMinimum { actual: usize, minimum: usize },
#[error("catalog has {actual} models, below {ratio} of reference count {reference}")]
#[error("has {actual} models, below {ratio} of reference count {reference}")]
#[strum_discriminants(strum(serialize = "shrunk"))]
Shrunk {
actual: usize,
reference: usize,
ratio: f64,
},
#[error("minimum reference ratio must be finite and between zero and one")]
InvalidRatio,
#[error("must be a JSON object")]
#[strum_discriminants(strum(serialize = "not_an_object"))]
NotAnObject,
#[error("invalid value")]
#[strum_discriminants(strum(serialize = "invalid"))]
Invalid(#[source] serde_json::Error),
#[error("not a field this block reads")]
#[strum_discriminants(strum(serialize = "unknown_block_key"))]
UnknownBlockKey,
#[error("required")]
#[strum_discriminants(strum(serialize = "missing_provider"))]
MissingProvider,
#[error("{zone:?} is not an IANA time zone")]
#[strum_discriminants(strum(serialize = "unknown_timezone"))]
UnknownTimezone { zone: String },
#[error("alias {alias:?} is already a model name")]
#[strum_discriminants(strum(serialize = "alias_shadows_model"))]
AliasShadowsModel { alias: String },
#[error("alias {alias:?} already belongs to {taken_by}")]
#[strum_discriminants(strum(serialize = "alias_taken"))]
AliasTaken { alias: String, taken_by: String },
#[error("{first} and {second} price the same meter under the same conditions")]
#[strum_discriminants(strum(serialize = "duplicate_price"))]
DuplicatePrice { first: String, second: String },
#[error("a price inside {block} takes no band or tier suffix")]
#[strum_discriminants(strum(serialize = "suffix_inside_block"))]
SuffixInsideBlock { block: &'static str },
#[error("a tier needs exactly one of range and max_results_range")]
#[strum_discriminants(strum(serialize = "tier_range"))]
TierRange,
#[error("the range ends below its start")]
#[strum_discriminants(strum(serialize = "tier_reversed"))]
TierReversed,
#[error("starts at {start}, but the tiers before it end at {expected}")]
#[strum_discriminants(strum(serialize = "tier_gap"))]
TierGap { start: u64, expected: u64 },
#[error("tiered_pricing mixes token ranges and result ranges")]
#[strum_discriminants(strum(serialize = "mixed_tier_basis"))]
MixedTierBasis,
#[error("a tier by result count only prices input_cost_per_query")]
#[strum_discriminants(strum(serialize = "not_priced_by_results"))]
NotPricedByResults,
#[error("tiered_pricing and prompt token bands are set in two places")]
#[strum_discriminants(strum(serialize = "bands_from_two_sources"))]
BandsFromTwoSources,
#[error("has no off-peak rate")]
#[strum_discriminants(strum(serialize = "not_an_off_peak_rate"))]
NotAnOffPeakRate,
#[error("off_peak_pricing has no prices")]
#[strum_discriminants(strum(serialize = "no_off_peak_prices"))]
NoOffPeakPrices,
#[error("off_peak_pricing needs hours_utc or windows")]
#[strum_discriminants(strum(serialize = "no_off_peak_hours"))]
NoOffPeakHours,
#[error("must list at least one entry")]
#[strum_discriminants(strum(serialize = "empty_list"))]
EmptyList,
#[error("expected a calendar date YYYY-MM-DD")]
#[strum_discriminants(strum(serialize = "invalid_date"))]
InvalidDate {
#[source]
cause: Option<DateCause>,
},
}
/// What a problem takes out of the served map
#[derive(Clone, Copy, Debug, PartialEq, Eq, VariantArray)]
pub enum Rejects {
Document,
Row,
Nothing,
}
impl Error {
pub fn kind(&self) -> ErrorKind {
ErrorKind::from(self.repr())
}
pub fn rejects(&self) -> Rejects {
match self.repr() {
Repr::Json(_) | Repr::Empty | Repr::BelowMinimum { .. } | Repr::Shrunk { .. } => {
Rejects::Document
}
Repr::NotAnObject
| Repr::Invalid(_)
| Repr::UnknownBlockKey
| Repr::UnknownTimezone { .. }
| Repr::AliasShadowsModel { .. }
| Repr::AliasTaken { .. }
| Repr::DuplicatePrice { .. }
| Repr::SuffixInsideBlock { .. }
| Repr::TierRange
| Repr::TierReversed
| Repr::TierGap { .. }
| Repr::MixedTierBasis
| Repr::NotPricedByResults
| Repr::BandsFromTwoSources
| Repr::NotAnOffPeakRate
| Repr::NoOffPeakPrices
| Repr::NoOffPeakHours
| Repr::EmptyList
| Repr::InvalidDate { .. } => Rejects::Row,
Repr::MissingProvider => Rejects::Nothing,
}
}
}
#[derive(Debug)]
pub struct Loaded<T> {
pub value: T,
pub errors: Vec<Error>,
}
impl From<Repr> for Error {
fn from(cause: Repr) -> Self {
Self(Box::new(Context::Bare(cause)))
}
}
impl From<DateCause> for Error {
fn from(cause: DateCause) -> Self {
Repr::InvalidDate { cause: Some(cause) }.into()
}
}
impl Error {
fn repr(&self) -> &Repr {
match &*self.0 {
Context::Bare(cause) | Context::At { cause, .. } => cause,
}
}
pub(crate) fn new(path: impl Into<String>, cause: Repr) -> Self {
Self(Box::new(Context::At {
path: path.into(),
cause,
}))
}
pub(crate) fn from_key(key: &str, cause: Repr) -> Self {
Self::new(pointer_for_key(key), cause)
}
pub(crate) fn from_path(path: &str, cause: Repr) -> Self {
Self::new(path_to_pointer(path), cause)
}
pub fn path(&self) -> &str {
match &*self.0 {
Context::Bare(_) => "",
Context::At { path, .. } => path,
}
}
pub fn key(&self) -> Option<String> {
pointer_root(self.path())
}
pub fn field(&self) -> Option<String> {
let segments: Vec<_> = self
.path()
.split('/')
.skip(2)
.map(unescape_pointer_segment)
.collect();
(!segments.is_empty()).then(|| segments.join("/"))
}
pub(crate) fn prepend(self, key: &str) -> Self {
let (path, cause) = match *self.0 {
Context::Bare(cause) => (String::new(), cause),
Context::At { path, cause } => (path, cause),
};
Self::new(format!("/{}{path}", escape_pointer_segment(key)), cause)
}
}
pub(crate) fn pointer_root(path: &str) -> Option<String> {
path.strip_prefix('/')
.and_then(|path| path.split('/').next())
.map(unescape_pointer_segment)
}
fn escape_pointer_segment(segment: &str) -> String {
segment.replace('~', "~0").replace('/', "~1")
}
pub(crate) fn pointer_for_key(key: &str) -> String {
format!("/{}", escape_pointer_segment(key))
}
pub(crate) fn pointer_child(parent: &str, key: &str) -> String {
format!("{parent}/{}", escape_pointer_segment(key))
}
fn unescape_pointer_segment(segment: &str) -> String {
segment.replace("~1", "/").replace("~0", "~")
}
fn path_to_pointer(key: &str) -> String {
let mut pointer = String::new();
let mut segment = String::new();
let mut chars = key.chars();
let mut push_segment = |segment: &mut String| {
if !segment.is_empty() {
pointer.push('/');
pointer.push_str(&escape_pointer_segment(segment));
segment.clear();
}
};
while let Some(character) = chars.next() {
match character {
'.' => push_segment(&mut segment),
'[' => {
push_segment(&mut segment);
let mut index = String::new();
for character in chars.by_ref() {
if character == ']' {
break;
}
index.push(character);
}
push_segment(&mut index);
}
character => segment.push(character),
}
}
push_segment(&mut segment);
pointer
}
fn display_path(path: &str) -> String {
if path.is_empty() {
return "catalog".to_owned();
}
path.split('/')
.skip(1)
.map(unescape_pointer_segment)
.enumerate()
.fold(String::new(), |mut display, (index, segment)| {
if index == 0 {
display.push_str(&segment);
} else if segment.bytes().all(|byte| byte.is_ascii_digit()) {
display.push('[');
display.push_str(&segment);
display.push(']');
} else {
display.push('.');
display.push_str(&segment);
}
display
})
}
#[cfg(test)]
mod tests {
use litellm_error::{Report, find};
use rstest::rstest;
use serde_json::Value;
use super::*;
use crate::{Catalog, wire::IsoDate};
#[rstest]
#[case::json(Repr::Json(serde_json::from_str::<Value>("{").unwrap_err()), Rejects::Document)]
#[case::empty(Repr::Empty, Rejects::Document)]
#[case::below_minimum(Repr::BelowMinimum { actual: 1, minimum: 2 }, Rejects::Document)]
#[case::shrunk(Repr::Shrunk { actual: 1, reference: 3, ratio: 0.5 }, Rejects::Document)]
#[case::not_an_object(Repr::NotAnObject, Rejects::Row)]
#[case::invalid(Repr::Invalid(serde_json::from_str::<Value>("x").unwrap_err()), Rejects::Row)]
#[case::unknown_block_key(Repr::UnknownBlockKey, Rejects::Row)]
#[case::missing_provider(Repr::MissingProvider, Rejects::Nothing)]
#[case::unknown_timezone(Repr::UnknownTimezone { zone: "invalid".into() }, Rejects::Row)]
#[case::alias_shadows_model(Repr::AliasShadowsModel { alias: "model".into() }, Rejects::Row)]
#[case::alias_taken(Repr::AliasTaken { alias: "alias".into(), taken_by: "owner".into() }, Rejects::Row)]
#[case::duplicate_price(Repr::DuplicatePrice { first: "a".into(), second: "b".into() }, Rejects::Row)]
#[case::suffix_inside_block(Repr::SuffixInsideBlock { block: "tiered_pricing" }, Rejects::Row)]
#[case::tier_range(Repr::TierRange, Rejects::Row)]
#[case::tier_reversed(Repr::TierReversed, Rejects::Row)]
#[case::tier_gap(Repr::TierGap { start: 1, expected: 0 }, Rejects::Row)]
#[case::mixed_tier_basis(Repr::MixedTierBasis, Rejects::Row)]
#[case::not_priced_by_results(Repr::NotPricedByResults, Rejects::Row)]
#[case::bands_from_two_sources(Repr::BandsFromTwoSources, Rejects::Row)]
#[case::not_an_off_peak_rate(Repr::NotAnOffPeakRate, Rejects::Row)]
#[case::no_off_peak_prices(Repr::NoOffPeakPrices, Rejects::Row)]
#[case::no_off_peak_hours(Repr::NoOffPeakHours, Rejects::Row)]
#[case::empty_list(Repr::EmptyList, Rejects::Row)]
#[case::invalid_date(Repr::InvalidDate { cause: None }, Rejects::Row)]
fn policy_classifies_each_error(#[case] error: Repr, #[case] expected: Rejects) {
assert_eq!(Error::from(error).rejects(), expected);
}
#[rstest]
#[case::root("", "x", "/x")]
#[case::empty_model("/", "x", "//x")]
#[case::empty_parent_segment("/m/", "x", "/m//x")]
#[case::escaped("/m~1n", "a~/b", "/m~1n/a~0~1b")]
fn pointers_preserve_segments(#[case] parent: &str, #[case] key: &str, #[case] expected: &str) {
assert_eq!(pointer_child(parent, key), expected);
}
#[rstest]
#[case::malformed(br#"{"#)]
#[case::bad_price(br#"{"m":{"litellm_provider":"test","input_cost_per_token":"free"}}"#)]
fn json_causes_are_typed_and_rendered_once(#[case] body: &[u8]) {
let error = match Catalog::parse(body) {
Ok(loaded) => loaded.errors.into_iter().next().expect("invalid price"),
Err(error) => error,
};
let cause = find::<serde_json::Error>(&error)
.expect("typed JSON cause")
.to_string();
let report = Report::new(&error).to_string();
assert_eq!(report.matches(&cause).count(), 1, "{report}");
assert!(!error.to_string().contains(&cause));
}
#[rstest]
#[case::year("xxxx-01-01")]
#[case::month("2026-xx-01")]
#[case::day("2026-01-xx")]
fn date_number_errors_keep_the_cause(#[case] value: &str) {
let error = value.parse::<IsoDate>().unwrap_err();
let cause = find::<ParseIntError>(&error)
.expect("typed integer cause")
.to_string();
assert_eq!(error.rejects(), Rejects::Row);
assert_eq!(Report::new(&error).to_string().matches(&cause).count(), 1);
assert!(!error.to_string().contains(&cause));
}
#[rstest]
#[case::month("2026-13-01")]
#[case::day("2026-02-30")]
fn date_range_errors_keep_the_cause(#[case] value: &str) {
let error = value.parse::<IsoDate>().unwrap_err();
let cause = find::<time::error::ComponentRange>(&error)
.expect("typed calendar cause")
.to_string();
assert_eq!(error.rejects(), Rejects::Row);
assert_eq!(Report::new(&error).to_string().matches(&cause).count(), 1);
assert!(!error.to_string().contains(&cause));
}
}

9
litellm-rust/crates/model-catalog/src/fallback.rs
View file

@ -1,10 +1,9 @@
use serde::{Deserialize, Serialize};
use serde_json::Value;
use std::collections::BTreeMap;
/// One regex rule generalizing unknown model ids to known families.
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Eq, Hash)]
pub struct FallbackRule {
pub name: String,
pub pattern: String,
@ -15,8 +14,8 @@ pub struct FallbackRule {
}
/// Regex rules that generalize unknown model ids to known families; not a model entry.
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Eq, Hash)]
#[serde(deny_unknown_fields)]
pub struct FallbackGeneralizations {
pub rules: Vec<FallbackRule>,

32
litellm-rust/crates/model-catalog/src/layers.rs Normal file
View file

@ -0,0 +1,32 @@
use std::sync::Arc;
use indexmap::IndexMap;
use serde_json::Value;
use crate::catalog::Catalog;
use crate::register::Registrations;
use crate::wire::Fields;
/// One field of a fetched row over the layer below, RFC 7386. An absent field inherits, JSON `null` is `Remove`
#[derive(Clone, Debug, PartialEq)]
pub enum Patch {
Set(Value),
Remove,
}
/// The four layers served rows are composed from, lowest first
#[derive(Clone, Debug)]
#[expect(
dead_code,
reason = "02 and 07 add the functions that read these layers"
)]
pub struct Layers {
/// 0, the wheel's embedded map
bundled: Arc<Catalog>,
/// 1, the last installed fetch, rejected rows left out
fetched: Option<Arc<Catalog>>,
/// 2, `register(persist=False)`, cleared by every install and refilled by the router replay
transient: IndexMap<String, Fields>,
/// 3, `register(persist=True)`, applied over every install
persisted: Registrations,
}

46
litellm-rust/crates/model-catalog/src/lib.rs
View file

@ -7,20 +7,18 @@
//! in `litellm-router-core`, which reads this catalog through its dynamic config.
//!
//! # Structure
//! - `Catalog`: the parsed map. `Catalog::parse` takes the JSON body and a `Provenance` (where
//! the body came from: the bundled copy in the repo or the remote copy, with revision and etag),
//! and keeps every entry in file order with its raw fields.
//! - `Catalog`: the parsed map. `Catalog::parse` takes the JSON body and returns `Loaded<Catalog>`,
//! keeping every entry in file order with its raw fields and collecting row and alias errors.
//! - `ModelEntry`: one model. `ModelEntry::fields` is the raw JSON object, so unknown fields are
//! never lost; `ModelEntry::info` is the typed mirror `ModelInfo` (pricing in `pricing`,
//! modalities and modes in `capabilities`).
//! never lost; `ModelEntry::model` is the optional typed `Model` with attributes, prices,
//! adjustments and capabilities. Invalid rows retain their raw fields without a typed model.
//! - Aliases: an entry may list `aliases`; each alias resolves to its canonical key. Problems
//! found while indexing them (bad lists, collisions) are kept as `AliasIssue`s instead of
//! failing the parse.
//! found while indexing them (bad lists, collisions) are kept in `Loaded::errors` instead of
//! failing the document.
//! - `FallbackGeneralizations` / `FallbackRule`: regex rules that map unknown model ids onto known
//! families. They are data in the file, not entries.
//! - Validation: `Catalog::validate` checks a catalog against `IntegrityLimits` (a remote copy that
//! lost most of its models is refused), and `validate_registry` checks every entry against the
//! checked-in contract.
//! - Validation: `Catalog::check_size` checks a catalog against `IntegrityLimits` (a remote copy
//! that lost most of its models is refused), and `Catalog::row_errors` validates one row.
//!
//! # Lookups
//! `Catalog::lookup` resolves a key exactly first (entry or alias), then case-insensitively, and
@ -31,21 +29,37 @@
//! Cost tracking, the router (through `RouterDynamicConfig`) and the gateway. Nothing in this crate
//! performs I/O; loading and refreshing the file is the caller's job.
mod capabilities;
macro_rules_attribute::attribute_alias! {
#[apply(wire_type)] =
#[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))];
}
mod bundled;
mod catalog;
mod error;
mod fallback;
mod layers;
mod lower;
mod model;
mod model_info;
mod pricing;
mod validation;
mod register;
mod resolve;
mod store;
mod vocabulary;
pub mod wire;
pub use capabilities::*;
pub use bundled::*;
pub use catalog::*;
pub use error::*;
pub use fallback::*;
pub use layers::*;
pub use model::*;
pub use model_info::*;
pub use pricing::*;
pub use validation::*;
pub use register::*;
pub use resolve::*;
pub use store::*;
pub use vocabulary::*;
#[cfg(feature = "schema")]
mod schema;

855
litellm-rust/crates/model-catalog/src/lower.rs Normal file
View file

@ -0,0 +1,855 @@
use std::collections::BTreeMap;
use std::collections::btree_map::Entry;
use std::num::NonZeroU64;
use litellm_cost::money::UnitPrice;
use litellm_cost::price::{
Adjustments, Band, BandEdge, Billing, CacheReadFallback, CacheReadMode, CacheTtl,
GuardrailUnit, InputUnit, Meter, OffPeak, OffPeakRates, OutputUnit, Price, Prices, Residency,
SearchContextSize, TextUnit, TierScope, TokenKind, Uplift, WallClock, WebSearchUnit, When,
};
use litellm_cost::schedule::{Schedule, Window};
use crate::error::{Error, Repr, pointer_child, pointer_for_key};
use crate::model::Model;
use crate::vocabulary::Mode;
use crate::wire::{EntryWire, FlatPrice, OffPeakBlock, TierRow, WebSearchBillingUnit};
impl TryFrom<&EntryWire> for Prices {
type Error = Vec<Error>;
fn try_from(wire: &EntryWire) -> Result<Self, Self::Error> {
let mut problems = Vec::new();
let sourced = flat_prices(&wire.prices)
.chain(tier_prices(wire, &mut problems))
.chain(search_context_prices(wire))
.chain(guardrail_prices(wire));
let rates = dedupe(sourced, &mut problems);
let off_peak = wire
.off_peak
.as_ref()
.and_then(|block| off_peak(block, &mut problems));
if problems.is_empty() {
let billing = billing(wire, &rates);
Ok(Prices::new(rates, off_peak, billing))
} else {
Err(problems)
}
}
}
fn billing(wire: &EntryWire, rates: &BTreeMap<Price, UnitPrice>) -> Billing {
let provider = wire.attributes.litellm_provider.as_deref();
let prices_meter = |priced: fn(&Meter) -> bool| rates.keys().any(|price| priced(&price.meter));
let prices_tokens = prices_meter(|meter| {
matches!(
meter,
Meter::Input(InputUnit::Token(_)) | Meter::Output(OutputUnit::Token(_))
)
});
let prices_characters = prices_meter(|meter| {
matches!(
meter,
Meter::Input(InputUnit::Character) | Meter::Output(OutputUnit::Character)
)
});
let prices_cache_read = prices_meter(|meter| {
matches!(
meter,
Meter::CacheRead {
kind: TokenKind::Text,
..
}
)
});
Billing {
text_unit: if prices_characters && !prices_tokens {
TextUnit::Character
} else {
TextUnit::Token
},
band_edge: if provider == Some("xai") {
BandEdge::Inclusive
} else {
BandEdge::Exclusive
},
cache_read_fallback: if provider == Some("fireworks_ai") && !prices_cache_read {
CacheReadFallback::HalfInputRate
} else {
CacheReadFallback::InputRate
},
wall_clock: match wire.attributes.mode {
None | Some(Mode::Chat | Mode::Completion | Mode::Embedding | Mode::Responses) => {
WallClock::ResponseTime
}
Some(Mode::AudioTranscription) => WallClock::AudioFile,
Some(_) => WallClock::Never,
},
web_search: match wire.attributes.web_search_billing_unit {
Some(WebSearchBillingUnit::PerQuery) => WebSearchUnit::PerQuery,
Some(WebSearchBillingUnit::PerPrompt) | None => WebSearchUnit::PerPrompt,
},
tier_scope: if provider == Some("dashscope") && wire.tier_rows.is_some() {
TierScope::WholeCall
} else {
TierScope::PerMeter
},
}
}
impl From<&EntryWire> for Adjustments {
fn from(wire: &EntryWire) -> Self {
Adjustments::new(
[
(Uplift::NonGlobalEndpoint, wire.regional_endpoint_uplift),
(Uplift::Residency(Residency::Eu), wire.residency_uplift_eu),
(Uplift::Residency(Residency::Us), wire.residency_uplift_us),
]
.into_iter()
.filter_map(|(uplift, factor)| Some((uplift, factor?))),
)
}
}
impl TryFrom<EntryWire> for Model {
type Error = Vec<Error>;
fn try_from(wire: EntryWire) -> Result<Self, Self::Error> {
let prices = Prices::try_from(&wire);
let adjustments = Adjustments::from(&wire);
let EntryWire {
attributes,
capabilities,
unknown,
problems,
..
} = wire;
let missing_provider = attributes.litellm_provider.is_none().then_some(Error::new(
pointer_for_key("litellm_provider"),
Repr::MissingProvider,
));
match prices {
Ok(prices) if problems.is_empty() && missing_provider.is_none() => Ok(Model {
attributes,
prices,
adjustments,
capabilities,
unrecognized: unknown.into_iter().map(|(key, _)| key).collect(),
}),
prices => Err(problems
.into_iter()
.chain(missing_provider)
.chain(prices.err().into_iter().flatten())
.collect()),
}
}
}
struct Sourced {
price: Price,
rate: UnitPrice,
source: String,
path: String,
}
fn flat_prices(prices: &[FlatPrice]) -> impl Iterator<Item = Sourced> + '_ {
prices.iter().map(|flat| {
let (channel, when) = flat.flat.channel_and_when();
Sourced {
price: Price {
meter: flat.flat.meter.clone(),
channel,
when,
},
rate: flat.price,
source: flat.key.clone(),
path: flat.path.clone(),
}
})
}
fn dedupe(
sourced: impl Iterator<Item = Sourced>,
problems: &mut Vec<Error>,
) -> BTreeMap<Price, UnitPrice> {
let mut seen: BTreeMap<Price, (UnitPrice, String, String)> = BTreeMap::new();
for Sourced {
price,
rate,
source,
path,
} in sourced
{
match seen.entry(price) {
Entry::Occupied(first) if first.get().0 == rate => {}
Entry::Occupied(first) => problems.push(Error::new(
path,
Repr::DuplicatePrice {
first: first.get().1.clone(),
second: source.clone(),
},
)),
Entry::Vacant(slot) => {
slot.insert((rate, source, path));
}
}
}
seen.into_iter()
.map(|(price, (rate, _, _))| (price, rate))
.collect()
}
#[derive(Clone, Copy, PartialEq, Eq)]
enum TierBasis {
PromptTokens,
Results,
}
fn tier_basis(row: &TierRow) -> Result<(TierBasis, [u64; 2]), Error> {
match (row.range, row.max_results_range) {
(Some(range), None) => Ok((TierBasis::PromptTokens, range)),
(None, Some(range)) => Ok((TierBasis::Results, range)),
_ => Err(Error::new(row.path.clone(), Repr::TierRange)),
}
}
fn tier_prices(wire: &EntryWire, problems: &mut Vec<Error>) -> Vec<Sourced> {
let Some(rows) = wire.tier_rows.as_deref().filter(|rows| !rows.is_empty()) else {
return Vec::new();
};
let based: Vec<_> = rows
.iter()
.filter_map(|row| match tier_basis(row) {
Ok((basis, range)) => Some((basis, range, row)),
Err(problem) => {
problems.push(problem);
None
}
})
.collect();
let Some(&(basis, ..)) = based.first() else {
return Vec::new();
};
if based.iter().any(|(other, ..)| *other != basis) {
problems.push(Error::from_key("tiered_pricing", Repr::MixedTierBasis));
return Vec::new();
}
if basis == TierBasis::PromptTokens {
problems.extend(
wire.prices
.iter()
.filter(|flat| flat.flat.band != Band::Base)
.map(|flat| Error::new(flat.path.clone(), Repr::BandsFromTwoSources)),
);
}
let mut ordered = based;
ordered.sort_by_key(|(_, [start, _], _)| *start);
problems.extend(ladder_problems(basis, &ordered));
ordered
.iter()
.flat_map(|(basis, [start, _], row)| {
row.prices.iter().map(move |flat| (*basis, *start, flat))
})
.filter_map(
|(basis, start, flat)| match tier_price(basis, start, flat) {
Ok(sourced) => Some(sourced),
Err(problem) => {
problems.push(problem);
None
}
},
)
.collect()
}
fn ladder_problems(basis: TierBasis, ordered: &[(TierBasis, [u64; 2], &TierRow)]) -> Vec<Error> {
let step = match basis {
TierBasis::PromptTokens => 0,
TierBasis::Results => 1,
};
let reversed = ordered
.iter()
.filter(|(_, [start, end], _)| end < start)
.map(|(.., row)| Error::new(row.path.clone(), Repr::TierReversed));
let first = ordered
.first()
.filter(|(_, [start, _], _)| *start > step)
.map(|(_, [start, _], row)| {
Error::new(
row.path.clone(),
Repr::TierGap {
start: *start,
expected: 0,
},
)
});
let gaps = ordered.windows(2).filter_map(|pair| {
let (_, [_, end], _) = pair[0];
let (_, [start, _], row) = pair[1];
(end.checked_add(step) != Some(start)).then(|| {
Error::new(
row.path.clone(),
Repr::TierGap {
start,
expected: end,
},
)
})
});
reversed.chain(first).chain(gaps).collect()
}
fn tier_price(basis: TierBasis, start: u64, flat: &FlatPrice) -> Result<Sourced, Error> {
if flat.flat.has_suffix() {
return Err(Error::new(
flat.path.clone(),
Repr::SuffixInsideBlock {
block: "tiered_pricing",
},
));
}
let (meter, band) = match (basis, &flat.flat.meter) {
(TierBasis::PromptTokens, meter) => (meter.clone(), Band::over_prompt_tokens(start)),
(TierBasis::Results, Meter::Input(InputUnit::Query { over_results: None })) => (
Meter::Input(InputUnit::Query {
over_results: NonZeroU64::new(start.saturating_sub(1)),
}),
Band::Base,
),
(TierBasis::Results, _) => {
return Err(Error::new(flat.path.clone(), Repr::NotPricedByResults));
}
};
Ok(Sourced {
price: Price::online(
meter,
When {
band,
..When::default()
},
),
rate: flat.price,
source: flat.key.clone(),
path: flat.path.clone(),
})
}
fn search_context_prices(wire: &EntryWire) -> impl Iterator<Item = Sourced> + '_ {
wire.search_context.iter().flat_map(|costs| {
[
(SearchContextSize::Low, "low", costs.search_context_size_low),
(
SearchContextSize::Medium,
"medium",
costs.search_context_size_medium,
),
(
SearchContextSize::High,
"high",
costs.search_context_size_high,
),
]
.into_iter()
.filter_map(|(size, name, rate)| {
Some(Sourced {
price: Price::online(Meter::WebSearch(size), When::default()),
rate: rate?,
source: format!("search_context_cost_per_query.search_context_size_{name}"),
path: pointer_child(
&pointer_for_key("search_context_cost_per_query"),
&format!("search_context_size_{name}"),
),
})
})
})
}
fn guardrail_prices(wire: &EntryWire) -> impl Iterator<Item = Sourced> + '_ {
wire.guardrail.iter().flatten().map(|(unit, rate)| Sourced {
price: Price::online(Meter::Guardrail(GuardrailUnit::new(unit)), When::default()),
rate: *rate,
source: format!("guardrail_cost_per_unit.{unit}"),
path: pointer_child(&pointer_for_key("guardrail_cost_per_unit"), unit),
})
}
fn off_peak(block: &OffPeakBlock, problems: &mut Vec<Error>) -> Option<OffPeak> {
let rates = off_peak_rates(&block.prices, problems);
let schedule = schedule(block, problems)?;
let off_peak = OffPeak::new(schedule, rates);
if off_peak.is_none() {
problems.push(Error::from_key("off_peak_pricing", Repr::NoOffPeakPrices));
}
off_peak
}
fn off_peak_rates(prices: &[FlatPrice], problems: &mut Vec<Error>) -> OffPeakRates {
let mut rates = OffPeakRates::default();
for flat in prices {
if flat.flat.has_suffix() {
problems.push(Error::new(
flat.path.clone(),
Repr::SuffixInsideBlock {
block: "off_peak_pricing",
},
));
continue;
}
let slot = match flat.flat.meter {
Meter::Input(InputUnit::Token(TokenKind::Text)) => &mut rates.input,
Meter::Output(OutputUnit::Token(TokenKind::Text)) => &mut rates.output,
Meter::Output(OutputUnit::ReasoningToken) => &mut rates.reasoning,
Meter::CacheRead {
kind: TokenKind::Text,
mode: CacheReadMode::Implicit,
} => &mut rates.cache_read,
Meter::CacheRead {
kind: TokenKind::Text,
mode: CacheReadMode::Explicit,
} => &mut rates.explicit_cache_read,
Meter::CacheWrite {
kind: TokenKind::Text,
ttl: CacheTtl::Default,
} => &mut rates.cache_write,
_ => {
problems.push(Error::new(flat.path.clone(), Repr::NotAnOffPeakRate));
continue;
}
};
*slot = Some(flat.price);
}
rates
}
fn schedule(block: &OffPeakBlock, problems: &mut Vec<Error>) -> Option<Schedule> {
let empty = |key: String| Error::from_path(&key, Repr::EmptyList);
let every_day = block
.hours_utc
.iter()
.map(|hours| ("off_peak_pricing.hours_utc".to_owned(), hours, None));
let windows = block
.windows
.iter()
.flatten()
.enumerate()
.map(|(index, window)| {
(
format!("off_peak_pricing.windows[{index}].hours_utc"),
&window.hours_utc,
Some((index, window.weekdays.as_ref())),
)
});
let windows: Vec<Window> = every_day
.chain(windows)
.map(|(key, hours, weekdays)| {
if hours.windows().is_empty() {
problems.push(empty(key));
}
if let Some((index, Some(days))) = weekdays
&& days.is_empty()
{
problems.push(empty(format!("off_peak_pricing.windows[{index}].weekdays")));
}
Window {
hours: hours.windows().to_vec(),
weekdays: weekdays.and_then(|(_, days)| days.cloned()),
}
})
.collect();
if block.windows.as_ref().is_some_and(Vec::is_empty) {
problems.push(empty("off_peak_pricing.windows".to_owned()));
}
if block.hours_utc.is_none() && block.windows.is_none() {
problems.push(Error::from_key("off_peak_pricing", Repr::NoOffPeakHours));
return None;
}
Some(Schedule {
windows,
weekday_timezone: block.weekday_timezone.clone(),
})
}
#[cfg(test)]
mod tests {
use litellm_cost::money::Multiplier;
use litellm_cost::price::{Channel, ServiceTier};
use litellm_cost::schedule::{UtcWindow, Weekday};
use rstest::rstest;
use serde_json::{Value, json};
use super::*;
const TEXT_INPUT: Meter = Meter::Input(InputUnit::Token(TokenKind::Text));
fn wire(entry: Value) -> EntryWire {
let Value::Object(fields) = entry else {
panic!("entry must be an object")
};
let wire = EntryWire::read(&fields);
assert!(wire.problems.is_empty(), "{:?}", wire.problems);
wire
}
fn prices(entry: Value) -> Result<Prices, Vec<Error>> {
Prices::try_from(&wire(entry))
}
fn online(meter: Meter, tier: ServiceTier, band: Band) -> Price {
Price::online(meter, When { tier, band })
}
#[rstest]
#[case::flat_key(json!({"input_cost_per_token": 1e-6}), online(TEXT_INPUT, ServiceTier::Standard, Band::Base))]
#[case::band_and_tier(
json!({"input_cost_per_token_above_272k_tokens_priority": 1e-6}),
online(TEXT_INPUT, ServiceTier::Priority, Band::over_prompt_tokens(272_000))
)]
#[case::batch(
json!({"input_cost_per_token_batches": 1e-6}),
Price { meter: TEXT_INPUT, channel: Channel::Batch, when: When::default() }
)]
#[case::first_token_tier_is_the_base_band(
json!({"tiered_pricing": [{"range": [0.0, 256000.0], "input_cost_per_token": 1e-6}]}),
online(TEXT_INPUT, ServiceTier::Standard, Band::Base)
)]
#[case::result_tier_stays_in_its_meter(
json!({"tiered_pricing": [
{"max_results_range": [0, 25], "input_cost_per_query": 2e-6},
{"max_results_range": [26, 100], "input_cost_per_query": 1e-6}
]}),
online(
Meter::Input(InputUnit::Query { over_results: NonZeroU64::new(25) }),
ServiceTier::Standard,
Band::Base
)
)]
#[case::search_context(
json!({"search_context_cost_per_query": {"search_context_size_low": 1e-6}}),
online(Meter::WebSearch(SearchContextSize::Low), ServiceTier::Standard, Band::Base)
)]
#[case::character_band_is_a_prompt_token_band(
json!({"input_cost_per_character_above_128k_tokens": 1e-6}),
online(Meter::Input(InputUnit::Character), ServiceTier::Standard, Band::over_prompt_tokens(128_000))
)]
#[case::guardrail(
json!({"guardrail_cost_per_unit": {"contentPolicyUnits": 1e-6}}),
online(Meter::Guardrail(GuardrailUnit::new("contentPolicyUnits")), ServiceTier::Standard, Band::Base)
)]
fn every_spelling_lowers_to_one_price(#[case] entry: Value, #[case] expected: Price) {
assert_eq!(prices(entry).unwrap().get(&expected), UnitPrice::new(1e-6));
}
#[rstest]
fn two_spellings_that_agree_are_one_price() {
let prices = prices(json!({
"input_cost_per_token": 1e-6,
"tiered_pricing": [
{"range": [0, 10], "input_cost_per_token": 1e-6},
{"range": [10, 20], "input_cost_per_token": 2e-6}
]
}))
.unwrap();
assert_eq!(
prices.get(&Price::online(TEXT_INPUT, When::default())),
UnitPrice::new(1e-6)
);
}
#[rstest]
fn token_tiers_lower_to_prompt_bands_at_each_start() {
let prices = prices(json!({"tiered_pricing": [
{"range": [0, 256000], "input_cost_per_token": 1e-6},
{"range": [256000, 1000000], "input_cost_per_token": 2e-6}
]}))
.unwrap();
assert_eq!(
prices.band(ServiceTier::Standard, 256_001),
Band::over_prompt_tokens(256_000)
);
assert_eq!(
prices
.rate(
&TEXT_INPUT,
ServiceTier::Standard,
Band::over_prompt_tokens(256_000)
)
.map(UnitPrice::get),
Some(2e-6)
);
}
#[rstest]
#[case::two_spellings_of_one_price(
json!({
"input_cost_per_token": 1e-6,
"tiered_pricing": [{"range": [0, 10], "input_cost_per_token": 2e-6}]
}),
"/tiered_pricing/0/input_cost_per_token",
"duplicate_price"
)]
#[case::zero_band_is_the_base_price(
json!({"input_cost_per_token": 1e-6, "input_cost_per_token_above_0k_tokens": 2e-6}),
"/input_cost_per_token_above_0k_tokens",
"duplicate_price"
)]
#[case::tier_without_range(
json!({"tiered_pricing": [{"input_cost_per_token": 1e-6}]}),
"/tiered_pricing/0",
"tier_range"
)]
#[case::tier_with_suffix(
json!({"tiered_pricing": [{"range": [0, 10], "input_cost_per_token_priority": 1e-6}]}),
"/tiered_pricing/0/input_cost_per_token_priority",
"suffix_inside_block"
)]
#[case::reversed_range(
json!({"tiered_pricing": [{"range": [0, 10], "input_cost_per_token": 1e-6}, {"range": [10, 5], "input_cost_per_token": 1e-6}]}),
"/tiered_pricing/1",
"tier_reversed"
)]
#[case::gap_between_tiers(
json!({"tiered_pricing": [
{"range": [0, 10], "input_cost_per_token": 1e-6},
{"range": [20, 30], "input_cost_per_token": 2e-6}
]}),
"/tiered_pricing/1",
"tier_gap"
)]
#[case::ladder_not_from_zero(
json!({"tiered_pricing": [{"range": [5, 10], "input_cost_per_token": 1e-6}]}),
"/tiered_pricing/0",
"tier_gap"
)]
#[case::result_ladder_overlaps(
json!({"tiered_pricing": [
{"max_results_range": [1, 10], "input_cost_per_query": 1e-6},
{"max_results_range": [10, 20], "input_cost_per_query": 2e-6}
]}),
"/tiered_pricing/1",
"tier_gap"
)]
#[case::mixed_basis(
json!({"tiered_pricing": [
{"range": [0, 10], "input_cost_per_token": 1e-6},
{"max_results_range": [11, 20], "input_cost_per_query": 2e-6}
]}),
"/tiered_pricing",
"mixed_tier_basis"
)]
#[case::result_tier_prices_a_token(
json!({"tiered_pricing": [{"max_results_range": [0, 10], "input_cost_per_token": 1e-6}]}),
"/tiered_pricing/0/input_cost_per_token",
"not_priced_by_results"
)]
#[case::two_band_sources(
json!({
"output_cost_per_token_above_200k_tokens": 1e-6,
"tiered_pricing": [{"range": [0, 10], "input_cost_per_token": 1e-6}]
}),
"/output_cost_per_token_above_200k_tokens",
"bands_from_two_sources"
)]
#[case::off_peak_without_hours(
json!({"off_peak_pricing": {"input_cost_per_token": 1e-6}}),
"/off_peak_pricing",
"no_off_peak_hours"
)]
#[case::off_peak_without_prices(
json!({"off_peak_pricing": {"hours_utc": "00:00-01:00"}}),
"/off_peak_pricing",
"no_off_peak_prices"
)]
#[case::off_peak_empty_windows(
json!({"off_peak_pricing": {"windows": [], "input_cost_per_token": 1e-6}}),
"/off_peak_pricing/windows",
"empty_list"
)]
#[case::off_peak_empty_hours(
json!({"off_peak_pricing": {"hours_utc": [], "input_cost_per_token": 1e-6}}),
"/off_peak_pricing/hours_utc",
"empty_list"
)]
#[case::off_peak_empty_weekdays(
json!({"off_peak_pricing": {
"windows": [{"hours_utc": "00:00-01:00", "weekdays": []}],
"input_cost_per_token": 1e-6
}}),
"/off_peak_pricing/windows/0/weekdays",
"empty_list"
)]
#[case::off_peak_image_price(
json!({"off_peak_pricing": {"hours_utc": "00:00-01:00", "output_cost_per_image": 1}}),
"/off_peak_pricing/output_cost_per_image",
"not_an_off_peak_rate"
)]
#[case::off_peak_band(
json!({"off_peak_pricing": {"hours_utc": "00:00-01:00", "input_cost_per_token_above_200k_tokens": 1}}),
"/off_peak_pricing/input_cost_per_token_above_200k_tokens",
"suffix_inside_block"
)]
fn rules_across_keys_name_where_each_price_came_from(
#[case] entry: Value,
#[case] expected_path: &str,
#[case] expected_variant: &str,
) {
let problems = prices(entry).unwrap_err();
assert!(
problems
.iter()
.any(|problem| problem.path() == expected_path
&& <&str>::from(problem.kind()) == expected_variant),
"{problems:?}"
);
}
#[rstest]
fn off_peak_hours_and_windows_form_one_schedule_with_its_rates() {
let prices = prices(json!({"off_peak_pricing": {
"hours_utc": ["00:00-01:00"],
"windows": [{"hours_utc": "10:00-12:00", "weekdays": [6, " sun "]}],
"weekday_timezone": "Asia/Shanghai",
"output_cost_per_token": 1e-6,
"explicit_cache_read_input_token_cost": 2e-6
}}))
.unwrap();
let off_peak = prices.off_peak().unwrap();
let window = |text: &str| text.parse::<UtcWindow>().unwrap();
assert_eq!(
off_peak.schedule(),
&Schedule {
windows: vec![
Window {
hours: vec![window("00:00-01:00")],
weekdays: None,
},
Window {
hours: vec![window("10:00-12:00")],
weekdays: Some(vec![Weekday::Saturday, Weekday::Sunday]),
},
],
weekday_timezone: Some("Asia/Shanghai".into()),
}
);
assert_eq!(
off_peak.rates(),
&OffPeakRates {
output: UnitPrice::new(1e-6),
explicit_cache_read: UnitPrice::new(2e-6),
..OffPeakRates::default()
}
);
assert_eq!(prices.iter().count(), 0);
}
#[rstest]
fn uplifts_are_adjustments_not_prices() {
let wire = wire(json!({
"regional_endpoint_uplift_multiplier": 1.1,
"regional_processing_uplift_multiplier_eu": "1.2"
}));
let adjustments = Adjustments::from(&wire);
assert_eq!(Prices::try_from(&wire).unwrap().iter().count(), 0);
assert_eq!(
adjustments
.get(&Uplift::NonGlobalEndpoint)
.map(Multiplier::get),
Some(1.1)
);
assert_eq!(
adjustments
.get(&Uplift::Residency(Residency::Eu))
.map(Multiplier::get),
Some(1.2)
);
assert_eq!(adjustments.get(&Uplift::Residency(Residency::Us)), None);
}
#[rstest]
fn a_model_reports_field_provider_and_price_problems_together() {
let Value::Object(fields) = json!({
"mode": "chatt",
"tiered_pricing": [{"input_cost_per_token": 1e-6}]
}) else {
unreachable!()
};
let problems: Vec<String> = Model::from_fields(&fields)
.unwrap_err()
.iter()
.map(|problem| litellm_error::Report::new(problem).to_string())
.collect();
assert_eq!(problems.len(), 3, "{problems:?}");
assert!(problems[0].starts_with("mode: invalid value: unknown variant `chatt`"));
assert_eq!(problems[1], "litellm_provider: required");
assert_eq!(
problems[2],
"tiered_pricing[0]: a tier needs exactly one of range and max_results_range"
);
}
#[rstest]
fn a_null_value_is_absent() {
let Value::Object(fields) = json!({
"litellm_provider": "openai",
"input_cost_per_token": null,
"mode": null
}) else {
unreachable!()
};
let model = Model::from_fields(&fields).unwrap();
assert_eq!(model.prices.iter().count(), 0);
assert_eq!(model.attributes.mode, None);
assert!(model.unrecognized.is_empty());
}
#[rstest]
#[case::tokens_win_over_characters(
json!({"litellm_provider": "vertex_ai-language-models", "input_cost_per_character": 1e-6, "input_cost_per_token": 1e-6}),
Billing::default()
)]
#[case::characters_only(
json!({"litellm_provider": "vertex_ai-language-models", "input_cost_per_character": 1e-6}),
Billing { text_unit: TextUnit::Character, ..Billing::default() }
)]
#[case::xai_band_is_inclusive(
json!({"litellm_provider": "xai", "input_cost_per_token": 1e-6}),
Billing { band_edge: BandEdge::Inclusive, ..Billing::default() }
)]
#[case::fireworks_without_cache_read_borrows_half(
json!({"litellm_provider": "fireworks_ai", "input_cost_per_token": 1e-6}),
Billing { cache_read_fallback: CacheReadFallback::HalfInputRate, ..Billing::default() }
)]
#[case::fireworks_with_cache_read_keeps_it(
json!({"litellm_provider": "fireworks_ai", "input_cost_per_token": 1e-6, "cache_read_input_token_cost": 1e-7}),
Billing::default()
)]
#[case::transcription_prices_the_audio_file(
json!({"litellm_provider": "openai", "mode": "audio_transcription", "input_cost_per_second": 1e-4}),
Billing { wall_clock: WallClock::AudioFile, ..Billing::default() }
)]
#[case::provisioned_chat_prices_response_time(
json!({"litellm_provider": "bedrock", "mode": "chat", "cost_per_second": 1e-2}),
Billing::default()
)]
#[case::image_generation_is_never_wall_clock(
json!({"litellm_provider": "openai", "mode": "image_generation"}),
Billing { wall_clock: WallClock::Never, ..Billing::default() }
)]
#[case::gemini_per_query_search(
json!({"litellm_provider": "gemini", "web_search_billing_unit": "per_query"}),
Billing { web_search: WebSearchUnit::PerQuery, ..Billing::default() }
)]
#[case::dashscope_tiers_bill_the_whole_call(
json!({"litellm_provider": "dashscope", "tiered_pricing": [{"range": [0, 32000], "input_cost_per_token": 1e-6}]}),
Billing { tier_scope: TierScope::WholeCall, ..Billing::default() }
)]
fn billing_is_lowered_from_the_row_and_its_provider(
#[case] entry: Value,
#[case] expected: Billing,
) {
assert_eq!(prices(entry).unwrap().billing(), expected);
}
}

22
litellm-rust/crates/model-catalog/src/model.rs Normal file
View file

@ -0,0 +1,22 @@
use std::collections::BTreeMap;
use litellm_cost::price::{Adjustments, Prices};
use crate::error::Error;
use crate::wire::{Attributes, EntryWire, Fields};
/// One catalog entry read into types.
#[derive(Clone, Debug, PartialEq)]
pub struct Model {
pub attributes: Attributes,
pub prices: Prices,
pub adjustments: Adjustments,
pub capabilities: BTreeMap<String, bool>,
pub unrecognized: Vec<String>,
}
impl Model {
pub fn from_fields(fields: &Fields) -> Result<Self, Vec<Error>> {
Self::try_from(EntryWire::read(fields))
}
}

734
litellm-rust/crates/model-catalog/src/model_info.rs
View file

@ -1,582 +1,194 @@
use crate::capabilities::{AudioFormat, InputModality, Mode, OutputModality, VertexAiAudioApi};
use crate::pricing::{OffPeakPricing, SearchContextCostPerQuery, TieredRate, WebSearchBillingUnit};
use litellm_llms_types::formats::chat_completions::ReasoningEffort;
use serde::{Deserialize, Serialize};
use indexmap::IndexMap;
use serde_json::Value;
use std::collections::BTreeMap;
/// Typed mirror of one catalog model entry.
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
pub struct ModelInfo {
#[serde(skip_serializing_if = "Option::is_none")]
pub annotation_cost_per_page: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub annotation_cost_per_page_batches: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub audio_transcription_config: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub bedrock_converse_supports_strict_tools: Option<bool>,
/// Highest reasoning effort the Bedrock output_config accepts for this model.
#[serde(skip_serializing_if = "Option::is_none")]
pub bedrock_output_config_effort_ceiling: Option<ReasoningEffort>,
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_creation_input_audio_token_cost: Option<f64>,
/// USD per token written to the provider's prompt cache.
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_creation_input_token_cost: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_creation_input_token_cost_above_32k_tokens: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_creation_input_token_cost_above_100k_tokens: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_creation_input_token_cost_above_100k_tokens_batches: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_creation_input_token_cost_above_128k_tokens: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_creation_input_token_cost_above_1hr: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_creation_input_token_cost_above_1hr_above_100k_tokens: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_creation_input_token_cost_above_1hr_above_200k_tokens: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_creation_input_token_cost_above_200k_tokens: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_creation_input_token_cost_above_200k_tokens_batches: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_creation_input_token_cost_above_256k_tokens: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_creation_input_token_cost_above_272k_tokens: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_creation_input_token_cost_above_272k_tokens_batches: Option<f64>,
/// Flex service-tier rate for the same-named base field.
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_creation_input_token_cost_above_272k_tokens_flex: Option<f64>,
/// Priority service-tier rate for the same-named base field.
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_creation_input_token_cost_above_272k_tokens_priority: Option<f64>,
/// Ultrafast service-tier rate for the same-named base field.
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_creation_input_token_cost_above_272k_tokens_ultrafast: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_creation_input_token_cost_batches: Option<f64>,
/// Flex service-tier rate for the same-named base field.
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_creation_input_token_cost_flex: Option<f64>,
/// Priority service-tier rate for the same-named base field.
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_creation_input_token_cost_priority: Option<f64>,
/// Ultrafast service-tier rate for the same-named base field.
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_creation_input_token_cost_ultrafast: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_read_input_audio_token_cost: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_read_input_image_token_cost: Option<f64>,
/// USD per prompt token served from the provider's prompt cache.
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_read_input_token_cost: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_read_input_token_cost_above_32k_tokens: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_read_input_token_cost_above_100k_tokens: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_read_input_token_cost_above_100k_tokens_batches: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_read_input_token_cost_above_128k_tokens: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_read_input_token_cost_above_200k_tokens: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_read_input_token_cost_above_200k_tokens_batches: Option<f64>,
/// Priority service-tier rate for the same-named base field.
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_read_input_token_cost_above_200k_tokens_priority: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_read_input_token_cost_above_256k_tokens: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_read_input_token_cost_above_272k_tokens: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_read_input_token_cost_above_272k_tokens_batches: Option<f64>,
/// Flex service-tier rate for the same-named base field.
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_read_input_token_cost_above_272k_tokens_flex: Option<f64>,
/// Priority service-tier rate for the same-named base field.
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_read_input_token_cost_above_272k_tokens_priority: Option<f64>,
/// Ultrafast service-tier rate for the same-named base field.
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_read_input_token_cost_above_272k_tokens_ultrafast: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_read_input_token_cost_above_512k_tokens: Option<f64>,
/// Balanced service-tier rate for the same-named base field.
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_read_input_token_cost_balanced: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_read_input_token_cost_batches: Option<f64>,
/// Flex service-tier rate for the same-named base field.
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_read_input_token_cost_flex: Option<f64>,
/// Priority service-tier rate for the same-named base field.
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_read_input_token_cost_priority: Option<f64>,
/// Ultrafast service-tier rate for the same-named base field.
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_read_input_token_cost_ultrafast: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub citation_cost_per_token: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub code_interpreter_cost_per_session: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub comment: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub computer_use_input_cost_per_1k_tokens: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub computer_use_output_cost_per_1k_tokens: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub cost_per_second: Option<f64>,
/// Reasoning effort the provider applies when the request omits reasoning_effort. Gates whether a non-default temperature or the top_p/logprobs sampling params are accepted, which hold only when the effort resolves to 'none'.
#[serde(skip_serializing_if = "Option::is_none")]
pub default_reasoning_effort: Option<ReasoningEffort>,
/// Date the provider deprecates the model, YYYY-MM-DD.
#[serde(skip_serializing_if = "Option::is_none")]
pub deprecation_date: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub file_search_cost_per_1k_calls: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub file_search_cost_per_gb_per_day: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub gemini_audio_only_live: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub gemini_native_audio: Option<bool>,
/// USD per Grounding with Google Maps request; billed per query or per prompt per web_search_billing_unit.
#[serde(skip_serializing_if = "Option::is_none")]
pub google_maps_grounding_cost_per_query: Option<f64>,
/// USD cost per billable guardrail unit, keyed by the provider's usage counter name (e.g. Bedrock's contentPolicyUnits).
#[serde(skip_serializing_if = "Option::is_none")]
pub guardrail_cost_per_unit: Option<BTreeMap<String, f64>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_audio_per_second: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_audio_per_second_above_128k_tokens: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_audio_token: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_audio_token_batches: Option<f64>,
/// Priority service-tier rate for the same-named base field.
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_audio_token_priority: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_character: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_character_above_128k_tokens: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_image: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_image_above_128k_tokens: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_image_token: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_image_token_batches: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_pixel: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_query: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_request: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_second: Option<f64>,
/// USD per prompt token.
#[serde(skip_serializing_if = "Option::is_none")]
/// The fields main's `ModelInfoBase(...)` copies (`litellm/utils.py` at c7c895db45), one typed field each,
/// read from the raw row without lowering it, so building one never fails
#[derive(Clone, Debug, Default, PartialEq)]
pub struct ModelInfoFields {
pub key: String,
pub max_tokens: Option<i64>,
pub max_input_tokens: Option<i64>,
pub max_output_tokens: Option<i64>,
pub input_cost_per_token: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_token_above_32k_tokens: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_token_above_100k_tokens: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_token_above_100k_tokens_batches: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_token_above_128k_tokens: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_token_above_200k_tokens: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_token_above_200k_tokens_batches: Option<f64>,
/// Priority service-tier rate for the same-named base field.
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_token_above_200k_tokens_priority: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_token_above_256k_tokens: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_token_above_272k_tokens: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_token_above_272k_tokens_batches: Option<f64>,
/// Flex service-tier rate for the same-named base field.
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_token_above_272k_tokens_flex: Option<f64>,
/// Priority service-tier rate for the same-named base field.
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_token_above_272k_tokens_priority: Option<f64>,
/// Ultrafast service-tier rate for the same-named base field.
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_token_above_272k_tokens_ultrafast: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_token_above_512k_tokens: Option<f64>,
/// Balanced service-tier rate for the same-named base field.
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_token_balanced: Option<f64>,
/// USD per prompt token via the provider's batch API.
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_token_batches: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_token_cache_hit: Option<f64>,
/// Flex service-tier rate for the same-named base field.
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_token_flex: Option<f64>,
/// Priority service-tier rate for the same-named base field.
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_token_priority: Option<f64>,
/// Ultrafast service-tier rate for the same-named base field.
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_token_balanced: Option<f64>,
pub input_cost_per_token_ultrafast: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_video_per_second: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_video_per_second_above_128k_tokens: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_video_per_second_above_15s_interval: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_video_per_second_above_8s_interval: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_creation_input_token_cost: Option<f64>,
pub cache_creation_input_token_cost_above_200k_tokens: Option<f64>,
pub cache_creation_input_token_cost_above_272k_tokens: Option<f64>,
pub cache_creation_input_token_cost_above_272k_tokens_priority: Option<f64>,
pub cache_creation_input_token_cost_above_272k_tokens_flex: Option<f64>,
pub cache_creation_input_token_cost_above_272k_tokens_ultrafast: Option<f64>,
pub cache_creation_input_token_cost_flex: Option<f64>,
pub cache_creation_input_token_cost_priority: Option<f64>,
pub cache_creation_input_token_cost_ultrafast: Option<f64>,
pub cache_read_input_token_cost: Option<f64>,
pub cache_read_input_audio_token_cost: Option<f64>,
pub cache_read_input_audio_token_cost_flex: Option<f64>,
pub cache_read_input_audio_token_cost_priority: Option<f64>,
pub cache_read_input_audio_token_cost_batches: Option<f64>,
pub explicit_cache_read_input_token_cost: Option<f64>,
pub prompt_cache_min_tokens: Option<i64>,
pub cache_read_input_token_cost_above_200k_tokens: Option<f64>,
pub cache_read_input_token_cost_above_200k_tokens_priority: Option<f64>,
pub cache_read_input_token_cost_above_272k_tokens: Option<f64>,
pub cache_read_input_token_cost_above_272k_tokens_priority: Option<f64>,
pub cache_read_input_token_cost_above_272k_tokens_flex: Option<f64>,
pub cache_read_input_token_cost_above_272k_tokens_ultrafast: Option<f64>,
pub cache_read_input_token_cost_above_512k_tokens: Option<f64>,
pub cache_read_input_token_cost_flex: Option<f64>,
pub cache_read_input_token_cost_priority: Option<f64>,
pub cache_read_input_token_cost_balanced: Option<f64>,
pub cache_read_input_token_cost_ultrafast: Option<f64>,
pub cache_read_input_token_cost_batches: Option<f64>,
pub cache_read_input_token_cost_above_200k_tokens_batches: Option<f64>,
pub cache_read_input_token_cost_above_272k_tokens_batches: Option<f64>,
pub cache_creation_input_token_cost_batches: Option<f64>,
pub cache_creation_input_token_cost_above_200k_tokens_batches: Option<f64>,
pub cache_creation_input_token_cost_above_272k_tokens_batches: Option<f64>,
pub cache_creation_input_token_cost_above_1hr: Option<f64>,
pub cache_creation_input_token_cost_above_12hr: Option<f64>,
pub cache_creation_input_token_cost_above_12hr_flex: Option<f64>,
pub cache_creation_input_token_cost_above_12hr_priority: Option<f64>,
pub cache_creation_input_token_cost_above_12hr_above_272k_tokens: Option<f64>,
pub cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex: Option<f64>,
pub cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority: Option<f64>,
pub cache_creation_input_token_cost_above_12hr_batches: Option<f64>,
pub cache_creation_input_token_cost_above_12hr_ultrafast: Option<f64>,
pub cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches: Option<f64>,
pub cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast: Option<f64>,
pub off_peak_pricing: Option<Value>,
pub input_cost_per_character: Option<f64>,
pub input_cost_per_token_above_128k_tokens: Option<f64>,
pub input_cost_per_token_above_200k_tokens: Option<f64>,
pub input_cost_per_token_above_200k_tokens_priority: Option<f64>,
pub input_cost_per_token_above_272k_tokens: Option<f64>,
pub input_cost_per_token_above_272k_tokens_priority: Option<f64>,
pub input_cost_per_token_above_272k_tokens_flex: Option<f64>,
pub input_cost_per_token_above_272k_tokens_ultrafast: Option<f64>,
pub input_cost_per_token_above_512k_tokens: Option<f64>,
pub input_cost_per_query: Option<f64>,
pub cost_per_second: Option<f64>,
pub input_cost_per_second: Option<f64>,
pub input_cost_per_text_message: Option<f64>,
pub input_cost_per_audio_token: Option<f64>,
pub input_cost_per_audio_token_flex: Option<f64>,
pub input_cost_per_audio_token_priority: Option<f64>,
pub input_cost_per_image_token: Option<f64>,
pub input_cost_per_video_token: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_audio_token_batches: Option<f64>,
pub input_cost_per_image_token_batches: Option<f64>,
pub input_cost_per_image: Option<f64>,
pub input_cost_per_audio_per_second: Option<f64>,
pub input_cost_per_video_per_second: Option<f64>,
pub input_cost_per_token_batches: Option<f64>,
pub input_cost_per_video_token_batches: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub input_dbu_cost_per_token: Option<f64>,
/// LiteLLM provider slug; one of https://docs.litellm.ai/docs/providers.
#[serde(skip_serializing_if = "Option::is_none")]
pub litellm_provider: Option<String>,
/// Maximum prompt/context tokens the model accepts.
#[serde(skip_serializing_if = "Option::is_none")]
pub max_input_tokens: Option<u64>,
/// Maximum tokens the model can generate in one response.
#[serde(skip_serializing_if = "Option::is_none")]
pub max_output_tokens: Option<u64>,
/// Legacy field: max output tokens if the provider specifies it, else max input tokens.
#[serde(skip_serializing_if = "Option::is_none")]
pub max_tokens: Option<u64>,
/// Free-form notes about the entry (e.g. pricing derivation).
#[serde(skip_serializing_if = "Option::is_none")]
pub metadata: Option<BTreeMap<String, Value>>,
/// Primary API surface / task type of the model.
#[serde(skip_serializing_if = "Option::is_none")]
pub mode: Option<Mode>,
#[serde(skip_serializing_if = "Option::is_none")]
pub ocr_cost_per_credit: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub ocr_cost_per_page: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub ocr_cost_per_page_batches: Option<f64>,
/// Rates that replace the same-named base fields while the request falls inside the stated UTC windows.
#[serde(skip_serializing_if = "Option::is_none")]
pub off_peak_pricing: Option<OffPeakPricing>,
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_audio_token: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_character: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_character_above_128k_tokens: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_image: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_image_1024: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_image_1536: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_image_512: Option<f64>,
#[serde(
rename = "output_cost_per_image_0.5K",
skip_serializing_if = "Option::is_none"
)]
pub output_cost_per_image_0_5k: Option<f64>,
#[serde(
rename = "output_cost_per_image_1K",
skip_serializing_if = "Option::is_none"
)]
pub output_cost_per_image_1k: Option<f64>,
#[serde(
rename = "output_cost_per_image_2K",
skip_serializing_if = "Option::is_none"
)]
pub output_cost_per_image_2k: Option<f64>,
#[serde(
rename = "output_cost_per_image_4K",
skip_serializing_if = "Option::is_none"
)]
pub output_cost_per_image_4k: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_image_token: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_pixel: Option<f64>,
/// USD per reasoning/thinking token, when billed separately.
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_reasoning_token: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_second: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_second_1080p: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_second_2k: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_second_480p: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_second_4k: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_second_720p: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_second_768p: Option<f64>,
/// USD per generated token.
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_token: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_token_above_32k_tokens: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_token_above_100k_tokens: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_token_above_100k_tokens_batches: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_token_above_128k_tokens: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_token_above_200k_tokens: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_token_above_200k_tokens_batches: Option<f64>,
/// Priority service-tier rate for the same-named base field.
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_token_above_200k_tokens_priority: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_token_above_256k_tokens: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_token_above_272k_tokens: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_token_above_272k_tokens_batches: Option<f64>,
/// Flex service-tier rate for the same-named base field.
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_token_above_272k_tokens_flex: Option<f64>,
/// Priority service-tier rate for the same-named base field.
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_token_above_272k_tokens_priority: Option<f64>,
/// Ultrafast service-tier rate for the same-named base field.
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_token_above_272k_tokens_ultrafast: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_token_above_512k_tokens: Option<f64>,
/// Balanced service-tier rate for the same-named base field.
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_token_balanced: Option<f64>,
/// USD per generated token via the provider's batch API.
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_token_above_200k_tokens_batches: Option<f64>,
pub input_cost_per_token_above_272k_tokens_batches: Option<f64>,
pub output_cost_per_token_batches: Option<f64>,
/// Flex service-tier rate for the same-named base field.
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_token_above_200k_tokens_batches: Option<f64>,
pub output_cost_per_token_above_272k_tokens_batches: Option<f64>,
pub output_cost_per_token: Option<f64>,
pub output_cost_per_token_flex: Option<f64>,
/// Priority service-tier rate for the same-named base field.
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_token_priority: Option<f64>,
/// Ultrafast service-tier rate for the same-named base field.
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_token_balanced: Option<f64>,
pub output_cost_per_token_ultrafast: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_video_per_second: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_video_token: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub output_dbu_cost_per_token: Option<f64>,
/// Embedding dimension for embedding models.
#[serde(skip_serializing_if = "Option::is_none")]
pub output_vector_size: Option<u64>,
/// Smallest prefix the provider will actually cache; absent means the provider default applies.
#[serde(skip_serializing_if = "Option::is_none")]
pub prompt_cache_min_tokens: Option<u64>,
/// Provider-internal routing hints (e.g. bedrock_invocation_schema).
#[serde(skip_serializing_if = "Option::is_none")]
pub provider_specific_entry: Option<BTreeMap<String, Value>>,
/// Exact reasoning_effort levels this deployment accepts; wins over supports_* flags.
#[serde(skip_serializing_if = "Option::is_none")]
pub reasoning_effort_levels: Option<Vec<ReasoningEffort>>,
/// Multiplier applied to all token costs when served from a non-global Vertex AI endpoint (e.g. 1.10 = +10%).
#[serde(skip_serializing_if = "Option::is_none")]
pub regional_endpoint_uplift_multiplier: Option<f64>,
/// Multiplier applied to all token costs for EU data residency (e.g. 1.10 = +10%).
#[serde(skip_serializing_if = "Option::is_none")]
pub regional_processing_uplift_multiplier_eu: Option<f64>,
/// Multiplier applied to all token costs for US data residency (e.g. 1.10 = +10%).
#[serde(skip_serializing_if = "Option::is_none")]
pub regional_processing_uplift_multiplier_us: Option<f64>,
/// Provider default requests-per-minute limit.
#[serde(skip_serializing_if = "Option::is_none")]
pub rpm: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub rules: Option<Vec<Value>>,
/// USD cost per web search query, keyed by search context size.
#[serde(skip_serializing_if = "Option::is_none")]
pub search_context_cost_per_query: Option<SearchContextCostPerQuery>,
/// URL of the provider pricing/model page this entry was taken from.
#[serde(skip_serializing_if = "Option::is_none")]
pub source: Option<String>,
/// Audio container formats the model can return.
#[serde(skip_serializing_if = "Option::is_none")]
pub supported_audio_formats: Option<Vec<AudioFormat>>,
/// OpenAI-style API routes this model can be called through, e.g. /v1/chat/completions.
#[serde(skip_serializing_if = "Option::is_none")]
pub regional_endpoint_uplift_multiplier: Option<f64>,
pub output_cost_per_audio_token: Option<f64>,
pub output_cost_per_character: Option<f64>,
pub output_cost_per_reasoning_token: Option<f64>,
pub output_cost_per_reasoning_token_flex: Option<f64>,
pub output_cost_per_reasoning_token_priority: Option<f64>,
pub output_cost_per_token_above_128k_tokens: Option<f64>,
pub output_cost_per_character_above_128k_tokens: Option<f64>,
pub output_cost_per_token_above_200k_tokens: Option<f64>,
pub output_cost_per_token_above_200k_tokens_priority: Option<f64>,
pub output_cost_per_token_above_272k_tokens: Option<f64>,
pub output_cost_per_token_above_272k_tokens_priority: Option<f64>,
pub output_cost_per_token_above_272k_tokens_flex: Option<f64>,
pub output_cost_per_token_above_272k_tokens_ultrafast: Option<f64>,
pub output_cost_per_token_above_512k_tokens: Option<f64>,
pub output_cost_per_second: Option<f64>,
pub output_cost_per_second_1080p: Option<f64>,
pub output_cost_per_second_480p: Option<f64>,
pub output_cost_per_second_720p: Option<f64>,
pub output_cost_per_second_768p: Option<f64>,
pub output_cost_per_second_2k: Option<f64>,
pub output_cost_per_second_4k: Option<f64>,
pub output_cost_per_video_per_second: Option<f64>,
pub output_cost_per_image: Option<f64>,
pub output_cost_per_pixel: Option<f64>,
pub output_cost_per_image_token: Option<f64>,
pub output_cost_per_image_token_batches: Option<f64>,
pub output_cost_per_video_token: Option<f64>,
pub output_vector_size: Option<i64>,
pub citation_cost_per_token: Option<f64>,
pub tiered_pricing: Option<Value>,
pub litellm_provider: Option<String>,
pub mode: Option<Value>,
pub supported_endpoints: Option<Vec<String>>,
/// Input modalities the model accepts.
#[serde(skip_serializing_if = "Option::is_none")]
pub supported_modalities: Option<Vec<InputModality>>,
/// Output modalities the model can produce.
#[serde(skip_serializing_if = "Option::is_none")]
pub supported_output_modalities: Option<Vec<OutputModality>>,
/// Cloud regions the model is available in ('global' or region ids).
#[serde(skip_serializing_if = "Option::is_none")]
pub supported_regions: Option<Vec<String>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_adaptive_thinking: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_anthropic_compaction: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_anthropic_thinking_payload: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_assistant_prefill: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_audio_input: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_audio_output: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_bedrock_runtime_chat_completions_response_format: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_bedrock_runtime_chat_completions_tools_with_reasoning: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_computer_use: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_embedding_image_input: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_fast_mode: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_forced_tool_use: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_function_calling: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_image_input: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_image_size: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_legacy_thinking: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_low_reasoning_effort: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_max_reasoning_effort: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_mid_conversation_system: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_minimal_reasoning_effort: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_multimodal: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_native_streaming: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_native_structured_output: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_none_reasoning_effort: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_nova_canvas_image_edit: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_output_config: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_parallel_function_calling: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_parallel_tool_use_config: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_pdf_input: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_prompt_cache_breakpoint: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_prompt_caching: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_reasoning: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_response_schema: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_sampling_params: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_speed: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_system_messages: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_thinking_cache_preservation: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_tool_choice: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_tool_search: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_url_context: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_video_input: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_response_schema: Option<bool>,
pub supports_vision: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_function_calling: Option<bool>,
pub supports_parallel_function_calling: Option<bool>,
pub supports_tool_choice: Option<bool>,
pub supports_assistant_prefill: Option<bool>,
pub supports_prompt_caching: Option<bool>,
pub supports_prompt_cache_breakpoint: Option<bool>,
pub supports_thinking_cache_preservation: Option<bool>,
pub supports_audio_input: Option<bool>,
pub supports_video_input: Option<bool>,
pub supports_box_questions: Option<bool>,
pub supports_audio_output: Option<bool>,
pub supports_pdf_input: Option<bool>,
pub supports_embedding_image_input: Option<bool>,
pub supports_native_streaming: Option<bool>,
pub supports_native_structured_output: Option<bool>,
pub supports_bedrock_native_structured_output: Option<bool>,
pub supports_web_search: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_xhigh_reasoning_effort: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_url_context: Option<bool>,
pub supports_reasoning: Option<bool>,
pub supports_adaptive_thinking: Option<bool>,
pub supports_legacy_thinking: Option<bool>,
pub thinking_always_on: Option<bool>,
/// Context-length or result-count tiered rates; each tier's costs apply within its range.
#[serde(skip_serializing_if = "Option::is_none")]
pub tiered_pricing: Option<Vec<TieredRate>>,
/// Provider default tokens-per-minute limit.
#[serde(skip_serializing_if = "Option::is_none")]
pub tpm: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub use_openai_responses_path: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub supports_tool_search: Option<bool>,
pub supports_mid_conversation_system: Option<bool>,
pub supports_anthropic_thinking_payload: Option<bool>,
pub supports_anthropic_compaction: Option<bool>,
pub supports_none_reasoning_effort: Option<bool>,
pub supports_minimal_reasoning_effort: Option<bool>,
pub supports_low_reasoning_effort: Option<bool>,
pub supports_xhigh_reasoning_effort: Option<bool>,
pub supports_max_reasoning_effort: Option<bool>,
pub reasoning_effort_levels: Option<Value>,
pub default_reasoning_effort: Option<Value>,
pub bedrock_output_config_effort_ceiling: Option<Value>,
pub bedrock_converse_supports_strict_tools: Option<bool>,
pub supports_regex_lookaround: Option<bool>,
pub supports_computer_use: Option<bool>,
pub search_context_cost_per_query: Option<Value>,
pub web_search_billing_unit: Option<Value>,
pub search_free_requests: Option<Value>,
pub google_maps_grounding_cost_per_query: Option<f64>,
pub tpm: Option<i64>,
pub rpm: Option<i64>,
pub ocr_cost_per_page: Option<f64>,
pub ocr_cost_per_page_batches: Option<f64>,
pub ocr_cost_per_credit: Option<f64>,
pub annotation_cost_per_page: Option<f64>,
pub annotation_cost_per_page_batches: Option<f64>,
pub provider_specific_entry: Option<Value>,
pub uses_embed_content: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub vector_store_cost_per_gb_per_day: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub vertex_ai_audio_api: Option<VertexAiAudioApi>,
/// Whether web search is billed per query or per prompt.
#[serde(skip_serializing_if = "Option::is_none")]
pub web_search_billing_unit: Option<WebSearchBillingUnit>,
pub supports_image_size: Option<bool>,
pub supported_audio_formats: Option<Value>,
pub token_count_framing: Option<i64>,
pub token_count_model: Option<Value>,
pub token_count_tokenizer: Option<Value>,
pub vertex_ai_audio_api: Option<Value>,
/// Every key matching `ABOVE_THRESHOLD_COST_KEY_PATTERN`, copied as written
pub above_threshold_costs: IndexMap<String, Value>,
}

97
litellm-rust/crates/model-catalog/src/pricing.rs
View file

@ -1,97 +0,0 @@
use serde::{Deserialize, Serialize};
/// Whether web search is billed per query or per prompt.
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(rename_all = "snake_case")]
pub enum WebSearchBillingUnit {
PerQuery,
PerPrompt,
}
/// UTC "HH:MM-HH:MM" window, or a list of them; a window may wrap past midnight.
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(untagged)]
pub enum UtcHours {
Single(String),
Multiple(Vec<String>),
}
/// ISO-8601 weekday number (1 = Monday .. 7 = Sunday) or English day name.
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(untagged)]
pub enum Weekday {
Number(u8),
Name(String),
}
/// One off-peak window entry inside `windows`.
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(deny_unknown_fields)]
pub struct OffPeakWindow {
pub hours_utc: UtcHours,
#[serde(skip_serializing_if = "Option::is_none")]
pub weekdays: Option<Vec<Weekday>>,
}
/// Rates that replace the same-named base fields inside the stated UTC windows.
#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(deny_unknown_fields)]
pub struct OffPeakPricing {
#[serde(skip_serializing_if = "Option::is_none")]
pub hours_utc: Option<UtcHours>,
#[serde(skip_serializing_if = "Option::is_none")]
pub windows: Option<Vec<OffPeakWindow>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub weekday_timezone: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_token: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_token: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_reasoning_token: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_read_input_token_cost: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_creation_input_token_cost: Option<f64>,
}
/// USD cost per web search query, keyed by search context size.
#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(deny_unknown_fields)]
pub struct SearchContextCostPerQuery {
#[serde(skip_serializing_if = "Option::is_none")]
pub search_context_size_low: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub search_context_size_medium: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub search_context_size_high: Option<f64>,
}
/// One tier of a context-length or result-count tiered rate.
#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(deny_unknown_fields)]
pub struct TieredRate {
#[serde(skip_serializing_if = "Option::is_none")]
pub range: Option<[f64; 2]>,
#[serde(skip_serializing_if = "Option::is_none")]
pub max_results_range: Option<[f64; 2]>,
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_token: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_token: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub output_cost_per_reasoning_token: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_read_input_token_cost: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_creation_input_token_cost: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub input_cost_per_query: Option<f64>,
}

26
litellm-rust/crates/model-catalog/src/register.rs Normal file
View file

@ -0,0 +1,26 @@
use std::collections::{BTreeSet, HashMap};
use indexmap::IndexMap;
use crate::wire::Fields;
/// Where one `register` call writes and which row it inherits from
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct RegistrationTarget {
/// Always the caller's key, never a case variant of it
pub write: String,
/// The first candidate that is an exact key whose row passes `provider_matches`
pub inherit_from: Option<String>,
}
/// Python's inheritance candidates in main's order, `get_model_info`'s then `_resolve_builtin_model_cost_entry`'s
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct RegistrationCandidates(pub Vec<String>);
/// Layer 3, and the fields `register` last wrote, which litellm owns again
#[derive(Clone, Debug, Default)]
#[expect(dead_code, reason = "07 adds the functions that read registrations")]
pub struct Registrations {
persisted: IndexMap<String, Fields>,
registered_fields: HashMap<String, BTreeSet<String>>,
}

18
litellm-rust/crates/model-catalog/src/resolve/image.rs Normal file
View file

@ -0,0 +1,18 @@
/// What an image call asks for. `resolve_image` builds main's eight key spellings from it
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ImageCandidates {
pub model: String,
pub provider: Option<String>,
/// Width and height, `1024 x 1024` when the request names none, as on main
pub size: Option<(u32, u32)>,
pub quality: Option<String>,
}
/// A size and quality keyed row, `{quality}/{W}-x-{H}/{model}`, parsed once per index build for fal's nearest match
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ImageKey {
pub quality: Option<String>,
pub width: u32,
pub height: u32,
pub model: String,
}

46
litellm-rust/crates/model-catalog/src/resolve/mod.rs Normal file
View file

@ -0,0 +1,46 @@
mod image;
pub use image::*;
/// The seven names main's `get_model_info` tries, built by Python, walked in this order by Rust
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct Candidates {
pub combined_model_name: Option<String>,
pub model: String,
pub region_free_combined_model_name: Option<String>,
pub split_model: Option<String>,
pub combined_stripped_model_name: Option<String>,
pub stripped_model_name: Option<String>,
pub provider_prefixed_model_name: Option<String>,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum CandidateSlot {
CombinedModelName,
Model,
RegionFreeCombinedModelName,
SplitModel,
CombinedStrippedModelName,
StrippedModelName,
ProviderPrefixedModelName,
}
/// Which step matched, so a test can name the step that changed
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Via {
pub slot: CandidateSlot,
/// Matched through the case-insensitive index rather than an exact key
pub case_insensitive: bool,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum Resolution {
Found {
key: String,
via: Via,
},
/// Every candidate tried, in order
Missing {
tried: Vec<String>,
},
}

197
litellm-rust/crates/model-catalog/src/schema.rs
View file

@ -1,9 +1,80 @@
use schemars::Schema;
use std::collections::BTreeMap;
use litellm_cost::money::{Multiplier, UnitPrice};
use schemars::{JsonSchema, Schema};
use serde_json::{Map, Value, json};
use crate::fallback::FallbackGeneralizations;
use crate::wire::{Attributes, OffPeakWindow, SearchContextCostPerQuery, UtcHours};
#[derive(JsonSchema)]
#[schemars(rename = "ModelEntry")]
struct ModelEntry {
#[serde(flatten)]
_attributes: Attributes,
#[serde(flatten)]
_blocks: PriceBlocks,
}
#[derive(JsonSchema)]
#[allow(dead_code)]
struct PriceBlocks {
/// USD cost per billable guardrail unit, keyed by the provider's usage counter name (e.g. Bedrock's contentPolicyUnits).
guardrail_cost_per_unit: Option<BTreeMap<String, UnitPrice>>,
/// Rates that replace the same-named base fields while the request falls inside the stated UTC windows.
off_peak_pricing: Option<OffPeakPricing>,
/// Multiplier applied to all token costs when served from a non-global Vertex AI endpoint (e.g. 1.10 = +10%).
regional_endpoint_uplift_multiplier: Option<Multiplier>,
/// Multiplier applied to all token costs for EU data residency (e.g. 1.10 = +10%).
regional_processing_uplift_multiplier_eu: Option<Multiplier>,
/// Multiplier applied to all token costs for US data residency (e.g. 1.10 = +10%).
regional_processing_uplift_multiplier_us: Option<Multiplier>,
/// USD cost per web search query, keyed by search context size.
search_context_cost_per_query: Option<SearchContextCostPerQuery>,
/// Context-length or result-count tiered rates; each tier's costs apply within its range.
tiered_pricing: Option<Vec<TieredRate>>,
}
/// Rates that replace the same-named base fields inside the stated UTC windows.
#[derive(JsonSchema)]
#[allow(dead_code)]
struct OffPeakPricing {
hours_utc: Option<UtcHours>,
windows: Option<Vec<OffPeakWindow>>,
weekday_timezone: Option<String>,
}
/// One tier of a context-length or result-count tiered rate.
#[derive(JsonSchema)]
#[allow(dead_code)]
struct TieredRate {
range: Option<[u64; 2]>,
max_results_range: Option<[u64; 2]>,
}
#[derive(JsonSchema)]
#[allow(dead_code)]
struct RegistryDocument {
sample_spec: Option<Map<String, Value>>,
fallback_generalizations: Option<FallbackGeneralizations>,
}
pub(crate) const FLAT_KEY_PATTERN: &str = concat!(
r"^(",
r"input_cost_per_((audio_|image_|video_)?token|character|image|pixel|query|request|second|text_message|audio_per_second|video_per_second(_above_\d+s_interval)?)",
r"|output_cost_per_((audio_|image_|video_|reasoning_)?token|character|image(_\d+(\.\d+)?K?)?|pixel|second(_\d+[pk])?|video_per_second)",
r"|cache_read_input_(audio_|image_|video_)?token_cost",
r"|explicit_cache_read_input_token_cost",
r"|cache_creation_input_(audio_|image_|video_)?token_cost(_above_1hr|_above_12hr)?",
r"|annotation_cost_per_page|citation_cost_per_token|code_interpreter_cost_per_session",
r"|google_maps_grounding_cost_per_query|ocr_cost_per_credit|ocr_cost_per_page|cost_per_second",
r"|browser_open_cost_per_request|cost_per_additional_result",
r")(_above_\d+k?_tokens)?(_(priority|flex|ultrafast|balanced|batches|decisions))?$",
);
/// JSON Schema for one model entry, including registry validation constraints.
pub fn model_entry_json_schema() -> Schema {
let mut schema = serde_json::to_value(schemars::schema_for!(crate::ModelInfo))
let mut schema = serde_json::to_value(schemars::schema_for!(ModelEntry))
.expect("derived model schema serializes");
remove_nullable_optional_fields(&mut schema);
decorate_model_entry(&mut schema);
@ -22,24 +93,21 @@ pub fn registry_json_schema() -> Schema {
entry.as_object_mut().unwrap().remove("$schema");
definitions.insert("modelEntry".into(), entry);
let mut fallback = serde_json::to_value(schemars::schema_for!(crate::FallbackGeneralizations))
.expect("derived fallback schema serializes");
remove_nullable_optional_fields(&mut fallback);
definitions.extend(take_definitions(&mut fallback));
fallback.as_object_mut().unwrap().remove("$schema");
let root = json!({
"$schema": "https://json-schema.org/draft/2020-12/schema",
"title": "LiteLLM model prices and context window registry",
"type": "object",
"properties": {
"sample_spec": {"type": "object"},
"fallback_generalizations": fallback
},
"additionalProperties": {"$ref": "#/$defs/modelEntry"},
"$defs": definitions
});
Schema::from(root.as_object().unwrap().clone())
let mut registry = serde_json::to_value(schemars::schema_for!(RegistryDocument))
.expect("derived registry schema serializes");
remove_nullable_optional_fields(&mut registry);
definitions.extend(take_definitions(&mut registry));
let root = registry.as_object_mut().unwrap();
root.insert(
"title".into(),
"LiteLLM model prices and context window registry".into(),
);
root.insert(
"additionalProperties".into(),
json!({"$ref": "#/$defs/modelEntry"}),
);
root.insert("$defs".into(), Value::Object(definitions));
Schema::from(root.clone())
}
fn take_definitions(schema: &mut Value) -> Map<String, Value> {
@ -80,42 +148,21 @@ fn remove_nullable_optional_fields(value: &mut Value) {
}
fn decorate_model_entry(schema: &mut Value) {
let rates = json!({FLAT_KEY_PATTERN: {"type": "number", "minimum": 0}});
let object = schema.as_object_mut().unwrap();
object.insert("required".into(), json!(["litellm_provider"]));
object.insert("additionalProperties".into(), Value::Bool(true));
let properties = object
.get_mut("properties")
.unwrap()
.as_object_mut()
.unwrap();
properties.insert(
"aliases".into(),
json!({"type": "array", "items": {"type": "string"}}),
object.insert(
"patternProperties".into(),
json!({
FLAT_KEY_PATTERN: {"type": "number", "minimum": 0},
"^supports_": {"type": "boolean"},
}),
);
properties.get_mut("deprecation_date").unwrap()["format"] = json!("date");
properties.get_mut("deprecation_date").unwrap()["pattern"] =
json!(r"^\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01])$");
properties.iter_mut().for_each(|(name, property)| {
if name.contains("cost") {
property["minimum"] = json!(0);
} else if name.contains("uplift_multiplier") {
property["minimum"] = json!(1);
}
});
properties.get_mut("guardrail_cost_per_unit").unwrap()["additionalProperties"]["minimum"] =
json!(0);
let definitions = object.get_mut("$defs").unwrap().as_object_mut().unwrap();
for definition in ["OffPeakPricing", "TieredRate", "SearchContextCostPerQuery"] {
let properties = definitions[definition]["properties"]
.as_object_mut()
.unwrap();
properties.iter_mut().for_each(|(name, property)| {
if name.contains("cost") || definition == "SearchContextCostPerQuery" {
property["minimum"] = json!(0);
}
});
for definition in ["OffPeakPricing", "TieredRate"] {
definitions[definition]["patternProperties"] = rates.clone();
definitions[definition]["additionalProperties"] = Value::Bool(false);
}
definitions["OffPeakPricing"]["anyOf"] = json!([
{"required": ["hours_utc"]},
@ -123,15 +170,43 @@ fn decorate_model_entry(schema: &mut Value) {
]);
definitions["OffPeakPricing"]["properties"]["windows"]["minItems"] = json!(1);
definitions["OffPeakWindow"]["properties"]["weekdays"]["minItems"] = json!(1);
definitions["TieredRate"]["properties"]["range"]["items"]["minimum"] = json!(0);
definitions["TieredRate"]["properties"]["max_results_range"]["items"]["minimum"] = json!(0);
definitions["Weekday"]["anyOf"][0]["minimum"] = json!(1);
definitions["Weekday"]["anyOf"][0]["maximum"] = json!(7);
definitions["Weekday"]["anyOf"][1]["pattern"] = json!(
r"(?i)^(mon|monday|tue|tues|tuesday|wed|wednesday|thu|thur|thurs|thursday|fri|friday|sat|saturday|sun|sunday)$"
);
let window_pattern = json!(r"^([01]\d|2[0-3]):[0-5]\d-([01]\d|2[0-3]):[0-5]\d$");
definitions["UtcHours"]["anyOf"][0]["pattern"] = window_pattern.clone();
definitions["UtcHours"]["anyOf"][1]["items"]["pattern"] = window_pattern;
definitions["UtcHours"]["anyOf"][1]["minItems"] = json!(1);
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use super::FLAT_KEY_PATTERN;
use crate::wire::FlatKey;
#[rstest]
#[case::input_token("input_cost_per_token")]
#[case::cache_write_one_hour_band(
"cache_creation_input_token_cost_above_1hr_above_200k_tokens"
)]
#[case::image_label("output_cost_per_image_0.5K")]
#[case::resolution_tier("output_cost_per_second_4k_flex")]
#[case::feature_batch("annotation_cost_per_page_batches")]
#[case::video_interval("input_cost_per_video_per_second_above_8s_interval")]
#[case::explicit_cache_read("explicit_cache_read_input_token_cost")]
#[case::wall_clock("cost_per_second")]
#[case::unused_meter("cost_per_token")]
#[case::other_currency("input_dbu_cost_per_token")]
#[case::typo("input_cost_per_tokne")]
#[case::read_has_no_ttl("cache_read_input_token_cost_above_1hr")]
#[case::interval_on_a_token_meter("input_cost_per_token_above_8s_interval")]
fn flat_key_pattern_agrees_with_the_parser(#[case] key: &str) {
let schema = serde_json::json!({
"type": "object",
"patternProperties": {FLAT_KEY_PATTERN: {}},
"additionalProperties": false,
});
let validator = jsonschema::validator_for(&schema).unwrap();
assert_eq!(
validator.is_valid(&serde_json::json!({key: 1})),
FlatKey::parse(key).is_some()
);
}
}

35
litellm-rust/crates/model-catalog/src/store/AGENTS.md Normal file
View file

@ -0,0 +1,35 @@
# Store
`Store<R: Rows>` is an in-memory store, a small database built for exactly three queries. It knows nothing about the catalog. A source picks its own `Rows::Row` type, how names fold for matching, and the group each key is listed under. The catalog's sources fold to lowercase and group by raw `litellm_provider`
| Query | Method | Real caller |
|---|---|---|
| row by exact key | `Store::get` | reading `litellm.model_cost["gpt-4o"]` |
| actual key for a name that folds to the same form | `Store::find_key` | `get_model_info("GPT-4o")` finding `gpt-4o` |
| keys in a group | `Store::keys_in` | the provider sets, `litellm.open_ai_chat_completion_models` |
## Rules
- nothing in `store/` imports from the rest of the crate. Wire types, case rules and field names such as `litellm_provider` belong to the source that implements `Rows`
- the index in `name_index.rs` is private to the store. Never build a folded-name map or a provider grouping anywhere else in the workspace. A new lookup pattern is a new `Store` method backed by `NameIndex`
- the index holds keys, never rows. Rows are always read from `Rows`, so a price edit is seen on the next read and never rebuilds the index
- `find_key` and `keys_in` never return a key the rows lack. A hit on a removed key rebuilds the index once and asks again
- `keys_in` matches the group exactly as the source wrote it. Provider aliases such as `vertex_ai` over `vertex_ai-language-models` are Python's job
- when two keys fold to the same name the later key in row order wins, like main's `_model_cost_lowercase_map`
## Stamps
`Rows::stamp` is how the store learns that the index may be stale. The store keeps the stamp it built the index at and rebuilds when the source returns a different one. A source must change its stamp whenever a key or a key's group changes, since those are the two things the index holds
| Source | Stamp | Blind spot |
|---|---|---|
| `Catalog`'s `CatalogRows` | `()`, every change goes through `Store::edit`, which drops the index | none |
| python-bridge's bound `litellm.model_cost` | the dict object and its `len` | a delete plus an add between two reads, or a `litellm_provider` rewrite on an existing key |
A source with a blind spot calls `Store::invalidate` when it knows keys changed. Until then a key added in the blind spot is not found, and a key moved to another provider stays listed under the old one. Removed keys are still never returned
## Tests
Tests live in `mod.rs`'s `tests` module and drive the store through `ModelCost`, a fake `litellm.model_cost` with real bundled keys whose row is just its `litellm_provider`. Each Python write is a method (`set`, `del`, `rebind`) and its stamp is what python-bridge can see. Add a scenario as a named test that reads like the Python user code it stands for
The two property tests compare every answer with a linear scan of the same dict. With a stamp that sees every key change, answers must match exactly. With the dict stamp, every answer must be sound (present, same folded name), and `invalidate` must make them match again. A store change that fails either is a bug in the store, not in the test

460
litellm-rust/crates/model-catalog/src/store/mod.rs Normal file
View file

@ -0,0 +1,460 @@
mod name_index;
use std::borrow::Cow;
use std::fmt;
use std::sync::{PoisonError, RwLock};
use name_index::NameIndex;
/// Where a `Store`'s rows live. The store rebuilds its index whenever `stamp` changes, so a source must change its
/// stamp whenever a key or a key's group changes, or call `Store::invalidate` when it cannot
pub trait Rows {
type Row: ToOwned + ?Sized;
type Stamp: PartialEq;
/// The name two spellings of one key share, which `Store::find_key` matches on
fn fold(name: &str) -> String;
fn stamp(&self) -> Self::Stamp;
fn contains(&self, key: &str) -> bool;
fn row(&self, key: &str) -> Option<Cow<'_, Self::Row>>;
/// Every key in row order with the group it is listed under
fn scan(&self, visit: &mut dyn FnMut(&str, Option<&str>));
}
/// An in-memory store for three queries (row by exact key, actual key for a name by its folded form, keys in a group)
/// that owns the only index behind them
pub struct Store<R: Rows> {
rows: R,
index: RwLock<Option<Built<R::Stamp>>>,
}
struct Built<S> {
stamp: S,
names: NameIndex,
}
impl<R: Rows> Store<R> {
pub fn new(rows: R) -> Self {
Self {
rows,
index: RwLock::new(None),
}
}
pub fn rows(&self) -> &R {
&self.rows
}
/// Changes the rows in place. The index is rebuilt on the next query
pub fn edit<T>(&mut self, edit: impl FnOnce(&mut R) -> T) -> T {
let edited = edit(&mut self.rows);
*self.index.get_mut().unwrap_or_else(PoisonError::into_inner) = None;
edited
}
pub fn get(&self, key: &str) -> Option<Cow<'_, R::Row>> {
self.rows.row(key)
}
/// The key itself when present, otherwise the actual key that folds to the same name. Never a key the rows lack
pub fn find_key(&self, name: &str) -> Option<String> {
if self.rows.contains(name) {
return Some(name.to_owned());
}
let folded = R::fold(name);
let hit = self.query(|names| names.key_for(&folded).map(str::to_owned))?;
if self.rows.contains(&hit) {
return Some(hit);
}
self.invalidate();
self.query(|names| names.key_for(&folded).map(str::to_owned))
.filter(|key| self.rows.contains(key))
}
/// Keys whose group is exactly `group`, in row order. Never a key the rows lack
pub fn keys_in(&self, group: &str) -> Vec<String> {
self.query(|names| names.keys_in(group).to_vec())
.into_iter()
.filter(|key| self.rows.contains(key))
.collect()
}
/// Drops the index, for a source whose stamp cannot see every key change
pub fn invalidate(&self) {
*self.index.write().unwrap_or_else(PoisonError::into_inner) = None;
}
fn query<T>(&self, read: impl Fn(&NameIndex) -> T) -> T {
let stamp = self.rows.stamp();
if let Some(built) = self
.index
.read()
.unwrap_or_else(PoisonError::into_inner)
.as_ref()
.filter(|built| built.stamp == stamp)
{
return read(&built.names);
}
let names = NameIndex::scan(&self.rows);
let answer = read(&names);
*self.index.write().unwrap_or_else(PoisonError::into_inner) = Some(Built { stamp, names });
answer
}
}
impl<R: Rows + Clone> Clone for Store<R> {
fn clone(&self) -> Self {
Self::new(self.rows.clone())
}
}
impl<R: Rows + fmt::Debug> fmt::Debug for Store<R> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Store")
.field("rows", &self.rows)
.finish_non_exhaustive()
}
}
#[cfg(test)]
mod tests {
use std::cell::{Cell, RefCell};
use indexmap::IndexMap;
use rstest::{fixture, rstest};
use super::*;
const BUNDLED: &[(&str, Option<&str>)] = &[
("gpt-4o", Some("openai")),
("o3-pro", Some("openai")),
("azure/gpt-4o", Some("azure")),
("claude-sonnet-4-5", Some("anthropic")),
("gemini-2.5-pro", Some("vertex_ai-language-models")),
(
"fireworks_ai/accounts/fireworks/models/llama-v3p1-8b-instruct",
Some("fireworks_ai"),
),
(
"anyscale/meta-llama/Meta-Llama-3-8B-Instruct",
Some("anyscale"),
),
("sample_free_row", None),
];
/// `litellm.model_cost` as Python sees it. The stamp is what python-bridge can observe for free, the dict's
/// identity and its length, so a write that keeps both hides a key change from the store
struct ModelCost {
dict: RefCell<IndexMap<String, Option<String>>>,
identity: Cell<u64>,
index_changes: Cell<u64>,
honest_stamp: bool,
scans: Cell<usize>,
}
impl ModelCost {
fn new(rows: &[(&str, Option<&str>)], honest_stamp: bool) -> Self {
Self {
dict: RefCell::new(
rows.iter()
.map(|(key, provider)| ((*key).to_owned(), provider.map(str::to_owned)))
.collect(),
),
identity: Cell::new(0),
index_changes: Cell::new(0),
honest_stamp,
scans: Cell::new(0),
}
}
/// `model_cost[key] = {"litellm_provider": provider, ...}`
fn set(&self, key: &str, provider: Option<&str>) {
let previous = self
.dict
.borrow_mut()
.insert(key.to_owned(), provider.map(str::to_owned));
if previous.as_ref().map(Option::as_deref) != Some(provider) {
self.index_changes.set(self.index_changes.get() + 1);
}
}
/// `del model_cost[key]`
fn del(&self, key: &str) {
if self.dict.borrow_mut().shift_remove(key).is_some() {
self.index_changes.set(self.index_changes.get() + 1);
}
}
/// `litellm.model_cost = {...}`, a new dict object
fn rebind(&self, rows: &[(&str, Option<&str>)]) {
*self.dict.borrow_mut() = rows
.iter()
.map(|(key, provider)| ((*key).to_owned(), provider.map(str::to_owned)))
.collect();
self.identity.set(self.identity.get() + 1);
self.index_changes.set(self.index_changes.get() + 1);
}
}
/// A row is its `litellm_provider`, which is also the group the store lists its key under
impl Rows for &ModelCost {
type Row = Option<String>;
type Stamp = (u64, u64);
fn fold(name: &str) -> String {
name.to_lowercase()
}
fn stamp(&self) -> Self::Stamp {
if self.honest_stamp {
(self.identity.get(), self.index_changes.get())
} else {
(self.identity.get(), self.dict.borrow().len() as u64)
}
}
fn contains(&self, key: &str) -> bool {
self.dict.borrow().contains_key(key)
}
fn row(&self, key: &str) -> Option<Cow<'_, Option<String>>> {
Some(Cow::Owned(self.dict.borrow().get(key)?.clone()))
}
fn scan(&self, visit: &mut dyn FnMut(&str, Option<&str>)) {
self.scans.set(self.scans.get() + 1);
for (key, provider) in self.dict.borrow().iter() {
visit(key, provider.as_deref());
}
}
}
#[fixture]
fn model_cost() -> ModelCost {
ModelCost::new(BUNDLED, false)
}
/// Rows only `Store::edit` can change, so the stamp never moves
#[derive(Debug)]
struct Owned(IndexMap<String, Option<String>>);
impl Rows for Owned {
type Row = Option<String>;
type Stamp = ();
fn fold(name: &str) -> String {
name.to_lowercase()
}
fn stamp(&self) -> Self::Stamp {}
fn contains(&self, key: &str) -> bool {
self.0.contains_key(key)
}
fn row(&self, key: &str) -> Option<Cow<'_, Option<String>>> {
self.0.get(key).map(Cow::Borrowed)
}
fn scan(&self, visit: &mut dyn FnMut(&str, Option<&str>)) {
for (key, provider) in &self.0 {
visit(key, provider.as_deref());
}
}
}
fn owned(rows: &[(&str, Option<&str>)]) -> Store<Owned> {
Store::new(Owned(
rows.iter()
.map(|(key, provider)| ((*key).to_owned(), provider.map(str::to_owned)))
.collect(),
))
}
#[rstest]
#[case::bundled_key_as_written("gpt-4o", Some("gpt-4o"))]
#[case::user_typed_upper_case("GPT-4o", Some("gpt-4o"))]
#[case::provider_prefixed("Azure/GPT-4o", Some("azure/gpt-4o"))]
#[case::fireworks_path_in_another_case(
"fireworks_ai/accounts/fireworks/models/Llama-v3p1-8B-Instruct",
Some("fireworks_ai/accounts/fireworks/models/llama-v3p1-8b-instruct")
)]
#[case::mixed_case_bundled_key(
"anyscale/meta-llama/meta-llama-3-8b-instruct",
Some("anyscale/meta-llama/Meta-Llama-3-8B-Instruct")
)]
#[case::unmapped_model("my-finetune", None)]
#[case::prefix_is_not_a_match("gpt-4", None)]
fn find_key_maps_a_requested_model_to_its_key(
model_cost: ModelCost,
#[case] model: &str,
#[case] expected: Option<&str>,
) {
assert_eq!(Store::new(&model_cost).find_key(model).as_deref(), expected);
}
#[rstest]
#[case::the_bundled_spelling("gpt-4o", "gpt-4o")]
#[case::the_registered_spelling("GPT-4o", "GPT-4o")]
#[case::another_spelling_takes_the_later_key("Gpt-4O", "GPT-4o")]
fn a_registered_case_variant_is_its_own_row(
model_cost: ModelCost,
#[case] model: &str,
#[case] expected: &str,
) {
model_cost.set("GPT-4o", Some("openai"));
assert_eq!(
Store::new(&model_cost).find_key(model).as_deref(),
Some(expected)
);
}
#[rstest]
#[case::openai_in_map_order("openai", &["gpt-4o", "o3-pro"])]
#[case::azure("azure", &["azure/gpt-4o"])]
#[case::raw_vertex_family("vertex_ai-language-models", &["gemini-2.5-pro"])]
#[case::provider_alias_is_python_s_job("vertex_ai", &[])]
#[case::matched_as_written("OpenAI", &[])]
#[case::unknown_provider("acme", &[])]
fn keys_in_lists_the_keys_a_raw_provider_names(
model_cost: ModelCost,
#[case] provider: &str,
#[case] expected: &[&str],
) {
assert_eq!(Store::new(&model_cost).keys_in(provider), expected);
}
#[rstest]
fn a_model_written_straight_into_the_dict_is_found_in_any_case(model_cost: ModelCost) {
let store = Store::new(&model_cost);
assert_eq!(store.find_key("mycorp/custom-model"), None);
model_cost.set("MyCorp/Custom-Model", Some("openai"));
assert_eq!(
store.find_key("mycorp/custom-model").as_deref(),
Some("MyCorp/Custom-Model")
);
assert_eq!(
store.keys_in("openai"),
["gpt-4o", "o3-pro", "MyCorp/Custom-Model"]
);
}
#[rstest]
fn a_price_edit_reuses_the_index(model_cost: ModelCost) {
let store = Store::new(&model_cost);
assert_eq!(store.find_key("GPT-4o").as_deref(), Some("gpt-4o"));
model_cost.set("gpt-4o", Some("openai"));
model_cost.set("o3-pro", Some("openai"));
model_cost.set("gpt-4o", Some("openai"));
assert_eq!(store.find_key("O3-PRO").as_deref(), Some("o3-pro"));
assert_eq!(store.keys_in("openai"), ["gpt-4o", "o3-pro"]);
assert_eq!(model_cost.scans.get(), 1);
}
#[rstest]
fn a_reload_that_rebinds_the_dict_is_seen_even_at_the_same_length(model_cost: ModelCost) {
let store = Store::new(&model_cost);
assert_eq!(store.find_key("GPT-4o").as_deref(), Some("gpt-4o"));
let mut reloaded = BUNDLED.to_vec();
reloaded[0] = ("gpt-4o-2024-11-20", Some("openai"));
model_cost.rebind(&reloaded);
assert_eq!(store.find_key("GPT-4o"), None);
assert_eq!(
store.find_key("GPT-4o-2024-11-20").as_deref(),
Some("gpt-4o-2024-11-20")
);
}
#[rstest]
fn a_deleted_model_is_never_returned(model_cost: ModelCost) {
let store = Store::new(&model_cost);
assert_eq!(store.find_key("O3-PRO").as_deref(), Some("o3-pro"));
model_cost.del("o3-pro");
assert_eq!(store.find_key("O3-PRO"), None);
assert_eq!(store.keys_in("openai"), ["gpt-4o"]);
}
#[rstest]
fn a_respelled_key_at_the_same_length_is_found_after_one_rebuild(model_cost: ModelCost) {
let store = Store::new(&model_cost);
assert_eq!(store.find_key("o3-PRO").as_deref(), Some("o3-pro"));
model_cost.del("o3-pro");
model_cost.set("O3-Pro", Some("openai"));
assert_eq!(store.find_key("o3-PRO").as_deref(), Some("O3-Pro"));
assert_eq!(model_cost.scans.get(), 2);
}
#[rstest]
fn a_swap_at_the_same_length_hides_the_new_key_until_invalidate(model_cost: ModelCost) {
let store = Store::new(&model_cost);
assert_eq!(store.find_key("O3-PRO").as_deref(), Some("o3-pro"));
model_cost.del("o3-pro");
model_cost.set("MyCorp/Custom-Model", Some("openai"));
assert_eq!(store.find_key("mycorp/custom-model"), None);
assert_eq!(store.keys_in("openai"), ["gpt-4o"]);
store.invalidate();
assert_eq!(
store.find_key("mycorp/custom-model").as_deref(),
Some("MyCorp/Custom-Model")
);
assert_eq!(store.keys_in("openai"), ["gpt-4o", "MyCorp/Custom-Model"]);
}
#[rstest]
fn a_provider_rewrite_on_an_existing_key_needs_invalidate(model_cost: ModelCost) {
let store = Store::new(&model_cost);
assert_eq!(store.keys_in("openai"), ["gpt-4o", "o3-pro"]);
model_cost.set("o3-pro", Some("azure"));
assert_eq!(store.keys_in("openai"), ["gpt-4o", "o3-pro"]);
store.invalidate();
assert_eq!(store.keys_in("openai"), ["gpt-4o"]);
assert_eq!(store.keys_in("azure"), ["o3-pro", "azure/gpt-4o"]);
}
#[rstest]
fn an_edit_on_owned_rows_is_seen_by_the_next_query() {
let mut store = owned(BUNDLED);
assert_eq!(store.find_key("GPT-4o").as_deref(), Some("gpt-4o"));
store.edit(|rows| {
rows.0.shift_remove("gpt-4o");
rows.0
.insert("MyCorp/Custom-Model".to_owned(), Some("openai".to_owned()));
});
assert_eq!(store.find_key("GPT-4o"), None);
assert_eq!(
store.find_key("mycorp/custom-model").as_deref(),
Some("MyCorp/Custom-Model")
);
assert_eq!(store.keys_in("openai"), ["o3-pro", "MyCorp/Custom-Model"]);
}
#[rstest]
fn get_reads_the_row_by_exact_key_only() {
let store = owned(BUNDLED);
assert_eq!(
store.get("azure/gpt-4o").as_deref(),
Some(&Some("azure".to_owned()))
);
assert_eq!(store.get("Azure/GPT-4o"), None);
}
}

35
litellm-rust/crates/model-catalog/src/store/name_index.rs Normal file
View file

@ -0,0 +1,35 @@
use std::collections::HashMap;
use super::Rows;
/// Keys only, never rows, so a value write never makes it stale. A later key wins when two keys fold to one name
#[derive(Debug, Default)]
pub(super) struct NameIndex {
by_folded: HashMap<String, String>,
by_group: HashMap<String, Vec<String>>,
}
impl NameIndex {
pub(super) fn scan<R: Rows>(rows: &R) -> Self {
let mut index = Self::default();
rows.scan(&mut |key, group| {
index.by_folded.insert(R::fold(key), key.to_owned());
if let Some(group) = group {
index
.by_group
.entry(group.to_owned())
.or_default()
.push(key.to_owned());
}
});
index
}
pub(super) fn key_for(&self, folded: &str) -> Option<&str> {
self.by_folded.get(folded).map(String::as_str)
}
pub(super) fn keys_in(&self, group: &str) -> &[String] {
self.by_group.get(group).map_or(&[], Vec::as_slice)
}
}

252
litellm-rust/crates/model-catalog/src/validation.rs
View file

@ -1,252 +0,0 @@
use std::collections::BTreeSet;
use serde_json::{Map, Value};
use thiserror::Error;
use crate::{AliasIssue, Catalog, ModelInfo, UtcHours, Weekday};
#[derive(Debug, Error)]
pub enum RegistryValidationError {
#[error("invalid registry entry {model:?}")]
Entry {
model: String,
#[source]
reason: EntryViolation,
},
#[error(transparent)]
Alias(AliasIssue),
}
#[derive(Debug, Error)]
pub enum EntryViolation {
#[error("entry must be an object")]
NotObject,
#[error("aliases must be an array")]
AliasesNotArray,
#[error("aliases must contain strings")]
AliasNotString,
#[error("entry does not match the model schema")]
Schema(#[source] serde_json::Error),
#[error("litellm_provider is required")]
MissingProvider,
#[error("invalid deprecation_date")]
DeprecationDate(#[source] time::error::Parse),
#[error("off_peak_pricing needs hours or windows")]
OffPeakWithoutHours,
#[error("off_peak_pricing.hours_utc is invalid")]
InvalidHours,
#[error("off_peak_pricing.windows is empty")]
EmptyWindows,
#[error("off_peak_pricing.weekdays is invalid")]
InvalidWeekdays,
#[error("typed entry could not be serialized")]
Serialize(#[source] serde_json::Error),
#[error("entry has an unknown field, null, or changed value: {}", fields.join(", "))]
ChangedFields { fields: Vec<String> },
#[error("{field} must be nonnegative")]
Negative { field: String },
#[error("{field} must be at least one")]
UpliftBelowOne { field: String },
}
pub fn validate_model_entry(model: &str, value: &Value) -> Result<(), RegistryValidationError> {
validate_entry_inner(value).map_err(|reason| RegistryValidationError::Entry {
model: model.to_owned(),
reason,
})
}
pub fn validate_registry(catalog: &Catalog) -> Result<(), RegistryValidationError> {
if let Some(issue) = catalog.alias_issues().first() {
return Err(RegistryValidationError::Alias(issue.clone()));
}
catalog.model_names().try_for_each(|name| {
let entry = catalog.lookup(name).expect("catalog name must resolve");
validate_model_entry(name, &Value::Object(entry.entry.fields().clone()))
})
}
fn json_eq(left: &Value, right: &Value) -> bool {
match (left, right) {
(Value::Number(left), Value::Number(right)) => left.as_f64() == right.as_f64(),
(Value::Array(left), Value::Array(right)) => {
left.len() == right.len() && left.iter().zip(right).all(|(a, b)| json_eq(a, b))
}
(Value::Object(left), Value::Object(right)) => {
left.len() == right.len()
&& left
.iter()
.all(|(key, value)| right.get(key).is_some_and(|other| json_eq(value, other)))
}
_ => left == right,
}
}
fn keys(value: &Map<String, Value>) -> BTreeSet<String> {
value.keys().cloned().collect()
}
fn symmetric_difference(left: &BTreeSet<String>, right: &BTreeSet<String>) -> Vec<String> {
left.symmetric_difference(right).cloned().collect()
}
fn validate_entry_inner(value: &Value) -> Result<(), EntryViolation> {
let object = value.as_object().ok_or(EntryViolation::NotObject)?;
if let Some(aliases) = object.get("aliases") {
let names = aliases.as_array().ok_or(EntryViolation::AliasesNotArray)?;
if names.iter().any(|name| !name.is_string()) {
return Err(EntryViolation::AliasNotString);
}
}
let info: ModelInfo = serde_json::from_value(value.clone()).map_err(EntryViolation::Schema)?;
if info.litellm_provider.is_none() {
return Err(EntryViolation::MissingProvider);
}
validate_dates_and_windows(&info)?;
let serialized = serde_json::to_value(info).map_err(EntryViolation::Serialize)?;
let mut expected = object.clone();
expected.remove("aliases");
if !json_eq(&Value::Object(expected.clone()), &serialized) {
let actual = serialized
.as_object()
.expect("ModelInfo serializes as an object");
return Err(EntryViolation::ChangedFields {
fields: symmetric_difference(&keys(&expected), &keys(actual)),
});
}
check_prices("", value)
}
fn validate_dates_and_windows(info: &ModelInfo) -> Result<(), EntryViolation> {
if let Some(date) = &info.deprecation_date {
let format = time::format_description::parse_borrowed::<2>("[year]-[month]-[day]").unwrap();
time::Date::parse(date, &format).map_err(EntryViolation::DeprecationDate)?;
}
let Some(pricing) = &info.off_peak_pricing else {
return Ok(());
};
if pricing.hours_utc.is_none() && pricing.windows.is_none() {
return Err(EntryViolation::OffPeakWithoutHours);
}
if let Some(hours) = &pricing.hours_utc {
validate_hours(hours)?;
}
if let Some(windows) = &pricing.windows {
if windows.is_empty() {
return Err(EntryViolation::EmptyWindows);
}
windows.iter().try_for_each(|window| {
validate_hours(&window.hours_utc)?;
if let Some(days) = &window.weekdays
&& (days.is_empty() || days.iter().any(|day| !valid_weekday(day)))
{
return Err(EntryViolation::InvalidWeekdays);
}
Ok(())
})?;
}
Ok(())
}
fn validate_hours(hours: &UtcHours) -> Result<(), EntryViolation> {
let values = match hours {
UtcHours::Single(value) => std::slice::from_ref(value),
UtcHours::Multiple(values) => values.as_slice(),
};
if values.is_empty() || values.iter().any(|value| !valid_utc_window(value)) {
return Err(EntryViolation::InvalidHours);
}
Ok(())
}
fn valid_utc_window(value: &str) -> bool {
let Some((start, end)) = value.split_once('-') else {
return false;
};
[start, end].into_iter().all(|clock| {
let Some((hour, minute)) = clock.split_once(':') else {
return false;
};
hour.len() == 2
&& minute.len() == 2
&& hour.parse::<u8>().is_ok_and(|hour| hour < 24)
&& minute.parse::<u8>().is_ok_and(|minute| minute < 60)
})
}
fn valid_weekday(day: &Weekday) -> bool {
match day {
Weekday::Number(number) => (1..=7).contains(number),
Weekday::Name(name) => matches!(
name.to_ascii_lowercase().as_str(),
"mon"
| "monday"
| "tue"
| "tues"
| "tuesday"
| "wed"
| "wednesday"
| "thu"
| "thur"
| "thurs"
| "thursday"
| "fri"
| "friday"
| "sat"
| "saturday"
| "sun"
| "sunday"
),
}
}
fn check_prices(path: &str, value: &Value) -> Result<(), EntryViolation> {
let Some(object) = value.as_object() else {
return Ok(());
};
object.iter().try_for_each(|(key, field)| {
let field_path = format!("{path}.{key}");
if (key.contains("cost")
|| path.ends_with(".guardrail_cost_per_unit")
|| path.ends_with(".search_context_cost_per_query"))
&& let Some(number) = field.as_f64()
&& number < 0.0
{
return Err(EntryViolation::Negative {
field: relative(&field_path),
});
}
if key.contains("uplift_multiplier")
&& let Some(number) = field.as_f64()
&& number < 1.0
{
return Err(EntryViolation::UpliftBelowOne {
field: relative(&field_path),
});
}
if matches!(key.as_str(), "range" | "max_results_range")
&& field.as_array().is_some_and(|values| {
values
.iter()
.any(|value| value.as_f64().is_some_and(|n| n < 0.0))
})
{
return Err(EntryViolation::Negative {
field: relative(&field_path),
});
}
if matches!(key.as_str(), "metadata" | "provider_specific_entry") {
return Ok(());
}
match field.as_array() {
Some(items) => items.iter().enumerate().try_for_each(|(index, item)| {
check_prices(&format!("{field_path}[{index}]"), item)
}),
None => check_prices(&field_path, field),
}
})
}
fn relative(field_path: &str) -> String {
field_path.trim_start_matches('.').to_owned()
}

126
litellm-rust/crates/model-catalog/src/vocabulary.rs Normal file
View file

@ -0,0 +1,126 @@
/// Primary API surface / task type of the model.
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq, Hash)]
#[serde(rename_all = "snake_case")]
pub enum Mode {
AudioSpeech,
AudioTranscription,
Chat,
Completion,
Decisions,
Embedding,
Evaluation,
Extraction,
Guardrail,
ImageEdit,
ImageGeneration,
Moderation,
Ocr,
Realtime,
Rerank,
Responses,
Search,
VectorStore,
VideoGeneration,
}
/// Gemini audio generation API the model is served through.
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq, Hash)]
#[serde(rename_all = "snake_case")]
pub enum VertexAiAudioApi {
LyriaPredict,
LyriaInteractions,
}
/// Audio container format the model can return.
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq, Hash)]
#[serde(rename_all = "snake_case")]
pub enum AudioFormat {
Mp3,
Wav,
}
/// Input modality the model accepts.
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq, Hash)]
#[serde(rename_all = "snake_case")]
pub enum InputModality {
Text,
Image,
Audio,
Video,
}
/// Output modality the model can produce.
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq, Hash)]
#[serde(rename_all = "snake_case")]
pub enum OutputModality {
Text,
Image,
Audio,
Video,
Code,
}
/// API route a model can be called through.
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq, Hash)]
pub enum Endpoint {
#[serde(rename = "/v1/chat/completions")]
ChatCompletions,
#[serde(rename = "/v1/completions")]
Completions,
#[serde(rename = "/v1/responses")]
Responses,
#[serde(rename = "/v1/messages")]
Messages,
#[serde(rename = "/v1/batch")]
Batch,
#[serde(rename = "/v1/embeddings")]
Embeddings,
#[serde(rename = "/v1/images/generations")]
ImageGenerations,
#[serde(rename = "/v1/images/edits")]
ImageEdits,
#[serde(rename = "/v1/audio/speech")]
AudioSpeech,
#[serde(rename = "/v1/audio/transcriptions")]
AudioTranscriptions,
#[serde(rename = "/v1/realtime")]
Realtime,
#[serde(rename = "/v1/realtime/transcription_sessions")]
RealtimeTranscriptionSessions,
#[serde(rename = "/v1/videos")]
Videos,
#[serde(rename = "/v1/ocr")]
Ocr,
#[serde(rename = "/v1/listen")]
Listen,
#[serde(rename = "/v1/systemone")]
SystemOne,
#[serde(rename = "/v1/decisions")]
Decisions,
#[serde(rename = "/v1beta/interactions")]
Interactions,
#[serde(rename = "/v1beta/models")]
Models,
#[serde(rename = "/vertex_ai/live")]
VertexAiLive,
}
/// Reasoning effort level a model accepts or applies.
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq, Hash)]
#[serde(rename_all = "snake_case")]
pub enum EffortLevel {
None,
Minimal,
Low,
Medium,
High,
Xhigh,
Max,
}

344
litellm-rust/crates/model-catalog/src/wire/attributes.rs Normal file
View file

@ -0,0 +1,344 @@
use std::collections::BTreeMap;
use std::fmt;
use std::str::FromStr;
use litellm_cost::money::UnitPrice;
use serde::de::{self, Deserializer};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use crate::error::{DateCause, Error, Repr};
use crate::vocabulary::{
AudioFormat, EffortLevel, Endpoint, InputModality, Mode, OutputModality, VertexAiAudioApi,
};
use crate::wire::lax::Count;
macro_rules! attributes {
($($(#[$doc:meta])* $name:ident: $ty:ty $(as $via:ty)?),* $(,)?) => {
/// The named, non-price fields of a catalog entry.
#[derive(Clone, Debug, Default, PartialEq)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
pub struct Attributes {
$($(#[$doc])* pub $name: Option<$ty>,)*
}
impl Attributes {
pub(crate) fn read(
&mut self,
key: &str,
value: &Value,
) -> Option<Result<(), serde_json::Error>> {
match key {
$(stringify!($name) => Some(
attributes!(@read value, $ty $(, $via)?).map(|read| self.$name = Some(read)),
),)*
_ => None,
}
}
}
};
(@read $value:ident, $ty:ty) => {
<$ty as Deserialize>::deserialize($value)
};
(@read $value:ident, $ty:ty, $via:ty) => {
serde_with::de::DeserializeAsWrap::<$ty, $via>::deserialize($value)
.map(serde_with::de::DeserializeAsWrap::into_inner)
};
}
attributes! {
/// Other names this entry is served under.
aliases: Vec<String>,
audio_transcription_config: String,
bedrock_converse_supports_strict_tools: bool,
/// Highest reasoning effort the Bedrock output_config accepts for this model.
bedrock_output_config_effort_ceiling: EffortLevel,
comment: String,
/// Reasoning effort the provider applies when the request omits reasoning_effort. Gates whether a non-default temperature or the top_p/logprobs sampling params are accepted, which hold only when the effort resolves to 'none'.
default_reasoning_effort: EffortLevel,
/// Date the provider deprecates the model, YYYY-MM-DD.
deprecation_date: IsoDate,
gemini_audio_only_live: bool,
gemini_native_audio: bool,
input_cost_per_token_cache_hit: UnitPrice,
/// Databricks units, not USD.
input_dbu_cost_per_token: Dbu,
/// LiteLLM provider slug; one of https://docs.litellm.ai/docs/providers.
litellm_provider: String,
/// Maximum prompt/context tokens the model accepts.
max_input_tokens: u64 as Count,
/// Maximum tokens the model can generate in one response.
max_output_tokens: u64 as Count,
/// Legacy field: max output tokens if the provider specifies it, else max input tokens.
max_tokens: u64 as Count,
/// Free-form notes about the entry (e.g. pricing derivation).
metadata: BTreeMap<String, Value>,
/// Primary API surface / task type of the model.
mode: Mode,
/// Databricks units, not USD.
output_dbu_cost_per_token: Dbu,
/// Embedding dimension for embedding models.
output_vector_size: u64 as Count,
/// Smallest prefix the provider will actually cache; absent means the provider default applies.
prompt_cache_min_tokens: u64 as Count,
/// Provider-internal routing hints (e.g. bedrock_invocation_schema).
provider_specific_entry: BTreeMap<String, Value>,
reasoning_contexts: Vec<String>,
/// Exact reasoning_effort levels this deployment accepts; wins over supports_* flags.
reasoning_effort_levels: Vec<EffortLevel>,
reasoning_modes: Vec<String>,
/// Provider default requests-per-minute limit.
rpm: u64 as Count,
/// Grounded search requests the provider includes for free before search is billed.
search_free_requests: SearchFreeRequests,
/// URL of the provider pricing/model page this entry was taken from.
source: String,
/// Audio container formats the model can return.
supported_audio_formats: Vec<AudioFormat>,
/// OpenAI-style API routes this model can be called through, e.g. /v1/chat/completions.
supported_endpoints: Vec<Endpoint>,
/// Input modalities the model accepts.
supported_modalities: Vec<InputModality>,
/// Output modalities the model can produce.
supported_output_modalities: Vec<OutputModality>,
/// Cloud regions the model is available in ('global' or region ids).
supported_regions: Vec<String>,
thinking_always_on: bool,
token_count_framing: u64 as Count,
token_count_model: TokenCountModel,
token_count_tokenizer: TokenCountTokenizer,
/// Provider default tokens-per-minute limit.
tpm: u64 as Count,
use_openai_responses_path: bool,
uses_embed_content: bool,
vertex_ai_audio_api: VertexAiAudioApi,
/// Whether web search is billed per query or per prompt.
web_search_billing_unit: WebSearchBillingUnit,
}
/// The model whose provider token counter counts tokens for this entry.
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Eq, Hash)]
#[serde(deny_unknown_fields)]
pub struct TokenCountModel {
pub model: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub litellm_provider: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub region: Option<String>,
}
/// Free grounded search requests per period, shared across the models of one group.
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Eq, Hash)]
#[serde(deny_unknown_fields)]
pub struct SearchFreeRequests {
pub count: u64,
pub period: FreeRequestPeriod,
pub shared_group: String,
}
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq, Hash)]
#[serde(rename_all = "snake_case")]
pub enum FreeRequestPeriod {
Day,
Month,
}
/// Local tokenizer that counts tokens for this entry.
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq, Hash)]
#[serde(rename_all = "snake_case")]
pub enum TokenCountTokenizer {
KimiK3,
}
/// Whether web search is billed per query or per prompt.
#[macro_rules_attribute::apply(crate::wire_type)]
#[derive(Copy, Eq, Hash)]
#[serde(rename_all = "snake_case")]
pub enum WebSearchBillingUnit {
PerQuery,
PerPrompt,
}
/// A non-negative amount of Databricks units.
#[derive(Clone, Copy, Debug, PartialEq, PartialOrd, Serialize)]
#[serde(transparent)]
pub struct Dbu(f64);
impl Dbu {
pub fn new(units: f64) -> Option<Self> {
(units.is_finite() && units >= 0.0).then_some(Self(units))
}
pub fn get(self) -> f64 {
self.0
}
}
impl<'de> Deserialize<'de> for Dbu {
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
let units = f64::deserialize(deserializer)?;
Self::new(units).ok_or_else(|| {
de::Error::invalid_value(
de::Unexpected::Float(units),
&"a non-negative number of Databricks units",
)
})
}
}
/// A calendar date written YYYY-MM-DD.
#[derive(
Clone,
Copy,
Debug,
PartialEq,
Eq,
PartialOrd,
Ord,
Hash,
serde_with::DeserializeFromStr,
serde_with::SerializeDisplay,
)]
pub struct IsoDate(pub time::Date);
impl FromStr for IsoDate {
type Err = Error;
fn from_str(date: &str) -> Result<Self, Self::Err> {
let mut parts = date.split('-');
let (Some(year), Some(month), Some(day), None) =
(parts.next(), parts.next(), parts.next(), parts.next())
else {
return Err(Repr::InvalidDate { cause: None }.into());
};
if year.len() != 4 || month.len() != 2 || day.len() != 2 {
return Err(Repr::InvalidDate { cause: None }.into());
}
let number = |part: &str| part.parse::<u8>().map_err(DateCause::from);
let month = time::Month::try_from(number(month)?).map_err(DateCause::from)?;
time::Date::from_calendar_date(year.parse().map_err(DateCause::from)?, month, number(day)?)
.map(Self)
.map_err(DateCause::from)
.map_err(Error::from)
}
}
impl fmt::Display for IsoDate {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
f,
"{:04}-{:02}-{:02}",
self.0.year(),
u8::from(self.0.month()),
self.0.day()
)
}
}
#[cfg(feature = "schema")]
mod schema {
use std::borrow::Cow;
use schemars::{JsonSchema, Schema, SchemaGenerator, json_schema};
use super::{Dbu, IsoDate};
impl JsonSchema for IsoDate {
fn inline_schema() -> bool {
true
}
fn schema_name() -> Cow<'static, str> {
"IsoDate".into()
}
fn json_schema(_: &mut SchemaGenerator) -> Schema {
json_schema!({
"type": "string",
"format": "date",
"pattern": r"^\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01])$"
})
}
}
impl JsonSchema for Dbu {
fn inline_schema() -> bool {
true
}
fn schema_name() -> Cow<'static, str> {
"Dbu".into()
}
fn json_schema(_: &mut SchemaGenerator) -> Schema {
json_schema!({"type": "number", "minimum": 0})
}
}
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use serde_json::json;
use super::*;
#[rstest]
#[case::leap_day("2028-02-29", true)]
#[case::not_a_leap_year("2026-02-29", false)]
#[case::impossible_day("2026-02-31", false)]
#[case::one_digit_month("2026-2-01", false)]
#[case::extra_part("2026-02-01-01", false)]
fn iso_dates_are_real_calendar_dates(#[case] text: &str, #[case] valid: bool) {
assert_eq!(text.parse::<IsoDate>().is_ok(), valid);
}
#[rstest]
#[case::count_written_as_text("max_tokens", json!("8192"), true)]
#[case::negative_count("max_tokens", json!(-1), false)]
#[case::known_mode("mode", json!("chat"), true)]
#[case::unknown_mode("mode", json!("chatt"), false)]
#[case::negative_dbu("input_dbu_cost_per_token", json!(-1), false)]
#[case::reasoning_contexts("reasoning_contexts", json!(["reasoning", "non_reasoning"]), true)]
#[case::reasoning_modes("reasoning_modes", json!(["none", "adaptive"]), true)]
#[case::reasoning_context_not_a_list("reasoning_contexts", json!("reasoning"), false)]
#[case::reasoning_mode_not_a_string("reasoning_modes", json!([1]), false)]
fn each_attribute_reads_its_own_type(
#[case] key: &str,
#[case] value: Value,
#[case] valid: bool,
) {
assert_eq!(
Attributes::default()
.read(key, &value)
.map(|read| read.is_ok()),
Some(valid)
);
}
#[rstest]
fn keys_that_are_not_attributes_are_left_to_the_caller() {
assert!(
Attributes::default()
.read("input_cost_per_token", &json!(1))
.is_none()
);
}
#[rstest]
fn reasoning_metadata_is_retained() {
let mut attributes = Attributes::default();
attributes
.read("reasoning_contexts", &json!(["reasoning", "non_reasoning"]))
.unwrap()
.unwrap();
attributes
.read("reasoning_modes", &json!(["none", "adaptive"]))
.unwrap()
.unwrap();
assert_eq!(
attributes.reasoning_contexts.unwrap(),
["reasoning", "non_reasoning"]
);
assert_eq!(attributes.reasoning_modes.unwrap(), ["none", "adaptive"]);
}
}

182
litellm-rust/crates/model-catalog/src/wire/blocks.rs Normal file
View file

@ -0,0 +1,182 @@
use litellm_cost::money::UnitPrice;
use litellm_cost::schedule::{UtcWindow, Weekday};
use serde::Deserialize;
use serde::de::DeserializeOwned;
use serde_json::{Map, Value};
use serde_with::de::DeserializeAsWrap;
use crate::error::{Error, Repr, pointer_child, pointer_for_key};
use crate::wire::flat_key::FlatKey;
use crate::wire::lax::Count;
/// A flat price key with its spelling, kept so problems can name it.
#[derive(Clone, Debug, PartialEq)]
pub struct FlatPrice {
pub key: String,
pub path: String,
pub flat: FlatKey,
pub price: UnitPrice,
}
#[derive(Clone, Debug, Default, PartialEq)]
pub struct TierRow {
pub path: String,
pub range: Option<[u64; 2]>,
pub max_results_range: Option<[u64; 2]>,
pub prices: Vec<FlatPrice>,
}
#[derive(Clone, Debug, Default, PartialEq)]
pub struct OffPeakBlock {
pub hours_utc: Option<UtcHours>,
pub windows: Option<Vec<OffPeakWindow>>,
pub weekday_timezone: Option<String>,
pub prices: Vec<FlatPrice>,
}
/// One UTC window or a list of them.
#[derive(Clone, Debug, PartialEq, Eq, Deserialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(untagged)]
pub enum UtcHours {
Single(UtcWindow),
Multiple(Vec<UtcWindow>),
}
/// One off-peak window entry inside `windows`.
#[derive(Clone, Debug, PartialEq, Eq, Deserialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(deny_unknown_fields)]
pub struct OffPeakWindow {
pub hours_utc: UtcHours,
pub weekdays: Option<Vec<Weekday>>,
}
/// USD cost per web search query, keyed by search context size.
#[derive(Clone, Debug, Default, PartialEq, Deserialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(deny_unknown_fields)]
pub struct SearchContextCostPerQuery {
pub search_context_size_low: Option<UnitPrice>,
pub search_context_size_medium: Option<UnitPrice>,
pub search_context_size_high: Option<UnitPrice>,
}
impl UtcHours {
pub fn windows(&self) -> &[UtcWindow] {
match self {
Self::Single(window) => std::slice::from_ref(window),
Self::Multiple(windows) => windows,
}
}
}
pub(crate) struct Reader<'a> {
pub(crate) problems: &'a mut Vec<Error>,
}
impl Reader<'_> {
pub(crate) fn value<T: DeserializeOwned>(&mut self, path: String, value: &Value) -> Option<T> {
T::deserialize(value)
.map_err(|error| self.problems.push(Error::new(path, Repr::Invalid(error))))
.ok()
}
pub(crate) fn price(
&mut self,
key: String,
path: String,
flat: FlatKey,
value: &Value,
) -> Option<FlatPrice> {
let price = self.value(path.clone(), value)?;
Some(FlatPrice {
key,
path,
flat,
price,
})
}
fn object<'v>(&mut self, path: &str, value: &'v Value) -> Option<&'v Map<String, Value>> {
match value {
Value::Object(fields) => Some(fields),
other => self
.value::<Map<String, Value>>(path.to_owned(), other)
.and(None),
}
}
fn block_price(
&mut self,
name: &str,
key: String,
path: String,
value: &Value,
) -> Option<FlatPrice> {
let Some(flat) = FlatKey::parse(name) else {
self.problems.push(Error::new(path, Repr::UnknownBlockKey));
return None;
};
self.price(key, path, flat, value)
}
pub(crate) fn tier_rows(&mut self, value: &Value) -> Option<Vec<TierRow>> {
let path = pointer_for_key("tiered_pricing");
let rows: Vec<Value> = self.value(path.clone(), value)?;
Some(
rows.iter()
.enumerate()
.filter_map(|(index, row)| self.tier_row(path.clone(), index, row))
.collect(),
)
}
fn tier_row(&mut self, parent: String, index: usize, value: &Value) -> Option<TierRow> {
let path = format!("{parent}/{index}");
let display_path = format!("tiered_pricing[{index}]");
let fields = self.object(&path, value)?;
let mut row = TierRow {
path: path.clone(),
..TierRow::default()
};
for (name, value) in fields.iter().filter(|(_, value)| !value.is_null()) {
let key = format!("{display_path}.{name}");
let child_path = pointer_child(&path, name);
match name.as_str() {
"range" => row.range = self.bounds(child_path, value),
"max_results_range" => row.max_results_range = self.bounds(child_path, value),
_ => row
.prices
.extend(self.block_price(name, key, child_path, value)),
}
}
Some(row)
}
fn bounds(&mut self, path: String, value: &Value) -> Option<[u64; 2]> {
DeserializeAsWrap::<[u64; 2], [Count; 2]>::deserialize(value)
.map(DeserializeAsWrap::into_inner)
.map_err(|error| self.problems.push(Error::new(path, Repr::Invalid(error))))
.ok()
}
pub(crate) fn off_peak(&mut self, value: &Value) -> Option<OffPeakBlock> {
let path = pointer_for_key("off_peak_pricing");
let fields = self.object(&path, value)?;
let mut block = OffPeakBlock::default();
for (name, value) in fields.iter().filter(|(_, value)| !value.is_null()) {
let key = format!("off_peak_pricing.{name}");
let child_path = pointer_child(&path, name);
match name.as_str() {
"hours_utc" => block.hours_utc = self.value(child_path, value),
"windows" => block.windows = self.value(child_path, value),
"weekday_timezone" => block.weekday_timezone = self.value(child_path, value),
_ => block
.prices
.extend(self.block_price(name, key, child_path, value)),
}
}
Some(block)
}
}

193
litellm-rust/crates/model-catalog/src/wire/entry.rs Normal file
View file

@ -0,0 +1,193 @@
use std::collections::BTreeMap;
use litellm_cost::money::{Multiplier, UnitPrice};
use serde_json::{Map, Value};
use crate::error::{Error, Repr, pointer_for_key};
use crate::wire::attributes::Attributes;
use crate::wire::blocks::{FlatPrice, OffPeakBlock, Reader, SearchContextCostPerQuery, TierRow};
use crate::wire::flat_key::FlatKey;
pub type Fields = Map<String, Value>;
/// One catalog entry as written. Every non-null key lands in exactly one field, and a known key
/// with a bad value is kept out and listed in `problems`.
#[derive(Debug, Default)]
pub struct EntryWire {
pub attributes: Attributes,
pub prices: Vec<FlatPrice>,
pub capabilities: BTreeMap<String, bool>,
pub tier_rows: Option<Vec<TierRow>>,
pub off_peak: Option<OffPeakBlock>,
pub search_context: Option<SearchContextCostPerQuery>,
pub guardrail: Option<BTreeMap<String, UnitPrice>>,
pub regional_endpoint_uplift: Option<Multiplier>,
pub residency_uplift_eu: Option<Multiplier>,
pub residency_uplift_us: Option<Multiplier>,
pub unknown: Fields,
pub problems: Vec<Error>,
}
impl EntryWire {
pub fn read(fields: &Fields) -> Self {
let mut problems = Vec::new();
let mut wire = Self::default();
let mut reader = Reader {
problems: &mut problems,
};
for (key, value) in fields.iter().filter(|(_, value)| !value.is_null()) {
wire.read_key(&mut reader, key, value);
}
Self { problems, ..wire }
}
fn read_key(&mut self, reader: &mut Reader<'_>, key: &str, value: &Value) {
let path = || pointer_for_key(key);
match key {
"tiered_pricing" => self.tier_rows = reader.tier_rows(value),
"off_peak_pricing" => self.off_peak = reader.off_peak(value),
"search_context_cost_per_query" => self.search_context = reader.value(path(), value),
"guardrail_cost_per_unit" => self.guardrail = reader.value(path(), value),
"regional_endpoint_uplift_multiplier" => {
self.regional_endpoint_uplift = reader.value(path(), value)
}
"regional_processing_uplift_multiplier_eu" => {
self.residency_uplift_eu = reader.value(path(), value)
}
"regional_processing_uplift_multiplier_us" => {
self.residency_uplift_us = reader.value(path(), value)
}
_ => self.read_open_key(reader, key, value),
}
}
fn read_open_key(&mut self, reader: &mut Reader<'_>, key: &str, value: &Value) {
if let Some(read) = self.attributes.read(key, value) {
if let Err(error) = read {
reader
.problems
.push(Error::from_key(key, Repr::Invalid(error)));
}
return;
}
if let Some(flat) = FlatKey::parse(key) {
self.prices
.extend(reader.price(key.to_owned(), pointer_for_key(key), flat, value));
return;
}
if key.starts_with("supports_") {
if let Some(flag) = reader.value(pointer_for_key(key), value) {
self.capabilities.insert(key.to_owned(), flag);
}
return;
}
self.unknown.insert(key.to_owned(), value.clone());
}
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use serde_json::json;
use super::*;
fn read(entry: Value) -> EntryWire {
let Value::Object(fields) = entry else {
panic!("entry must be an object")
};
EntryWire::read(&fields)
}
fn problem_keys(wire: &EntryWire) -> Vec<String> {
wire.problems
.iter()
.map(|problem| problem.path().to_owned())
.collect()
}
#[rstest]
fn every_key_lands_in_exactly_one_place() {
let wire = read(json!({
"litellm_provider": "openai",
"max_tokens": "8192",
"input_cost_per_token": "0.000001",
"tiered_pricing": [{"range": [0, 10], "output_cost_per_token": 2e-6}],
"supports_vision": true,
"id": "deployment-1",
"mode": null
}));
assert_eq!(wire.attributes.litellm_provider.as_deref(), Some("openai"));
assert_eq!(wire.attributes.max_tokens, Some(8192));
assert_eq!(wire.attributes.mode, None);
assert_eq!(
wire.prices
.iter()
.map(|price| price.key.as_str())
.collect::<Vec<_>>(),
["input_cost_per_token"]
);
assert_eq!(wire.tier_rows.unwrap()[0].prices.len(), 1);
assert_eq!(
wire.capabilities,
BTreeMap::from([("supports_vision".to_owned(), true)])
);
assert_eq!(wire.unknown.keys().collect::<Vec<_>>(), ["id"]);
assert!(wire.problems.is_empty());
}
#[rstest]
fn every_bad_value_is_reported_by_its_key_at_once() {
let wire = read(json!({
"mode": "chatt",
"max_tokens": -1,
"input_cost_per_token": "x",
"output_cost_per_image_0.5K": "x",
"supports_vision": "yes",
"regional_endpoint_uplift_multiplier": 0.5,
"tiered_pricing": [
{"range": [0, 10], "input_cost_per_token": 1e-6},
{"range": [10.5, 20], "typo_cost": 1, "output_cost_per_token": -1}
],
"off_peak_pricing": {"hours_utc": "25:00-01:00", "input_cost_per_token": 1e-6}
}));
assert_eq!(
problem_keys(&wire)
.into_iter()
.collect::<std::collections::BTreeSet<_>>(),
[
"/input_cost_per_token",
"/max_tokens",
"/mode",
"/off_peak_pricing/hours_utc",
"/output_cost_per_image_0.5K",
"/regional_endpoint_uplift_multiplier",
"/supports_vision",
"/tiered_pricing/1/output_cost_per_token",
"/tiered_pricing/1/range",
"/tiered_pricing/1/typo_cost",
]
.map(str::to_owned)
.into_iter()
.collect()
);
}
#[rstest]
#[case::tiers_not_a_list(json!({"tiered_pricing": {"range": [0, 1]}}), "/tiered_pricing")]
#[case::row_not_an_object(json!({"tiered_pricing": [3]}), "/tiered_pricing/0")]
#[case::off_peak_not_an_object(json!({"off_peak_pricing": true}), "/off_peak_pricing")]
#[case::unknown_search_size(
json!({"search_context_cost_per_query": {"search_context_size_huge": 1}}),
"/search_context_cost_per_query"
)]
#[case::negative_guardrail(
json!({"guardrail_cost_per_unit": {"contentPolicyUnits": -1}}),
"/guardrail_cost_per_unit"
)]
fn a_bad_block_is_reported_at_the_block(#[case] entry: Value, #[case] expected: &str) {
assert_eq!(problem_keys(&read(entry)), [expected]);
}
}

359
litellm-rust/crates/model-catalog/src/wire/flat_key.rs Normal file
View file

@ -0,0 +1,359 @@
use std::num::NonZeroU64;
use litellm_cost::price::{
Band, CacheReadMode, CacheTtl, Channel, Feature, ImageSize, InputUnit, Meter, OutputUnit,
Resolution, ServiceTier, TokenKind, When,
};
/// A flat price key as the grammar reads it: `{meter}[_above_{N}k_tokens][_{tier}]`.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct FlatKey {
pub meter: Meter,
pub band: Band,
pub tier: Option<TierSuffix>,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum TierSuffix {
Service(ServiceTier),
Batches,
Decisions,
}
impl FlatKey {
pub fn parse(key: &str) -> Option<Self> {
let (rest, tier) = split_tier(key);
let (base, suffix) = split_above(rest)?;
let (meter, band) = match suffix {
Above::None => (meter(base)?, Band::Base),
Above::PromptTokens(tokens) => (meter(base)?, Band::over_prompt_tokens(tokens)),
Above::Seconds(seconds) => match meter(base)? {
Meter::Input(InputUnit::VideoSecond { over_seconds: None }) => (
Meter::Input(InputUnit::VideoSecond {
over_seconds: NonZeroU64::new(seconds),
}),
Band::Base,
),
_ => return None,
},
};
Some(Self { meter, band, tier })
}
pub fn has_suffix(&self) -> bool {
self.band != Band::Base || self.tier.is_some()
}
pub fn channel_and_when(&self) -> (Channel, When) {
let (channel, tier) = match self.tier {
None => (Channel::Online, ServiceTier::Standard),
Some(TierSuffix::Service(tier)) => (Channel::Online, tier),
Some(TierSuffix::Batches) => (Channel::Batch, ServiceTier::Standard),
Some(TierSuffix::Decisions) => (Channel::Decisions, ServiceTier::Standard),
};
(
channel,
When {
tier,
band: self.band,
},
)
}
}
enum Above {
None,
PromptTokens(u64),
Seconds(u64),
}
fn split_tier(key: &str) -> (&str, Option<TierSuffix>) {
let Some((head, tail)) = key.rsplit_once('_') else {
return (key, None);
};
let tier = match tail {
"priority" => TierSuffix::Service(ServiceTier::Priority),
"flex" => TierSuffix::Service(ServiceTier::Flex),
"ultrafast" => TierSuffix::Service(ServiceTier::Ultrafast),
"balanced" => TierSuffix::Service(ServiceTier::Balanced),
"batches" => TierSuffix::Batches,
"decisions" => TierSuffix::Decisions,
_ => return (key, None),
};
(head, Some(tier))
}
fn split_above(key: &str) -> Option<(&str, Above)> {
let Some((head, tail)) = key.rsplit_once("_above_") else {
return Some((key, Above::None));
};
if tail == "1hr" || tail == "12hr" {
return Some((key, Above::None));
}
if let Some(thousands) = tail.strip_suffix("k_tokens") {
return Some((
head,
Above::PromptTokens(number(thousands)?.checked_mul(1000)?),
));
}
if let Some(tokens) = tail.strip_suffix("_tokens") {
return Some((head, Above::PromptTokens(number(tokens)?)));
}
let seconds = tail.strip_suffix("s_interval")?;
Some((head, Above::Seconds(number(seconds)?)))
}
fn number(digits: &str) -> Option<u64> {
(!digits.is_empty() && digits.bytes().all(|byte| byte.is_ascii_digit()))
.then(|| digits.parse().ok())
.flatten()
}
fn meter(base: &str) -> Option<Meter> {
let fixed = match base {
"cost_per_second" => Some(Meter::WallClockSecond),
"explicit_cache_read_input_token_cost" => Some(Meter::CacheRead {
kind: TokenKind::Text,
mode: CacheReadMode::Explicit,
}),
"annotation_cost_per_page" => Some(Meter::Feature(Feature::AnnotationPage)),
"citation_cost_per_token" => Some(Meter::Feature(Feature::CitationToken)),
"code_interpreter_cost_per_session" => {
Some(Meter::Feature(Feature::CodeInterpreterSession))
}
"google_maps_grounding_cost_per_query" => {
Some(Meter::Feature(Feature::GoogleMapsGroundingQuery))
}
"ocr_cost_per_credit" => Some(Meter::Feature(Feature::OcrCredit)),
"ocr_cost_per_page" => Some(Meter::Feature(Feature::OcrPage)),
"browser_open_cost_per_request" => Some(Meter::Feature(Feature::BrowserOpen)),
"cost_per_additional_result" => Some(Meter::Feature(Feature::AdditionalResult)),
_ => None,
};
if fixed.is_some() {
return fixed;
}
if let Some(unit) = base.strip_prefix("input_cost_per_") {
return input_unit(unit).map(Meter::Input);
}
if let Some(unit) = base.strip_prefix("output_cost_per_") {
return output_unit(unit).map(Meter::Output);
}
if let Some(kind) = base
.strip_prefix("cache_read_input_")
.and_then(|rest| rest.strip_suffix("_cost"))
{
return Some(Meter::CacheRead {
kind: token_kind(kind)?,
mode: CacheReadMode::Implicit,
});
}
let rest = base.strip_prefix("cache_creation_input_")?;
let (kind, ttl) = if let Some(kind) = rest.strip_suffix("_cost_above_1hr") {
(kind, CacheTtl::OneHour)
} else if let Some(kind) = rest.strip_suffix("_cost_above_12hr") {
(kind, CacheTtl::TwelveHours)
} else {
(rest.strip_suffix("_cost")?, CacheTtl::Default)
};
Some(Meter::CacheWrite {
kind: token_kind(kind)?,
ttl,
})
}
fn token_kind(unit: &str) -> Option<TokenKind> {
match unit {
"token" => Some(TokenKind::Text),
"audio_token" => Some(TokenKind::Audio),
"image_token" => Some(TokenKind::Image),
"video_token" => Some(TokenKind::Video),
_ => None,
}
}
fn input_unit(unit: &str) -> Option<InputUnit> {
if let Some(kind) = token_kind(unit) {
return Some(InputUnit::Token(kind));
}
match unit {
"character" => Some(InputUnit::Character),
"image" => Some(InputUnit::Image),
"pixel" => Some(InputUnit::Pixel),
"query" => Some(InputUnit::Query { over_results: None }),
"request" => Some(InputUnit::Request),
"second" => Some(InputUnit::Second),
"text_message" => Some(InputUnit::TextMessage),
"audio_per_second" => Some(InputUnit::AudioSecond),
"video_per_second" => Some(InputUnit::VideoSecond { over_seconds: None }),
_ => None,
}
}
fn output_unit(unit: &str) -> Option<OutputUnit> {
if let Some(kind) = token_kind(unit) {
return Some(OutputUnit::Token(kind));
}
match unit.split_once('_') {
Some(("image", label)) => {
return ImageSize::new(label).map(|size| OutputUnit::Image(Some(size)));
}
Some(("second", label)) => {
return Resolution::new(label).map(|resolution| OutputUnit::Second(Some(resolution)));
}
_ => {}
}
match unit {
"reasoning_token" => Some(OutputUnit::ReasoningToken),
"character" => Some(OutputUnit::Character),
"image" => Some(OutputUnit::Image(None)),
"pixel" => Some(OutputUnit::Pixel),
"second" => Some(OutputUnit::Second(None)),
"video_per_second" => Some(OutputUnit::VideoSecond),
_ => None,
}
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use super::*;
const TEXT_INPUT: Meter = Meter::Input(InputUnit::Token(TokenKind::Text));
const TEXT_OUTPUT: Meter = Meter::Output(OutputUnit::Token(TokenKind::Text));
fn key(meter: Meter, band: Band, tier: Option<TierSuffix>) -> FlatKey {
FlatKey { meter, band, tier }
}
#[rstest]
#[case::input_token("input_cost_per_token", key(TEXT_INPUT, Band::Base, None))]
#[case::band_and_tier(
"output_cost_per_token_above_272k_tokens_priority",
key(
TEXT_OUTPUT,
Band::over_prompt_tokens(272_000),
Some(TierSuffix::Service(ServiceTier::Priority))
)
)]
#[case::band_without_k(
"input_cost_per_token_above_128_tokens",
key(TEXT_INPUT, Band::over_prompt_tokens(128), None)
)]
#[case::zero_band_is_base(
"input_cost_per_token_above_0k_tokens",
key(TEXT_INPUT, Band::Base, None)
)]
#[case::batch(
"input_cost_per_token_batches",
key(TEXT_INPUT, Band::Base, Some(TierSuffix::Batches))
)]
#[case::reasoning(
"output_cost_per_reasoning_token",
key(Meter::Output(OutputUnit::ReasoningToken), Band::Base, None)
)]
#[case::cache_read_audio(
"cache_read_input_audio_token_cost",
key(Meter::CacheRead { kind: TokenKind::Audio, mode: CacheReadMode::Implicit }, Band::Base, None)
)]
#[case::explicit_cache_read(
"explicit_cache_read_input_token_cost",
key(Meter::CacheRead { kind: TokenKind::Text, mode: CacheReadMode::Explicit }, Band::Base, None)
)]
#[case::cache_write_one_hour_band(
"cache_creation_input_token_cost_above_1hr_above_200k_tokens",
key(
Meter::CacheWrite { kind: TokenKind::Text, ttl: CacheTtl::OneHour },
Band::over_prompt_tokens(200_000),
None
)
)]
#[case::cache_write_twelve_hours_priority(
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority",
key(
Meter::CacheWrite { kind: TokenKind::Text, ttl: CacheTtl::TwelveHours },
Band::over_prompt_tokens(272_000),
Some(TierSuffix::Service(ServiceTier::Priority))
)
)]
#[case::image_label(
"output_cost_per_image_0.5K",
key(
Meter::Output(OutputUnit::Image(ImageSize::new("0.5K"))),
Band::Base,
None
)
)]
#[case::image_token_is_a_token(
"output_cost_per_image_token",
key(Meter::Output(OutputUnit::Token(TokenKind::Image)), Band::Base, None)
)]
#[case::resolution(
"output_cost_per_second_720p",
key(
Meter::Output(OutputUnit::Second(Resolution::new("720p"))),
Band::Base,
None
)
)]
#[case::video_interval(
"input_cost_per_video_per_second_above_8s_interval",
key(Meter::Input(InputUnit::VideoSecond { over_seconds: NonZeroU64::new(8) }), Band::Base, None)
)]
#[case::feature_batch(
"ocr_cost_per_page_batches",
key(
Meter::Feature(Feature::OcrPage),
Band::Base,
Some(TierSuffix::Batches)
)
)]
#[case::wall_clock("cost_per_second", key(Meter::WallClockSecond, Band::Base, None))]
#[case::browser_open(
"browser_open_cost_per_request",
key(Meter::Feature(Feature::BrowserOpen), Band::Base, None)
)]
#[case::additional_result(
"cost_per_additional_result",
key(Meter::Feature(Feature::AdditionalResult), Band::Base, None)
)]
fn parses_every_key_shape(#[case] text: &str, #[case] expected: FlatKey) {
assert_eq!(FlatKey::parse(text), Some(expected));
}
#[rstest]
#[case::typo("input_cost_per_tokne")]
#[case::unused_meter("cost_per_token")]
#[case::other_currency("input_dbu_cost_per_token")]
#[case::cache_hit_alias("input_cost_per_token_cache_hit")]
#[case::non_numeric_band("input_cost_per_token_above_xk_tokens")]
#[case::interval_on_a_token_meter("input_cost_per_token_above_8s_interval")]
#[case::read_has_no_ttl("cache_read_input_token_cost_above_1hr")]
#[case::cache_unit_must_be_token("cache_read_input_character_cost")]
#[case::bad_image_size("output_cost_per_image_big")]
#[case::bad_resolution("output_cost_per_second_hd")]
#[case::unknown_feature("telepathy_cost_per_page")]
#[case::empty("")]
fn rejects_keys_outside_the_grammar(#[case] text: &str) {
assert_eq!(FlatKey::parse(text), None);
}
#[rstest]
#[case::plain("input_cost_per_token", (Channel::Online, When::default()))]
#[case::service_tier(
"input_cost_per_token_flex",
(Channel::Online, When { tier: ServiceTier::Flex, band: Band::Base })
)]
#[case::batch(
"input_cost_per_token_above_200k_tokens_batches",
(Channel::Batch, When { tier: ServiceTier::Standard, band: Band::over_prompt_tokens(200_000) })
)]
#[case::decisions("input_cost_per_token_decisions", (Channel::Decisions, When::default()))]
fn batch_and_decisions_pick_a_channel_not_a_tier(
#[case] text: &str,
#[case] expected: (Channel, When),
) {
assert_eq!(FlatKey::parse(text).unwrap().channel_and_when(), expected);
}
}

128
litellm-rust/crates/model-catalog/src/wire/lax.rs Normal file
View file

@ -0,0 +1,128 @@
use std::fmt;
use serde::Deserializer;
use serde::de::{self, Visitor};
use serde_with::DeserializeAs;
pub(crate) struct LaxI64;
pub(crate) struct Count;
impl<'de> DeserializeAs<'de, i64> for LaxI64 {
fn deserialize_as<D: Deserializer<'de>>(deserializer: D) -> Result<i64, D::Error> {
deserializer.deserialize_any(Self)
}
}
impl Visitor<'_> for LaxI64 {
type Value = i64;
fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("an integer in the i64 range")
}
fn visit_i64<E: de::Error>(self, value: i64) -> Result<i64, E> {
Ok(value)
}
fn visit_u64<E: de::Error>(self, value: u64) -> Result<i64, E> {
i64::try_from(value).map_err(E::custom)
}
fn visit_f64<E: de::Error>(self, value: f64) -> Result<i64, E> {
integral_float(value).ok_or_else(|| E::custom("expected an integer in the i64 range"))
}
fn visit_str<E: de::Error>(self, value: &str) -> Result<i64, E> {
integer_string(value.trim())
.ok_or_else(|| E::custom("expected an integer in the i64 range"))
}
fn visit_bool<E: de::Error>(self, value: bool) -> Result<i64, E> {
Ok(i64::from(value))
}
}
impl<'de> DeserializeAs<'de, u64> for Count {
fn deserialize_as<D: Deserializer<'de>>(deserializer: D) -> Result<u64, D::Error> {
let count = LaxI64::deserialize_as(deserializer)?;
u64::try_from(count).map_err(|_| {
de::Error::invalid_value(de::Unexpected::Signed(count), &"a count of zero or more")
})
}
}
fn integer_string(value: &str) -> Option<i64> {
let integer = match value.split_once('.') {
Some((integer, fraction)) => {
if fraction.is_empty() || !fraction.bytes().all(|byte| byte == b'0') {
return None;
}
integer
}
None => value,
};
if integer.starts_with('_') || integer.ends_with('_') || integer.contains("__") {
return None;
}
let digits = integer.strip_prefix(['+', '-']).unwrap_or(integer);
if digits.is_empty()
|| digits.starts_with('_')
|| !digits
.bytes()
.all(|byte| byte.is_ascii_digit() || byte == b'_')
{
return None;
}
integer.replace('_', "").parse().ok()
}
fn integral_float(value: f64) -> Option<i64> {
(value.is_finite()
&& value.fract() == 0.0
&& value >= i64::MIN as f64
&& value < -(i64::MIN as f64))
.then_some(value as i64)
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use serde_json::{Value, json};
use serde_with::de::DeserializeAsWrap;
use super::*;
fn count(value: Value) -> Option<u64> {
DeserializeAsWrap::<u64, Count>::deserialize(value)
.ok()
.map(DeserializeAsWrap::into_inner)
}
use serde::Deserialize;
#[rstest]
#[case::integer(json!(128000), Some(128_000))]
#[case::whole_float(json!(128000.0), Some(128_000))]
#[case::string(json!(" 8192 "), Some(8192))]
#[case::separated(json!("1_000"), Some(1000))]
#[case::trailing_zeros(json!("2.000"), Some(2))]
#[case::boolean(json!(true), Some(1))]
#[case::maximum(json!(i64::MAX.to_string()), Some(i64::MAX as u64))]
#[case::negative(json!(-5), None)]
#[case::fraction(json!(2.5), None)]
#[case::precise_fraction(json!("1.0000000000000001"), None)]
#[case::exponent(json!("1e3"), None)]
#[case::missing_fraction(json!("2."), None)]
#[case::repeated_separator(json!("2__0"), None)]
#[case::above_i64(json!(u64::MAX), None)]
#[case::float_above_i64(json!(9_223_372_036_854_775_808.0), None)]
#[case::word(json!("8k"), None)]
#[case::object(json!({}), None)]
fn counts_accept_whole_numbers_however_they_are_written(
#[case] written: Value,
#[case] expected: Option<u64>,
) {
assert_eq!(count(written), expected);
}
}

10
litellm-rust/crates/model-catalog/src/wire/mod.rs Normal file
View file

@ -0,0 +1,10 @@
mod attributes;
mod blocks;
mod entry;
mod flat_key;
mod lax;
pub use attributes::*;
pub use blocks::*;
pub use entry::*;
pub use flat_key::*;

632
litellm-rust/crates/model-catalog/tests/catalog.rs
View file

@ -1,303 +1,405 @@
use std::path::{Path, PathBuf};
use std::collections::BTreeSet;
use litellm_model_catalog::{AliasIssue, Catalog, Error, IntegrityLimits, Mode, Provenance};
use litellm_model_catalog::{
BUNDLED_REGISTRY, Catalog, Error, ErrorKind, IntegrityLimits, Mode, Rejects, Written,
};
use rstest::{fixture, rstest};
use serde_json::json;
use serde_json::{Map, Value, json};
const ALPHA_FIXTURE: &[u8] = br#"{
"sample_spec":{"explanation":"example"},
"sample_spec":{"explanation":"example","input_cost_per_token":"not checked"},
"fallback_generalizations":{"rules":[{"name":"family","pattern":"^new-","model_info":{"mode":"chat"}}]},
"Alpha":{"litellm_provider":"test","aliases":["short"],"price":0,"enabled":false,
"optional":null,"unknown":{"nested":[1,{"x":true}]}}
}"#;
#[fixture]
fn repo_root() -> PathBuf {
Path::new(env!("CARGO_MANIFEST_DIR")).join("../../..")
}
#[fixture]
fn current_catalog(repo_root: PathBuf) -> Catalog {
let body = std::fs::read(repo_root.join("model_prices_and_context_window.json")).unwrap();
Catalog::parse(&body, Provenance::default()).unwrap()
}
#[fixture]
fn backup_catalog(repo_root: PathBuf) -> Catalog {
let body = std::fs::read(repo_root.join("litellm/model_prices_and_context_window_backup.json"))
.unwrap();
Catalog::parse(&body, Provenance::default()).unwrap()
}
#[fixture]
fn fixture_catalog() -> Catalog {
Catalog::parse(
ALPHA_FIXTURE,
Provenance {
source: Some("fixture".into()),
revision: Some("rev".into()),
etag: None,
},
)
.unwrap()
Catalog::parse(ALPHA_FIXTURE).unwrap().value
}
fn loaded(document: Value) -> litellm_model_catalog::Loaded<Catalog> {
Catalog::parse(document.to_string().as_bytes()).unwrap()
}
fn catalog(document: Value) -> Catalog {
loaded(document).value
}
fn written(catalog: &Catalog) -> String {
let document: Map<String, Value> = catalog
.written()
.map(|(key, value)| {
let value = match value {
Written::Entry(fields) => serde_json::to_value(fields).unwrap(),
Written::Section(value) => value.clone(),
};
(key.to_owned(), value)
})
.collect();
serde_json::to_string(&document).unwrap()
}
fn report(error: &Error) -> String {
litellm_error::Report::new(error).to_string()
}
fn error_text(document: Value) -> Vec<String> {
loaded(document).errors.iter().map(report).collect()
}
#[rstest]
#[case::decisions("decisions", Mode::Decisions)]
#[case::legacy_evaluation("evaluation", Mode::Evaluation)]
fn parses_decision_modes(#[case] mode: &str, #[case] expected: Mode) {
let source = json!({"Test":{"litellm_provider":"test","mode":mode}}).to_string();
let catalog = Catalog::parse(source.as_bytes(), Provenance::default()).unwrap();
let catalog = catalog(json!({"Test": {"litellm_provider": "test", "mode": mode}}));
assert_eq!(
catalog.lookup("Test").unwrap().entry.info().mode,
catalog
.lookup("Test")
.unwrap()
.entry
.model()
.unwrap()
.attributes
.mode,
Some(expected)
);
assert_eq!(serde_json::to_value(expected).unwrap(), json!(mode));
}
#[rstest]
#[ignore]
fn preserves_fields_and_metadata(fixture_catalog: Catalog) {
let catalog = fixture_catalog;
let entry = catalog.lookup("SHORT").unwrap();
assert_eq!(entry.canonical_key, "Alpha");
assert_eq!(entry.matched_key, "short");
assert_eq!(entry.entry.field("price"), Some(&json!(0)));
assert_eq!(entry.entry.field("enabled"), Some(&json!(false)));
assert_eq!(entry.entry.field("optional"), Some(&json!(null)));
assert_eq!(entry.entry.field("missing"), None);
fn unknown_keys_are_kept_raw_and_listed_as_unrecognized(fixture_catalog: Catalog) {
let entry = fixture_catalog.lookup("Alpha").unwrap().entry;
assert_eq!(
entry.entry.field("unknown"),
entry.field("unknown"),
Some(&json!({"nested":[1,{"x":true}]}))
);
assert_eq!(entry.entry.field("aliases"), None);
assert_eq!(entry.entry.info().litellm_provider.as_deref(), Some("test"));
assert_eq!(
catalog.sample_spec(),
Some(&json!({"explanation":"example"}))
);
assert_eq!(catalog.fallback_rules().unwrap().len(), 1);
assert_eq!(catalog.provenance().revision.as_deref(), Some("rev"));
assert_eq!(catalog.model_count(), 1);
}
#[rstest]
#[ignore]
fn snapshot_does_not_borrow_source() {
let mut source = ALPHA_FIXTURE.to_vec();
let catalog = Catalog::parse(&source, Provenance::default()).unwrap();
source.fill(b' ');
let entry = catalog.lookup("short").unwrap();
assert_eq!(entry.canonical_key, "Alpha");
assert_eq!(entry.entry.field("price"), Some(&json!(0)));
}
#[rstest]
#[case("Shared", "First")]
#[case("Second", "Second")]
#[case("shared", "Second")]
#[case("FIRST", "First")]
#[case("sHaReD", "Second")]
#[ignore]
fn alias_collisions_and_case_fallback_follow_entry_order(
#[case] lookup: &str,
#[case] expected: &str,
) {
let catalog = Catalog::parse(
br#"{
"First":{"aliases":["Shared","Second","first"],"value":1},
"Second":{"aliases":["Shared","sHaReD"],"value":2},
"SHARED":{"value":3}
}"#,
Provenance::default(),
)
.unwrap();
assert_eq!(catalog.lookup(lookup).unwrap().canonical_key, expected);
assert_eq!(catalog.alias_count(), 3);
assert!(
catalog
.alias_issues()
.contains(&AliasIssue::CanonicalCollision {
model: "First".into(),
alias: "Second".into(),
})
);
assert!(
catalog
.alias_issues()
.contains(&AliasIssue::AliasCollision {
model: "Second".into(),
alias: "Shared".into(),
})
fixture_catalog.unrecognized().collect::<BTreeSet<_>>(),
BTreeSet::from([
("Alpha", "enabled"),
("Alpha", "price"),
("Alpha", "unknown")
])
);
}
#[test]
#[ignore]
fn json_entry_order_controls_alias_ownership_and_case_fallback() {
let forward = Catalog::parse(
br#"{
"Alpha":{"aliases":["shared"]},
"Beta":{"aliases":["shared"]},
"Foo":{},
"fOO":{}
}"#,
Provenance::default(),
)
.unwrap();
let reversed = Catalog::parse(
br#"{
"fOO":{},
"Foo":{},
"Beta":{"aliases":["shared"]},
"Alpha":{"aliases":["shared"]}
}"#,
Provenance::default(),
)
.unwrap();
assert_eq!(forward.lookup("shared").unwrap().canonical_key, "Alpha");
assert_eq!(reversed.lookup("shared").unwrap().canonical_key, "Beta");
assert_eq!(forward.lookup("foo").unwrap().canonical_key, "fOO");
assert_eq!(reversed.lookup("foo").unwrap().canonical_key, "Foo");
}
#[derive(Debug)]
enum ValidationOutcome {
Ok,
Shrunk,
BelowMinimum,
InvalidRatio,
}
#[rstest]
#[case(
IntegrityLimits {
reference_model_count: 2,
min_model_count: 1,
min_reference_ratio: 0.5,
},
ValidationOutcome::Ok
)]
#[case(
IntegrityLimits {
reference_model_count: 3,
min_model_count: 1,
min_reference_ratio: 0.5,
},
ValidationOutcome::Shrunk
)]
#[case(
IntegrityLimits {
reference_model_count: 0,
min_model_count: 2,
min_reference_ratio: 0.5,
},
ValidationOutcome::BelowMinimum
)]
#[case(
IntegrityLimits {
reference_model_count: 0,
min_model_count: 0,
min_reference_ratio: f64::NAN,
},
ValidationOutcome::InvalidRatio
)]
#[ignore]
fn integrity_uses_canonical_count_and_strict_shrink_boundary(
#[case] limits: IntegrityLimits,
#[case] expected: ValidationOutcome,
) {
let catalog = Catalog::parse(
br#"{"sample_spec":{},"fallback_generalizations":{"rules":[]},"a":{"aliases":["b","c"]}}"#,
Provenance::default(),
)
.unwrap();
let actual = catalog.validate(limits);
match expected {
ValidationOutcome::Ok => assert!(actual.is_ok()),
ValidationOutcome::Shrunk => {
assert!(matches!(actual, Err(Error::Shrunk { actual: 1, .. })))
}
ValidationOutcome::BelowMinimum => {
assert!(matches!(actual, Err(Error::BelowMinimum { actual: 1, .. })))
}
ValidationOutcome::InvalidRatio => assert!(matches!(actual, Err(Error::InvalidRatio))),
}
}
#[derive(Debug)]
enum MalformedOutcome {
Empty,
Json,
EntryNotObject,
}
#[rstest]
#[case::empty(b"{}", MalformedOutcome::Empty)]
#[case::invalid_json(b"{", MalformedOutcome::Json)]
#[case::entry_not_object(br#"{"a":1}"#, MalformedOutcome::EntryNotObject)]
#[case::fallback_rules_missing(
br#"{"fallback_generalizations":{},"a":{}}"#,
MalformedOutcome::Json
)]
#[ignore]
fn malformed_input_and_aliases_have_typed_outcomes(
#[case] body: &[u8],
#[case] expected: MalformedOutcome,
) {
let actual = Catalog::parse(body, Provenance::default());
match expected {
MalformedOutcome::Empty => assert!(matches!(actual, Err(Error::Empty))),
MalformedOutcome::Json => assert!(matches!(actual, Err(Error::Json(_)))),
MalformedOutcome::EntryNotObject => {
assert!(matches!(actual, Err(Error::EntryNotObject { .. })))
}
}
}
#[rstest]
#[ignore]
fn invalid_aliases_are_reported_not_fatal() {
let catalog = Catalog::parse(
br#"{"a":{"aliases":"bad"},"b":{"aliases":[9,"ok"]}}"#,
Provenance::default(),
)
.unwrap();
fn sections_aliases_and_raw_fields_are_read_in_their_document_views(fixture_catalog: Catalog) {
let found = fixture_catalog.lookup("SHORT").unwrap();
assert_eq!(found.canonical_key, "Alpha");
assert_eq!(found.matched_key, "short");
assert_eq!(found.entry.field("optional"), Some(&json!(null)));
assert_eq!(found.entry.field("aliases"), Some(&json!(["short"])));
assert_eq!(
catalog.alias_issues(),
&[
AliasIssue::InvalidList { model: "a".into() },
AliasIssue::InvalidName { model: "b".into() },
]
fixture_catalog.sample_spec(),
Some(&json!({"explanation":"example","input_cost_per_token":"not checked"}))
);
assert_eq!(fixture_catalog.fallback_rules().unwrap()[0].name, "family");
assert_eq!(fixture_catalog.model_count(), 1);
assert_eq!(
fixture_catalog
.served()
.map(|(key, _)| key)
.collect::<Vec<_>>(),
["Alpha", "short"]
);
assert_eq!(catalog.lookup("ok").unwrap().canonical_key, "b");
assert!(catalog.lookup("missing").is_none());
}
#[rstest]
#[ignore]
fn parses_current_and_packaged_catalogs_against_independent_baseline(
current_catalog: Catalog,
backup_catalog: Catalog,
) {
assert!(current_catalog.model_count() > 0);
assert!(backup_catalog.model_count() > 0);
assert!(current_catalog.sample_spec().is_some());
assert!(backup_catalog.sample_spec().is_some());
// Snapshot from 2026-09-23; the backup file mirrors the current file and cannot detect shrinkage.
const REFERENCE_MODEL_COUNT: usize = 4303;
current_catalog
.validate(IntegrityLimits {
reference_model_count: REFERENCE_MODEL_COUNT,
min_model_count: 50,
min_reference_ratio: 0.9,
})
.unwrap();
assert!(current_catalog.model_names().all(|name| {
let entry = current_catalog.lookup(name).unwrap().entry;
entry.info().litellm_provider.is_some() == entry.field("litellm_provider").is_some()
fn a_bad_entry_is_reported_by_model_and_key_and_kept_raw() {
let parsed = loaded(json!({
"Good": {"litellm_provider": "test"},
"Bad": {"litellm_provider": "test", "mode": "chatt", "input_cost_per_token": "free"},
"Worse": {"mode": "chat"}
}));
let catalog = parsed.value;
let problems: Vec<String> = parsed.errors.iter().map(report).collect();
assert_eq!(problems.len(), 3, "{problems:?}");
assert!(problems[0].starts_with("Bad.mode: invalid value: unknown variant `chatt`"));
assert!(problems[1].starts_with("Bad.input_cost_per_token: invalid value"));
assert_eq!(problems[2], "Worse.litellm_provider: required");
assert!(catalog.lookup("Good").unwrap().entry.model().is_some());
assert!(catalog.lookup("Bad").unwrap().entry.model().is_none());
assert_eq!(catalog.model_count(), 3);
assert!(
parsed
.errors
.iter()
.find(|error| error.rejects() == Rejects::Document)
.is_none()
);
}
#[rstest]
#[case::alias_names_a_model(
json!({"First": {"litellm_provider": "t", "aliases": ["Second"]}, "Second": {"litellm_provider": "t"}}),
r#"First: alias "Second" is already a model name"#
)]
#[case::alias_claimed_twice(
json!({"First": {"litellm_provider": "t", "aliases": ["shared"]}, "Second": {"litellm_provider": "t", "aliases": ["shared"]}}),
r#"Second: alias "shared" already belongs to First"#
)]
#[case::aliases_not_a_list(
json!({"a": {"litellm_provider": "t", "aliases": "bad"}}),
"a.aliases: invalid value: invalid type: string \"bad\", expected a sequence"
)]
#[case::entry_not_object(
json!({"a": 1, "valid": {"litellm_provider": "test"}}),
"a: must be a JSON object"
)]
#[case::fallback_rules_missing(
json!({"fallback_generalizations": {}, "a": {"litellm_provider": "t"}}),
"fallback_generalizations: invalid value: missing field `rules`"
)]
#[case::unknown_timezone(
json!({"a": {"litellm_provider": "t", "off_peak_pricing": {
"windows": [{"hours_utc": "00:00-01:00"}],
"weekday_timezone": "Not/AZone",
"input_cost_per_token": 1e-6
}}}),
r#"a.off_peak_pricing.weekday_timezone: "Not/AZone" is not an IANA time zone"#
)]
fn document_level_problems_are_reported(#[case] document: Value, #[case] expected: &str) {
assert_eq!(error_text(document), [expected]);
}
#[rstest]
fn escaped_row_paths_and_row_validation_share_the_parse_rules() {
let document = json!({
"open/router~v1": {"litellm_provider": "test", "input_cost_per_token": -1}
});
let parsed = loaded(document);
let fields = parsed
.value
.lookup("open/router~v1")
.unwrap()
.entry
.fields();
let row_errors = Catalog::row_errors("open/router~v1", fields);
assert_eq!(
row_errors
.iter()
.map(|error| (error.path(), report(error)))
.collect::<Vec<_>>(),
parsed
.errors
.iter()
.map(|error| (error.path(), report(error)))
.collect::<Vec<_>>()
);
assert_eq!(
row_errors[0].path(),
"/open~1router~0v1/input_cost_per_token"
);
assert!(row_errors[0].kind() == ErrorKind::Invalid);
}
#[rstest]
fn malformed_fallback_is_collected_while_models_are_read() {
let parsed = loaded(json!({
"fallback_generalizations": {"rules": "bad"},
"a": {"litellm_provider": "test"}
}));
assert_eq!(parsed.value.model_count(), 1);
assert_eq!(parsed.errors.len(), 1);
assert_eq!(parsed.errors[0].path(), "/fallback_generalizations");
}
#[rstest]
fn rejected_rows_and_their_aliases_are_removed_without_disturbing_written_order() {
let body = r#"{"before":{"litellm_provider":"test","aliases":["before-alias"]},"bad":{"litellm_provider":"test","aliases":["bad-alias"],"mode":"chatt"},"not-object":4,"sample_spec":{"keep":true},"after":{"litellm_provider":"test"}}"#;
let parsed = Catalog::parse(body.as_bytes()).unwrap();
let (catalog, removed) = parsed.value.without_rejected(&parsed.errors);
assert_eq!(removed.len(), 2);
assert_eq!(
catalog.lookup("before-alias").unwrap().canonical_key,
"before"
);
assert_eq!(
catalog.lookup("BEFORE-ALIAS").unwrap().matched_key,
"before-alias"
);
assert!(catalog.lookup("bad").is_none());
assert!(catalog.lookup("BAD").is_none());
assert!(catalog.lookup("Bad-Alias").is_none());
assert_eq!(catalog.model_count(), 2);
assert_eq!(
written(&catalog),
r#"{"before":{"litellm_provider":"test","aliases":["before-alias"]},"sample_spec":{"keep":true},"after":{"litellm_provider":"test"}}"#
);
}
#[rstest]
fn invalid_empty_key_rows_are_removed() {
let parsed = loaded(json!({
"": {"litellm_provider": "test", "mode": "chatt"},
"good": {"litellm_provider": "test"}
}));
assert_eq!(parsed.errors[0].path(), "//mode");
let (catalog, removed) = parsed.value.without_rejected(&parsed.errors);
assert_eq!(catalog.model_count(), 1);
assert!(catalog.lookup("").is_none());
assert!(catalog.lookup("good").is_some());
assert_eq!(removed.len(), 1);
}
#[rstest]
fn timezone_errors_reject_providerless_rows_without_rejecting_other_providerless_rows() {
let parsed = loaded(json!({
"good": {"mode": "chat"},
"bad": {"off_peak_pricing": {
"windows": [{"hours_utc": "00:00-01:00"}],
"weekday_timezone": "Not/AZone",
"input_cost_per_token": 1e-6
}}
}));
assert_eq!(parsed.errors.len(), 3);
assert!(
parsed
.errors
.iter()
.any(|error| error.kind() == ErrorKind::UnknownTimezone)
);
let (catalog, removed) = parsed.value.without_rejected(&parsed.errors);
assert_eq!(catalog.model_count(), 1);
assert!(catalog.lookup("good").is_some());
assert!(catalog.lookup("bad").is_none());
assert_eq!(removed.len(), 2);
}
#[rstest]
fn bundled_per_row_errors_match_document_errors() {
let parsed = Catalog::parse(BUNDLED_REGISTRY).unwrap();
let row_errors: Vec<_> = parsed
.value
.written()
.flat_map(|(key, written)| match written {
Written::Entry(fields) => Catalog::row_errors(key, fields),
Written::Section(_) => Vec::new(),
})
.map(|error| (error.path().to_owned(), report(&error)))
.collect();
let document_errors: Vec<_> = parsed
.errors
.iter()
.map(|error| (error.path().to_owned(), report(error)))
.collect();
assert_eq!(row_errors, document_errors);
}
#[rstest]
fn three_hundred_rows_filter_one_row_with_two_invalid_prices() {
let valid_rows: Map<String, Value> = (0..299)
.map(|index| {
(
format!("valid-{index:03}"),
json!({"litellm_provider": "test"}),
)
})
.collect();
let mut document = valid_rows;
document.insert(
"bad".to_owned(),
json!({
"litellm_provider": "test",
"input_cost_per_token": -1,
"output_cost_per_token": -1
}),
);
let parsed = Catalog::parse(serde_json::to_vec(&document).unwrap().as_slice()).unwrap();
assert_eq!(parsed.value.model_count(), 300);
assert_eq!(parsed.errors.len(), 2);
let (catalog, removed) = parsed.value.without_rejected(&parsed.errors);
assert_eq!(catalog.model_count(), 299);
assert_eq!(removed.len(), 2);
}
#[rstest]
fn integrity_limits_count_canonical_models_not_aliases() {
let fetched = catalog(json!({"a": {"litellm_provider": "t", "aliases": ["b", "c"]}}));
assert!(
fetched
.check_size(2, IntegrityLimits::new(1, 0.5).unwrap())
.is_ok()
);
assert_eq!(
fetched
.check_size(3, IntegrityLimits::new(1, 0.5).unwrap())
.unwrap_err()
.kind(),
ErrorKind::Shrunk
);
assert_eq!(
fetched
.check_size(2, IntegrityLimits::new(2, 0.5).unwrap())
.unwrap_err()
.kind(),
ErrorKind::BelowMinimum
);
}
#[rstest]
#[case::empty(b"{}", "empty")]
#[case::only_sections(br#"{"sample_spec":{}}"#, "empty")]
#[case::invalid_json(b"{", "json")]
#[case::array_root_is_json_error(b"[]", "json")]
fn malformed_documents_are_rejected_whole(#[case] body: &[u8], #[case] kind: &str) {
let error = Catalog::parse(body).unwrap_err();
assert_eq!(error.rejects(), Rejects::Document);
assert_eq!(<&str>::from(error.kind()), kind);
}
#[rstest]
fn written_is_the_document_in_order_including_sections_and_bad_entries() {
let body = concat!(
r#"{"b":{"litellm_provider":"t","z":1,"a":null},"sample_spec":{"x":"y"},"n":4,"#,
r#""fallback_generalizations":{"rules":"bad"},"a":{"mode":"chatt","price":1.0}}"#
);
let catalog = Catalog::parse(body.as_bytes()).unwrap().value;
assert_eq!(written(&catalog), body);
assert_eq!(catalog.model_count(), 2);
}
#[rstest]
fn the_bundled_registry_parses_with_no_problems_and_every_key_understood() {
let loaded = Catalog::parse(BUNDLED_REGISTRY).unwrap();
let catalog = loaded.value;
assert!(loaded.errors.is_empty(), "{:#?}", loaded.errors);
let unrecognized: Vec<_> = catalog.unrecognized().collect();
assert!(unrecognized.is_empty(), "{unrecognized:#?}");
assert!(
catalog
.fallback_rules()
.is_some_and(|rules| !rules.is_empty())
);
const REFERENCE_MODEL_COUNT: usize = 4303;
assert!(catalog.model_count() as f64 >= REFERENCE_MODEL_COUNT as f64 * 0.9);
}
#[rstest]
fn the_bundled_registry_is_written_back_as_it_was_read() {
let document: Value = serde_json::from_slice(BUNDLED_REGISTRY).unwrap();
assert_eq!(
written(&Catalog::parse(BUNDLED_REGISTRY).unwrap().value),
serde_json::to_string(&document).unwrap()
);
}
#[rstest]
fn a_bad_value_keeps_its_json_error_as_the_source_and_names_its_key_and_field() {
let parsed =
loaded(json!({"open/router": {"litellm_provider": "t", "input_cost_per_token": "free"}}));
let error = &parsed.errors[0];
let cause = litellm_error::find::<serde_json::Error>(error)
.expect("the JSON error is in the chain")
.to_string();
assert_eq!(report(error).matches(&cause).count(), 1);
assert!(!error.to_string().contains(&cause));
assert_eq!(error.key().as_deref(), Some("open/router"));
assert_eq!(error.field().as_deref(), Some("input_cost_per_token"));
}

91
litellm-rust/crates/model-catalog/tests/generate_price_lookup_reference.py Normal file
View file

@ -0,0 +1,91 @@
import itertools
import json
import re
import subprocess
from pathlib import Path
from typing import Final
from litellm.litellm_core_utils.llm_cost_calc.utils import _get_token_base_cost
from litellm.types.utils import Usage
REPO_ROOT: Final = Path(__file__).resolve().parents[4]
TOKEN_RATE_KEY: Final = re.compile(
r"^(input_cost_per_token|output_cost_per_token|cache_read_input_token_cost"
r"|cache_creation_input_token_cost(_above_1hr)?)"
r"(_above_(\d+)(k?)_tokens)?(_(priority|flex|ultrafast|balanced|batches|decisions))?$"
)
SERVICE_TIERS: Final = ("priority", "flex", "ultrafast", "balanced")
LARGEST_PROMPT: Final = 10_000_000
INCLUSIVE_PROVIDERS: Final = frozenset({"xai"})
def token_rates(entry: dict[str, object]) -> dict[str, object]:
return {
key: value
for key, value in entry.items()
if value is not None and (key == "tiered_pricing" or TOKEN_RATE_KEY.match(key))
}
def band_floors(info: dict[str, object]) -> frozenset[int]:
matches: Final = (TOKEN_RATE_KEY.match(key) for key in info)
flat: Final = frozenset(
int(match.group(4)) * (1000 if match.group(5) else 1)
for match in matches
if match is not None and match.group(4) is not None
)
rows: Final = info.get("tiered_pricing")
tiers: Final = (
frozenset(int(row["range"][0]) for row in rows if isinstance(row, dict) and "range" in row)
if isinstance(rows, list)
else frozenset()
)
return flat | tiers
def tiers_in(info: dict[str, object]) -> tuple[str | None, ...]:
return (None, *(tier for tier in SERVICE_TIERS if any(key.endswith(f"_{tier}") for key in info)))
def prompts(info: dict[str, object]) -> tuple[int, ...]:
floors: Final = band_floors(info)
return tuple(sorted(({1, LARGEST_PROMPT} | floors | {floor + 1 for floor in floors}) - {0}))
def check(info: dict[str, object], tier: str | None, prompt: int, inclusive: bool) -> list[object]:
usage: Final = Usage(prompt_tokens=prompt, completion_tokens=1, total_tokens=prompt + 1)
rates: Final = _get_token_base_cost(info, usage, service_tier=tier, threshold_is_inclusive=inclusive)
one_hour: Final = rates.cache_creation_rates_by_ttl.get("1hr", rates.cache_creation_rate)
return [tier, prompt, [rates.input_rate, rates.output_rate, rates.cache_creation_rate, one_hour, rates.cache_read_rate]]
def shape(info: dict[str, object], inclusive: bool) -> str:
rows: Final = info.get("tiered_pricing")
row_keys: Final = [sorted(row) for row in rows if isinstance(row, dict)] if isinstance(rows, list) else []
return json.dumps([sorted(info), row_keys, inclusive])
def checks(info: dict[str, object], inclusive: bool) -> list[list[object]]:
return [check(info, tier, prompt, inclusive) for tier, prompt in itertools.product(tiers_in(info), prompts(info))]
def main() -> None:
registry: Final = json.loads((REPO_ROOT / "model_prices_and_context_window.json").read_text())
entries: Final = tuple(
(token_rates(entry), entry.get("litellm_provider") in INCLUSIVE_PROVIDERS)
for name, entry in registry.items()
if name not in ("sample_spec", "fallback_generalizations") and isinstance(entry, dict)
)
by_shape: Final = {shape(info, inclusive): (info, inclusive) for info, inclusive in reversed(entries) if info}
models: Final = tuple(
json.dumps({"info": info, "inclusive": inclusive, "checks": checks(info, inclusive)})
for info, inclusive in by_shape.values()
)
revision: Final = subprocess.check_output(("git", "rev-parse", "HEAD"), text=True, cwd=REPO_ROOT).strip()
Path(__file__).with_name("price_lookup_reference.json").write_text(
f'{{"python_reference_commit": "{revision}", "models": [\n' + ",\n".join(models) + "\n]}\n"
)
if __name__ == "__main__":
main()

89
litellm-rust/crates/model-catalog/tests/price_lookup.rs Normal file
View file

@ -0,0 +1,89 @@
use litellm_cost::money::UnitPrice;
use litellm_cost::price::{
CacheReadMode, CacheTtl, InputUnit, Meter, OutputUnit, Prices, ServiceTier, TokenKind,
};
use litellm_model_catalog::wire::{EntryWire, Fields};
use rstest::rstest;
use serde::Deserialize;
#[derive(Deserialize)]
struct Reference {
models: Vec<ReferenceModel>,
}
#[derive(Deserialize)]
struct ReferenceModel {
info: Fields,
inclusive: bool,
checks: Vec<(Option<String>, u64, [f64; 5])>,
}
const METERS: [Meter; 5] = [
Meter::Input(InputUnit::Token(TokenKind::Text)),
Meter::Output(OutputUnit::Token(TokenKind::Text)),
Meter::CacheWrite {
kind: TokenKind::Text,
ttl: CacheTtl::Default,
},
Meter::CacheWrite {
kind: TokenKind::Text,
ttl: CacheTtl::OneHour,
},
Meter::CacheRead {
kind: TokenKind::Text,
mode: CacheReadMode::Implicit,
},
];
fn service_tier(name: Option<&str>) -> ServiceTier {
match name {
None => ServiceTier::Standard,
Some("priority") => ServiceTier::Priority,
Some("flex") => ServiceTier::Flex,
Some("ultrafast") => ServiceTier::Ultrafast,
Some("balanced") => ServiceTier::Balanced,
Some(other) => panic!("the reference uses an unknown service tier {other}"),
}
}
fn mismatch(
prices: &Prices,
(tier_name, prompt, python): &(Option<String>, u64, [f64; 5]),
) -> Option<String> {
let tier = service_tier(tier_name.as_deref());
let band = prices.band(tier, *prompt);
let rust = METERS
.each_ref()
.map(|meter| prices.rate(meter, tier, band).map_or(0.0, UnitPrice::get));
(rust != *python)
.then(|| format!("tier={tier_name:?} prompt={prompt}: rust {rust:?}, python {python:?}"))
}
#[rstest]
fn band_and_rate_match_python_token_base_cost() {
let reference: Reference =
serde_json::from_str(include_str!("price_lookup_reference.json")).unwrap();
let mismatches: Vec<String> = reference
.models
.iter()
.flat_map(|model| {
let provider = if model.inclusive { "xai" } else { "openai" };
let fields: Fields = model
.info
.clone()
.into_iter()
.chain([("litellm_provider".to_owned(), provider.into())])
.collect();
let prices = Prices::try_from(&EntryWire::read(&fields)).unwrap();
model
.checks
.iter()
.filter_map(move |check| mismatch(&prices, check))
.map(move |found| format!("{:?} {found}", model.info))
})
.collect();
assert!(reference.models.len() > 50);
assert!(mismatches.is_empty(), "{mismatches:#?}");
}

89
litellm-rust/crates/model-catalog/tests/price_lookup_reference.json Normal file
View file

@ -0,0 +1,89 @@
{"python_reference_commit": "c7c895db456eda349a7be9661b4b21f578f9b589", "models": [
{"info": {"cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-05, "cache_read_input_token_cost": 1e-06, "cache_read_input_token_cost_above_272k_tokens": 2e-06, "input_cost_per_token": 1e-05, "input_cost_per_token_above_272k_tokens": 2e-05, "output_cost_per_token": 5e-05, "output_cost_per_token_above_272k_tokens": 7.5e-05}, "inclusive": false, "checks": [[null, 1, [1e-05, 5e-05, 1.25e-05, 1.25e-05, 1e-06]], [null, 272000, [1e-05, 5e-05, 1.25e-05, 1.25e-05, 1e-06]], [null, 272001, [2e-05, 7.5e-05, 2.5e-05, 2.5e-05, 2e-06]], [null, 10000000, [2e-05, 7.5e-05, 2.5e-05, 2.5e-05, 2e-06]]]},
{"info": {"cache_read_input_token_cost": 1e-07, "input_cost_per_token": 1e-06, "output_cost_per_token": 3e-06}, "inclusive": false, "checks": [[null, 1, [1e-06, 3e-06, 1e-06, 1e-06, 1e-07]], [null, 10000000, [1e-06, 3e-06, 1e-06, 1e-06, 1e-07]]]},
{"info": {"input_cost_per_token": 1.25e-05, "output_cost_per_token": 1.25e-05}, "inclusive": false, "checks": [[null, 1, [1.25e-05, 1.25e-05, 1.25e-05, 1.25e-05, 1.25e-05]], [null, 10000000, [1.25e-05, 1.25e-05, 1.25e-05, 1.25e-05, 1.25e-05]]]},
{"info": {"cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05, "cache_creation_input_token_cost_flex": 1.25e-06, "cache_creation_input_token_cost_priority": 5e-06, "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_272k_tokens": 4e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 2e-07, "cache_read_input_token_cost_above_272k_tokens_priority": 8e-07, "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_above_272k_tokens_batches": 2e-07, "cache_creation_input_token_cost_batches": 1.25e-06, "cache_creation_input_token_cost_above_272k_tokens_batches": 2.5e-06, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 4e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_above_272k_tokens": 4e-06, "input_cost_per_token_above_272k_tokens_flex": 2e-06, "input_cost_per_token_above_272k_tokens_priority": 8e-06, "input_cost_per_token_batches": 1e-06, "input_cost_per_token_above_272k_tokens_batches": 2e-06, "input_cost_per_token_flex": 1e-06, "input_cost_per_token_priority": 4e-06, "output_cost_per_token": 1e-05, "output_cost_per_token_above_272k_tokens": 1.5e-05, "output_cost_per_token_above_272k_tokens_flex": 7.5e-06, "output_cost_per_token_above_272k_tokens_priority": 3e-05, "output_cost_per_token_batches": 5e-06, "output_cost_per_token_above_272k_tokens_batches": 7.5e-06, "output_cost_per_token_flex": 5e-06, "output_cost_per_token_priority": 2e-05}, "inclusive": false, "checks": [[null, 1, [2e-06, 1e-05, 2.5e-06, 2.5e-06, 2e-07]], [null, 272000, [2e-06, 1e-05, 2.5e-06, 2.5e-06, 2e-07]], [null, 272001, [4e-06, 1.5e-05, 5e-06, 5e-06, 4e-07]], [null, 10000000, [4e-06, 1.5e-05, 5e-06, 5e-06, 4e-07]], ["priority", 1, [4e-06, 2e-05, 5e-06, 5e-06, 4e-07]], ["priority", 272000, [4e-06, 2e-05, 5e-06, 5e-06, 4e-07]], ["priority", 272001, [8e-06, 3e-05, 1e-05, 1e-05, 8e-07]], ["priority", 10000000, [8e-06, 3e-05, 1e-05, 1e-05, 8e-07]], ["flex", 1, [1e-06, 5e-06, 1.25e-06, 1.25e-06, 1e-07]], ["flex", 272000, [1e-06, 5e-06, 1.25e-06, 1.25e-06, 1e-07]], ["flex", 272001, [2e-06, 7.5e-06, 2.5e-06, 2.5e-06, 2e-07]], ["flex", 10000000, [2e-06, 7.5e-06, 2.5e-06, 2.5e-06, 2e-07]]]},
{"info": {"cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_creation_input_token_cost_batches": 6.25e-07, "cache_read_input_token_cost": 1e-07, "cache_read_input_token_cost_batches": 5e-08, "input_cost_per_token": 1e-06, "input_cost_per_token_batches": 5e-07, "output_cost_per_token": 5e-06, "output_cost_per_token_batches": 2.5e-06}, "inclusive": false, "checks": [[null, 1, [1e-06, 5e-06, 1.25e-06, 2e-06, 1e-07]], [null, 10000000, [1e-06, 5e-06, 1.25e-06, 2e-06, 1e-07]]]},
{"info": {"input_cost_per_token_above_100k_tokens_batches": 2.5e-07, "output_cost_per_token_above_100k_tokens_batches": 1.25e-06, "cache_creation_input_token_cost_above_100k_tokens_batches": 3.125e-07, "cache_read_input_token_cost_above_100k_tokens_batches": 2.5e-08, "cache_creation_input_token_cost": 1.25e-07, "cache_creation_input_token_cost_above_1hr": 2e-07, "cache_creation_input_token_cost_batches": 6.25e-08, "cache_read_input_token_cost": 1e-08, "cache_read_input_token_cost_batches": 5e-09, "input_cost_per_token": 1e-07, "input_cost_per_token_batches": 5e-08, "output_cost_per_token": 5e-07, "output_cost_per_token_batches": 2.5e-07, "input_cost_per_token_above_100k_tokens": 5e-07, "output_cost_per_token_above_100k_tokens": 2.5e-06, "cache_creation_input_token_cost_above_100k_tokens": 6.25e-07, "cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1e-06, "cache_read_input_token_cost_above_100k_tokens": 5e-08}, "inclusive": false, "checks": [[null, 1, [1e-07, 5e-07, 1.25e-07, 2e-07, 1e-08]], [null, 100000, [1e-07, 5e-07, 1.25e-07, 2e-07, 1e-08]], [null, 100001, [5e-07, 2.5e-06, 6.25e-07, 1e-06, 5e-08]], [null, 10000000, [5e-07, 2.5e-06, 6.25e-07, 1e-06, 5e-08]]]},
{"info": {"input_cost_per_token": 1.0000000000000001e-07, "output_cost_per_token": 5e-07, "cache_read_input_token_cost": 1e-08, "cache_creation_input_token_cost": 1.25e-07, "tiered_pricing": [{"range": [0, 272000], "input_cost_per_token": 1.0000000000000001e-07, "output_cost_per_token": 5e-07, "cache_read_input_token_cost": 1e-08, "cache_creation_input_token_cost": 1.25e-07}, {"range": [272000, 1050000], "input_cost_per_token": 2.0000000000000002e-07, "output_cost_per_token": 7.5e-07, "cache_read_input_token_cost": 2e-08, "cache_creation_input_token_cost": 2.5e-07}]}, "inclusive": false, "checks": [[null, 1, [1.0000000000000001e-07, 5e-07, 1.25e-07, 1.25e-07, 1e-08]], [null, 272000, [1.0000000000000001e-07, 5e-07, 1.25e-07, 1.25e-07, 1e-08]], [null, 272001, [2.0000000000000002e-07, 7.5e-07, 2.5e-07, 2.5e-07, 2e-08]], [null, 10000000, [2.0000000000000002e-07, 7.5e-07, 2.5e-07, 2.5e-07, 2e-08]]]},
{"info": {"cache_creation_input_token_cost": 1e-06, "cache_read_input_token_cost": 8e-08, "input_cost_per_token": 8e-07, "output_cost_per_token": 4e-06}, "inclusive": false, "checks": [[null, 1, [8e-07, 4e-06, 1e-06, 1e-06, 8e-08]], [null, 10000000, [8e-07, 4e-06, 1e-06, 1e-06, 8e-08]]]},
{"info": {"input_cost_per_token": 2e-06, "output_cost_per_token": 6e-06, "cache_read_input_token_cost": 5e-07, "tiered_pricing": [{"range": [0, 200000], "input_cost_per_token": 2e-06, "output_cost_per_token": 6e-06, "cache_read_input_token_cost": 5e-07}, {"range": [200000, 500000], "input_cost_per_token": 4e-06, "output_cost_per_token": 1.2e-05, "cache_read_input_token_cost": 1e-06}]}, "inclusive": false, "checks": [[null, 1, [2e-06, 6e-06, 2e-06, 2e-06, 5e-07]], [null, 200000, [2e-06, 6e-06, 2e-06, 2e-06, 5e-07]], [null, 200001, [4e-06, 1.2e-05, 4e-06, 4e-06, 1e-06]], [null, 10000000, [4e-06, 1.2e-05, 4e-06, 4e-06, 1e-06]]]},
{"info": {"cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_priority": 1.5e-07, "input_cost_per_token": 7.5e-07, "input_cost_per_token_priority": 1.5e-06, "output_cost_per_token": 4.5e-06, "output_cost_per_token_priority": 9e-06}, "inclusive": false, "checks": [[null, 1, [7.5e-07, 4.5e-06, 7.5e-07, 7.5e-07, 7.5e-08]], [null, 10000000, [7.5e-07, 4.5e-06, 7.5e-07, 7.5e-07, 7.5e-08]], ["priority", 1, [1.5e-06, 9e-06, 1.5e-06, 1.5e-06, 1.5e-07]], ["priority", 10000000, [1.5e-06, 9e-06, 1.5e-06, 1.5e-06, 1.5e-07]]]},
{"info": {"cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "input_cost_per_token": 1.25e-06, "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06}, "inclusive": false, "checks": [[null, 1, [1.25e-06, 2.5e-06, 1.25e-06, 1.25e-06, 2e-07]], [null, 200000, [1.25e-06, 2.5e-06, 1.25e-06, 1.25e-06, 2e-07]], [null, 200001, [2.5e-06, 5e-06, 2.5e-06, 2.5e-06, 4e-07]], [null, 10000000, [2.5e-06, 5e-06, 2.5e-06, 2.5e-06, 4e-07]]]},
{"info": {"cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 1e-06, "output_cost_per_token": 5e-06}, "inclusive": false, "checks": [[null, 1, [1e-06, 5e-06, 1.25e-06, 2e-06, 1e-07]], [null, 10000000, [1e-06, 5e-06, 1.25e-06, 2e-06, 1e-07]]]},
{"info": {"cache_read_input_token_cost": 8.25e-08, "cache_read_input_token_cost_flex": 4.125e-08, "cache_read_input_token_cost_priority": 1.44375e-07, "input_cost_per_token": 3.3e-07, "input_cost_per_token_batches": 1.65e-07, "input_cost_per_token_flex": 1.65e-07, "input_cost_per_token_priority": 5.775e-07, "output_cost_per_token": 2.75e-06, "output_cost_per_token_batches": 1.375e-06, "output_cost_per_token_flex": 1.375e-06, "output_cost_per_token_priority": 4.8125e-06}, "inclusive": false, "checks": [[null, 1, [3.3e-07, 2.75e-06, 3.3e-07, 3.3e-07, 8.25e-08]], [null, 10000000, [3.3e-07, 2.75e-06, 3.3e-07, 3.3e-07, 8.25e-08]], ["priority", 1, [5.775e-07, 4.8125e-06, 5.775e-07, 5.775e-07, 1.44375e-07]], ["priority", 10000000, [5.775e-07, 4.8125e-06, 5.775e-07, 5.775e-07, 1.44375e-07]], ["flex", 1, [1.65e-07, 1.375e-06, 1.65e-07, 1.65e-07, 4.125e-08]], ["flex", 10000000, [1.65e-07, 1.375e-06, 1.65e-07, 1.65e-07, 4.125e-08]]]},
{"info": {"cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "cache_read_input_token_cost_priority": 1.5e-07, "input_cost_per_token": 7.5e-07, "output_cost_per_token": 4.5e-06, "input_cost_per_token_batches": 3.75e-07, "input_cost_per_token_flex": 3.75e-07, "input_cost_per_token_priority": 1.5e-06, "output_cost_per_token_batches": 2.25e-06, "output_cost_per_token_flex": 2.25e-06, "output_cost_per_token_priority": 9e-06}, "inclusive": false, "checks": [[null, 1, [7.5e-07, 4.5e-06, 7.5e-07, 7.5e-07, 7.5e-08]], [null, 10000000, [7.5e-07, 4.5e-06, 7.5e-07, 7.5e-07, 7.5e-08]], ["priority", 1, [1.5e-06, 9e-06, 1.5e-06, 1.5e-06, 1.5e-07]], ["priority", 10000000, [1.5e-06, 9e-06, 1.5e-06, 1.5e-06, 1.5e-07]], ["flex", 1, [3.75e-07, 2.25e-06, 3.75e-07, 3.75e-07, 3.75e-08]], ["flex", 10000000, [3.75e-07, 2.25e-06, 3.75e-07, 3.75e-07, 3.75e-08]]]},
{"info": {"cache_read_input_token_cost": 5.5e-09, "input_cost_per_token": 5.5e-08, "input_cost_per_token_batches": 2.75e-08, "output_cost_per_token": 4.4e-07, "output_cost_per_token_batches": 2.2e-07, "cache_read_input_token_cost_batches": 2.75e-09}, "inclusive": false, "checks": [[null, 1, [5.5e-08, 4.4e-07, 5.5e-08, 5.5e-08, 5.5e-09]], [null, 10000000, [5.5e-08, 4.4e-07, 5.5e-08, 5.5e-08, 5.5e-09]]]},
{"info": {"output_cost_per_token": 0.0}, "inclusive": false, "checks": [[null, 1, [0.0, 0.0, 0.0, 0.0, 0.0]], [null, 10000000, [0.0, 0.0, 0.0, 0.0, 0.0]]]},
{"info": {"input_cost_per_token_above_100k_tokens": 5e-07, "output_cost_per_token_above_100k_tokens": 2.5e-06, "cache_creation_input_token_cost_above_100k_tokens": 6.25e-07, "cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1e-06, "cache_read_input_token_cost_above_100k_tokens": 5e-08, "cache_creation_input_token_cost": 1.25e-07, "cache_creation_input_token_cost_above_1hr": 2e-07, "cache_creation_input_token_cost_batches": 6.25e-08, "cache_read_input_token_cost": 1e-08, "cache_read_input_token_cost_batches": 5e-09, "input_cost_per_token": 1e-07, "input_cost_per_token_batches": 5e-08, "output_cost_per_token": 5e-07, "output_cost_per_token_batches": 2.5e-07}, "inclusive": false, "checks": [[null, 1, [1e-07, 5e-07, 1.25e-07, 2e-07, 1e-08]], [null, 100000, [1e-07, 5e-07, 1.25e-07, 2e-07, 1e-08]], [null, 100001, [5e-07, 2.5e-06, 6.25e-07, 1e-06, 5e-08]], [null, 10000000, [5e-07, 2.5e-06, 6.25e-07, 1e-06, 5e-08]]]},
{"info": {"input_cost_per_token_above_100k_tokens": 5.5e-07, "output_cost_per_token_above_100k_tokens": 2.75e-06, "cache_creation_input_token_cost_above_100k_tokens": 6.875e-07, "cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1.1e-06, "cache_read_input_token_cost_above_100k_tokens": 5.5e-08, "cache_creation_input_token_cost": 1.375e-07, "cache_creation_input_token_cost_above_1hr": 2.2e-07, "cache_read_input_token_cost": 1.1e-08, "input_cost_per_token": 1.1e-07, "output_cost_per_token": 5.5e-07}, "inclusive": false, "checks": [[null, 1, [1.1e-07, 5.5e-07, 1.375e-07, 2.2e-07, 1.1e-08]], [null, 100000, [1.1e-07, 5.5e-07, 1.375e-07, 2.2e-07, 1.1e-08]], [null, 100001, [5.5e-07, 2.75e-06, 6.875e-07, 1.1e-06, 5.5e-08]], [null, 10000000, [5.5e-07, 2.75e-06, 6.875e-07, 1.1e-06, 5.5e-08]]]},
{"info": {"cache_creation_input_token_cost": 4.125e-06, "cache_creation_input_token_cost_flex": 2.0625e-06, "cache_creation_input_token_cost_priority": 7.21875e-06, "cache_read_input_token_cost": 3.3e-07, "cache_read_input_token_cost_flex": 1.65e-07, "cache_read_input_token_cost_priority": 5.775e-07, "input_cost_per_token": 3.3e-06, "input_cost_per_token_flex": 1.65e-06, "input_cost_per_token_priority": 5.775e-06, "output_cost_per_token": 1.65e-05, "output_cost_per_token_flex": 8.25e-06, "output_cost_per_token_priority": 2.8875e-05}, "inclusive": false, "checks": [[null, 1, [3.3e-06, 1.65e-05, 4.125e-06, 4.125e-06, 3.3e-07]], [null, 10000000, [3.3e-06, 1.65e-05, 4.125e-06, 4.125e-06, 3.3e-07]], ["priority", 1, [5.775e-06, 2.8875e-05, 7.21875e-06, 7.21875e-06, 5.775e-07]], ["priority", 10000000, [5.775e-06, 2.8875e-05, 7.21875e-06, 7.21875e-06, 5.775e-07]], ["flex", 1, [1.65e-06, 8.25e-06, 2.0625e-06, 2.0625e-06, 1.65e-07]], ["flex", 10000000, [1.65e-06, 8.25e-06, 2.0625e-06, 2.0625e-06, 1.65e-07]]]},
{"info": {"cache_read_input_token_cost": 1.375e-07, "cache_read_input_token_cost_priority": 2.75e-07, "input_cost_per_token": 1.375e-06, "input_cost_per_token_batches": 6.875e-07, "input_cost_per_token_priority": 2.75e-06, "output_cost_per_token": 1.1e-05, "output_cost_per_token_batches": 5.5e-06, "output_cost_per_token_priority": 2.2e-05, "cache_read_input_token_cost_batches": 6.875e-08}, "inclusive": false, "checks": [[null, 1, [1.375e-06, 1.1e-05, 1.375e-06, 1.375e-06, 1.375e-07]], [null, 10000000, [1.375e-06, 1.1e-05, 1.375e-06, 1.375e-06, 1.375e-07]], ["priority", 1, [2.75e-06, 2.2e-05, 2.75e-06, 2.75e-06, 2.75e-07]], ["priority", 10000000, [2.75e-06, 2.2e-05, 2.75e-06, 2.75e-06, 2.75e-07]]]},
{"info": {"input_cost_per_token": 5e-06, "input_cost_per_token_batches": 2.5e-06, "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06}, "inclusive": false, "checks": [[null, 1, [5e-06, 1.5e-05, 5e-06, 5e-06, 5e-06]], [null, 10000000, [5e-06, 1.5e-05, 5e-06, 5e-06, 5e-06]]]},
{"info": {"cache_creation_input_token_cost": 2.1e-06, "cache_creation_input_token_cost_batches": 1.05e-06, "cache_creation_input_token_cost_flex": 1.05e-06, "cache_creation_input_token_cost_priority": 3.675e-06, "cache_read_input_token_cost": 3.12e-07, "cache_read_input_token_cost_batches": 1.56e-07, "cache_read_input_token_cost_flex": 1.56e-07, "cache_read_input_token_cost_priority": 5.46e-07, "input_cost_per_token": 1.68e-06, "input_cost_per_token_batches": 8.4e-07, "input_cost_per_token_flex": 8.4e-07, "input_cost_per_token_priority": 2.94e-06, "output_cost_per_token": 5.28e-06, "output_cost_per_token_batches": 2.64e-06, "output_cost_per_token_flex": 2.64e-06, "output_cost_per_token_priority": 9.24e-06}, "inclusive": false, "checks": [[null, 1, [1.68e-06, 5.28e-06, 2.1e-06, 2.1e-06, 3.12e-07]], [null, 10000000, [1.68e-06, 5.28e-06, 2.1e-06, 2.1e-06, 3.12e-07]], ["priority", 1, [2.94e-06, 9.24e-06, 3.675e-06, 3.675e-06, 5.46e-07]], ["priority", 10000000, [2.94e-06, 9.24e-06, 3.675e-06, 3.675e-06, 5.46e-07]], ["flex", 1, [8.4e-07, 2.64e-06, 1.05e-06, 1.05e-06, 1.56e-07]], ["flex", 10000000, [8.4e-07, 2.64e-06, 1.05e-06, 1.05e-06, 1.56e-07]]]},
{"info": {"input_cost_per_token": 1.1e-05, "input_cost_per_token_above_272k_tokens": 2.2e-05, "cache_creation_input_token_cost": 1.375e-05, "cache_creation_input_token_cost_above_272k_tokens": 2.75e-05, "cache_creation_input_token_cost_above_272k_tokens_ultrafast": 0.000165, "cache_creation_input_token_cost_ultrafast": 8.25e-05, "cache_read_input_token_cost": 1.1e-06, "cache_read_input_token_cost_above_272k_tokens": 2.2e-06, "cache_read_input_token_cost_above_272k_tokens_ultrafast": 1.32e-05, "cache_read_input_token_cost_ultrafast": 6.6e-06, "input_cost_per_token_above_272k_tokens_ultrafast": 0.000132, "input_cost_per_token_ultrafast": 6.6e-05, "output_cost_per_token": 5.5e-05, "output_cost_per_token_above_272k_tokens": 8.25e-05, "output_cost_per_token_above_272k_tokens_ultrafast": 0.000495, "output_cost_per_token_ultrafast": 0.00033}, "inclusive": false, "checks": [[null, 1, [1.1e-05, 5.5e-05, 1.375e-05, 1.375e-05, 1.1e-06]], [null, 272000, [1.1e-05, 5.5e-05, 1.375e-05, 1.375e-05, 1.1e-06]], [null, 272001, [2.2e-05, 8.25e-05, 2.75e-05, 2.75e-05, 2.2e-06]], [null, 10000000, [2.2e-05, 8.25e-05, 2.75e-05, 2.75e-05, 2.2e-06]], ["ultrafast", 1, [6.6e-05, 0.00033, 8.25e-05, 8.25e-05, 6.6e-06]], ["ultrafast", 272000, [6.6e-05, 0.00033, 8.25e-05, 8.25e-05, 6.6e-06]], ["ultrafast", 272001, [0.000132, 0.000495, 0.000165, 0.000165, 1.32e-05]], ["ultrafast", 10000000, [0.000132, 0.000495, 0.000165, 0.000165, 1.32e-05]]]},
{"info": {"cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-05, "cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05, "cache_creation_input_token_cost_above_272k_tokens_priority": 5e-05, "cache_creation_input_token_cost_flex": 6.25e-06, "cache_creation_input_token_cost_priority": 2.5e-05, "cache_read_input_token_cost": 1e-06, "cache_read_input_token_cost_above_272k_tokens": 2e-06, "cache_read_input_token_cost_above_272k_tokens_flex": 1e-06, "cache_read_input_token_cost_above_272k_tokens_priority": 4e-06, "cache_read_input_token_cost_batches": 5e-07, "cache_read_input_token_cost_above_272k_tokens_batches": 1e-06, "cache_creation_input_token_cost_batches": 6.25e-06, "cache_creation_input_token_cost_above_272k_tokens_batches": 1.25e-05, "cache_creation_input_token_cost_above_272k_tokens_ultrafast": 0.00015, "cache_creation_input_token_cost_ultrafast": 7.5e-05, "cache_read_input_token_cost_above_272k_tokens_ultrafast": 1.2e-05, "cache_read_input_token_cost_flex": 5e-07, "cache_read_input_token_cost_priority": 2e-06, "cache_read_input_token_cost_ultrafast": 6e-06, "input_cost_per_token": 1e-05, "input_cost_per_token_above_272k_tokens": 2e-05, "input_cost_per_token_above_272k_tokens_flex": 1e-05, "input_cost_per_token_above_272k_tokens_priority": 4e-05, "input_cost_per_token_batches": 5e-06, "input_cost_per_token_above_272k_tokens_batches": 1e-05, "input_cost_per_token_above_272k_tokens_ultrafast": 0.00012, "input_cost_per_token_flex": 5e-06, "input_cost_per_token_priority": 2e-05, "input_cost_per_token_ultrafast": 6e-05, "output_cost_per_token": 5e-05, "output_cost_per_token_above_272k_tokens": 7.5e-05, "output_cost_per_token_above_272k_tokens_flex": 3.75e-05, "output_cost_per_token_above_272k_tokens_priority": 0.00015, "output_cost_per_token_batches": 2.5e-05, "output_cost_per_token_above_272k_tokens_batches": 3.75e-05, "output_cost_per_token_above_272k_tokens_ultrafast": 0.00045, "output_cost_per_token_flex": 2.5e-05, "output_cost_per_token_priority": 0.0001, "output_cost_per_token_ultrafast": 0.0003}, "inclusive": false, "checks": [[null, 1, [1e-05, 5e-05, 1.25e-05, 1.25e-05, 1e-06]], [null, 272000, [1e-05, 5e-05, 1.25e-05, 1.25e-05, 1e-06]], [null, 272001, [2e-05, 7.5e-05, 2.5e-05, 2.5e-05, 2e-06]], [null, 10000000, [2e-05, 7.5e-05, 2.5e-05, 2.5e-05, 2e-06]], ["priority", 1, [2e-05, 0.0001, 2.5e-05, 2.5e-05, 2e-06]], ["priority", 272000, [2e-05, 0.0001, 2.5e-05, 2.5e-05, 2e-06]], ["priority", 272001, [4e-05, 0.00015, 5e-05, 5e-05, 4e-06]], ["priority", 10000000, [4e-05, 0.00015, 5e-05, 5e-05, 4e-06]], ["flex", 1, [5e-06, 2.5e-05, 6.25e-06, 6.25e-06, 5e-07]], ["flex", 272000, [5e-06, 2.5e-05, 6.25e-06, 6.25e-06, 5e-07]], ["flex", 272001, [1e-05, 3.75e-05, 1.25e-05, 1.25e-05, 1e-06]], ["flex", 10000000, [1e-05, 3.75e-05, 1.25e-05, 1.25e-05, 1e-06]], ["ultrafast", 1, [6e-05, 0.0003, 7.5e-05, 7.5e-05, 6e-06]], ["ultrafast", 272000, [6e-05, 0.0003, 7.5e-05, 7.5e-05, 6e-06]], ["ultrafast", 272001, [0.00012, 0.00045, 0.00015, 0.00015, 1.2e-05]], ["ultrafast", 10000000, [0.00012, 0.00045, 0.00015, 0.00015, 1.2e-05]]]},
{"info": {"input_cost_per_token": 2.2e-06, "output_cost_per_token": 6.6e-06, "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_flex": 2.75e-07, "cache_read_input_token_cost_priority": 9.625e-07, "input_cost_per_token_flex": 1.1e-06, "input_cost_per_token_priority": 3.85e-06, "output_cost_per_token_flex": 3.3e-06, "output_cost_per_token_priority": 1.155e-05}, "inclusive": false, "checks": [[null, 1, [2.2e-06, 6.6e-06, 2.2e-06, 2.2e-06, 5.5e-07]], [null, 10000000, [2.2e-06, 6.6e-06, 2.2e-06, 2.2e-06, 5.5e-07]], ["priority", 1, [3.85e-06, 1.155e-05, 3.85e-06, 3.85e-06, 9.625e-07]], ["priority", 10000000, [3.85e-06, 1.155e-05, 3.85e-06, 3.85e-06, 9.625e-07]], ["flex", 1, [1.1e-06, 3.3e-06, 1.1e-06, 1.1e-06, 2.75e-07]], ["flex", 10000000, [1.1e-06, 3.3e-06, 1.1e-06, 1.1e-06, 2.75e-07]]]},
{"info": {"cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, "input_cost_per_token": 3e-05, "input_cost_per_token_above_272k_tokens": 6e-05, "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027}, "inclusive": false, "checks": [[null, 1, [3e-05, 0.00018, 3e-05, 3e-05, 3e-06]], [null, 272000, [3e-05, 0.00018, 3e-05, 3e-05, 3e-06]], [null, 272001, [6e-05, 0.00027, 6e-05, 6e-05, 6e-06]], [null, 10000000, [6e-05, 0.00027, 6e-05, 6e-05, 6e-06]]]},
{"info": {"input_cost_per_token": 1.5e-07, "input_cost_per_token_batches": 1.2e-07, "output_cost_per_token": 0}, "inclusive": false, "checks": [[null, 1, [1.5e-07, 0.0, 1.5e-07, 1.5e-07, 1.5e-07]], [null, 10000000, [1.5e-07, 0.0, 1.5e-07, 1.5e-07, 1.5e-07]]]},
{"info": {"cache_read_input_token_cost_priority": 6e-07, "input_cost_per_token_priority": 1.2e-06, "output_cost_per_token_priority": 1.2e-06}, "inclusive": false, "checks": [[null, 1, [0.0, 0.0, 0.0, 0.0, 0.0]], [null, 10000000, [0.0, 0.0, 0.0, 0.0, 0.0]], ["priority", 1, [1.2e-06, 1.2e-06, 1.2e-06, 1.2e-06, 6e-07]], ["priority", 10000000, [1.2e-06, 1.2e-06, 1.2e-06, 1.2e-06, 6e-07]]]},
{"info": {"cache_read_input_token_cost": 2e-07, "input_cost_per_token": 1.25e-06, "output_cost_per_token": 2.5e-06, "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07}, "inclusive": true, "checks": [[null, 1, [1.25e-06, 2.5e-06, 1.25e-06, 1.25e-06, 2e-07]], [null, 200000, [2.5e-06, 5e-06, 2.5e-06, 2.5e-06, 4e-07]], [null, 200001, [2.5e-06, 5e-06, 2.5e-06, 2.5e-06, 4e-07]], [null, 10000000, [2.5e-06, 5e-06, 2.5e-06, 2.5e-06, 4e-07]]]},
{"info": {"cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_batches": 1.6e-07, "input_cost_per_token": 1.25e-06, "input_cost_per_token_batches": 1e-06, "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 2e-06, "input_cost_per_token_above_200k_tokens": 2.5e-06, "input_cost_per_token_above_200k_tokens_batches": 2e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "output_cost_per_token_above_200k_tokens_batches": 4e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_batches": 3.2e-07}, "inclusive": true, "checks": [[null, 1, [1.25e-06, 2.5e-06, 1.25e-06, 1.25e-06, 2e-07]], [null, 200000, [2.5e-06, 5e-06, 2.5e-06, 2.5e-06, 4e-07]], [null, 200001, [2.5e-06, 5e-06, 2.5e-06, 2.5e-06, 4e-07]], [null, 10000000, [2.5e-06, 5e-06, 2.5e-06, 2.5e-06, 4e-07]]]},
{"info": {"input_cost_per_token": 5e-06}, "inclusive": false, "checks": [[null, 1, [5e-06, 0.0, 5e-06, 5e-06, 5e-06]], [null, 10000000, [5e-06, 0.0, 5e-06, 5e-06, 5e-06]]]},
{"info": {"cache_read_input_token_cost": 1.5e-08, "input_cost_per_token": 6e-08, "input_cost_per_token_batches": 3e-08, "output_cost_per_token": 2.4e-07, "output_cost_per_token_batches": 1.2e-07}, "inclusive": false, "checks": [[null, 1, [6e-08, 2.4e-07, 6e-08, 6e-08, 1.5e-08]], [null, 10000000, [6e-08, 2.4e-07, 6e-08, 6e-08, 1.5e-08]]]},
{"info": {"input_cost_per_token": 3.3e-05, "input_cost_per_token_above_272k_tokens": 6.6e-05, "input_cost_per_token_above_272k_tokens_batches": 3.3e-05, "input_cost_per_token_batches": 1.65e-05, "output_cost_per_token": 0.000198, "output_cost_per_token_above_272k_tokens": 0.000297, "output_cost_per_token_above_272k_tokens_batches": 0.0001485, "output_cost_per_token_batches": 9.9e-05}, "inclusive": false, "checks": [[null, 1, [3.3e-05, 0.000198, 3.3e-05, 3.3e-05, 3.3e-05]], [null, 272000, [3.3e-05, 0.000198, 3.3e-05, 3.3e-05, 3.3e-05]], [null, 272001, [6.6e-05, 0.000297, 6.6e-05, 6.6e-05, 6.6e-05]], [null, 10000000, [6.6e-05, 0.000297, 6.6e-05, 6.6e-05, 6.6e-05]]]},
{"info": {"cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_priority": 1.3125e-07, "input_cost_per_token": 3e-07, "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_priority": 5.25e-07, "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 1.25e-06, "output_cost_per_token_priority": 4.375e-06}, "inclusive": false, "checks": [[null, 1, [3e-07, 2.5e-06, 3e-07, 3e-07, 7.5e-08]], [null, 10000000, [3e-07, 2.5e-06, 3e-07, 3e-07, 7.5e-08]], ["priority", 1, [5.25e-07, 4.375e-06, 5.25e-07, 5.25e-07, 1.3125e-07]], ["priority", 10000000, [5.25e-07, 4.375e-06, 5.25e-07, 5.25e-07, 1.3125e-07]]]},
{"info": {"input_cost_per_token": 1.25e-06, "output_cost_per_token": 2.5e-06, "cache_read_input_token_cost": 2e-07, "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06}, "inclusive": false, "checks": [[null, 1, [1.25e-06, 2.5e-06, 1.25e-06, 1.25e-06, 2e-07]], [null, 200000, [1.25e-06, 2.5e-06, 1.25e-06, 1.25e-06, 2e-07]], [null, 200001, [2.5e-06, 5e-06, 2.5e-06, 2.5e-06, 2e-07]], [null, 10000000, [2.5e-06, 5e-06, 2.5e-06, 2.5e-06, 2e-07]]]},
{"info": {"input_cost_per_token": 5e-06, "output_cost_per_token": 3e-05, "cache_read_input_token_cost": 5e-07, "input_cost_per_token_above_272k_tokens": 1e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05}, "inclusive": false, "checks": [[null, 1, [5e-06, 3e-05, 5e-06, 5e-06, 5e-07]], [null, 272000, [5e-06, 3e-05, 5e-06, 5e-06, 5e-07]], [null, 272001, [1e-05, 4.5e-05, 1e-05, 1e-05, 5e-07]], [null, 10000000, [1e-05, 4.5e-05, 1e-05, 1e-05, 5e-07]]]},
{"info": {"input_cost_per_token": 1.5e-07, "input_cost_per_token_batches": 7.5e-08, "input_cost_per_token_flex": 7.5e-08, "input_cost_per_token_priority": 2.625e-07, "output_cost_per_token": 6e-07, "output_cost_per_token_batches": 3e-07, "output_cost_per_token_flex": 3e-07, "output_cost_per_token_priority": 1.05e-06}, "inclusive": false, "checks": [[null, 1, [1.5e-07, 6e-07, 1.5e-07, 1.5e-07, 1.5e-07]], [null, 10000000, [1.5e-07, 6e-07, 1.5e-07, 1.5e-07, 1.5e-07]], ["priority", 1, [2.625e-07, 1.05e-06, 2.625e-07, 2.625e-07, 2.625e-07]], ["priority", 10000000, [2.625e-07, 1.05e-06, 2.625e-07, 2.625e-07, 2.625e-07]], ["flex", 1, [7.5e-08, 3e-07, 7.5e-08, 7.5e-08, 7.5e-08]], ["flex", 10000000, [7.5e-08, 3e-07, 7.5e-08, 7.5e-08, 7.5e-08]]]},
{"info": {"tiered_pricing": [{"input_cost_per_token": 1.2e-06, "output_cost_per_token": 6e-06, "range": [0, 32000.0]}, {"input_cost_per_token": 2.4e-06, "output_cost_per_token": 1.2e-05, "range": [32000.0, 128000.0]}, {"input_cost_per_token": 3e-06, "output_cost_per_token": 1.5e-05, "range": [128000.0, 252000.0]}]}, "inclusive": false, "checks": [[null, 1, [1.2e-06, 6e-06, 1.2e-06, 1.2e-06, 1.2e-06]], [null, 32000, [1.2e-06, 6e-06, 1.2e-06, 1.2e-06, 1.2e-06]], [null, 32001, [2.4e-06, 1.2e-05, 2.4e-06, 2.4e-06, 2.4e-06]], [null, 128000, [2.4e-06, 1.2e-05, 2.4e-06, 2.4e-06, 2.4e-06]], [null, 128001, [3e-06, 1.5e-05, 3e-06, 3e-06, 3e-06]], [null, 10000000, [3e-06, 1.5e-05, 3e-06, 3e-06, 3e-06]]]},
{"info": {"cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 1e-06, "output_cost_per_token": 5e-06, "input_cost_per_token_batches": 5e-07, "output_cost_per_token_batches": 2.5e-06}, "inclusive": false, "checks": [[null, 1, [1e-06, 5e-06, 1.25e-06, 2e-06, 1e-07]], [null, 10000000, [1e-06, 5e-06, 1.25e-06, 2e-06, 1e-07]]]},
{"info": {"cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, "cache_read_input_token_cost_priority": 3.6e-07, "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, "input_cost_per_token_batches": 1e-06, "input_cost_per_token_priority": 3.6e-06, "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "output_cost_per_token_batches": 6e-06, "output_cost_per_token_priority": 2.16e-05, "cache_read_input_token_cost_flex": 2e-07, "input_cost_per_token_flex": 1e-06, "output_cost_per_token_flex": 6e-06}, "inclusive": false, "checks": [[null, 1, [2e-06, 1.2e-05, 2e-06, 2e-06, 2e-07]], [null, 200000, [2e-06, 1.2e-05, 2e-06, 2e-06, 2e-07]], [null, 200001, [4e-06, 1.8e-05, 4e-06, 4e-06, 4e-07]], [null, 10000000, [4e-06, 1.8e-05, 4e-06, 4e-06, 4e-07]], ["priority", 1, [3.6e-06, 2.16e-05, 3.6e-06, 3.6e-06, 3.6e-07]], ["priority", 200000, [3.6e-06, 2.16e-05, 3.6e-06, 3.6e-06, 3.6e-07]], ["priority", 200001, [7.2e-06, 3.24e-05, 7.2e-06, 7.2e-06, 7.2e-07]], ["priority", 10000000, [7.2e-06, 3.24e-05, 7.2e-06, 7.2e-06, 7.2e-07]], ["flex", 1, [1e-06, 6e-06, 1e-06, 1e-06, 2e-07]], ["flex", 200000, [1e-06, 6e-06, 1e-06, 1e-06, 2e-07]], ["flex", 200001, [4e-06, 1.8e-05, 4e-06, 4e-06, 4e-07]], ["flex", 10000000, [4e-06, 1.8e-05, 4e-06, 4e-06, 4e-07]]]},
{"info": {"input_cost_per_token": 5e-06, "input_cost_per_token_batches": 2.5e-06, "input_cost_per_token_priority": 8.75e-06, "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, "output_cost_per_token_priority": 2.625e-05}, "inclusive": false, "checks": [[null, 1, [5e-06, 1.5e-05, 5e-06, 5e-06, 5e-06]], [null, 10000000, [5e-06, 1.5e-05, 5e-06, 5e-06, 5e-06]], ["priority", 1, [8.75e-06, 2.625e-05, 8.75e-06, 8.75e-06, 8.75e-06]], ["priority", 10000000, [8.75e-06, 2.625e-05, 8.75e-06, 8.75e-06, 8.75e-06]]]},
{"info": {"cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, "cache_read_input_token_cost_batches": 1e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_priority": 3.6e-06, "input_cost_per_token_above_200k_tokens": 4e-06, "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, "input_cost_per_token_batches": 1e-06, "output_cost_per_token": 1.2e-05, "output_cost_per_token_priority": 2.16e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "output_cost_per_token_batches": 6e-06}, "inclusive": false, "checks": [[null, 1, [2e-06, 1.2e-05, 2e-06, 2e-06, 2e-07]], [null, 200000, [2e-06, 1.2e-05, 2e-06, 2e-06, 2e-07]], [null, 200001, [4e-06, 1.8e-05, 4e-06, 4e-06, 4e-07]], [null, 10000000, [4e-06, 1.8e-05, 4e-06, 4e-06, 4e-07]], ["priority", 1, [3.6e-06, 2.16e-05, 3.6e-06, 3.6e-06, 3.6e-07]], ["priority", 200000, [3.6e-06, 2.16e-05, 3.6e-06, 3.6e-06, 3.6e-07]], ["priority", 200001, [7.2e-06, 3.24e-05, 7.2e-06, 7.2e-06, 7.2e-07]], ["priority", 10000000, [7.2e-06, 3.24e-05, 7.2e-06, 7.2e-06, 7.2e-07]]]},
{"info": {"cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 3.6e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, "input_cost_per_token_batches": 1e-06, "input_cost_per_token_flex": 1e-06, "input_cost_per_token_priority": 3.6e-06, "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "output_cost_per_token_batches": 6e-06, "output_cost_per_token_flex": 6e-06, "output_cost_per_token_priority": 2.16e-05}, "inclusive": false, "checks": [[null, 1, [2e-06, 1.2e-05, 2e-06, 2e-06, 2e-07]], [null, 200000, [2e-06, 1.2e-05, 2e-06, 2e-06, 2e-07]], [null, 200001, [4e-06, 1.8e-05, 4e-06, 4e-06, 4e-07]], [null, 10000000, [4e-06, 1.8e-05, 4e-06, 4e-06, 4e-07]], ["priority", 1, [3.6e-06, 2.16e-05, 3.6e-06, 3.6e-06, 3.6e-07]], ["priority", 200000, [3.6e-06, 2.16e-05, 3.6e-06, 3.6e-06, 3.6e-07]], ["priority", 200001, [7.2e-06, 3.24e-05, 7.2e-06, 7.2e-06, 7.2e-07]], ["priority", 10000000, [7.2e-06, 3.24e-05, 7.2e-06, 7.2e-06, 7.2e-07]], ["flex", 1, [1e-06, 6e-06, 1e-06, 1e-06, 1e-07]], ["flex", 200000, [1e-06, 6e-06, 1e-06, 1e-06, 1e-07]], ["flex", 200001, [4e-06, 1.8e-05, 4e-06, 4e-06, 4e-07]], ["flex", 10000000, [4e-06, 1.8e-05, 4e-06, 4e-06, 4e-07]]]},
{"info": {"input_cost_per_token": 3e-07, "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, "output_cost_per_token": 2.5e-06, "output_cost_per_token_batches": 1.25e-06, "output_cost_per_token_flex": 1.25e-06}, "inclusive": false, "checks": [[null, 1, [3e-07, 2.5e-06, 3e-07, 3e-07, 3e-07]], [null, 10000000, [3e-07, 2.5e-06, 3e-07, 3e-07, 3e-07]], ["flex", 1, [1.5e-07, 1.25e-06, 1.5e-07, 1.5e-07, 1.5e-07]], ["flex", 10000000, [1.5e-07, 1.25e-06, 1.5e-07, 1.5e-07, 1.5e-07]]]},
{"info": {"cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.2e-05, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, "input_cost_per_token_above_200k_tokens": 6e-06, "output_cost_per_token_above_200k_tokens": 2.25e-05, "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, "cache_read_input_token_cost_above_200k_tokens": 6e-07, "cache_read_input_token_cost_batches": 1.5e-07, "input_cost_per_token_batches": 1.5e-06, "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, "cache_creation_input_token_cost_batches": 1.88e-06}, "inclusive": false, "checks": [[null, 1, [3e-06, 1.5e-05, 3.75e-06, 6e-06, 3e-07]], [null, 200000, [3e-06, 1.5e-05, 3.75e-06, 6e-06, 3e-07]], [null, 200001, [6e-06, 2.25e-05, 7.5e-06, 1.2e-05, 6e-07]], [null, 10000000, [6e-06, 2.25e-05, 7.5e-06, 1.2e-05, 6e-07]]]},
{"info": {"input_cost_per_token": 7.4e-07, "input_cost_per_token_flex": 3.7e-07, "output_cost_per_token": 2.22e-06, "output_cost_per_token_flex": 1.11e-06}, "inclusive": false, "checks": [[null, 1, [7.4e-07, 2.22e-06, 7.4e-07, 7.4e-07, 7.4e-07]], [null, 10000000, [7.4e-07, 2.22e-06, 7.4e-07, 7.4e-07, 7.4e-07]], ["flex", 1, [3.7e-07, 1.11e-06, 3.7e-07, 3.7e-07, 3.7e-07]], ["flex", 10000000, [3.7e-07, 1.11e-06, 3.7e-07, 3.7e-07, 3.7e-07]]]},
{"info": {"cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, "input_cost_per_token_above_200k_tokens": 6e-06, "output_cost_per_token_above_200k_tokens": 2.25e-05, "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, "cache_read_input_token_cost_above_200k_tokens": 6e-07, "output_cost_per_token": 1.5e-05}, "inclusive": false, "checks": [[null, 1, [3e-06, 1.5e-05, 3.75e-06, 3.75e-06, 3e-07]], [null, 200000, [3e-06, 1.5e-05, 3.75e-06, 3.75e-06, 3e-07]], [null, 200001, [6e-06, 2.25e-05, 7.5e-06, 7.5e-06, 6e-07]], [null, 10000000, [6e-06, 2.25e-05, 7.5e-06, 7.5e-06, 6e-07]]]},
{"info": {"cache_creation_input_token_cost": 4.5e-06, "cache_creation_input_token_cost_above_1hr": 7.2e-06, "cache_read_input_token_cost": 3.6e-07, "input_cost_per_token": 3.6e-06, "input_cost_per_token_above_200k_tokens": 7.2e-06, "output_cost_per_token_above_200k_tokens": 2.7e-05, "cache_creation_input_token_cost_above_200k_tokens": 9e-06, "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.44e-05, "cache_read_input_token_cost_above_200k_tokens": 7.2e-07, "output_cost_per_token": 1.8e-05}, "inclusive": false, "checks": [[null, 1, [3.6e-06, 1.8e-05, 4.5e-06, 7.2e-06, 3.6e-07]], [null, 200000, [3.6e-06, 1.8e-05, 4.5e-06, 7.2e-06, 3.6e-07]], [null, 200001, [7.2e-06, 2.7e-05, 9e-06, 1.44e-05, 7.2e-07]], [null, 10000000, [7.2e-06, 2.7e-05, 9e-06, 1.44e-05, 7.2e-07]]]},
{"info": {"cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.2e-05, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, "input_cost_per_token_above_200k_tokens": 6e-06, "output_cost_per_token_above_200k_tokens": 2.25e-05, "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, "cache_read_input_token_cost_above_200k_tokens": 6e-07, "output_cost_per_token": 1.5e-05, "input_cost_per_token_batches": 1.5e-06, "output_cost_per_token_batches": 7.5e-06}, "inclusive": false, "checks": [[null, 1, [3e-06, 1.5e-05, 3.75e-06, 6e-06, 3e-07]], [null, 200000, [3e-06, 1.5e-05, 3.75e-06, 6e-06, 3e-07]], [null, 200001, [6e-06, 2.25e-05, 7.5e-06, 1.2e-05, 6e-07]], [null, 10000000, [6e-06, 2.25e-05, 7.5e-06, 1.2e-05, 6e-07]]]},
{"info": {"input_cost_per_token": 1.25e-06, "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token": 1e-05, "output_cost_per_token_above_200k_tokens": 1.5e-05}, "inclusive": false, "checks": [[null, 1, [1.25e-06, 1e-05, 1.25e-06, 1.25e-06, 1.25e-06]], [null, 200000, [1.25e-06, 1e-05, 1.25e-06, 1.25e-06, 1.25e-06]], [null, 200001, [2.5e-06, 1.5e-05, 2.5e-06, 2.5e-06, 2.5e-06]], [null, 10000000, [2.5e-06, 1.5e-05, 2.5e-06, 2.5e-06, 2.5e-06]]]},
{"info": {"cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 5e-06}, "inclusive": false, "checks": [[null, 1, [5e-06, 0.0, 5e-06, 5e-06, 1.25e-06]], [null, 10000000, [5e-06, 0.0, 5e-06, 5e-06, 1.25e-06]]]},
{"info": {"input_cost_per_token": 3e-07, "input_cost_per_token_above_512k_tokens": 6e-07, "output_cost_per_token": 1.2e-06, "output_cost_per_token_above_512k_tokens": 2.4e-06, "cache_read_input_token_cost": 6e-08, "cache_read_input_token_cost_above_512k_tokens": 1.2e-07}, "inclusive": false, "checks": [[null, 1, [3e-07, 1.2e-06, 3e-07, 3e-07, 6e-08]], [null, 512000, [3e-07, 1.2e-06, 3e-07, 3e-07, 6e-08]], [null, 512001, [6e-07, 2.4e-06, 6e-07, 6e-07, 1.2e-07]], [null, 10000000, [6e-07, 2.4e-06, 6e-07, 6e-07, 1.2e-07]]]},
{"info": {"cache_read_input_token_cost": 1.25e-06, "cache_read_input_token_cost_batches": 6.25e-07, "input_cost_per_token": 5e-06, "input_cost_per_token_batches": 2.5e-06}, "inclusive": false, "checks": [[null, 1, [5e-06, 0.0, 5e-06, 5e-06, 1.25e-06]], [null, 10000000, [5e-06, 0.0, 5e-06, 5e-06, 1.25e-06]]]},
{"info": {"cache_read_input_token_cost": 5e-09, "cache_read_input_token_cost_batches": 2.5e-09, "cache_read_input_token_cost_flex": 2.5e-09, "input_cost_per_token": 5e-08, "input_cost_per_token_batches": 2.5e-08, "input_cost_per_token_flex": 2.5e-08, "input_cost_per_token_priority": 2.5e-06, "output_cost_per_token": 4e-07, "output_cost_per_token_batches": 2e-07, "output_cost_per_token_flex": 2e-07}, "inclusive": false, "checks": [[null, 1, [5e-08, 4e-07, 5e-08, 5e-08, 5e-09]], [null, 10000000, [5e-08, 4e-07, 5e-08, 5e-08, 5e-09]], ["priority", 1, [2.5e-06, 4e-07, 2.5e-06, 2.5e-06, 5e-09]], ["priority", 10000000, [2.5e-06, 4e-07, 2.5e-06, 2.5e-06, 5e-09]], ["flex", 1, [2.5e-08, 2e-07, 2.5e-08, 2.5e-08, 2.5e-09]], ["flex", 10000000, [2.5e-08, 2e-07, 2.5e-08, 2.5e-08, 2.5e-09]]]},
{"info": {"cache_read_input_token_cost": 2e-08, "cache_read_input_token_cost_batches": 1e-08, "cache_read_input_token_cost_flex": 1e-08, "input_cost_per_token": 2e-07, "output_cost_per_token": 1.25e-06, "input_cost_per_token_batches": 1e-07, "input_cost_per_token_flex": 1e-07, "output_cost_per_token_batches": 6.25e-07, "output_cost_per_token_flex": 6.25e-07}, "inclusive": false, "checks": [[null, 1, [2e-07, 1.25e-06, 2e-07, 2e-07, 2e-08]], [null, 10000000, [2e-07, 1.25e-06, 2e-07, 2e-07, 2e-08]], ["flex", 1, [1e-07, 6.25e-07, 1e-07, 1e-07, 1e-08]], ["flex", 10000000, [1e-07, 6.25e-07, 1e-07, 1e-07, 1e-08]]]},
{"info": {"input_cost_per_token": 3e-05, "input_cost_per_token_above_272k_tokens": 6e-05, "input_cost_per_token_flex": 1.5e-05, "input_cost_per_token_batches": 1.5e-05, "input_cost_per_token_above_272k_tokens_batches": 3e-05, "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, "output_cost_per_token_above_272k_tokens_batches": 0.000135, "input_cost_per_token_above_272k_tokens_flex": 3e-05, "output_cost_per_token_above_272k_tokens_flex": 0.000135}, "inclusive": false, "checks": [[null, 1, [3e-05, 0.00018, 3e-05, 3e-05, 3e-05]], [null, 272000, [3e-05, 0.00018, 3e-05, 3e-05, 3e-05]], [null, 272001, [6e-05, 0.00027, 6e-05, 6e-05, 6e-05]], [null, 10000000, [6e-05, 0.00027, 6e-05, 6e-05, 6e-05]], ["flex", 1, [1.5e-05, 9e-05, 1.5e-05, 1.5e-05, 1.5e-05]], ["flex", 272000, [1.5e-05, 9e-05, 1.5e-05, 1.5e-05, 1.5e-05]], ["flex", 272001, [3e-05, 0.000135, 3e-05, 3e-05, 3e-05]], ["flex", 10000000, [3e-05, 0.000135, 3e-05, 3e-05, 3e-05]]]},
{"info": {"cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, "cache_read_input_token_cost_batches": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens_batches": 5e-07, "cache_read_input_token_cost_flex": 2.5e-07, "cache_read_input_token_cost_priority": 1.25e-06, "input_cost_per_token": 5e-06, "input_cost_per_token_above_272k_tokens": 1e-05, "input_cost_per_token_flex": 2.5e-06, "input_cost_per_token_batches": 2.5e-06, "input_cost_per_token_above_272k_tokens_batches": 5e-06, "input_cost_per_token_priority": 1.25e-05, "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, "output_cost_per_token_flex": 1.5e-05, "output_cost_per_token_batches": 1.5e-05, "output_cost_per_token_above_272k_tokens_batches": 2.25e-05, "output_cost_per_token_priority": 7.5e-05, "input_cost_per_token_above_272k_tokens_flex": 5e-06, "output_cost_per_token_above_272k_tokens_flex": 2.25e-05, "cache_read_input_token_cost_above_272k_tokens_flex": 5e-07}, "inclusive": false, "checks": [[null, 1, [5e-06, 3e-05, 5e-06, 5e-06, 5e-07]], [null, 272000, [5e-06, 3e-05, 5e-06, 5e-06, 5e-07]], [null, 272001, [1e-05, 4.5e-05, 1e-05, 1e-05, 1e-06]], [null, 10000000, [1e-05, 4.5e-05, 1e-05, 1e-05, 1e-06]], ["priority", 1, [1.25e-05, 7.5e-05, 1.25e-05, 1.25e-05, 1.25e-06]], ["priority", 272000, [1.25e-05, 7.5e-05, 1.25e-05, 1.25e-05, 1.25e-06]], ["priority", 272001, [1e-05, 4.5e-05, 1e-05, 1e-05, 1e-06]], ["priority", 10000000, [1e-05, 4.5e-05, 1e-05, 1e-05, 1e-06]], ["flex", 1, [2.5e-06, 1.5e-05, 2.5e-06, 2.5e-06, 2.5e-07]], ["flex", 272000, [2.5e-06, 1.5e-05, 2.5e-06, 2.5e-06, 2.5e-07]], ["flex", 272001, [5e-06, 2.25e-05, 5e-06, 5e-06, 5e-07]], ["flex", 10000000, [5e-06, 2.25e-05, 5e-06, 5e-06, 5e-07]]]},
{"info": {"input_cost_per_token": 3e-05, "input_cost_per_token_above_272k_tokens": 6e-05, "input_cost_per_token_flex": 1.5e-05, "input_cost_per_token_batches": 1.5e-05, "input_cost_per_token_above_272k_tokens_batches": 3e-05, "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, "output_cost_per_token_above_272k_tokens_batches": 0.000135}, "inclusive": false, "checks": [[null, 1, [3e-05, 0.00018, 3e-05, 3e-05, 3e-05]], [null, 272000, [3e-05, 0.00018, 3e-05, 3e-05, 3e-05]], [null, 272001, [6e-05, 0.00027, 6e-05, 6e-05, 6e-05]], [null, 10000000, [6e-05, 0.00027, 6e-05, 6e-05, 6e-05]], ["flex", 1, [1.5e-05, 9e-05, 1.5e-05, 1.5e-05, 1.5e-05]], ["flex", 272000, [1.5e-05, 9e-05, 1.5e-05, 1.5e-05, 1.5e-05]], ["flex", 272001, [6e-05, 0.00027, 6e-05, 6e-05, 6e-05]], ["flex", 10000000, [6e-05, 0.00027, 6e-05, 6e-05, 6e-05]]]},
{"info": {"cache_creation_input_token_cost": 1.25e-07, "cache_creation_input_token_cost_decisions": 1.25e-07, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-07, "cache_creation_input_token_cost_above_272k_tokens_decisions": 2.5e-07, "cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-07, "cache_creation_input_token_cost_above_272k_tokens_priority": 5e-07, "cache_creation_input_token_cost_flex": 6.25e-08, "cache_creation_input_token_cost_priority": 2.5e-07, "cache_read_input_token_cost": 1e-08, "cache_read_input_token_cost_decisions": 1e-08, "cache_read_input_token_cost_above_272k_tokens": 2e-08, "cache_read_input_token_cost_above_272k_tokens_decisions": 2e-08, "cache_read_input_token_cost_above_272k_tokens_flex": 1e-08, "cache_read_input_token_cost_above_272k_tokens_priority": 4e-08, "cache_read_input_token_cost_batches": 5e-09, "cache_read_input_token_cost_above_272k_tokens_batches": 1e-08, "cache_creation_input_token_cost_batches": 6.25e-08, "cache_creation_input_token_cost_above_272k_tokens_batches": 1.25e-07, "cache_read_input_token_cost_flex": 5e-09, "cache_read_input_token_cost_priority": 2e-08, "input_cost_per_token": 1e-07, "input_cost_per_token_decisions": 1e-07, "input_cost_per_token_above_272k_tokens": 2e-07, "input_cost_per_token_above_272k_tokens_decisions": 2e-07, "input_cost_per_token_above_272k_tokens_flex": 1e-07, "input_cost_per_token_above_272k_tokens_priority": 4e-07, "input_cost_per_token_batches": 5e-08, "input_cost_per_token_above_272k_tokens_batches": 1e-07, "input_cost_per_token_flex": 5e-08, "input_cost_per_token_priority": 2e-07, "output_cost_per_token": 5e-07, "output_cost_per_token_decisions": 0.0, "output_cost_per_token_above_272k_tokens": 7.5e-07, "output_cost_per_token_above_272k_tokens_decisions": 0.0, "output_cost_per_token_above_272k_tokens_flex": 3.75e-07, "output_cost_per_token_above_272k_tokens_priority": 1.5e-06, "output_cost_per_token_batches": 2.5e-07, "output_cost_per_token_above_272k_tokens_batches": 3.75e-07, "output_cost_per_token_flex": 2.5e-07, "output_cost_per_token_priority": 1e-06}, "inclusive": false, "checks": [[null, 1, [1e-07, 5e-07, 1.25e-07, 1.25e-07, 1e-08]], [null, 272000, [1e-07, 5e-07, 1.25e-07, 1.25e-07, 1e-08]], [null, 272001, [2e-07, 7.5e-07, 2.5e-07, 2.5e-07, 2e-08]], [null, 10000000, [2e-07, 7.5e-07, 2.5e-07, 2.5e-07, 2e-08]], ["priority", 1, [2e-07, 1e-06, 2.5e-07, 2.5e-07, 2e-08]], ["priority", 272000, [2e-07, 1e-06, 2.5e-07, 2.5e-07, 2e-08]], ["priority", 272001, [4e-07, 1.5e-06, 5e-07, 5e-07, 4e-08]], ["priority", 10000000, [4e-07, 1.5e-06, 5e-07, 5e-07, 4e-08]], ["flex", 1, [5e-08, 2.5e-07, 6.25e-08, 6.25e-08, 5e-09]], ["flex", 272000, [5e-08, 2.5e-07, 6.25e-08, 6.25e-08, 5e-09]], ["flex", 272001, [1e-07, 3.75e-07, 1.25e-07, 1.25e-07, 1e-08]], ["flex", 10000000, [1e-07, 3.75e-07, 1.25e-07, 1.25e-07, 1e-08]]]},
{"info": {"cache_creation_input_token_cost": 1.25e-07, "cache_creation_input_token_cost_above_100k_tokens": 6.25e-07, "cache_read_input_token_cost": 1e-08, "cache_read_input_token_cost_above_100k_tokens": 5e-08, "input_cost_per_token": 1e-07, "input_cost_per_token_above_100k_tokens": 5e-07, "output_cost_per_token": 5e-07, "output_cost_per_token_above_100k_tokens": 2.5e-06}, "inclusive": false, "checks": [[null, 1, [1e-07, 5e-07, 1.25e-07, 1.25e-07, 1e-08]], [null, 100000, [1e-07, 5e-07, 1.25e-07, 1.25e-07, 1e-08]], [null, 100001, [5e-07, 2.5e-06, 6.25e-07, 6.25e-07, 5e-08]], [null, 10000000, [5e-07, 2.5e-06, 6.25e-07, 6.25e-07, 5e-08]]]},
{"info": {"input_cost_per_token": 0, "input_cost_per_token_above_128k_tokens": 0, "output_cost_per_token": 0, "output_cost_per_token_above_128k_tokens": 0}, "inclusive": false, "checks": [[null, 1, [0.0, 0.0, 0.0, 0.0, 0.0]], [null, 128000, [0.0, 0.0, 0.0, 0.0, 0.0]], [null, 128001, [0.0, 0.0, 0.0, 0.0, 0.0]], [null, 10000000, [0.0, 0.0, 0.0, 0.0, 0.0]]]},
{"info": {"cache_read_input_token_cost": 5e-08, "cache_read_input_token_cost_priority": 9e-08, "input_cost_per_token": 5e-07, "input_cost_per_token_batches": 2.5e-07, "input_cost_per_token_flex": 2.5e-07, "input_cost_per_token_priority": 9e-07, "output_cost_per_token": 3e-06, "output_cost_per_token_batches": 1.5e-06, "output_cost_per_token_flex": 1.5e-06, "output_cost_per_token_priority": 5.4e-06}, "inclusive": false, "checks": [[null, 1, [5e-07, 3e-06, 5e-07, 5e-07, 5e-08]], [null, 10000000, [5e-07, 3e-06, 5e-07, 5e-07, 5e-08]], ["priority", 1, [9e-07, 5.4e-06, 9e-07, 9e-07, 9e-08]], ["priority", 10000000, [9e-07, 5.4e-06, 9e-07, 9e-07, 9e-08]], ["flex", 1, [2.5e-07, 1.5e-06, 2.5e-07, 2.5e-07, 5e-08]], ["flex", 10000000, [2.5e-07, 1.5e-06, 2.5e-07, 2.5e-07, 5e-08]]]},
{"info": {"cache_read_input_token_cost": 3e-08, "input_cost_per_token": 3e-07, "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, "input_cost_per_token_priority": 5.4e-07, "output_cost_per_token": 2.5e-06}, "inclusive": false, "checks": [[null, 1, [3e-07, 2.5e-06, 3e-07, 3e-07, 3e-08]], [null, 10000000, [3e-07, 2.5e-06, 3e-07, 3e-07, 3e-08]], ["priority", 1, [5.4e-07, 2.5e-06, 5.4e-07, 5.4e-07, 3e-08]], ["priority", 10000000, [5.4e-07, 2.5e-06, 5.4e-07, 5.4e-07, 3e-08]], ["flex", 1, [1.5e-07, 2.5e-06, 1.5e-07, 1.5e-07, 3e-08]], ["flex", 10000000, [1.5e-07, 2.5e-06, 1.5e-07, 1.5e-07, 3e-08]]]},
{"info": {"cache_read_input_token_cost": 8.3e-08, "input_cost_per_token": 1.65e-07, "input_cost_per_token_batches": 8.3e-08, "output_cost_per_token": 6.6e-07}, "inclusive": false, "checks": [[null, 1, [1.65e-07, 6.6e-07, 1.65e-07, 1.65e-07, 8.3e-08]], [null, 10000000, [1.65e-07, 6.6e-07, 1.65e-07, 1.65e-07, 8.3e-08]]]},
{"info": {"cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_creation_input_token_cost_above_200k_tokens": 2.5e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, "input_cost_per_token_batches": 1e-06, "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, "output_cost_per_token_batches": 6e-06, "input_cost_per_token_priority": 3.6e-06, "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, "output_cost_per_token_priority": 2.16e-05, "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07}, "inclusive": false, "checks": [[null, 1, [2e-06, 1.2e-05, 2e-06, 2e-06, 2e-07]], [null, 200000, [2e-06, 1.2e-05, 2e-06, 2e-06, 2e-07]], [null, 200001, [4e-06, 1.8e-05, 2.5e-07, 2.5e-07, 4e-07]], [null, 10000000, [4e-06, 1.8e-05, 2.5e-07, 2.5e-07, 4e-07]], ["priority", 1, [3.6e-06, 2.16e-05, 3.6e-06, 3.6e-06, 3.6e-07]], ["priority", 200000, [3.6e-06, 2.16e-05, 3.6e-06, 3.6e-06, 3.6e-07]], ["priority", 200001, [7.2e-06, 3.24e-05, 2.5e-07, 2.5e-07, 7.2e-07]], ["priority", 10000000, [7.2e-06, 3.24e-05, 2.5e-07, 2.5e-07, 7.2e-07]]]},
{"info": {"cache_creation_input_token_cost_above_200k_tokens": 2.5e-07, "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, "cache_read_input_token_cost_priority": 3.6e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, "input_cost_per_token_batches": 1e-06, "input_cost_per_token_flex": 1e-06, "input_cost_per_token_priority": 3.6e-06, "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "output_cost_per_token_batches": 6e-06, "output_cost_per_token_flex": 6e-06, "output_cost_per_token_priority": 2.16e-05}, "inclusive": false, "checks": [[null, 1, [2e-06, 1.2e-05, 2e-06, 2e-06, 2e-07]], [null, 200000, [2e-06, 1.2e-05, 2e-06, 2e-06, 2e-07]], [null, 200001, [4e-06, 1.8e-05, 2.5e-07, 2.5e-07, 4e-07]], [null, 10000000, [4e-06, 1.8e-05, 2.5e-07, 2.5e-07, 4e-07]], ["priority", 1, [3.6e-06, 2.16e-05, 3.6e-06, 3.6e-06, 3.6e-07]], ["priority", 200000, [3.6e-06, 2.16e-05, 3.6e-06, 3.6e-06, 3.6e-07]], ["priority", 200001, [7.2e-06, 3.24e-05, 2.5e-07, 2.5e-07, 7.2e-07]], ["priority", 10000000, [7.2e-06, 3.24e-05, 2.5e-07, 2.5e-07, 7.2e-07]], ["flex", 1, [1e-06, 6e-06, 1e-06, 1e-06, 2e-07]], ["flex", 200000, [1e-06, 6e-06, 1e-06, 1e-06, 2e-07]], ["flex", 200001, [4e-06, 1.8e-05, 2.5e-07, 2.5e-07, 4e-07]], ["flex", 10000000, [4e-06, 1.8e-05, 2.5e-07, 2.5e-07, 4e-07]]]},
{"info": {"cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_creation_input_token_cost_above_200k_tokens": 2.5e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, "input_cost_per_token_batches": 1e-06, "output_cost_per_token": 1.2e-05, "output_cost_per_token_above_200k_tokens": 1.8e-05, "output_cost_per_token_batches": 6e-06}, "inclusive": false, "checks": [[null, 1, [2e-06, 1.2e-05, 2e-06, 2e-06, 2e-07]], [null, 200000, [2e-06, 1.2e-05, 2e-06, 2e-06, 2e-07]], [null, 200001, [4e-06, 1.8e-05, 2.5e-07, 2.5e-07, 4e-07]], [null, 10000000, [4e-06, 1.8e-05, 2.5e-07, 2.5e-07, 4e-07]]]},
{"info": {"cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_above_200k_tokens": 2.5e-07, "cache_creation_input_token_cost_above_200k_tokens": 2.5e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 4.5e-07, "cache_read_input_token_cost_flex": 1.25e-07, "cache_read_input_token_cost_priority": 2.25e-07, "input_cost_per_token": 1.25e-06, "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token": 1e-05, "output_cost_per_token_above_200k_tokens": 1.5e-05, "input_cost_per_token_above_200k_tokens_priority": 4.5e-06, "input_cost_per_token_batches": 6.25e-07, "input_cost_per_token_flex": 6.25e-07, "input_cost_per_token_priority": 2.25e-06, "output_cost_per_token_above_200k_tokens_priority": 2.7e-05, "output_cost_per_token_batches": 5e-06, "output_cost_per_token_flex": 5e-06, "output_cost_per_token_priority": 1.8e-05}, "inclusive": false, "checks": [[null, 1, [1.25e-06, 1e-05, 1.25e-06, 1.25e-06, 1.25e-07]], [null, 200000, [1.25e-06, 1e-05, 1.25e-06, 1.25e-06, 1.25e-07]], [null, 200001, [2.5e-06, 1.5e-05, 2.5e-07, 2.5e-07, 2.5e-07]], [null, 10000000, [2.5e-06, 1.5e-05, 2.5e-07, 2.5e-07, 2.5e-07]], ["priority", 1, [2.25e-06, 1.8e-05, 2.25e-06, 2.25e-06, 2.25e-07]], ["priority", 200000, [2.25e-06, 1.8e-05, 2.25e-06, 2.25e-06, 2.25e-07]], ["priority", 200001, [4.5e-06, 2.7e-05, 2.5e-07, 2.5e-07, 4.5e-07]], ["priority", 10000000, [4.5e-06, 2.7e-05, 2.5e-07, 2.5e-07, 4.5e-07]], ["flex", 1, [6.25e-07, 5e-06, 6.25e-07, 6.25e-07, 1.25e-07]], ["flex", 200000, [6.25e-07, 5e-06, 6.25e-07, 6.25e-07, 1.25e-07]], ["flex", 200001, [2.5e-06, 1.5e-05, 2.5e-07, 2.5e-07, 2.5e-07]], ["flex", 10000000, [2.5e-06, 1.5e-05, 2.5e-07, 2.5e-07, 2.5e-07]]]},
{"info": {"cache_creation_input_token_cost": 1.875e-06, "input_cost_per_token": 3.75e-06, "output_cost_per_token": 1.5e-05}, "inclusive": false, "checks": [[null, 1, [3.75e-06, 1.5e-05, 1.875e-06, 1.875e-06, 3.75e-06]], [null, 10000000, [3.75e-06, 1.5e-05, 1.875e-06, 1.875e-06, 3.75e-06]]]},
{"info": {"cache_read_input_token_cost": 2.1e-07, "cache_read_input_token_cost_flex": 1.05e-07, "input_cost_per_token": 8.4e-07, "input_cost_per_token_flex": 4.2e-07, "output_cost_per_token": 3.36e-06, "output_cost_per_token_flex": 1.68e-06}, "inclusive": false, "checks": [[null, 1, [8.4e-07, 3.36e-06, 8.4e-07, 8.4e-07, 2.1e-07]], [null, 10000000, [8.4e-07, 3.36e-06, 8.4e-07, 8.4e-07, 2.1e-07]], ["flex", 1, [4.2e-07, 1.68e-06, 4.2e-07, 4.2e-07, 1.05e-07]], ["flex", 10000000, [4.2e-07, 1.68e-06, 4.2e-07, 4.2e-07, 1.05e-07]]]},
{"info": {"input_cost_per_token": 2.5e-06, "output_cost_per_token": 1.25e-05, "cache_read_input_token_cost": 2.5e-07, "input_cost_per_token_balanced": 2e-06, "output_cost_per_token_balanced": 1e-05, "cache_read_input_token_cost_balanced": 2e-07, "input_cost_per_token_flex": 1.25e-06, "output_cost_per_token_flex": 6.25e-06, "cache_read_input_token_cost_flex": 1.5e-07}, "inclusive": false, "checks": [[null, 1, [2.5e-06, 1.25e-05, 2.5e-06, 2.5e-06, 2.5e-07]], [null, 10000000, [2.5e-06, 1.25e-05, 2.5e-06, 2.5e-06, 2.5e-07]], ["flex", 1, [1.25e-06, 6.25e-06, 1.25e-06, 1.25e-06, 1.5e-07]], ["flex", 10000000, [1.25e-06, 6.25e-06, 1.25e-06, 1.25e-06, 1.5e-07]], ["balanced", 1, [2e-06, 1e-05, 2e-06, 2e-06, 2e-07]], ["balanced", 10000000, [2e-06, 1e-05, 2e-06, 2e-06, 2e-07]]]},
{"info": {"tiered_pricing": [{"input_cost_per_query": 0.00166, "max_results_range": [1, 10]}, {"input_cost_per_query": 0.00332, "max_results_range": [11, 20]}, {"input_cost_per_query": 0.00498, "max_results_range": [21, 30]}, {"input_cost_per_query": 0.00664, "max_results_range": [31, 40]}, {"input_cost_per_query": 0.0083, "max_results_range": [41, 50]}, {"input_cost_per_query": 0.00996, "max_results_range": [51, 60]}, {"input_cost_per_query": 0.01162, "max_results_range": [61, 70]}, {"input_cost_per_query": 0.01328, "max_results_range": [71, 80]}, {"input_cost_per_query": 0.01494, "max_results_range": [81, 90]}, {"input_cost_per_query": 0.0166, "max_results_range": [91, 100]}]}, "inclusive": false, "checks": [[null, 1, [0.0, 0.0, 0.0, 0.0, 0.0]], [null, 10000000, [0.0, 0.0, 0.0, 0.0, 0.0]]]},
{"info": {"tiered_pricing": [{"input_cost_per_query": 0.005, "max_results_range": [0, 25]}, {"input_cost_per_query": 0.025, "max_results_range": [26, 100]}]}, "inclusive": false, "checks": [[null, 1, [0.0, 0.0, 0.0, 0.0, 0.0]], [null, 10000000, [0.0, 0.0, 0.0, 0.0, 0.0]]]},
{"info": {"tiered_pricing": [{"cache_read_input_token_cost": 8e-08, "input_cost_per_token": 4e-07, "output_cost_per_token": 1.6e-06, "range": [0, 256000.0]}, {"cache_read_input_token_cost": 2.4e-07, "input_cost_per_token": 1.2e-06, "output_cost_per_token": 4.8e-06, "range": [256000.0, 1000000.0]}]}, "inclusive": false, "checks": [[null, 1, [4e-07, 1.6e-06, 4e-07, 4e-07, 8e-08]], [null, 256000, [4e-07, 1.6e-06, 4e-07, 4e-07, 8e-08]], [null, 256001, [1.2e-06, 4.8e-06, 1.2e-06, 1.2e-06, 2.4e-07]], [null, 10000000, [1.2e-06, 4.8e-06, 1.2e-06, 1.2e-06, 2.4e-07]]]},
{"info": {"tiered_pricing": [{"input_cost_per_token": 5e-08, "output_cost_per_token": 4e-07, "range": [0, 256000.0]}, {"input_cost_per_token": 2.5e-07, "output_cost_per_token": 2e-06, "range": [256000.0, 1000000.0]}]}, "inclusive": false, "checks": [[null, 1, [5e-08, 4e-07, 5e-08, 5e-08, 5e-08]], [null, 256000, [5e-08, 4e-07, 5e-08, 5e-08, 5e-08]], [null, 256001, [2.5e-07, 2e-06, 2.5e-07, 2.5e-07, 2.5e-07]], [null, 10000000, [2.5e-07, 2e-06, 2.5e-07, 2.5e-07, 2.5e-07]]]},
{"info": {"tiered_pricing": [{"input_cost_per_token": 3e-07, "output_cost_per_token": 1.5e-06, "range": [0, 32000.0]}, {"input_cost_per_token": 5e-07, "output_cost_per_token": 2.5e-06, "range": [32000.0, 128000.0]}, {"input_cost_per_token": 8e-07, "output_cost_per_token": 4e-06, "range": [128000.0, 256000.0]}, {"input_cost_per_token": 1.6e-06, "output_cost_per_token": 9.6e-06, "range": [256000.0, 1000000.0]}]}, "inclusive": false, "checks": [[null, 1, [3e-07, 1.5e-06, 3e-07, 3e-07, 3e-07]], [null, 32000, [3e-07, 1.5e-06, 3e-07, 3e-07, 3e-07]], [null, 32001, [5e-07, 2.5e-06, 5e-07, 5e-07, 5e-07]], [null, 128000, [5e-07, 2.5e-06, 5e-07, 5e-07, 5e-07]], [null, 128001, [8e-07, 4e-06, 8e-07, 8e-07, 8e-07]], [null, 256000, [8e-07, 4e-06, 8e-07, 8e-07, 8e-07]], [null, 256001, [1.6e-06, 9.6e-06, 1.6e-06, 1.6e-06, 1.6e-06]], [null, 10000000, [1.6e-06, 9.6e-06, 1.6e-06, 1.6e-06, 1.6e-06]]]},
{"info": {"tiered_pricing": [{"cache_read_input_token_cost": 8e-08, "input_cost_per_token": 3e-07, "output_cost_per_token": 1.5e-06, "range": [0, 32000.0]}, {"cache_read_input_token_cost": 1.2e-07, "input_cost_per_token": 5e-07, "output_cost_per_token": 2.5e-06, "range": [32000.0, 128000.0]}, {"cache_read_input_token_cost": 2e-07, "input_cost_per_token": 8e-07, "output_cost_per_token": 4e-06, "range": [128000.0, 256000.0]}, {"cache_read_input_token_cost": 4e-07, "input_cost_per_token": 1.6e-06, "output_cost_per_token": 9.6e-06, "range": [256000.0, 1000000.0]}]}, "inclusive": false, "checks": [[null, 1, [3e-07, 1.5e-06, 3e-07, 3e-07, 8e-08]], [null, 32000, [3e-07, 1.5e-06, 3e-07, 3e-07, 8e-08]], [null, 32001, [5e-07, 2.5e-06, 5e-07, 5e-07, 1.2e-07]], [null, 128000, [5e-07, 2.5e-06, 5e-07, 5e-07, 1.2e-07]], [null, 128001, [8e-07, 4e-06, 8e-07, 8e-07, 2e-07]], [null, 256000, [8e-07, 4e-06, 8e-07, 8e-07, 2e-07]], [null, 256001, [1.6e-06, 9.6e-06, 1.6e-06, 1.6e-06, 4e-07]], [null, 10000000, [1.6e-06, 9.6e-06, 1.6e-06, 1.6e-06, 4e-07]]]},
{"info": {"tiered_pricing": [{"input_cost_per_token": 4e-07, "output_cost_per_reasoning_token": 4e-06, "output_cost_per_token": 1.2e-06, "range": [0, 256000.0]}, {"input_cost_per_token": 1.2e-06, "output_cost_per_reasoning_token": 1.2e-05, "output_cost_per_token": 3.6e-06, "range": [256000.0, 1000000.0]}]}, "inclusive": false, "checks": [[null, 1, [4e-07, 1.2e-06, 4e-07, 4e-07, 4e-07]], [null, 256000, [4e-07, 1.2e-06, 4e-07, 4e-07, 4e-07]], [null, 256001, [1.2e-06, 3.6e-06, 1.2e-06, 1.2e-06, 1.2e-06]], [null, 10000000, [1.2e-06, 3.6e-06, 1.2e-06, 1.2e-06, 1.2e-06]]]},
{"info": {"cache_creation_input_token_cost": 1.375e-06, "cache_read_input_token_cost": 1.1e-07, "input_cost_per_token": 1.1e-06, "output_cost_per_token": 5.5e-06, "input_cost_per_token_batches": 5.5e-07, "output_cost_per_token_batches": 2.75e-06}, "inclusive": false, "checks": [[null, 1, [1.1e-06, 5.5e-06, 1.375e-06, 1.375e-06, 1.1e-07]], [null, 10000000, [1.1e-06, 5.5e-06, 1.375e-06, 1.375e-06, 1.1e-07]]]},
{"info": {"cache_read_input_token_cost": 2.75e-07, "cache_read_input_token_cost_above_272k_tokens": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens_batches": 2.75e-07, "cache_read_input_token_cost_batches": 1.43e-07, "cache_read_input_token_cost_priority": 5.5e-07, "input_cost_per_token": 2.75e-06, "input_cost_per_token_priority": 5.5e-06, "output_cost_per_token": 1.65e-05, "output_cost_per_token_priority": 3.3e-05, "input_cost_per_token_above_272k_tokens": 5.5e-06, "input_cost_per_token_above_272k_tokens_batches": 2.75e-06, "input_cost_per_token_batches": 1.375e-06, "output_cost_per_token_above_272k_tokens": 2.475e-05, "output_cost_per_token_above_272k_tokens_batches": 1.2375e-05, "output_cost_per_token_batches": 8.25e-06}, "inclusive": false, "checks": [[null, 1, [2.75e-06, 1.65e-05, 2.75e-06, 2.75e-06, 2.75e-07]], [null, 272000, [2.75e-06, 1.65e-05, 2.75e-06, 2.75e-06, 2.75e-07]], [null, 272001, [5.5e-06, 2.475e-05, 5.5e-06, 5.5e-06, 5.5e-07]], [null, 10000000, [5.5e-06, 2.475e-05, 5.5e-06, 5.5e-06, 5.5e-07]], ["priority", 1, [5.5e-06, 3.3e-05, 5.5e-06, 5.5e-06, 5.5e-07]], ["priority", 272000, [5.5e-06, 3.3e-05, 5.5e-06, 5.5e-06, 5.5e-07]], ["priority", 272001, [5.5e-06, 2.475e-05, 5.5e-06, 5.5e-06, 5.5e-07]], ["priority", 10000000, [5.5e-06, 2.475e-05, 5.5e-06, 5.5e-06, 5.5e-07]]]},
{"info": {"cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, "cache_read_input_token_cost_above_272k_tokens_batches": 5e-07, "cache_read_input_token_cost_priority": 1.25e-06, "cache_read_input_token_cost_above_272k_tokens_priority": 2e-06, "cache_read_input_token_cost_batches": 2.5e-07, "cache_read_input_token_cost_flex": 2.5e-07, "input_cost_per_token": 5e-06, "input_cost_per_token_above_272k_tokens": 1e-05, "input_cost_per_token_above_272k_tokens_batches": 5e-06, "input_cost_per_token_above_272k_tokens_flex": 5e-06, "input_cost_per_token_priority": 1.25e-05, "input_cost_per_token_above_272k_tokens_priority": 2e-05, "input_cost_per_token_batches": 2.5e-06, "input_cost_per_token_flex": 2.5e-06, "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, "output_cost_per_token_above_272k_tokens_batches": 2.25e-05, "output_cost_per_token_priority": 7.5e-05, "output_cost_per_token_above_272k_tokens_priority": 9e-05, "output_cost_per_token_batches": 1.5e-05}, "inclusive": false, "checks": [[null, 1, [5e-06, 3e-05, 5e-06, 5e-06, 5e-07]], [null, 272000, [5e-06, 3e-05, 5e-06, 5e-06, 5e-07]], [null, 272001, [1e-05, 4.5e-05, 1e-05, 1e-05, 1e-06]], [null, 10000000, [1e-05, 4.5e-05, 1e-05, 1e-05, 1e-06]], ["priority", 1, [1.25e-05, 7.5e-05, 1.25e-05, 1.25e-05, 1.25e-06]], ["priority", 272000, [1.25e-05, 7.5e-05, 1.25e-05, 1.25e-05, 1.25e-06]], ["priority", 272001, [2e-05, 9e-05, 2e-05, 2e-05, 2e-06]], ["priority", 10000000, [2e-05, 9e-05, 2e-05, 2e-05, 2e-06]], ["flex", 1, [2.5e-06, 3e-05, 2.5e-06, 2.5e-06, 2.5e-07]], ["flex", 272000, [2.5e-06, 3e-05, 2.5e-06, 2.5e-06, 2.5e-07]], ["flex", 272001, [5e-06, 4.5e-05, 5e-06, 5e-06, 1e-06]], ["flex", 10000000, [5e-06, 4.5e-05, 5e-06, 5e-06, 1e-06]]]},
{"info": {"cache_creation_input_token_cost": 5e-06, "cache_creation_input_token_cost_above_272k_tokens": 1e-05, "cache_creation_input_token_cost_priority": 1e-05, "cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05, "cache_read_input_token_cost": 4e-07, "cache_read_input_token_cost_above_272k_tokens": 8e-07, "cache_read_input_token_cost_priority": 8e-07, "cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06, "input_cost_per_token": 4e-06, "input_cost_per_token_above_272k_tokens": 8e-06, "input_cost_per_token_priority": 8e-06, "input_cost_per_token_above_272k_tokens_priority": 1.6e-05, "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, "output_cost_per_token_priority": 4e-05, "output_cost_per_token_above_272k_tokens_priority": 6e-05}, "inclusive": false, "checks": [[null, 1, [4e-06, 2e-05, 5e-06, 5e-06, 4e-07]], [null, 272000, [4e-06, 2e-05, 5e-06, 5e-06, 4e-07]], [null, 272001, [8e-06, 3e-05, 1e-05, 1e-05, 8e-07]], [null, 10000000, [8e-06, 3e-05, 1e-05, 1e-05, 8e-07]], ["priority", 1, [8e-06, 4e-05, 1e-05, 1e-05, 8e-07]], ["priority", 272000, [8e-06, 4e-05, 1e-05, 1e-05, 8e-07]], ["priority", 272001, [1.6e-05, 6e-05, 2e-05, 2e-05, 1.6e-06]], ["priority", 10000000, [1.6e-05, 6e-05, 2e-05, 2e-05, 1.6e-06]]]},
{"info": {"cache_creation_input_token_cost": 5e-06, "cache_creation_input_token_cost_above_272k_tokens": 1e-05, "cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06, "cache_creation_input_token_cost_priority": 1e-05, "cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05, "cache_creation_input_token_cost_flex": 2.5e-06, "cache_read_input_token_cost": 4e-07, "cache_read_input_token_cost_above_272k_tokens": 8e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 4e-07, "cache_read_input_token_cost_priority": 8e-07, "cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06, "cache_read_input_token_cost_flex": 2e-07, "input_cost_per_token": 4e-06, "input_cost_per_token_above_272k_tokens": 8e-06, "input_cost_per_token_above_272k_tokens_flex": 4e-06, "input_cost_per_token_priority": 8e-06, "input_cost_per_token_above_272k_tokens_priority": 1.6e-05, "input_cost_per_token_flex": 2e-06, "output_cost_per_token": 2e-05, "output_cost_per_token_above_272k_tokens": 3e-05, "output_cost_per_token_above_272k_tokens_flex": 1.5e-05, "output_cost_per_token_priority": 4e-05, "output_cost_per_token_above_272k_tokens_priority": 6e-05, "output_cost_per_token_flex": 1e-05}, "inclusive": false, "checks": [[null, 1, [4e-06, 2e-05, 5e-06, 5e-06, 4e-07]], [null, 272000, [4e-06, 2e-05, 5e-06, 5e-06, 4e-07]], [null, 272001, [8e-06, 3e-05, 1e-05, 1e-05, 8e-07]], [null, 10000000, [8e-06, 3e-05, 1e-05, 1e-05, 8e-07]], ["priority", 1, [8e-06, 4e-05, 1e-05, 1e-05, 8e-07]], ["priority", 272000, [8e-06, 4e-05, 1e-05, 1e-05, 8e-07]], ["priority", 272001, [1.6e-05, 6e-05, 2e-05, 2e-05, 1.6e-06]], ["priority", 10000000, [1.6e-05, 6e-05, 2e-05, 2e-05, 1.6e-06]], ["flex", 1, [2e-06, 1e-05, 2.5e-06, 2.5e-06, 2e-07]], ["flex", 272000, [2e-06, 1e-05, 2.5e-06, 2.5e-06, 2e-07]], ["flex", 272001, [4e-06, 1.5e-05, 5e-06, 5e-06, 4e-07]], ["flex", 10000000, [4e-06, 1.5e-05, 5e-06, 5e-06, 4e-07]]]},
{"info": {"cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, "input_cost_per_token": 3e-05, "input_cost_per_token_above_272k_tokens": 6e-05, "input_cost_per_token_above_272k_tokens_batches": 3e-05, "input_cost_per_token_above_272k_tokens_flex": 3e-05, "input_cost_per_token_batches": 1.5e-05, "input_cost_per_token_flex": 1.5e-05, "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_above_272k_tokens_batches": 0.000135, "output_cost_per_token_above_272k_tokens_flex": 0.000135, "output_cost_per_token_batches": 9e-05, "output_cost_per_token_flex": 9e-05}, "inclusive": false, "checks": [[null, 1, [3e-05, 0.00018, 3e-05, 3e-05, 3e-06]], [null, 272000, [3e-05, 0.00018, 3e-05, 3e-05, 3e-06]], [null, 272001, [6e-05, 0.00027, 6e-05, 6e-05, 6e-06]], [null, 10000000, [6e-05, 0.00027, 6e-05, 6e-05, 6e-06]], ["flex", 1, [1.5e-05, 9e-05, 1.5e-05, 1.5e-05, 3e-06]], ["flex", 272000, [1.5e-05, 9e-05, 1.5e-05, 1.5e-05, 3e-06]], ["flex", 272001, [3e-05, 0.000135, 3e-05, 3e-05, 6e-06]], ["flex", 10000000, [3e-05, 0.000135, 3e-05, 3e-05, 6e-06]]]},
{"info": {"cache_read_input_token_cost": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens": 5e-07, "cache_read_input_token_cost_above_272k_tokens_batches": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 2.5e-07, "cache_read_input_token_cost_priority": 5e-07, "cache_read_input_token_cost_above_272k_tokens_priority": 1e-06, "cache_read_input_token_cost_batches": 1.3e-07, "input_cost_per_token": 2.5e-06, "input_cost_per_token_above_272k_tokens": 5e-06, "input_cost_per_token_priority": 5e-06, "input_cost_per_token_above_272k_tokens_priority": 1e-05, "output_cost_per_token": 1.5e-05, "output_cost_per_token_above_272k_tokens": 2.25e-05, "output_cost_per_token_priority": 3e-05, "output_cost_per_token_above_272k_tokens_priority": 4.5e-05, "input_cost_per_token_above_272k_tokens_batches": 2.5e-06, "input_cost_per_token_above_272k_tokens_flex": 2.5e-06, "input_cost_per_token_batches": 1.25e-06, "input_cost_per_token_flex": 1.25e-06, "output_cost_per_token_above_272k_tokens_batches": 1.125e-05, "output_cost_per_token_above_272k_tokens_flex": 1.125e-05, "output_cost_per_token_batches": 7.5e-06, "output_cost_per_token_flex": 7.5e-06}, "inclusive": false, "checks": [[null, 1, [2.5e-06, 1.5e-05, 2.5e-06, 2.5e-06, 2.5e-07]], [null, 272000, [2.5e-06, 1.5e-05, 2.5e-06, 2.5e-06, 2.5e-07]], [null, 272001, [5e-06, 2.25e-05, 5e-06, 5e-06, 5e-07]], [null, 10000000, [5e-06, 2.25e-05, 5e-06, 5e-06, 5e-07]], ["priority", 1, [5e-06, 3e-05, 5e-06, 5e-06, 5e-07]], ["priority", 272000, [5e-06, 3e-05, 5e-06, 5e-06, 5e-07]], ["priority", 272001, [1e-05, 4.5e-05, 1e-05, 1e-05, 1e-06]], ["priority", 10000000, [1e-05, 4.5e-05, 1e-05, 1e-05, 1e-06]], ["flex", 1, [1.25e-06, 7.5e-06, 1.25e-06, 1.25e-06, 2.5e-07]], ["flex", 272000, [1.25e-06, 7.5e-06, 1.25e-06, 1.25e-06, 2.5e-07]], ["flex", 272001, [2.5e-06, 1.125e-05, 2.5e-06, 2.5e-06, 2.5e-07]], ["flex", 10000000, [2.5e-06, 1.125e-05, 2.5e-06, 2.5e-06, 2.5e-07]]]},
{"info": {"cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, "cache_read_input_token_cost_priority": 1e-06, "cache_read_input_token_cost_above_272k_tokens_priority": 2e-06, "input_cost_per_token": 5e-06, "input_cost_per_token_above_272k_tokens": 1e-05, "input_cost_per_token_priority": 1e-05, "input_cost_per_token_above_272k_tokens_priority": 2e-05, "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, "output_cost_per_token_priority": 6e-05, "output_cost_per_token_above_272k_tokens_priority": 9e-05}, "inclusive": false, "checks": [[null, 1, [5e-06, 3e-05, 5e-06, 5e-06, 5e-07]], [null, 272000, [5e-06, 3e-05, 5e-06, 5e-06, 5e-07]], [null, 272001, [1e-05, 4.5e-05, 1e-05, 1e-05, 1e-06]], [null, 10000000, [1e-05, 4.5e-05, 1e-05, 1e-05, 1e-06]], ["priority", 1, [1e-05, 6e-05, 1e-05, 1e-05, 1e-06]], ["priority", 272000, [1e-05, 6e-05, 1e-05, 1e-05, 1e-06]], ["priority", 272001, [2e-05, 9e-05, 2e-05, 2e-05, 2e-06]], ["priority", 10000000, [2e-05, 9e-05, 2e-05, 2e-05, 2e-06]]]},
{"info": {"cache_read_input_token_cost": 3.125e-07, "input_cost_per_token": 1.25e-06, "input_cost_per_token_batches": 6.25e-07, "input_cost_per_token_priority": 2.1875e-06, "output_cost_per_token": 1e-05, "output_cost_per_token_batches": 5e-06, "output_cost_per_token_priority": 1.75e-05}, "inclusive": false, "checks": [[null, 1, [1.25e-06, 1e-05, 1.25e-06, 1.25e-06, 3.125e-07]], [null, 10000000, [1.25e-06, 1e-05, 1.25e-06, 1.25e-06, 3.125e-07]], ["priority", 1, [2.1875e-06, 1.75e-05, 2.1875e-06, 2.1875e-06, 3.125e-07]], ["priority", 10000000, [2.1875e-06, 1.75e-05, 2.1875e-06, 2.1875e-06, 3.125e-07]]]}
]}

93
litellm-rust/crates/model-catalog/tests/registry_validation.rs
View file

@ -1,93 +0,0 @@
use std::path::{Path, PathBuf};
use litellm_model_catalog::{
Catalog, FallbackGeneralizations, Provenance, validate_model_entry, validate_registry,
};
use rstest::{fixture, rstest};
use serde_json::{Map, Value};
#[fixture]
fn repo_root() -> PathBuf {
Path::new(env!("CARGO_MANIFEST_DIR")).join("../../..")
}
#[rstest]
#[case("model_prices_and_context_window.json")]
#[case("litellm/model_prices_and_context_window_backup.json")]
#[ignore]
fn checked_in_registry_passes_strict_validation(repo_root: PathBuf, #[case] filename: &str) {
let body = std::fs::read(repo_root.join(filename)).unwrap();
let catalog = Catalog::parse(&body, Provenance::default()).unwrap();
validate_registry(&catalog).unwrap();
}
#[rstest]
#[ignore]
fn fallback_generalizations_are_typed(repo_root: PathBuf) {
let body = std::fs::read(repo_root.join("model_prices_and_context_window.json")).unwrap();
let document: Map<String, Value> = serde_json::from_slice(&body).unwrap();
let Some(raw_rules) = document.get("fallback_generalizations") else {
return;
};
let _: FallbackGeneralizations = serde_json::from_value(raw_rules.clone()).unwrap();
let catalog = Catalog::parse(&body, Provenance::default()).unwrap();
assert!(
catalog
.fallback_rules()
.is_some_and(|rules| !rules.is_empty())
);
}
#[rstest]
#[case::missing_provider(serde_json::json!({"mode": "chat"}), "litellm_provider")]
#[case::unknown_field(serde_json::json!({"litellm_provider": "test", "typo": true}), "unknown field")]
#[case::negative_price(serde_json::json!({"litellm_provider": "test", "input_cost_per_token": -1}), "nonnegative")]
#[case::negative_nested_price(serde_json::json!({"litellm_provider": "test", "guardrail_cost_per_unit": {"unit": -1}}), "nonnegative")]
#[case::invalid_mode(serde_json::json!({"litellm_provider": "test", "mode": "invalid"}), "unknown variant")]
#[case::invalid_date(serde_json::json!({"litellm_provider": "test", "deprecation_date": "2026-02-31"}), "deprecation_date")]
#[case::invalid_hours(serde_json::json!({"litellm_provider": "test", "off_peak_pricing": {"hours_utc": "25:00-01:00"}}), "hours_utc")]
#[case::empty_windows(serde_json::json!({"litellm_provider": "test", "off_peak_pricing": {"windows": []}}), "windows is empty")]
#[case::invalid_weekday(serde_json::json!({"litellm_provider": "test", "off_peak_pricing": {"windows": [{"hours_utc": "00:00-01:00", "weekdays": [0]}]}}), "weekdays is invalid")]
#[case::invalid_aliases(serde_json::json!({"litellm_provider": "test", "aliases": ["good", 7]}), "aliases must contain strings")]
#[case::null_aliases(serde_json::json!({"litellm_provider": "test", "aliases": null}), "aliases must be an array")]
#[ignore]
fn registry_validation_rejects_malformed_entries(#[case] entry: Value, #[case] expected: &str) {
let error = validate_model_entry("test", &entry).unwrap_err();
assert!(
litellm_error::Report::new(&error)
.to_string()
.contains(expected)
);
}
#[test]
#[ignore]
fn checked_in_catalog_and_backup_match() {
let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../..");
let current = std::fs::read(root.join("model_prices_and_context_window.json")).unwrap();
let backup =
std::fs::read(root.join("litellm/model_prices_and_context_window_backup.json")).unwrap();
assert_eq!(current, backup);
let catalog = Catalog::parse(&current, Provenance::default()).unwrap();
assert!(
catalog.alias_issues().is_empty(),
"invalid registry aliases"
);
}
#[rstest]
#[case::input("input_cost_per_token_above_100k_tokens")]
#[case::input_batches("input_cost_per_token_above_100k_tokens_batches")]
#[case::output("output_cost_per_token_above_100k_tokens")]
#[case::output_batches("output_cost_per_token_above_100k_tokens_batches")]
#[case::cache_creation("cache_creation_input_token_cost_above_100k_tokens")]
#[case::cache_creation_batches("cache_creation_input_token_cost_above_100k_tokens_batches")]
#[case::cache_creation_1hr("cache_creation_input_token_cost_above_1hr_above_100k_tokens")]
#[case::cache_read("cache_read_input_token_cost_above_100k_tokens")]
#[case::cache_read_batches("cache_read_input_token_cost_above_100k_tokens_batches")]
fn registry_validation_keeps_above_100k_tier_rates(#[case] field: &str) {
let mut entry = Map::new();
entry.insert("litellm_provider".into(), "anthropic".into());
entry.insert(field.into(), 5e-7.into());
validate_model_entry("test", &Value::Object(entry)).unwrap();
}

73
litellm-rust/crates/model-catalog/tests/schema.rs
View file

@ -1,16 +1,17 @@
#![cfg(feature = "schema")]
use std::collections::BTreeSet;
use std::path::Path;
use litellm_model_catalog::{model_entry_json_schema, registry_json_schema};
use litellm_model_catalog::{BUNDLED_REGISTRY, model_entry_json_schema, registry_json_schema};
use rstest::rstest;
use serde_json::{Value, json};
use serde_json::{Map, Value, json};
fn schema() -> Value {
serde_json::to_value(model_entry_json_schema()).expect("generated schema serializes")
}
fn checked_in_catalog() -> Value {
serde_json::from_slice(BUNDLED_REGISTRY).unwrap()
}
fn registry_validator() -> jsonschema::Validator {
let schema = serde_json::to_value(registry_json_schema()).unwrap();
jsonschema::options()
@ -19,26 +20,21 @@ fn registry_validator() -> jsonschema::Validator {
.expect("generated registry schema is valid")
}
#[rstest]
#[case("model_prices_and_context_window.json")]
#[case("litellm/model_prices_and_context_window_backup.json")]
#[ignore]
fn generated_registry_schema_validates_checked_in_catalog(#[case] path: &str) {
let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../..");
let catalog: Value = serde_json::from_slice(&std::fs::read(root.join(path)).unwrap()).unwrap();
#[test]
fn generated_registry_schema_validates_checked_in_catalog() {
let catalog = checked_in_catalog();
let validator = registry_validator();
let errors: Vec<_> = validator
.iter_errors(&catalog)
.map(|error| error.to_string())
.collect();
assert!(errors.is_empty(), "{path}: {errors:?}");
assert!(errors.is_empty(), "{errors:?}");
}
#[rstest]
#[case(json!({"example": {"litellm_provider": "test"}}))]
#[case(json!({"example": {"litellm_provider": "test", "future_field": true}}))]
#[case(json!({"sample_spec": {"litellm_provider": "placeholder"}}))]
#[ignore]
fn generated_registry_schema_keeps_reader_compatibility(#[case] document: Value) {
assert!(registry_validator().is_valid(&document));
}
@ -59,47 +55,55 @@ fn generated_registry_schema_keeps_reader_compatibility(#[case] document: Value)
#[case::invalid_weekday(json!({"litellm_provider": "test", "off_peak_pricing": {"windows": [{"hours_utc": "00:00-01:00", "weekdays": [0]}]}}))]
#[case::invalid_aliases(json!({"litellm_provider": "test", "aliases": "wrong"}))]
#[case::non_object_model(json!(4))]
#[ignore]
fn generated_registry_schema_rejects_invalid_entries(#[case] entry: Value) {
assert!(!registry_validator().is_valid(&json!({"example": entry})));
}
#[rstest]
#[case("model_prices_and_context_window.json")]
#[case("litellm/model_prices_and_context_window_backup.json")]
#[ignore]
fn generated_schema_covers_catalog_fields(#[case] path: &str) {
let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../..");
let catalog: Value = serde_json::from_slice(&std::fs::read(root.join(path)).unwrap()).unwrap();
#[test]
fn generated_schema_covers_catalog_fields() {
let catalog = checked_in_catalog();
let schema = schema();
let properties = schema["properties"]
let any_value = |keys: &Value| -> Map<String, Value> {
keys.as_object()
.expect("schema keys are an object")
.keys()
.map(|key| (key.clone(), json!({})))
.collect()
};
let keys_only = json!({
"type": "object",
"properties": any_value(&schema["properties"]),
"patternProperties": any_value(&schema["patternProperties"]),
"additionalProperties": false,
});
let registry = serde_json::to_value(registry_json_schema()).unwrap();
let sections = registry["properties"]
.as_object()
.expect("ModelInfo schema has properties");
let fields: BTreeSet<&str> = catalog
.expect("registry has sections");
let fields: Map<String, Value> = catalog
.as_object()
.expect("catalog is an object")
.iter()
.filter(|(name, _)| *name != "sample_spec" && *name != "fallback_generalizations")
.filter(|(name, _)| !sections.contains_key(*name))
.flat_map(|(_, entry)| entry.as_object().expect("model entry is an object").keys())
.map(String::as_str)
.filter(|name| *name != "aliases")
.map(|name| (name.clone(), Value::Null))
.collect();
let missing: Vec<_> = fields
.into_iter()
.filter(|name| !properties.contains_key(*name))
let missing: Vec<_> = jsonschema::validator_for(&keys_only)
.unwrap()
.iter_errors(&Value::Object(fields))
.map(|error| error.to_string())
.collect();
assert!(
missing.is_empty(),
"{path}: fields missing from schema: {missing:?}"
"fields missing from schema: {missing:?}"
);
}
#[rstest]
#[case("Mode", "chat")]
#[case("ReasoningEffort", "high")]
#[case("EffortLevel", "high")]
#[case("InputModality", "image")]
#[ignore]
fn generated_schema_includes_enum_values(#[case] definition: &str, #[case] value: &str) {
let schema = schema();
let variants = schema["$defs"][definition]["enum"]
@ -110,7 +114,6 @@ fn generated_schema_includes_enum_values(#[case] definition: &str, #[case] value
}
#[test]
#[ignore]
fn generated_schema_includes_nested_pricing_types() {
let schema = schema();
let definitions = schema["$defs"].as_object().expect("schema has definitions");

6
litellm-rust/crates/python-bridge/src/routes/ocr/host.rs
View file

@ -13,7 +13,7 @@ use pyo3::{
use super::{
errors::route_failure,
project::{OcrHostHandles, SETTINGS_ERROR_MARKER, project_request},
project::{OcrHostHandles, ROUTE_HOST_MODULE, SETTINGS_ERROR_MARKER, project_request},
};
use crate::{marshal::public_response, routes::RequestView};
@ -77,7 +77,7 @@ impl OcrPythonHost {
_ => "",
};
let mapped = py
.import("litellm.rust_bridge.host.routes.ocr")
.import(ROUTE_HOST_MODULE)
.and_then(|module| module.getattr("map_failure"))
.and_then(|map| map.call1((error.value(py), self.view.request(py)?, provider)))
.and_then(|mapped| mapped.extract::<Py<PyBaseException>>().map_err(PyErr::from));
@ -115,7 +115,7 @@ impl PythonBinding for OcrPythonHost {
py: Python<'_>,
response: LiteLLMOcrResponse,
) -> PyResult<Py<PyAny>> {
public_response(py, "litellm.rust_bridge.host.routes.ocr", &response)
public_response(py, ROUTE_HOST_MODULE, &response)
}
fn encode_stream_head(

30
litellm-rust/crates/python-bridge/src/routes/ocr/project.rs
View file

@ -5,7 +5,7 @@ use litellm_inference_ocr::{
types::{LiteLLMOcrRequest, OcrDocumentInput},
wire::{OcrWireRequest, consumed_optional_params, decode_document, decode_request_input},
};
use litellm_llms::base_llm::ocr::error::Error;
use litellm_llms::base_llm::ocr::{error::Error, transformation::DROP_PARAMS_PARAM};
use pyo3::{prelude::*, types::PyDict};
use serde_json::{Map, Value};
@ -19,6 +19,8 @@ use crate::{
python_settings::{PythonSettings, Snapshot},
};
pub(super) const ROUTE_HOST_MODULE: &str = "litellm.rust_bridge.host.routes.ocr";
/// What the host keeps after projection: the caller's token callable that answers the
/// token operation, and the provider name the failure mapping reports.
pub(super) struct OcrHostHandles {
@ -139,6 +141,15 @@ pub(super) fn project_request(
Some(defaults) => fold_module_globals(bound.py(), optional_params, &globals, &defaults)?,
None => optional_params,
};
let optional_params = if names.contains(&DROP_PARAMS_PARAM) {
fold_drop_params(
bound.py(),
optional_params,
present(kwargs, bound, DROP_PARAMS_PARAM)?,
)?
} else {
optional_params
};
let input_sources = request_input_sources(
kwargs,
names
@ -216,6 +227,23 @@ fn fold_module_globals(
.collect())
}
fn fold_drop_params(
py: Python<'_>,
optional_params: Map<String, Value>,
requested: Option<Bound<'_, PyAny>>,
) -> PyResult<Map<String, Value>> {
let drops: bool = py
.import(ROUTE_HOST_MODULE)?
.getattr("drops_unsupported_params")?
.call1((requested,))?
.extract()?;
Ok(optional_params
.into_iter()
.filter(|(name, _)| name != DROP_PARAMS_PARAM)
.chain(drops.then(|| (DROP_PARAMS_PARAM.to_string(), Value::Bool(true))))
.collect())
}
/// A misconfigured litellm setting is the operator's error, not the request's or the
/// provider's, so the host raises it as is instead of mapping it onto a provider failure.
pub(super) const SETTINGS_ERROR_MARKER: &str = "native_settings_error";

5
litellm/__init__.py
View file

@ -681,6 +681,7 @@ moonshot_models: Set = set()
publicai_models: Set = set()
darkbloom_models: Set = set()
openinfer_models: Final[Set[str]] = set()
streamlake_models: Final[Set[str]] = set()
zerogpu_models: Final[Set[str]] = set()
v0_models: Set = set()
morph_models: Set = set()
@ -970,6 +971,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
darkbloom_models.add(key)
elif value.get("litellm_provider") == "openinfer":
openinfer_models.add(key)
elif value.get("litellm_provider") == "streamlake":
streamlake_models.add(key)
elif value.get("litellm_provider") == "zerogpu":
zerogpu_models.add(key)
elif value.get("litellm_provider") == "v0":
@ -1145,6 +1148,7 @@ model_list = list(
| publicai_models
| darkbloom_models
| openinfer_models
| streamlake_models
| zerogpu_models
| v0_models
| morph_models
@ -1265,6 +1269,7 @@ def _build_models_by_provider() -> dict:
"publicai": publicai_models,
"darkbloom": darkbloom_models,
"openinfer": openinfer_models,
"streamlake": streamlake_models,
"zerogpu": zerogpu_models,
"v0": v0_models,
"morph": morph_models,

2
litellm/constants.py
View file

@ -1064,6 +1064,7 @@ openai_compatible_endpoints: Final[list[str]] = [
"https://api.reka.ai/v1",
"https://gigachat.devices.sberbank.ru/api/v1",
"https://api.openinfer.ai/v1",
"https://vanchin.streamlake.ai/api/gateway/v1/endpoints",
]
@ -1145,6 +1146,7 @@ openai_compatible_providers: Final[list[str]] = [
"prism",
"sail",
"reka",
"streamlake",
]
OPENAI_AUDIO_TRANSCRIPTION_PROVIDERS: Final = frozenset({"openai"} | frozenset(openai_compatible_providers))

2
litellm/cost_calculator.py
View file

@ -688,7 +688,7 @@ def cost_per_token(
elif call_type == "atranscription" or call_type == "transcription":
if _transcription_usage_has_token_details(usage_block):
return generic_cost_per_token(
model=model_without_prefix,
model=model,
usage=usage_block,
custom_llm_provider=custom_llm_provider,
service_tier=service_tier,

1
litellm/litellm_core_utils/get_litellm_params.py
View file

@ -77,6 +77,7 @@ OPTIONAL_KWARGS_KEYS: Final = (
"otpm",
"tpd",
"rpd",
"daily_reset_time",
"use_xai_oauth",
"chatgpt_auth_file",
"github_copilot_auth_type",

3
litellm/litellm_core_utils/litellm_logging.py
View file

@ -1130,9 +1130,6 @@ class Logging(LiteLLMLoggingBaseClass):
}
self.litellm_request_debug = litellm_params.get("litellm_request_debug", False)
self.logger_fn = litellm_params.get("logger_fn", None)
if is_debugging_on() or self.litellm_request_debug:
verbose_logger.debug("self.optional_params: %s", self.optional_params)
self.model_call_details.update(
{
"model": self.model,

11
litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py
View file

@ -27,13 +27,18 @@ class TranscriptionUsageObjectTransformation:
if isinstance(usage_object, TranscriptionUsageDurationObject):
return None
elif isinstance(usage_object, TranscriptionUsageTokensObject):
input_token_details: Final = usage_object.input_token_details
return Usage(
prompt_tokens=usage_object.input_tokens,
completion_tokens=usage_object.output_tokens,
total_tokens=usage_object.total_tokens,
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=usage_object.input_token_details.text_tokens,
audio_tokens=usage_object.input_token_details.audio_tokens,
prompt_tokens_details=(
None
if input_token_details is None
else PromptTokensDetailsWrapper(
text_tokens=input_token_details.text_tokens,
audio_tokens=input_token_details.audio_tokens,
)
),
)
return None

14
litellm/litellm_core_utils/streaming_handler.py
View file

@ -92,6 +92,18 @@ def _next_sync_or_exhausted(it: Iterator[object]) -> object:
return _SYNC_ITER_EXHAUSTED
def _stamp_usage_cost(
usage_chunk: ModelResponse | TextCompletionResponse | ModelResponseStream,
complete_response: ModelResponse | TextCompletionResponse | ModelResponseStream,
) -> None:
bill: Final = (get_hidden_params(complete_response) or {}).get("response_cost")
usage: Final[object] = getattr(usage_chunk, "usage", None)
if usage is None or isinstance(bill, bool) or not isinstance(bill, (int, float)):
return
if bill > 0 or isinstance(getattr(usage, "cost", None), (int, float)):
setattr(usage, "cost", float(bill)) # noqa: B010 # cost is an optional extra on the usage model
def _stamp_served_service_tier(response: ModelResponseStream, complete_streaming_response: ModelResponse) -> None:
served_tier: Final = complete_streaming_response.model_dump().get("service_tier")
if isinstance(served_tier, str) and served_tier:
@ -2141,6 +2153,7 @@ class CustomStreamWrapper:
getattr(complete_streaming_response, "usage"),
)
_stamp_served_service_tier(response, complete_streaming_response)
_stamp_usage_cost(response, complete_streaming_response)
try:
_cache_copy = complete_streaming_response.model_copy(deep=True)
_log_copy = complete_streaming_response.model_copy(deep=True)
@ -2361,6 +2374,7 @@ class CustomStreamWrapper:
getattr(complete_streaming_response, "usage"),
)
_stamp_served_service_tier(response, complete_streaming_response)
_stamp_usage_cost(response, complete_streaming_response)
try:
_copy = complete_streaming_response.model_copy(deep=True)
except RuntimeError:

8
litellm/llms/azure/common_utils.py
View file

@ -621,7 +621,7 @@ class BaseAzureLLM(BaseOpenAILLM):
# For Azure v1 API, use standard OpenAI client instead of AzureOpenAI
# See: https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#api-specs
if self._is_azure_v1_api_version(api_version):
if self.is_azure_v1_api_version(api_version):
# Extract only params that OpenAI client accepts
# Always use /openai/v1/ regardless of whether user passed "v1", "latest", or "preview"
# The OpenAI client accepts a callable for `api_key` and re-invokes it
@ -922,7 +922,7 @@ class BaseAzureLLM(BaseOpenAILLM):
else:
new_url = api_base
if BaseAzureLLM._is_azure_v1_api_version(api_version):
if BaseAzureLLM.is_azure_v1_api_version(api_version):
# ensure the request go to /openai/v1 and not just /openai
if "/openai/v1" not in new_url:
parsed_url: Final = httpx.URL(new_url)
@ -952,7 +952,7 @@ class BaseAzureLLM(BaseOpenAILLM):
Returns None when ``api_version`` is a dated one, which still uses the deployment route.
"""
if not BaseAzureLLM._is_azure_v1_api_version(api_version):
if not BaseAzureLLM.is_azure_v1_api_version(api_version):
return None
base_url: Final = httpx.URL(api_base)
@ -970,7 +970,7 @@ class BaseAzureLLM(BaseOpenAILLM):
)
@staticmethod
def _is_azure_v1_api_version(api_version: str | None) -> bool:
def is_azure_v1_api_version(api_version: str | None) -> bool:
if api_version is None:
return False
return api_version in {"preview", "latest", "v1"}

28
litellm/llms/azure/responses/transformation.py
View file

@ -54,6 +54,16 @@ def _resolve_responses_api_version(api_base: str | None, litellm_params: Mapping
return api_version
def _v1_responses_url(api_base: str) -> httpx.URL:
parsed_url: Final = httpx.URL(api_base)
path: Final = parsed_url.path.rstrip("/")
root: Final = next(
(path.removesuffix(route) for route in ("/openai/v1/responses", "/openai/responses") if path.endswith(route)),
path,
)
return parsed_url.copy_with(path=f"{root}/openai/v1/responses", query=None)
def _with_responses_api_version(api_base: str | None, litellm_params: Mapping[str, object]) -> Mapping[str, object]:
return MappingProxyType({**litellm_params, "api_version": _resolve_responses_api_version(api_base, litellm_params)})
@ -247,20 +257,20 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
if api_base is None:
raise ValueError("api_base is required for Azure WebSocket")
parsed_url: Final = httpx.URL(api_base)
path = parsed_url.path.rstrip("/")
# Strip existing /openai/responses path if the api_base already contains it
for suffix in ("/openai/v1/responses", "/openai/responses"):
if path.endswith(suffix):
path = path[: -len(suffix)]
break
scheme: Final = "wss" if parsed_url.scheme == "https" else "ws"
return str(parsed_url.copy_with(scheme=scheme, path=f"{path}/openai/v1/responses", query=None))
v1_url: Final = _v1_responses_url(api_base)
return str(v1_url.copy_with(scheme="wss" if v1_url.scheme == "https" else "ws"))
def model_in_websocket_url(self) -> bool:
# Azure sends the model in the response.create body, not the URL
return False
def compact_url(self, api_base: str) -> str:
params: Final = httpx.URL(api_base).params
v1_params: Final = (
params if BaseAzureLLM.is_azure_v1_api_version(params.get("api-version")) else params.remove("api-version")
)
return super().compact_url(str(_v1_responses_url(api_base).copy_with(params=v1_params)))
#########################################################
########## DELETE RESPONSE API TRANSFORMATION ##############
#########################################################

9
litellm/llms/azure_ai/common_utils.py
View file

@ -140,6 +140,15 @@ def get_azure_ai_auth_headers(
AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY: Final = "azure_model_router_selected_model"
FOUNDRY_CLASSIC_AGENT_ID_PREFIX: Final = "asst_"
def is_foundry_project_agent(model: str | None) -> bool:
if model is None or AzureFoundryModelInfo.get_azure_ai_route(model) != "agents":
return False
return not model.split("agents/", 1)[1].startswith(FOUNDRY_CLASSIC_AGENT_ID_PREFIX)
def azure_ai_supports_native_responses(model: str | None, api_base: str | None) -> bool:
resolved_base: Final = AzureFoundryModelInfo.get_api_base(api_base)
if resolved_base is not None and not is_azure_openai_v1_host(resolved_base):

263
litellm/llms/azure_ai/responses/transformation.py
View file

@ -1,26 +1,64 @@
from typing import Final
import os
from collections.abc import Collection, Mapping, Sequence
from typing import TYPE_CHECKING, Final, NoReturn
import httpx
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
import litellm
from litellm.exceptions import UnsupportedParamsError
from litellm.llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig
from litellm.llms.azure_ai.chat.transformation import AzureAIGPT5Config
from litellm.llms.azure_ai.common_utils import (
AZURE_AI_AGENTS_SCOPE,
AzureFoundryModelInfo,
api_key_header_for_base,
get_azure_ai_auth_headers,
)
from litellm.types.llms.azure_ai import FoundryAgentReference
from litellm.types.llms.openai import (
ResponseInputParam,
ResponsesAPIOptionalRequestParams,
ResponsesAPIResponse,
ResponsesAPIStreamingResponse,
)
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import LlmProviders
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
_PROJECT_PATH_PREFIX: Final = ("api", "projects")
_RESPONSES_PATH: Final = ("openai", "v1", "responses")
_AGENT_OWNED_PARAMS: Final = frozenset({"temperature", "top_p", "tools", "text", "reasoning"})
_CALLER_AGENT_KEYS: Final = ("agent", "agent_reference")
_RESPONSE_FIELDS: Final = TypeAdapter(dict[str, object])
def _project_root(base_url: httpx.URL) -> tuple[str, ...]:
segments: Final = tuple(segment for segment in base_url.path.split("/") if segment)
return segments[:3] if segments[:2] == _PROJECT_PATH_PREFIX else ()
def _responses_url(api_base: str) -> str:
base_url: Final = httpx.URL(api_base)
segments: Final = tuple(segment for segment in base_url.path.split("/") if segment)
project_root: Final = segments[:3] if segments[:2] == _PROJECT_PATH_PREFIX else ()
return str(base_url.copy_with(path="/" + "/".join((*project_root, *_RESPONSES_PATH)), query=None))
return str(base_url.copy_with(path="/" + "/".join((*_project_root(base_url), *_RESPONSES_PATH)), query=None))
def _without_agent_instructions(response: ResponsesAPIResponse) -> ResponsesAPIResponse:
if (response.model_extra or {}).get("agent_reference") is None:
return response
return response.model_copy(update={"instructions": None})
def _event_without_agent_instructions(event: Mapping[str, object]) -> Mapping[str, object]:
response: Final = event.get("response")
if not isinstance(response, Mapping):
return event
fields: Final = _RESPONSE_FIELDS.validate_python(response)
if fields.get("agent_reference") is None:
return event
return {**event, "response": {**fields, "instructions": None}}
class AzureAIResponsesAPIConfig(AzureOpenAIResponsesAPIConfig):
@ -42,10 +80,19 @@ class AzureAIResponsesAPIConfig(AzureOpenAIResponsesAPIConfig):
def validate_environment(self, headers: dict, model: str, litellm_params: GenericLiteLLMParams | None) -> dict:
params: Final = litellm_params or GenericLiteLLMParams()
api_base: Final = AzureFoundryModelInfo.get_api_base(params.api_base)
on_project: Final = api_base is not None and bool(_project_root(httpx.URL(api_base)))
auth_params: Final = (
params.model_copy(
update={"azure_scope": params.azure_scope or os.getenv("AZURE_SCOPE") or AZURE_AI_AGENTS_SCOPE}
)
if on_project
else params
)
auth_headers: Final = get_azure_ai_auth_headers(
api_key=AzureFoundryModelInfo.get_api_key(params.api_key),
litellm_params=params.model_dump(),
api_key_header=api_key_header_for_base(AzureFoundryModelInfo.get_api_base(params.api_base)),
litellm_params=auth_params.model_dump(),
api_key_header="Authorization" if on_project else api_key_header_for_base(api_base),
)
return {
**headers,
@ -64,3 +111,207 @@ class AzureAIResponsesAPIConfig(AzureOpenAIResponsesAPIConfig):
"Set the api_base parameter or the AZURE_AI_API_BASE environment variable."
)
return _responses_url(resolved_base)
def transform_response_api_response(
self, model: str, raw_response: httpx.Response, logging_obj: "LiteLLMLoggingObj"
) -> ResponsesAPIResponse:
return _without_agent_instructions(super().transform_response_api_response(model, raw_response, logging_obj))
def transform_streaming_response(
self,
model: str,
parsed_chunk: Mapping[str, object],
logging_obj: "LiteLLMLoggingObj",
) -> ResponsesAPIStreamingResponse:
return super().transform_streaming_response( # pyright: ignore[reportUnknownMemberType] # parent types parsed_chunk as a bare dict
model=model, parsed_chunk=dict(_event_without_agent_instructions(parsed_chunk)), logging_obj=logging_obj
)
def transform_get_response_api_response(
self, raw_response: httpx.Response, logging_obj: "LiteLLMLoggingObj"
) -> ResponsesAPIResponse:
return _without_agent_instructions(super().transform_get_response_api_response(raw_response, logging_obj))
def transform_cancel_response_api_response(
self, raw_response: httpx.Response, logging_obj: "LiteLLMLoggingObj"
) -> ResponsesAPIResponse:
return _without_agent_instructions(super().transform_cancel_response_api_response(raw_response, logging_obj))
def _agent_reference(model: str) -> FoundryAgentReference:
name, _, version = model.split("agents/", 1)[1].partition(":")
if version:
return FoundryAgentReference(type="agent_reference", name=name, version=version)
return FoundryAgentReference(type="agent_reference", name=name)
class _CallerAgentReference(BaseModel):
model_config = ConfigDict(frozen=True, coerce_numbers_to_str=True)
name: str
version: str | None = None
def _caller_agent_reference(value: object) -> _CallerAgentReference | None:
try:
return _CallerAgentReference.model_validate(value)
except ValidationError:
return None
def _requested_agent(
deployment_agent: FoundryAgentReference, caller_agents: Sequence[_CallerAgentReference | None]
) -> FoundryAgentReference:
requested_versions: Final = frozenset(
agent.version
for agent in caller_agents
if agent is not None and agent.name == deployment_agent["name"] and agent.version is not None
)
if "version" in deployment_agent or len(requested_versions) != 1:
return deployment_agent
return FoundryAgentReference(
type="agent_reference", name=deployment_agent["name"], version=next(iter(requested_versions))
)
def _caller_agent_fields(fields: Mapping[str, object]) -> tuple[tuple[str, object], ...]:
return tuple((key, fields[key]) for key in _CALLER_AGENT_KEYS if key in fields)
def _deployment_agent(
model: str, caller_fields: Sequence[tuple[str, object]], litellm_params: GenericLiteLLMParams
) -> FoundryAgentReference:
caller_keys: Final = tuple(key for key, _ in caller_fields)
caller_agents: Final = tuple(_caller_agent_reference(value) for _, value in caller_fields)
agent: Final = _requested_agent(_agent_reference(model), caller_agents)
other_agents: Final = tuple(
key
for key, caller_agent in zip(caller_keys, caller_agents)
if caller_agent is None
or caller_agent.name != agent["name"]
or caller_agent.version not in (None, agent.get("version"))
)
if other_agents and not (litellm.drop_params is True or litellm_params.drop_params is True):
raise UnsupportedParamsError(
message=(
f"This model entry always calls Foundry agent {agent['name']!r}, so `{other_agents[0]}` "
"in the request cannot pick another agent or version. Remove it, call the model entry for that "
"agent, or set `drop_params: true` to drop it"
),
llm_provider=LlmProviders.AZURE_AI.value,
model=model,
)
return agent
_INPUT_ITEMS: Final = TypeAdapter(list[object])
def _with_system_prompt(instructions: str, request_input: object) -> Sequence[object]:
conversation: Final = (
[{"type": "message", "role": "user", "content": request_input}]
if isinstance(request_input, str)
else _INPUT_ITEMS.validate_python(request_input)
)
return [{"type": "message", "role": "system", "content": instructions}, *conversation]
class AzureAIAgentResponsesAPIConfig(AzureAIResponsesAPIConfig):
def should_fake_stream(
self, model: str | None, stream: bool | None, custom_llm_provider: str | None = None
) -> bool:
return False
def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: override contract
inherited: Final[list[str]] = super().get_supported_openai_params(model) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # parent returns a bare list # mutable-ok: same
return [param for param in inherited if param not in _AGENT_OWNED_PARAMS]
def map_openai_params_for_deployment(
self,
response_api_optional_params: ResponsesAPIOptionalRequestParams,
model: str,
drop_params: bool,
*,
declared_reasoning_efforts: Collection[str],
) -> Mapping[str, object]:
mapped: Final = super().map_openai_params_for_deployment(
response_api_optional_params, model, drop_params, declared_reasoning_efforts=declared_reasoning_efforts
)
resends_instructions: Final = bool(mapped.get("instructions")) and bool(mapped.get("previous_response_id"))
if resends_instructions and not drop_params:
raise UnsupportedParamsError(
message=(
"azure_ai Foundry agents store a turn's system message with the conversation, so `instructions` "
"with `previous_response_id` would add another copy each turn. Send `instructions` on the first "
"turn only, or set `drop_params: true` to drop it"
),
llm_provider=LlmProviders.AZURE_AI.value,
model=model,
)
dropped: Final = _AGENT_OWNED_PARAMS | (
frozenset({"instructions"}) if resends_instructions else frozenset[str]()
)
return {key: value for key, value in mapped.items() if key not in dropped}
def transform_responses_api_request(
self,
model: str,
input: str | ResponseInputParam,
response_api_optional_request_params: dict[str, object], # mutable-ok: signature fixed by the override contract
litellm_params: GenericLiteLLMParams,
headers: dict[str, str], # mutable-ok: same
) -> dict[str, object]: # mutable-ok: same
request: Final[dict[str, object]] = super().transform_responses_api_request( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # parent returns a bare dict # mutable-ok: same
model=model,
input=input,
response_api_optional_request_params=response_api_optional_request_params,
litellm_params=litellm_params,
headers=headers,
)
instructions: Final = request.get("instructions")
return {
**{key: value for key, value in request.items() if key not in ("model", "instructions")},
**(
{"input": _with_system_prompt(instructions, request.get("input"))}
if isinstance(instructions, str) and instructions
else {}
),
"agent_reference": _deployment_agent(
model, _caller_agent_fields(litellm_params.model_extra or {}), litellm_params
),
}
def transform_extra_body(
self,
extra_body: Mapping[str, object],
request: Mapping[str, object],
model: str,
litellm_params: GenericLiteLLMParams,
) -> Mapping[str, object]:
passthrough: Final = {key: value for key, value in extra_body.items() if key not in _CALLER_AGENT_KEYS}
if len(passthrough) == len(extra_body):
return passthrough
return {
**passthrough,
"agent_reference": _deployment_agent(
model, (*_caller_agent_fields(request), *_caller_agent_fields(extra_body)), litellm_params
),
}
def transform_compact_response_api_request(
self,
model: str,
input: str | ResponseInputParam,
response_api_optional_request_params: dict[str, object], # mutable-ok: signature fixed by the override contract
api_base: str,
litellm_params: GenericLiteLLMParams,
headers: dict[str, str], # mutable-ok: same
) -> NoReturn:
raise UnsupportedParamsError(
message=(
"azure_ai Foundry agents cannot be compacted, because the project's /responses/compact endpoint "
"takes a model deployment, not an agent. Call compact on a model entry for that deployment instead"
),
llm_provider=LlmProviders.AZURE_AI.value,
model=model,
)

75
litellm/llms/bedrock/audio_transcription/__init__.py
View file

@ -1,7 +1,11 @@
import base64
import math
from collections.abc import Mapping
from dataclasses import dataclass
from typing import Final
import httpx
from pydantic import BaseModel, ConfigDict
from litellm.litellm_core_utils.audio_utils.utils import process_audio_file
from litellm.rust_bridge import runtime
@ -14,7 +18,72 @@ from litellm.rust_bridge.transcription.native import (
RustAtranscription,
RustTranscription,
)
from litellm.types.utils import FileTypes, TranscriptionResponse
from litellm.secret_managers.main import get_secret_str
from litellm.types.utils import FileTypes, TranscriptionResponse, TranscriptionUsageTokensObject
SUPPORTED_OPENAI_TRANSCRIPTION_PARAMS: Final = ("language", "prompt", "temperature", "response_format")
TEXT_RESPONSE_FORMATS: Final = frozenset({"json", "text"})
def _as_float(value: object) -> float | None:
if isinstance(value, bool) or not isinstance(value, (int, float, str)):
return None
try:
number: Final = float(value)
except ValueError:
return None
return number if math.isfinite(number) else None
def _rejection(key: str, value: object) -> str | None:
if key not in SUPPORTED_OPENAI_TRANSCRIPTION_PARAMS:
return f"bedrock transcription does not support {key}."
if key == "response_format" and value not in TEXT_RESPONSE_FORMATS:
return f"bedrock transcription returns only the transcript text, so response_format={value!r} is not supported."
if key == "temperature" and _as_float(value) is None:
return f"bedrock transcription needs a number for temperature, got {value!r}."
return None
@dataclass(frozen=True, slots=True)
class UnsupportedTranscriptionParams:
message: str
def map_openai_transcription_params(
non_default_params: Mapping[str, object], drop_params: bool
) -> Mapping[str, object] | UnsupportedTranscriptionParams:
rejections: Final = {
key: reason for key, value in non_default_params.items() if (reason := _rejection(key, value)) is not None
}
if rejections and not drop_params:
return UnsupportedTranscriptionParams(
message=f"{' '.join(rejections.values())} To drop it from the call, set `litellm.drop_params = True`."
)
return {
key: _as_float(value) if key == "temperature" else value
for key, value in non_default_params.items()
if key not in rejections
}
class _RustTranscriptionResult(BaseModel):
model_config = ConfigDict(frozen=True)
text: str
usage: TranscriptionUsageTokensObject | None = None
def _transcription_response(
result: Mapping[str, object], optional_params: Mapping[str, object]
) -> TranscriptionResponse:
parsed: Final = _RustTranscriptionResult.model_validate(result)
response: Final = TranscriptionResponse(text=parsed.text).model_copy(update={"usage": parsed.usage})
region_name: Final = (
optional_params.get("aws_region_name") or get_secret_str("AWS_REGION_NAME") or get_secret_str("AWS_REGION")
)
response.hidden_params = {**response.hidden_params, "region_name": region_name}
return response
class BedrockAudioTranscriptionRustDispatch:
@ -64,7 +133,7 @@ class BedrockAudioTranscriptionRustDispatch:
"timeout_seconds": timeout_to_seconds(timeout),
}
call: Final = NativeCall(args=(), kwargs=fields, base={})
return TranscriptionResponse(**rust(call))
return _transcription_response(rust(call), optional_params)
return runtime.run(
RouteContext(Route.TRANSCRIPTION, provider=custom_llm_provider, model=model),
@ -97,7 +166,7 @@ class BedrockAudioTranscriptionRustDispatch:
"timeout_seconds": timeout_to_seconds(timeout),
}
call: Final = NativeCall(args=(), kwargs=fields, base={})
return TranscriptionResponse(**await rust(call))
return _transcription_response(await rust(call), optional_params)
return await runtime.arun(
RouteContext(Route.TRANSCRIPTION, provider=custom_llm_provider, model=model),

192
litellm/llms/openai/responses/guardrail_translation/handler.py
View file

@ -32,11 +32,11 @@ import copy
import json
import time
import uuid
from collections.abc import Mapping, MutableSequence, Sequence
from collections.abc import Iterable, Mapping, MutableSequence, Sequence
from dataclasses import dataclass
from itertools import accumulate, chain, repeat
from itertools import accumulate, chain, groupby, repeat
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, NamedTuple, Protocol, Union, cast
from typing import TYPE_CHECKING, Any, Final, NamedTuple, Protocol, TypeVar, Union, cast
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
@ -55,7 +55,9 @@ from litellm.llms.base_llm.guardrail_translation.base_translation import (
)
from litellm.llms.base_llm.guardrail_translation.utils import (
blocked_responses_stream_usage,
effective_scan_only_tool_results_for_guardrail,
effective_skip_system_message_for_guardrail,
effective_skip_tool_message_for_guardrail,
list_field,
merge_guardrailed_scoped_messages,
role_out_of_guardrail_scope,
@ -65,7 +67,10 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
stream_item_items,
unappliable_request_rewrite,
)
from litellm.llms.openai.responses.guardrail_translation.tool_merge import merge_guardrailed_tools
from litellm.llms.openai.responses.guardrail_translation.tool_merge import (
append_returned_tools,
merge_guardrailed_tools,
)
from litellm.responses.litellm_completion_transformation.compaction_items import (
compaction_summary,
decode_compaction_item,
@ -74,7 +79,9 @@ from litellm.responses.litellm_completion_transformation.compaction_items import
from litellm.responses.litellm_completion_transformation.custom_tools import without_malformed_tool_flags
from litellm.responses.litellm_completion_transformation.tool_search import is_server_tool_search_tool
from litellm.responses.litellm_completion_transformation.transformation import (
TOOL_CALL_OUTPUT_ITEM_TYPES,
LiteLLMCompletionResponsesConfig,
tool_output_to_chat_content,
)
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.llms.openai import (
@ -270,6 +277,7 @@ _TOOL_CALL_PAYLOAD_EVENT_TYPES: Final = _TOOL_CALL_PAYLOAD_DELTA_EVENT_TYPES | f
)
_OUTPUT_ITEM_EVENT_TYPES: Final = frozenset({"response.output_item.added", "response.output_item.done"})
_OUTPUT_TEXT_EVENT_TYPES: Final = frozenset({"response.output_text.delta", "response.output_text.done"})
_TOOL_OUTPUT_TEXT_PART_TYPES: Final = frozenset({"input_text", "output_text", "text"})
_PATCHABLE_ITEM_FIELDS: Final[Mapping[str, str]] = MappingProxyType(
{"function_call_output": "output", "custom_tool_call_output": "output", "message": "content"}
)
@ -519,10 +527,109 @@ def scannable_instructions(data: Mapping[str, object], *, skip_system: bool = Fa
def _input_item_role(item: object) -> str:
if stream_item_field(item, "type") in TOOL_CALL_OUTPUT_ITEM_TYPES:
return "tool"
role: Final = item.get("role") if isinstance(item, Mapping) else None
return role.lower() if isinstance(role, str) else ""
def _scanned_text_field(item: Mapping[str, object]) -> str:
return "output" if item.get("type") in TOOL_CALL_OUTPUT_ITEM_TYPES else "content"
def _tool_output_part_text(part: object) -> str | None:
if not isinstance(part, dict):
return None
fields: Final = _item_fields_adapter.validate_python(part)
text: Final = fields.get("text")
return text if fields.get("type") in _TOOL_OUTPUT_TEXT_PART_TYPES and isinstance(text, str) else None
def tool_output_texts(item: object) -> tuple[tuple[int | None, str], ...]:
if not stream_item_field(item, "call_id") or stream_item_field(item, "type") == "computer_call_output":
return ()
parts: Final = stream_item_items(item, "output")
part_texts: Final = tuple(
(index, text) for index, part in enumerate(parts) if (text := _tool_output_part_text(part))
)
if part_texts:
return part_texts
tool_content: Final = tool_output_to_chat_content(stream_item_field(item, "output"))
return ((None, tool_content),) if isinstance(tool_content, str) and tool_content else ()
_Grouped: Final = TypeVar("_Grouped")
def _pair_call_id(pair: tuple[str, object]) -> str:
return pair[0]
def _grouped_by_call_id(pairs: Iterable[tuple[str, _Grouped]]) -> Mapping[str, tuple[_Grouped, ...]]:
ordered: Final = sorted(pairs, key=_pair_call_id)
return MappingProxyType(
{call_id: tuple(value for _, value in group) for call_id, group in groupby(ordered, key=_pair_call_id)}
)
def _texts_by_tool_row(
messages: Sequence[Mapping[str, object]], input_data: "str | ResponseInputParam"
) -> Mapping[int, tuple[str, ...]]:
if isinstance(input_data, str):
return MappingProxyType({})
output_texts_by_call_id: Final = _grouped_by_call_id(
(str(stream_item_field(item, "call_id")), tuple(text for _, text in tool_output_texts(item)))
for item in input_data
if stream_item_field(item, "type") in TOOL_CALL_OUTPUT_ITEM_TYPES
)
row_indices_by_call_id: Final = _grouped_by_call_id(
(str(message.get("tool_call_id")), index)
for index, message in enumerate(messages)
if message.get("role") == "tool"
)
return MappingProxyType(
dict(
chain.from_iterable(
zip(indices, output_texts_by_call_id[call_id], strict=True)
for call_id, indices in row_indices_by_call_id.items()
if len(indices) == len(output_texts_by_call_id.get(call_id, ()))
)
)
)
def _is_string_tool_row(message: Mapping[str, object]) -> bool:
return message.get("role") == "tool" and isinstance(message.get("content"), str)
def _split_into_text_parts(message: AllMessageValues, texts: tuple[str, ...]) -> AllMessageValues:
if not _is_string_tool_row(message) or len(texts) < 2:
return message
return cast( # cast-ok: a tool row whose content is its text parts as chat text blocks
"AllMessageValues", {**message, "content": [{"type": "text", "text": text} for text in texts]}
)
def _with_split_rows_restored(
rows: Sequence[AllMessageValues], split_rows: Sequence[tuple[AllMessageValues, AllMessageValues]]
) -> Sequence[AllMessageValues]:
if not isinstance(rows, list):
return rows
return [next((original for split, original in split_rows if row == split), row) for row in rows]
def _is_textless_tool_row(message: Mapping[str, object], texts: tuple[str, ...] | None) -> bool:
return _is_string_tool_row(message) and (not message.get("content") or texts == ())
def _rewritten_whole_text(item: object, field: str, guardrail_response: str) -> object | None:
if isinstance(stream_item_field(item, field), str):
return guardrail_response
if tool_output_texts(item) == ((None, guardrail_response),):
return None
return guardrail_response
def _patched_request_fields(
raw_input: object,
instructions: object,
@ -636,17 +743,28 @@ class OpenAIResponsesHandler(BaseTranslation):
Handles both string input and list of message objects.
"""
input_data: Final[str | ResponseInputParam | None] = data.get("input")
if not isinstance(input_data, (str, list)):
raw_input: Final[str | ResponseInputParam | None] = data.get("input")
if not isinstance(raw_input, (str, list)):
return data
input_data: Final = cast("str | ResponseInputParam", raw_input) # cast-ok: fields read via stream_item_field
skip_system: Final = effective_skip_system_message_for_guardrail(guardrail_to_apply)
skip_tool: Final = effective_skip_tool_message_for_guardrail(guardrail_to_apply)
scan_only: Final = effective_scan_only_tool_results_for_guardrail(guardrail_to_apply)
structured_messages: Final = self.get_structured_messages(data)
scoped_indices: Final = scoped_structured_message_indices(
structured_messages or [], scan_only_tool_results=False, skip_system=skip_system, skip_tool=False
all_messages: Final = structured_messages or []
texts_by_tool_row: Final = _texts_by_tool_row(all_messages, input_data)
scoped_indices: Final = tuple(
index
for index in scoped_structured_message_indices(
all_messages, scan_only_tool_results=scan_only, skip_system=skip_system, skip_tool=skip_tool
)
if not _is_textless_tool_row(all_messages[index], texts_by_tool_row.get(index))
)
scoped_structured_messages: Final = (
[structured_messages[index] for index in scoped_indices] if structured_messages else None
split_rows: Final = tuple(
(_split_into_text_parts(all_messages[index], texts_by_tool_row.get(index, ())), all_messages[index])
for index in scoped_indices
)
scoped_structured_messages: Final = [split for split, _ in split_rows] if structured_messages else None
raw_tools: Final = data.get("tools")
original_tools: Final[tuple[Mapping[str, object], ...]] = (
tuple(raw_tools) if isinstance(raw_tools, list) else ()
@ -661,7 +779,12 @@ class OpenAIResponsesHandler(BaseTranslation):
)
)
extracted: Final = self._extract_guardrail_inputs(
data, input_data, flattened_tool_groups, skip_system=skip_system
data,
input_data,
() if scan_only else flattened_tool_groups,
skip_system=skip_system,
skip_tool=skip_tool,
scan_only=scan_only,
)
if not extracted.inputs.get("texts"):
return data
@ -674,13 +797,14 @@ class OpenAIResponsesHandler(BaseTranslation):
logging_obj=litellm_logging_obj,
)
self._apply_guardrailed_tools_to_data(
data, original_tools, flattened_tool_groups, guardrailed_inputs.get("tools")
data, original_tools, flattened_tool_groups, guardrailed_inputs.get("tools"), scan_only=scan_only
)
written_back: Final = self._written_back_request_fields(
data,
structured_messages or (),
scoped_indices,
scoped_structured_messages,
tuple((split, original) for split, original in split_rows if split is not original),
guardrail_to_apply,
guardrailed_inputs,
)
@ -730,8 +854,10 @@ class OpenAIResponsesHandler(BaseTranslation):
flattened_tool_groups: Sequence[Sequence[Mapping[str, object]]],
*,
skip_system: bool = False,
skip_tool: bool = False,
scan_only: bool = False,
) -> _ExtractedInputs:
instructions: Final = scannable_instructions(data, skip_system=skip_system)
instructions: Final = scannable_instructions(data, skip_system=skip_system or scan_only)
texts_to_check: Final[list[str]] = [] if instructions is None else [instructions]
images_to_check: Final[list[str]] = []
task_mappings: Final[list[tuple[int, int | None]]] = []
@ -745,11 +871,15 @@ class OpenAIResponsesHandler(BaseTranslation):
)
)
if isinstance(input_data, str):
texts_to_check.append(input_data)
if not scan_only:
texts_to_check.append(input_data)
else:
for msg_idx, message in enumerate(input_data):
if role_out_of_guardrail_scope(
_input_item_role(message), skip_system_message=skip_system, skip_tool_message=False
_input_item_role(message),
skip_system_message=skip_system,
skip_tool_message=skip_tool,
scan_only_tool_results=scan_only,
):
continue
self._extract_input_text_and_images(
@ -775,12 +905,14 @@ class OpenAIResponsesHandler(BaseTranslation):
structured_messages: Sequence[AllMessageValues],
scoped_indices: Sequence[int],
scoped_structured_messages: Sequence[AllMessageValues] | None,
split_rows: Sequence[tuple[AllMessageValues, AllMessageValues]],
guardrail_to_apply: "CustomGuardrail",
guardrailed_inputs: GenericGuardrailAPIInputs,
) -> _RequestFields | None:
guardrailed: Final = guardrailed_inputs.get("structured_messages")
if guardrailed is None or guardrailed is scoped_structured_messages:
returned: Final = guardrailed_inputs.get("structured_messages")
if returned is None or returned is scoped_structured_messages:
return None
guardrailed: Final = _with_split_rows_restored(returned, split_rows)
covers_full_request: Final = len(scoped_indices) == len(structured_messages) or (
guardrail_to_apply.structured_messages_cover_full_request() and len(guardrailed) == len(structured_messages)
)
@ -814,11 +946,14 @@ class OpenAIResponsesHandler(BaseTranslation):
original_tools: Sequence[Mapping[str, object]],
flattened_tool_groups: Sequence[Sequence[Mapping[str, object]]],
guardrailed_tools: Sequence[ChatCompletionToolParam] | None,
*,
scan_only: bool,
) -> None:
if guardrailed_tools is None:
return
merge: Final = append_returned_tools if scan_only else merge_guardrailed_tools
data["tools"] = list( # rebind-ok: in-place request rewrite
merge_guardrailed_tools(original_tools, flattened_tool_groups, guardrailed_tools)
merge(original_tools, flattened_tool_groups, guardrailed_tools)
)
def _extract_input_text_and_images(
@ -839,6 +974,11 @@ class OpenAIResponsesHandler(BaseTranslation):
texts_to_check.append(summary)
task_mappings.append((msg_idx, None))
return
if message.get("type") in TOOL_CALL_OUTPUT_ITEM_TYPES:
for content_idx, text in tool_output_texts(message):
texts_to_check.append(text)
task_mappings.append((msg_idx, content_idx))
return
content: Final = message.get("content", None)
if content is None:
return
@ -890,18 +1030,20 @@ class OpenAIResponsesHandler(BaseTranslation):
)
continue
content = messages[msg_idx].get("content", None)
field = _scanned_text_field(messages[msg_idx])
content = messages[msg_idx].get(field, None)
if content is None:
continue
if isinstance(content, str) and content_idx_optional is None:
# Replace string content with guardrail response
messages[msg_idx]["content"] = guardrail_response
if content_idx_optional is None:
rewritten = _rewritten_whole_text(messages[msg_idx], field, guardrail_response)
if rewritten is not None:
messages[msg_idx][field] = rewritten
elif isinstance(content, list) and content_idx_optional is not None:
elif isinstance(content, list):
# Replace specific text item in list content
if isinstance(messages[msg_idx]["content"][content_idx_optional], dict):
messages[msg_idx]["content"][content_idx_optional]["text"] = guardrail_response
if isinstance(messages[msg_idx][field][content_idx_optional], dict):
messages[msg_idx][field][content_idx_optional]["text"] = guardrail_response
async def process_output_response(
self,

16
litellm/llms/openai/responses/guardrail_translation/tool_merge.py
View file

@ -6,6 +6,7 @@ from typing import Final, TypeAlias
from pydantic import BaseModel, TypeAdapter, ValidationError
from litellm._logging import verbose_logger
from litellm.llms.base_llm.guardrail_translation.utils import merge_returned_tools_into_request_tools, openai_tool_name
from litellm.responses.litellm_completion_transformation.custom_tools import custom_tool_grammar_suffix
from litellm.responses.litellm_completion_transformation.transformation import (
NAMESPACE_DESCRIPTION_SEPARATOR,
@ -183,3 +184,18 @@ def merge_guardrailed_tools(
tuple(tool for key, tool in zip(guardrailed_keys, guardrailed) if key not in owned_keys)
)
return tuple(chain(merged_originals, appended))
def append_returned_tools(
original_tools: Sequence[Tool],
flattened_groups: Sequence[Sequence[Tool]],
returned_tools: Iterable[object],
) -> tuple[Tool, ...]:
request_chat_tools: Final = tuple(chain.from_iterable(flattened_groups))
additions: Final = merge_returned_tools_into_request_tools(
request_tools=request_chat_tools, returned_tools=_validated_tools(returned_tools), tool_name=openai_tool_name
)[len(request_chat_tools) :]
return (
*original_tools,
*LiteLLMCompletionResponsesConfig.transform_chat_completion_tool_params_to_responses_api_tools(additions),
)

53
litellm/llms/openai/responses/transformation.py
View file

@ -66,6 +66,29 @@ _PROVIDERS_VALIDATING_TOOL_CALL_ITEM_IDS: Final = frozenset({LlmProviders.AZURE,
_PROVIDERS_REPLAYING_ONLY_THEIR_OWN_REASONING: Final = _PROVIDERS_VALIDATING_TOOL_CALL_ITEM_IDS
_ROLES_READING_PROMPT_CACHE_BREAKPOINTS: Final = frozenset({"system", "developer", "user"})
_PART_TYPES_READING_PROMPT_CACHE_BREAKPOINTS: Final = frozenset({"input_text", "input_image", "input_file"})
_COMPACT_TRACKING_PARAMS: Final = frozenset({"metadata", "user"})
_COMPACT_REJECTED_PARAMS: Final = frozenset(
{
"background",
"context_management",
"max_output_tokens",
"max_tool_calls",
"moderation",
"partial_images",
"prompt",
"safety_identifier",
"store",
"stream_options",
"temperature",
"tool_choice",
"top_logprobs",
"top_p",
"truncation",
}
)
_PROVIDERS_WITH_STRICT_COMPACT_PARAMS: Final = frozenset(
{LlmProviders.OPENAI, LlmProviders.AZURE, LlmProviders.AZURE_AI}
)
_INPUT_ITEM: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
_CONTENT_PARTS: Final[TypeAdapter[Sequence[Mapping[str, object]]]] = TypeAdapter(Sequence[Mapping[str, object]])
@ -1182,6 +1205,17 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
#########################################################
########## COMPACT RESPONSE API TRANSFORMATION ##########
#########################################################
def compact_url(self, api_base: str) -> str:
parsed_url: Final = httpx.URL(api_base)
return str(parsed_url.copy_with(path=parsed_url.path.rstrip("/") + "/compact"))
def compact_dropped_params(self, litellm_params: GenericLiteLLMParams) -> frozenset[str]:
if self.custom_llm_provider not in _PROVIDERS_WITH_STRICT_COMPACT_PARAMS:
return frozenset()
if _drops_params(litellm_params):
return _COMPACT_TRACKING_PARAMS | _COMPACT_REJECTED_PARAMS
return _COMPACT_TRACKING_PARAMS
def transform_compact_response_api_request(
self,
model: str,
@ -1197,10 +1231,8 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
OpenAI API expects the following request
- POST /v1/responses/compact
"""
# Preserve query params (e.g., api-version) while appending /compact.
parsed_url: Final = httpx.URL(api_base)
compact_path: Final = parsed_url.path.rstrip("/") + "/compact"
url: Final = str(parsed_url.copy_with(path=compact_path))
url: Final = self.compact_url(api_base)
dropped_params: Final = self.compact_dropped_params(litellm_params)
request_params: Final = cast( # cast-ok: the base signature leaves the params bag untyped
"Mapping[str, object]", response_api_optional_request_params
@ -1214,14 +1246,11 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
)
if sanitized_tools is not None:
response_api_optional_request_params["tools"] = sanitized_tools
data: Final = cast( # cast-ok: typed dict spread is inferred as Unknown by basedpyright
dict[str, object],
{
"model": model,
"input": replay_safe_input,
**response_api_optional_request_params,
},
)
data: Final = {
"model": model,
"input": replay_safe_input,
**{key: value for key, value in request_params.items() if key not in dropped_params},
}
return url, data

11
litellm/llms/openai_like/dynamic_config.py
View file

@ -55,6 +55,15 @@ def create_config_class(provider: SimpleProviderConfig):
else:
return super()._transform_messages(messages=messages, model=model, is_async=False)
def _should_preserve_cache_control_for_endpoint(
self,
custom_llm_provider: str | None,
api_base: str | None,
) -> bool:
return provider.special_handling.get(
"preserve_cache_control", False
) or super()._should_preserve_cache_control_for_endpoint(custom_llm_provider, api_base)
def _get_openai_compatible_provider_info(
self, api_base: str | None, api_key: str | None
) -> tuple[str | None, str | None]:
@ -177,7 +186,7 @@ def create_config_class(provider: SimpleProviderConfig):
return JSONProviderConfig
_responses_config_cache: Final[dict] = {}
_responses_config_cache: Final[dict[str, "type[OpenAILikeResponsesConfig]"]] = {}
def create_responses_config_class(provider: SimpleProviderConfig) -> "type[OpenAILikeResponsesConfig]":

10
litellm/llms/openai_like/json_loader.py
View file

@ -3,6 +3,7 @@ JSON-based provider configuration loader for OpenAI-compatible providers.
"""
import json
from collections.abc import Mapping
from pathlib import Path
from typing import Final
@ -12,6 +13,7 @@ from litellm._logging import verbose_logger
from litellm.types.utils import ProviderSpecificModelInfo
_PROVIDER_INFO_ADAPTER: Final = TypeAdapter(ProviderSpecificModelInfo)
_SPECIAL_HANDLING_ADAPTER: Final = TypeAdapter(Mapping[str, bool])
class SimpleProviderConfig:
@ -20,12 +22,14 @@ class SimpleProviderConfig:
def __init__(self, slug: str, data: dict):
self.slug = slug
self.base_url = data["base_url"]
self.api_key_env = data["api_key_env"]
self.api_base_env = data.get("api_base_env")
self.api_key_env: str = data["api_key_env"]
self.api_base_env: str | None = data.get("api_base_env")
self.base_class = data.get("base_class", "openai_gpt")
self.param_mappings = data.get("param_mappings", {})
self.constraints = data.get("constraints", {})
self.special_handling = data.get("special_handling", {})
self.special_handling: Mapping[str, bool] = _SPECIAL_HANDLING_ADAPTER.validate_python(
data.get("special_handling", {})
)
self.supported_endpoints = data.get("supported_endpoints", [])
self.provider_info: ProviderSpecificModelInfo = (
_PROVIDER_INFO_ADAPTER.validate_python(data["provider_info"]) if "provider_info" in data else {}

16
litellm/llms/openai_like/providers.json
View file

@ -269,5 +269,21 @@
"/v1/chat/completions",
"/v1/responses"
]
},
"streamlake": {
"base_url": "https://vanchin.streamlake.ai/api/gateway/v1/endpoints",
"api_key_env": "STREAMLAKE_API_KEY",
"api_base_env": "STREAMLAKE_API_BASE",
"param_mappings": {
"max_tokens": "max_completion_tokens"
},
"provider_info": {
"supports_function_calling": true,
"supports_reasoning": true
},
"special_handling": {
"preserve_cache_control": true
},
"supported_endpoints": ["/v1/chat/completions"]
}
}

481
litellm/model_prices_and_context_window_backup.json
View file

@ -6101,6 +6101,7 @@
"text",
"audio"
],
"supports_audio_output": true,
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
@ -6133,6 +6134,7 @@
"text",
"audio"
],
"supports_audio_output": true,
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
@ -6164,6 +6166,7 @@
"text",
"audio"
],
"supports_audio_output": true,
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
@ -6230,6 +6233,7 @@
"text",
"audio"
],
"supports_audio_output": true,
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
@ -6293,6 +6297,7 @@
"text",
"audio"
],
"supports_audio_output": true,
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
@ -6369,6 +6374,7 @@
"text",
"audio"
],
"supports_audio_output": true,
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
@ -41933,6 +41939,120 @@
"supports_system_messages": true,
"supports_native_structured_output": true
},
"bedrock/ap-southeast-2/mistral.voxtral-mini-3b-2507": {
"deprecation_date": "2027-03-30",
"input_cost_per_token": 4.12e-08,
"input_cost_per_token_batches": 2.06e-08,
"input_cost_per_token_flex": 2.06e-08,
"input_cost_per_token_priority": 7.21e-08,
"litellm_provider": "bedrock",
"max_input_tokens": 32768,
"max_output_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 4.12e-08,
"output_cost_per_token_batches": 2.06e-08,
"output_cost_per_token_flex": 2.06e-08,
"output_cost_per_token_priority": 7.21e-08,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-southeast-2/index.json",
"supports_audio_input": true,
"supports_system_messages": true,
"supports_native_structured_output": true
},
"bedrock/ap-southeast-2/mistral.voxtral-small-24b-2507": {
"deprecation_date": "2027-03-30",
"input_cost_per_token": 1.03e-07,
"input_cost_per_token_batches": 5.15e-08,
"input_cost_per_token_flex": 5.15e-08,
"input_cost_per_token_priority": 1.8025e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 32768,
"max_output_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 3.09e-07,
"output_cost_per_token_batches": 1.545e-07,
"output_cost_per_token_flex": 1.545e-07,
"output_cost_per_token_priority": 5.4075e-07,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-southeast-2/index.json",
"supports_audio_input": true,
"supports_system_messages": true,
"supports_native_structured_output": true
},
"bedrock/ap-northeast-1/mistral.voxtral-mini-3b-2507": {
"deprecation_date": "2027-03-30",
"input_cost_per_token": 5e-08,
"input_cost_per_token_batches": 3e-08,
"input_cost_per_token_flex": 3e-08,
"input_cost_per_token_priority": 9e-08,
"litellm_provider": "bedrock",
"max_input_tokens": 32768,
"max_output_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 5e-08,
"output_cost_per_token_batches": 3e-08,
"output_cost_per_token_flex": 3e-08,
"output_cost_per_token_priority": 9e-08,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-northeast-1/index.json",
"supports_audio_input": true,
"supports_system_messages": true,
"supports_native_structured_output": true
},
"bedrock/ap-northeast-1/mistral.voxtral-small-24b-2507": {
"deprecation_date": "2027-03-30",
"input_cost_per_token": 1.2e-07,
"input_cost_per_token_batches": 6e-08,
"input_cost_per_token_flex": 6e-08,
"input_cost_per_token_priority": 2.1e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 32768,
"max_output_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 3.6e-07,
"output_cost_per_token_batches": 1.8e-07,
"output_cost_per_token_flex": 1.8e-07,
"output_cost_per_token_priority": 6.3e-07,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-northeast-1/index.json",
"supports_audio_input": true,
"supports_system_messages": true,
"supports_native_structured_output": true
},
"bedrock/eu-west-2/mistral.voxtral-mini-3b-2507": {
"deprecation_date": "2027-03-30",
"input_cost_per_token": 6e-08,
"input_cost_per_token_batches": 3e-08,
"input_cost_per_token_flex": 3e-08,
"input_cost_per_token_priority": 1.1e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 32768,
"max_output_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 6e-08,
"output_cost_per_token_batches": 3e-08,
"output_cost_per_token_flex": 3e-08,
"output_cost_per_token_priority": 1.1e-07,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/eu-west-2/index.json",
"supports_audio_input": true,
"supports_system_messages": true,
"supports_native_structured_output": true
},
"bedrock/eu-west-2/mistral.voxtral-small-24b-2507": {
"deprecation_date": "2027-03-30",
"input_cost_per_token": 1.6e-07,
"input_cost_per_token_batches": 8e-08,
"input_cost_per_token_flex": 8e-08,
"input_cost_per_token_priority": 2.8e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 32768,
"max_output_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 4.7e-07,
"output_cost_per_token_batches": 2.3e-07,
"output_cost_per_token_flex": 2.3e-07,
"output_cost_per_token_priority": 8.2e-07,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/eu-west-2/index.json",
"supports_audio_input": true,
"supports_system_messages": true,
"supports_native_structured_output": true
},
"mistral/codestral-2508": {
"cache_read_input_token_cost": 3e-08,
"input_cost_per_token": 3e-07,
@ -61269,6 +61389,359 @@
"supports_tool_choice": true,
"supports_vision": true
},
"streamlake/deepseek-v4-flash-0731": {
"cache_read_input_token_cost": 1.4e-08,
"input_cost_per_token": 4.4e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 1048576,
"max_output_tokens": 384000,
"mode": "chat",
"output_cost_per_token": 1.32e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/DeepSeek-V4-Pro-0813": {
"cache_read_input_token_cost": 4.4e-08,
"input_cost_per_token": 1.32e-06,
"litellm_provider": "streamlake",
"max_input_tokens": 1048566,
"max_output_tokens": 384000,
"mode": "chat",
"output_cost_per_token": 3.96e-06,
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/DeepSeek-V3": {
"cache_read_input_token_cost": 1.14e-07,
"input_cost_per_token": 2.86e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 56000,
"max_output_tokens": 8000,
"mode": "chat",
"output_cost_per_token": 1.143e-06,
"supports_function_calling": true,
"supports_reasoning": false,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/GLM-5.3": {
"cache_read_input_token_cost": 2.6e-07,
"input_cost_per_token": 1.4e-06,
"litellm_provider": "streamlake",
"max_input_tokens": 1048576,
"max_output_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 4.4e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_assistant_prefill": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/GLM-5.3-Flash": {
"cache_read_input_token_cost": 3e-08,
"input_cost_per_token": 1.5e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 1048576,
"max_output_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 5e-07,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_vision": true,
"supports_assistant_prefill": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/Kimi-K2-Instruct-0905": {
"cache_read_input_token_cost": 1.43e-07,
"input_cost_per_token": 5.71e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 224000,
"max_output_tokens": 32000,
"mode": "chat",
"output_cost_per_token": 2.286e-06,
"supports_function_calling": true,
"supports_reasoning": false,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/deepseek-r1": {
"cache_read_input_token_cost": 2.29e-07,
"input_cost_per_token": 5.71e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 98304,
"max_output_tokens": 16000,
"mode": "chat",
"output_cost_per_token": 2.286e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/deepseek-v3.1": {
"cache_read_input_token_cost": 2.29e-07,
"input_cost_per_token": 5.71e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 98304,
"max_output_tokens": 32000,
"mode": "chat",
"output_cost_per_token": 1.714e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/deepseek-v4-flash": {
"cache_read_input_token_cost": 2.8e-08,
"input_cost_per_token": 1.4e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 1048576,
"max_output_tokens": 384000,
"mode": "chat",
"output_cost_per_token": 2.8e-07,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/deepseek-v4-pro": {
"cache_read_input_token_cost": 1.45e-07,
"input_cost_per_token": 1.74e-06,
"litellm_provider": "streamlake",
"max_input_tokens": 1048576,
"max_output_tokens": 384000,
"mode": "chat",
"output_cost_per_token": 3.48e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/deepseek-v4.1-flash-1017": {
"cache_read_input_token_cost": 6e-09,
"input_cost_per_token": 3e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 1048576,
"max_output_tokens": 384000,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_vision": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/deepseek-v4.1-flash-260910": {
"cache_read_input_token_cost": 6e-09,
"input_cost_per_token": 3e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 1048568,
"max_output_tokens": 384000,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_vision": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/glm-5": {
"cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 1e-06,
"litellm_provider": "streamlake",
"max_input_tokens": 202752,
"max_output_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 3.2e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/glm-5.1": {
"cache_read_input_token_cost": 2.6e-07,
"input_cost_per_token": 1.4e-06,
"litellm_provider": "streamlake",
"max_input_tokens": 202752,
"max_output_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 4.4e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/glm-5.2": {
"cache_read_input_token_cost": 2.6e-07,
"input_cost_per_token": 1.4e-06,
"litellm_provider": "streamlake",
"max_input_tokens": 1048566,
"max_output_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 4.4e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/kimi-k2.6": {
"cache_read_input_token_cost": 1.6e-07,
"input_cost_per_token": 9.5e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 262144,
"max_output_tokens": 256000,
"mode": "chat",
"output_cost_per_token": 4e-06,
"supports_function_calling": true,
"supports_vision": true,
"supports_reasoning": false,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/kimi-k2.7-code": {
"cache_read_input_token_cost": 1.9e-07,
"input_cost_per_token": 9.5e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 262144,
"max_output_tokens": 32000,
"mode": "chat",
"output_cost_per_token": 4e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_vision": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/mimo-v2.6-flash": {
"cache_read_input_token_cost": 2.8e-09,
"input_cost_per_token": 1.4e-07,
"litellm_provider": "streamlake",
"mode": "chat",
"output_cost_per_token": 2.8e-07,
"supports_vision": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/mimo-v2.6-pro": {
"cache_read_input_token_cost": 3.6e-09,
"input_cost_per_token": 4.35e-07,
"litellm_provider": "streamlake",
"mode": "chat",
"output_cost_per_token": 8.7e-07,
"supports_vision": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/minimax-m2.5": {
"cache_creation_input_token_cost": 3.75e-07,
"cache_read_input_token_cost": 3e-08,
"input_cost_per_token": 3e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 196608,
"max_output_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/minimax-m3": {
"cache_read_input_token_cost": 1.2e-07,
"cache_read_input_token_cost_above_512k_tokens": 2.4e-07,
"input_cost_per_token": 6e-07,
"input_cost_per_token_above_512k_tokens": 1.2e-06,
"litellm_provider": "streamlake",
"max_input_tokens": 1000000,
"max_output_tokens": 512000,
"mode": "chat",
"output_cost_per_token": 2.4e-06,
"output_cost_per_token_above_512k_tokens": 4.8e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_vision": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/qwen3.5-397b-a17b": {
"cache_read_input_token_cost": 1.2e-07,
"input_cost_per_token": 6e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 258048,
"max_output_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 3.6e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_vision": true,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/qwen3.6-27b": {
"input_cost_per_token": 6e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 229376,
"max_output_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 3.6e-06,
"supports_function_calling": true,
"supports_vision": true,
"supports_reasoning": false,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/qwen3.6-35b-a3b": {
"input_cost_per_token": 3.75e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 229376,
"max_output_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 2.25e-06,
"supports_function_calling": true,
"supports_vision": true,
"supports_reasoning": false,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/qwen3.8-2.4t-a95b": {
"cache_read_input_token_cost": 2.5e-07,
"input_cost_per_token": 2e-06,
"litellm_provider": "streamlake",
"max_input_tokens": 983616,
"max_output_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 6e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_vision": true,
"cache_creation_input_token_cost": 2.5e-06,
"explicit_cache_read_input_token_cost": 1.7e-07,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"streamlake/qwen3.8-27b": {
"cache_read_input_token_cost": 1e-07,
"input_cost_per_token": 5e-07,
"litellm_provider": "streamlake",
"max_input_tokens": 983616,
"max_output_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 3e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_vision": true,
"cache_creation_input_token_cost": 6.25e-07,
"explicit_cache_read_input_token_cost": 5e-08,
"supports_prompt_caching": true,
"source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
},
"deepseek-v4-flash": {
"cache_creation_input_token_cost": 0.0,
"cache_read_input_token_cost": 6e-09,
@ -62584,6 +63057,14 @@
"litellm_provider": "anthropic"
}
},
{
"name": "typesafe-jev-ids",
"pattern": "^jev-(?:latest|preview|1\\.13\\.0|1\\.14\\.0)$",
"description": "A bare Jev id that has a typesafe/ price entry (jev-latest, jev-preview, jev-1.13.0, jev-1.14.0), the name the /typesafe pass-through forwards to TypeSafe. Listed by name, matched case-insensitively like every rule here, so foo/jev-latest or a model group named like a Jev id TypeSafe does not serve, such as jev-1.0 or jev-internal-router, never resolves to typesafe. A new Jev version goes here together with its price entry. Provider inference feeds the proxy's provider/* wildcard access checks, so this rule is what lets typesafe/* on a key, team or project allowlist cover a jev-latest request.",
"model_info": {
"litellm_provider": "typesafe"
}
},
{
"name": "claude-family-baseline",
"pattern": "claude-[a-z]+-\\d+(?:[-.]\\d+)?(?:-\\d{8})?",

26
litellm/provider_endpoints_support_backup.json
View file

@ -3785,6 +3785,32 @@
"generateContent": true
}
},
"streamlake": {
"display_name": "StreamLake (`streamlake`)",
"url": "https://docs.litellm.ai/docs/providers/openai_compatible",
"endpoints": {
"chat_completions": true,
"responses": true,
"text_completion": true,
"embeddings": false,
"image_generations": false,
"image_edits": false,
"audio_transcriptions": false,
"audio_speech": false,
"rerank": false,
"ocr": false,
"search": false,
"moderations": false,
"realtime": false,
"video_generations": false,
"batches": false,
"messages": true,
"interactions": true,
"systemone": false,
"a2a": true,
"generateContent": true
}
},
"tencent": {
"display_name": "Tencent TokenHub (`tencent`)",
"url": "https://docs.litellm.ai/docs/providers/tencent",

1
litellm/proxy/auth/auth_checks.py
View file

@ -8075,7 +8075,6 @@ async def vector_store_access_check(
rag_ids: Final = (rag_vector_store_id,) if rag_vector_store_id is not None else ()
vector_store_ids_to_run: Final = tuple(dict.fromkeys((*registry_ids, *rag_ids)))
if not vector_store_ids_to_run:
verbose_proxy_logger.debug("Vector store to run not found, skipping vector store access check")
return True
#########################################################

5
litellm/proxy/common_request_processing.py
View file

@ -2581,7 +2581,6 @@ class ProxyBaseLLMRequestProcessing:
# have mutated `self.data` in place, and the audit-trail snapshot taken in
# add_litellm_data_to_request predates that mutation.
refresh_proxy_server_request_body_snapshot(self.data, guardrails_applied=True)
verbose_proxy_logger.debug("receiving data: %s", self.data)
if "messages" in self.data and self.data["messages"]:
logging_obj.update_messages(self.data["messages"])
@ -4234,7 +4233,7 @@ class ProxyBaseLLMRequestProcessing:
e = transformed_exception
litellm_debug_info: Final = getattr(e, "litellm_debug_info", "")
verbose_proxy_logger.debug(
"\033[1;31mAn error occurred: %s %s\n\n Debug this by setting `--debug`, e.g. `litellm --model gpt-3.5-turbo --debug`",
"An error occurred: %s %s",
e,
litellm_debug_info,
)
@ -4597,7 +4596,7 @@ class ProxyBaseLLMRequestProcessing:
if transformed_exception is not None:
e = transformed_exception
verbose_proxy_logger.debug(
"\x1b[1;31mAn error occurred: %s\n\n Debug this by setting `--debug`, e.g. `litellm --model gpt-3.5-turbo --debug`",
"An error occurred: %s",
e,
)

2
litellm/proxy/common_utils/http_parsing_utils.py
View file

@ -845,5 +845,3 @@ def _add_vector_store_id_from_path(request_data: dict, request: Request) -> None
"populate_request_with_path_params: Updated request_data with vector_store_ids=%s",
request_data.get("vector_store_ids"),
)
else:
verbose_proxy_logger.debug("populate_request_with_path_params: No vector_store_id present in path=%s", path)

3
litellm/proxy/example_config_yaml/custom_callbacks.py
View file

@ -28,7 +28,7 @@ class MyCustomHandler(CustomLogger):
def __init__(self):
blue_color_code: Final = "\033[94m"
reset_color_code: Final = "\033[0m"
print_verbose(f"{blue_color_code}Initialized LiteLLM custom logger")
print_verbose(f"{blue_color_code}Initialized LiteLLM custom logger{reset_color_code}")
try:
print_verbose("Logger Initialized with following methods:")
methods: Final = [
@ -40,7 +40,6 @@ class MyCustomHandler(CustomLogger):
# Pretty print_verbose the methods
for method in methods:
print_verbose(f" - {method}")
print_verbose(f"{reset_color_code}")
except Exception:
pass

14
litellm/proxy/guardrails/_content_utils.py
View file

@ -62,6 +62,20 @@ def is_non_conversational_call_type(call_type: str) -> bool:
return call_type in NON_CONVERSATIONAL_CALL_TYPES
_PASSTHROUGH_EMBEDDINGS_ENDPOINT: Final = re.compile(r"/?(?:[\w-]+/)*embeddings/?")
def is_non_conversational_request(call_type: str, data: Mapping[str, object]) -> bool:
if is_non_conversational_call_type(call_type):
return True
endpoint: Final = data.get("endpoint")
return (
call_type == "allm_passthrough_route"
and isinstance(endpoint, str)
and _PASSTHROUGH_EMBEDDINGS_ENDPOINT.fullmatch(endpoint) is not None
)
TEXT_PART_TYPES: Final[frozenset[str]] = frozenset(
{"text", "input_text", "output_text", "summary_text", "reasoning_text"}
)

6
litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py
View file

@ -49,6 +49,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, bedrock_bearer_token, run_aws_signing
from litellm.llms.bedrock.guardrail_attachments import find_request_attachments
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
get_async_httpx_client,
httpxSpecialProvider,
)
@ -286,9 +287,12 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
streaming_sampling_rate: int | None = None,
streaming_end_of_stream_only: bool | None = None,
streaming_buffer_release_on_scan: bool | None = None,
async_handler: AsyncHTTPHandler | None = None,
**kwargs,
):
self.async_handler = get_async_httpx_client(llm_provider=httpxSpecialProvider.GuardrailCallback)
self.async_handler = async_handler or get_async_httpx_client(
llm_provider=httpxSpecialProvider.GuardrailCallback
)
self._set_streaming_params(
BedrockGuardrailStreamingParams.from_extras(
MappingProxyType(

27
litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/example_config.yaml
View file

@ -6,20 +6,19 @@ model_list:
model: gpt-4
api_key: os.environ/OPENAI_API_KEY
litellm_settings:
guardrails:
- guardrail_name: "my-generic-guardrail"
litellm_params:
guardrail: generic_guardrail_api
mode: pre_call # Options: pre_call, post_call, during_call, [pre_call, post_call]
api_key: os.environ/GENERIC_GUARDRAIL_API_KEY # Optional if using Bearer auth
api_base: http://localhost:8080 # Required. Endpoint /beta/litellm_basic_guardrail_api is automatically appended
unreachable_fallback: fail_closed # Options: fail_closed (default, raise), fail_open (proceed if endpoint unreachable or upstream returns 502/503/504)
default_on: false # Set to true to apply to all requests by default
additional_provider_specific_params:
# Any additional parameters your guardrail API needs
api_version: "v1"
custom_param: "value"
guardrails:
- guardrail_name: "my-generic-guardrail"
litellm_params:
guardrail: generic_guardrail_api
mode: pre_call # Options: pre_call, post_call, during_call, [pre_call, post_call]
api_key: os.environ/GENERIC_GUARDRAIL_API_KEY # Optional if using Bearer auth
api_base: http://localhost:8080 # Required. Endpoint /beta/litellm_basic_guardrail_api is automatically appended
unreachable_fallback: fail_closed # Options: fail_closed (default, raise), fail_open (proceed if endpoint unreachable or upstream returns 502/503/504)
default_on: false # Set to true to apply to all requests by default
additional_provider_specific_params:
# Any additional parameters your guardrail API needs
api_version: "v1"
custom_param: "value"
# Usage examples:

29
litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py
View file

@ -30,6 +30,7 @@ from litellm.integrations.custom_guardrail import (
from litellm.llms.base_llm.guardrail_translation.utils import (
effective_scan_only_tool_results_for_guardrail,
effective_skip_system_message_for_guardrail,
effective_skip_tool_message_for_guardrail,
role_out_of_guardrail_scope,
)
from litellm.llms.custom_httpx.http_handler import (
@ -37,12 +38,16 @@ from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
)
from litellm.llms.openai.responses.guardrail_translation.handler import scannable_instructions
from litellm.llms.openai.responses.guardrail_translation.handler import (
scannable_instructions,
tool_output_texts,
)
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.common_utils.callback_utils import (
add_guardrail_scan_id,
add_guardrail_to_applied_guardrails_header,
)
from litellm.responses.litellm_completion_transformation.transformation import TOOL_CALL_OUTPUT_ITEM_TYPES
from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.llms.openai import AllMessageValues
@ -112,12 +117,16 @@ class _ResponsesInputItem(LiteLLMBaseModel):
type: str | None = None
role: str | None = None
content: str | tuple[_ResponsesContentPart, ...] | None = None
output: object = None
call_id: object = None
def text_count(self, *, skip_system: bool) -> int:
if role_out_of_guardrail_scope(
(self.role or "").lower(), skip_system_message=skip_system, skip_tool_message=False
):
def text_count(self, *, skip_system: bool, skip_tool: bool) -> int:
is_tool_output: Final = self.type in TOOL_CALL_OUTPUT_ITEM_TYPES
role: Final = "tool" if is_tool_output else (self.role or "").lower()
if role_out_of_guardrail_scope(role, skip_system_message=skip_system, skip_tool_message=skip_tool):
return 0
if is_tool_output:
return len(tool_output_texts(self))
if isinstance(self.content, str):
return 1
if self.content is None:
@ -1692,13 +1701,14 @@ class PanwPrismaAirsHandler(CustomGuardrail):
request_data: Mapping[str, object],
*,
skip_system: bool,
skip_tool: bool,
) -> frozenset[int] | None:
"""Return the ``texts`` indices flattened from Responses ``reasoning`` input items.
The Responses translation handler gives those model-authored items the default
``user`` role, so the latest-turn selection must not mistake one for a human turn.
Empty for requests without a Responses ``input`` item list; None when the raw items
(after the leading ``instructions`` text, both minus whatever ``skip_system`` drops)
(after the leading ``instructions`` text, both minus whatever ``skip_system`` and ``skip_tool`` drop)
do not account for every entry of ``texts``.
"""
try:
@ -1708,7 +1718,7 @@ class PanwPrismaAirsHandler(CustomGuardrail):
if not isinstance(raw_input, tuple):
return frozenset()
offset: Final = 0 if scannable_instructions(request_data, skip_system=skip_system) is None else 1
counts: Final = tuple(item.text_count(skip_system=skip_system) for item in raw_input)
counts: Final = tuple(item.text_count(skip_system=skip_system, skip_tool=skip_tool) for item in raw_input)
if offset + sum(counts) != len(texts):
return None
starts: Final = itertools.accumulate(counts, initial=offset)
@ -1738,7 +1748,10 @@ class PanwPrismaAirsHandler(CustomGuardrail):
if sources is None:
return None
reasoning: Final = self._reasoning_item_text_indices(
texts, request_data, skip_system=effective_skip_system_message_for_guardrail(self)
texts,
request_data,
skip_system=effective_skip_system_message_for_guardrail(self),
skip_tool=effective_skip_tool_message_for_guardrail(self),
)
if reasoning is None:
return None

68
litellm/proxy/health_check.py
View file

@ -131,6 +131,52 @@ def _should_inject_health_check_max_tokens(model_info: Mapping[str, object], mod
return (mode or "chat") in _MAX_TOKEN_SUPPORT_MODES
_AUDIO_OUTPUT_PROBE_PROVIDERS: Final[frozenset[str]] = frozenset({"openai", "azure"})
def _health_check_requires_audio_output(
model_info: Mapping[str, object], probe_params: Mapping[str, object], mode: str | None, deployment_model: str | None
) -> bool:
if (mode or "chat") != "chat":
return False
operator_modalities: Final = probe_params.get("modalities")
if isinstance(operator_modalities, list) and "audio" not in operator_modalities:
return False
model: Final = probe_params.get("model")
if not isinstance(model, str):
return False
provider_param: Final = probe_params.get("custom_llm_provider")
try:
_, provider, _, _ = litellm.get_llm_provider(
model=model, custom_llm_provider=provider_param if isinstance(provider_param, str) else None
)
except litellm.exceptions.BadRequestError:
return False
if provider not in _AUDIO_OUTPUT_PROBE_PROVIDERS:
return False
base_model: Final = model_info.get("base_model") or probe_params.get("base_model")
checked_model: Final = base_model if isinstance(base_model, str) and model == deployment_model else model
checked_model_prefix: Final = checked_model.partition("/")[0]
return litellm.supports_audio_output(
model=checked_model,
custom_llm_provider=checked_model_prefix if checked_model_prefix in _AUDIO_OUTPUT_PROBE_PROVIDERS else provider,
)
def _health_check_audio_output_params(
model_info: Mapping[str, object], probe_params: Mapping[str, object], mode: str | None, deployment_model: str | None
) -> Mapping[str, object]:
if not _health_check_requires_audio_output(model_info, probe_params, mode, deployment_model):
return MappingProxyType({})
voice: Final = model_info.get("health_check_voice")
return MappingProxyType(
{
"modalities": ["text", "audio"],
"audio": {"voice": voice if isinstance(voice, str) else "alloy", "format": "wav"},
}
)
# Health-check modes that forward `reasoning_effort` to the provider (chat-style calls).
_HEALTH_CHECK_MODES_SUPPORTING_REASONING_EFFORT: Final = frozenset((None, "chat", "completion"))
@ -782,9 +828,12 @@ def _resolve_health_check_max_tokens(model_info: dict, litellm_params: dict) ->
_PROBE_PARAMS_ADAPTER: Final = TypeAdapter(dict[str, object])
_MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object])
def update_litellm_params_for_health_check(model_info: dict, litellm_params: dict) -> dict:
def update_litellm_params_for_health_check(
model_info: dict, litellm_params: dict, deployment_model: str | None = None
) -> dict:
"""
Update the litellm params for health check.
@ -800,7 +849,15 @@ def update_litellm_params_for_health_check(model_info: dict, litellm_params: dic
- updates the `model` param with the `health_check_model` if it exists Doc: https://docs.litellm.ai/docs/proxy/health#wildcard-routes
- updates the `voice` param with the `health_check_voice` for `audio_speech` mode if it exists Doc: https://docs.litellm.ai/docs/proxy/health#text-to-speech-models
- for Bedrock models with region routing (bedrock/region/model), strips the litellm routing prefix but preserves the model ID, and pins `custom_llm_provider` to `bedrock` (only when the deployment hasn't already set one, so an explicit `bedrock_converse` survives) so the bare model id still resolves to the provider (e.g. cross-region ids like `us.cohere.embed-v4:0`)
- asks OpenAI and Azure chat models that support audio output for `modalities` text + audio and a wav
`audio` in the `health_check_voice` (default "alloy"), since those models reject text-only requests.
`base_model` (from `model_info`, then `litellm_params`) decides it only while the probe still targets
`deployment_model` (default: the incoming `litellm_params` model), so a `health_check_model` or a `model`
in `health_check_params` is judged on its own. Keys already set through `health_check_params` are kept,
and `modalities` set there without "audio" turns the audio fields off
"""
incoming_model: Final = _PROBE_PARAMS_ADAPTER.validate_python(litellm_params).get("model")
base_model_owner: Final = deployment_model or (incoming_model if isinstance(incoming_model, str) else None)
mode: Final = resolve_health_check_mode(
model_info,
litellm_params, # any-ok: untyped router config dict
@ -875,7 +932,14 @@ def update_litellm_params_for_health_check(model_info: dict, litellm_params: dic
)
probe_params: Final = _PROBE_PARAMS_ADAPTER.validate_python(litellm_params)
return {key: value for key, value in probe_params.items() if not is_model_capability_field(key)}
audio_output_params: Final = _health_check_audio_output_params(
_MODEL_INFO_ADAPTER.validate_python(model_info), probe_params, mode, base_model_owner
)
return {
key: value
for key, value in {**audio_output_params, **probe_params}.items()
if not is_model_capability_field(key)
}
_update_litellm_params_for_health_check: Final = update_litellm_params_for_health_check

2
litellm/proxy/health_endpoints/_health_endpoints.py
View file

@ -2330,9 +2330,11 @@ async def test_model_connection(
stored_params=_OBJECT_MAPPING.validate_python(config_litellm_params),
request_params=_OBJECT_MAPPING.validate_python(request_litellm_params),
)
stored_model: Final = _OBJECT_MAPPING.validate_python(config_litellm_params).get("model")
litellm_params = update_litellm_params_for_health_check(
model_info=dict(probe_model_info),
litellm_params=litellm_params,
deployment_model=stored_model if isinstance(stored_model, str) else None,
)
## Auth check, on the final probe params so health_check_params cannot retarget it afterwards

Some files were not shown because too many files have changed in this diff Show more