refactor(model-cost): load litellm.model_cost through the Rust catalog

The Rust catalog parses, validates and installs the cost map into the dict
bound to litellm.model_cost. Fetched rows patch the bundled rows, rejected
rows fall back to bundled, and fields the user wrote stay theirs.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Yujong Lee 2026-10-27 23:31:51 -07:00
parent 21acd3ce29
commit 18665e09be
46 changed files with 1665 additions and 754 deletions

View file

@ -5,6 +5,9 @@ checks: the files parse and the backup copy matches the root file. A pull reques
skips them, since merging it keeps the base branch's copies and its head tree only carries whatever state the
branch was cut from. Pull requests from the cost map sync bot (branches named litellm_cost_map_sync_*) always
get the file checks and additionally may only touch those two files and may only add or update models.
Installed wheels layer a fetched map over their bundled one, so a key or a price or ``supports_*`` field the map
leaves out is still served from the wheel. Every pull request that drops one must write it as ``null`` instead.
"""
from __future__ import annotations
@ -91,6 +94,35 @@ def _entries(cost_map: CostMap) -> dict[str, dict[str, object]]:
return {key: entry for key, entry in cost_map.items() if isinstance(entry, dict)}
def _retired_by_null(field: str) -> bool:
return "cost" in field or field in ("tiered_pricing", "off_peak_pricing") or field.startswith("supports_")
def _silent_removal_failures(base: Snapshot, head_map: CostMap) -> tuple[str, ...]:
base_map: Final = _parse_object(base.cost_map, COST_MAP_PATH)
if isinstance(base_map, str):
return ()
head_entries: Final = _entries(head_map)
removed_fields: Final = tuple(
f"{key}.{field}"
for key, entry in _entries(base_map).items()
if key in head_entries
for field in entry
if _retired_by_null(field) and field not in head_entries[key]
)
return (
*(
f"{key} is removed without a null; write it as null so older wheels stop serving it"
for key in base_map
if key not in head_map and key not in SPECIAL_ROOT_KEYS
),
*(
f"{ref} is removed without a null; write it as null so older wheels stop serving it"
for ref in removed_fields
),
)
def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str]) -> tuple[str, ...]:
base_map: Final = _parse_object(base.cost_map, COST_MAP_PATH)
if isinstance(base_map, str):
@ -147,6 +179,7 @@ def guard_failures(
*_file_failures(
head, head_map, evaluation_output_price_sources=evaluation_output_price_sources
),
*_silent_removal_failures(base, head_map),
*(_bot_failures(base, head_map, changed_files) if bot else ()),
)

View file

@ -4268,6 +4268,7 @@ dependencies = [
"criterion",
"fancy-regex 0.19.2",
"futures-util",
"indexmap 2.14.0",
"litellm-auth",
"litellm-auth-aws",
"litellm-cache",
@ -4287,6 +4288,7 @@ dependencies = [
"litellm-inference-transcription",
"litellm-llms",
"litellm-llms-types",
"litellm-model-catalog",
"litellm-router-types",
"litellm-secrets",
"litellm-secrets-aws",
@ -4304,6 +4306,7 @@ dependencies = [
"serde",
"serde_json",
"serde_with",
"sha1 0.10.7",
"sqlx",
"strum",
"thiserror 2.0.19",

View file

@ -62,6 +62,7 @@ pub struct Catalog {
aliases: IndexMap<String, String>,
lowercase_keys: HashMap<String, String>,
fallback_generalizations: Option<FallbackGeneralizations>,
removals: Vec<String>,
}
impl IntegrityLimits {
@ -96,7 +97,17 @@ impl Catalog {
let mut sections = Vec::new();
let mut entries = IndexMap::with_capacity(document.len());
let mut alias_lists = Vec::new();
let mut removals = Vec::new();
for (index, (name, value)) in document.into_iter().enumerate() {
if value.is_null() && !is_section(&name) {
removals.push(name.clone());
sections.push(Section {
index,
key: name,
value,
});
continue;
}
let fields = match value {
Value::Object(fields) if !is_section(&name) => fields,
value => {
@ -169,6 +180,7 @@ impl Catalog {
aliases,
lowercase_keys,
fallback_generalizations,
removals,
},
errors: problems,
})
@ -280,6 +292,45 @@ impl Catalog {
(catalog, removed_errors)
}
/// Rows replaced by the same key's row in `bundled` when it has one, and left out otherwise (decision 15)
pub fn with_rejected_replaced<'a>(
self,
errors: &'a [Located],
bundled: &Catalog,
) -> (Self, Vec<Rejection<'a>>) {
let (mut catalog, removed) = self.without_rejected(errors);
let mut rejected: IndexMap<String, Vec<&'a Located>> = IndexMap::new();
for error in removed {
if let Some(key) = crate::error::pointer_root(&error.path) {
rejected.entry(key).or_default().push(error);
}
}
let rejections = rejected
.into_iter()
.map(|(key, errors)| {
let bundled_entry = bundled.entries.get(&key).cloned();
let replaced_by_bundled = bundled_entry.is_some();
if let Some(entry) = bundled_entry {
catalog
.lowercase_keys
.insert(key.to_lowercase(), key.clone());
catalog.entries.insert(key.clone(), entry);
}
Rejection {
key,
errors,
replaced_by_bundled,
}
})
.collect();
(catalog, rejections)
}
/// Keys written as `null`, which a fetched document uses to retire a row it no longer serves (decision 23)
pub fn removals(&self) -> impl Iterator<Item = &str> {
self.removals.iter().map(String::as_str)
}
pub fn unrecognized(&self) -> impl Iterator<Item = (&str, &str)> {
self.entries.iter().flat_map(|(name, entry)| {
entry
@ -370,3 +421,55 @@ fn unknown_timezone(wire: &EntryWire) -> Option<Located> {
)
})
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use serde_json::json;
use super::*;
fn parse(document: Value) -> Loaded<Catalog> {
Catalog::parse(document.to_string().as_bytes()).unwrap()
}
#[rstest]
fn a_rejected_row_is_replaced_by_bundled_or_left_out() {
let bundled =
parse(json!({"gpt-4o": {"litellm_provider": "openai", "input_cost_per_token": 1.0}}))
.value;
let fetched = parse(json!({
"gpt-4o": {"litellm_provider": "openai", "input_cost_per_token": -1},
"acme/new": {"litellm_provider": "acme", "input_cost_per_token": "free"},
"kept": {"litellm_provider": "openai", "input_cost_per_token": 2.0}
}));
let (catalog, rejections) = fetched
.value
.with_rejected_replaced(&fetched.errors, &bundled);
let replaced: Vec<(&str, bool)> = rejections
.iter()
.map(|rejection| (rejection.key.as_str(), rejection.replaced_by_bundled))
.collect();
assert_eq!(replaced, [("gpt-4o", true), ("acme/new", false)]);
assert_eq!(
catalog
.lookup("gpt-4o")
.unwrap()
.entry
.field("input_cost_per_token"),
Some(&json!(1.0))
);
assert!(catalog.lookup("acme/new").is_none());
assert!(catalog.lookup("kept").is_some());
}
#[rstest]
fn a_null_row_is_a_removal_not_an_error() {
let loaded = parse(json!({"gpt-4o": {"litellm_provider": "openai"}, "retired": null}));
assert!(loaded.errors.is_empty(), "{:?}", loaded.errors);
assert_eq!(loaded.value.removals().collect::<Vec<_>>(), ["retired"]);
assert_eq!(loaded.value.served().count(), 1);
}
}

View file

@ -7,6 +7,8 @@ use crate::catalog::Catalog;
use crate::register::Registrations;
use crate::wire::Fields;
const ALIASES: &str = "aliases";
/// One field of a fetched row over the layer below, RFC 7386. An absent field inherits, JSON `null` is `Remove`
#[derive(Clone, Debug, PartialEq)]
pub enum Patch {
@ -16,10 +18,6 @@ pub enum Patch {
/// The four layers served rows are composed from, lowest first (decisions 23 and 25)
#[derive(Clone, Debug)]
#[expect(
dead_code,
reason = "02 and 07 add the functions that read these layers"
)]
pub struct Layers {
/// 0, the wheel's embedded map
bundled: Arc<Catalog>,
@ -28,5 +26,138 @@ pub struct Layers {
/// 2, `register(persist=False)`, cleared by every install and refilled by the router replay
transient: IndexMap<String, Fields>,
/// 3, `register(persist=True)`, applied over every install
#[expect(dead_code, reason = "07 applies registrations over the fetched layer")]
persisted: Registrations,
}
impl Patch {
pub fn from_value(value: &Value) -> Self {
match value {
Value::Null => Self::Remove,
value => Self::Set(value.clone()),
}
}
}
impl Layers {
pub fn new(bundled: Arc<Catalog>) -> Self {
Self {
bundled,
fetched: None,
transient: IndexMap::new(),
persisted: Registrations::default(),
}
}
/// Install replaces layer 1 and clears layer 2, which the router replay refills
pub fn replace_fetched(&mut self, fetched: Arc<Catalog>) {
self.fetched = Some(fetched);
self.transient.clear();
}
/// What install writes. A fetched row patches the bundled row field by field, a fetched `null` row removes it
pub fn compose(&self) -> IndexMap<String, Fields> {
let mut rows: IndexMap<String, Fields> = self
.bundled
.served()
.map(|(key, fields)| (key.to_owned(), without_aliases(fields)))
.collect();
let Some(fetched) = &self.fetched else {
return rows;
};
for key in fetched.removals() {
rows.shift_remove(key);
}
for (key, fields) in fetched.served() {
let row = rows.entry(key.to_owned()).or_default();
for (field, value) in fields.iter().filter(|(field, _)| *field != ALIASES) {
match Patch::from_value(value) {
Patch::Set(value) => {
row.insert(field.clone(), value);
}
Patch::Remove => {
row.shift_remove(field);
}
}
}
}
rows
}
/// The canonical key an alias is served under, from the fetched layer first
pub fn alias_owner(&self, key: &str) -> Option<&str> {
self.fetched
.iter()
.chain([&self.bundled])
.find_map(|catalog| catalog.aliases().get(key))
.map(String::as_str)
}
}
fn without_aliases(fields: &Fields) -> Fields {
fields
.iter()
.filter(|(field, value)| *field != ALIASES && !value.is_null())
.map(|(field, value)| (field.clone(), value.clone()))
.collect()
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use serde_json::json;
use super::*;
fn catalog(document: Value) -> Arc<Catalog> {
Arc::new(
Catalog::parse(document.to_string().as_bytes())
.unwrap()
.value,
)
}
fn bundled() -> Arc<Catalog> {
catalog(json!({
"gpt-4o": {"litellm_provider": "openai", "input_cost_per_token": 1.0, "max_tokens": 16384},
"retired": {"litellm_provider": "openai", "input_cost_per_token": 1.0}
}))
}
fn composed(fetched: Value) -> IndexMap<String, Fields> {
let mut layers = Layers::new(bundled());
layers.replace_fetched(catalog(fetched));
layers.compose()
}
#[rstest]
fn a_field_the_fetched_row_lacks_is_served_from_bundled() {
let rows = composed(
json!({"gpt-4o": {"litellm_provider": "openai", "input_cost_per_token": 2.0}}),
);
assert_eq!(rows["gpt-4o"]["input_cost_per_token"], json!(2.0));
assert_eq!(rows["gpt-4o"]["max_tokens"], json!(16384));
}
#[rstest]
fn a_null_field_removes_it() {
let rows = composed(json!({"gpt-4o": {"litellm_provider": "openai", "max_tokens": null}}));
assert!(!rows["gpt-4o"].contains_key("max_tokens"));
}
#[rstest]
fn a_key_the_fetched_map_lacks_stays_and_a_null_key_goes() {
let kept = composed(json!({"gpt-4o": {"litellm_provider": "openai"}}));
let retired = composed(json!({"gpt-4o": {"litellm_provider": "openai"}, "retired": null}));
assert!(kept.contains_key("retired"));
assert!(!retired.contains_key("retired"));
}
#[rstest]
fn aliases_are_served_under_every_name_without_the_aliases_field() {
let rows =
composed(json!({"new": {"litellm_provider": "openai", "aliases": ["new-alias"]}}));
assert_eq!(rows["new-alias"], rows["new"]);
assert!(!rows["new"].contains_key("aliases"));
}
}

View file

@ -380,6 +380,7 @@ fn the_bundled_registry_parses_with_no_problems_and_every_key_understood() {
assert!(loaded.errors.is_empty(), "{:#?}", loaded.errors);
let unrecognized: Vec<_> = catalog.unrecognized().collect();
assert!(unrecognized.is_empty(), "{unrecognized:#?}");
assert!(catalog.removals().next().is_none());
assert!(
catalog
.fallback_rules()

View file

@ -48,6 +48,9 @@ litellm-secrets-types.workspace = true
litellm-llms-types.workspace = true
litellm-router-types.workspace = true
litellm-host-python.workspace = true
litellm-model-catalog.workspace = true
indexmap = "2.14.0"
sha1.workspace = true
litellm-token-counter = { path = "../token-counter", default-features = false }
pyo3.workspace = true
prost.workspace = true

View file

@ -8,6 +8,7 @@ mod execution;
mod http;
mod logger;
mod marshal;
mod model_registry;
mod preflight;
mod python_settings;
mod routes;
@ -18,7 +19,10 @@ mod tokenizer;
mod _native {
#[pymodule_export]
use litellm_host_python::{ForkedAfterNativeRuntimeStarted, ProcessReservedForForking};
use pyo3::{prelude::*, types::PyModule};
use pyo3::{
prelude::*,
types::{PyModule, PyTuple},
};
#[cfg(feature = "panic-test")]
#[pymodule_export]
@ -30,6 +34,8 @@ mod _native {
#[pymodule_export]
use crate::logger::NativeDiagnosticProcessor;
#[pymodule_export]
use crate::model_registry::{CatalogHandle, Fetched, Installed, RowError, Unavailable};
#[pymodule_export]
use crate::routes::audio_transcription::{atranscription, transcription};
#[pymodule_export]
use crate::routes::chat_completions::{acompletion, completion};
@ -66,7 +72,18 @@ mod _native {
dict.set_item(
"_SecretManagerRuntime",
py.get_type::<crate::secrets::runtime::NativeSecretManager>(),
)
)?;
let typing = py.import("typing")?;
let literal = typing.getattr("Literal")?;
let row_error_kinds = PyTuple::new(
py,
<crate::model_registry::RowErrorKind as strum::VariantArray>::VARIANTS
.iter()
.map(|kind| literal.get_item(<&'static str>::from(*kind)))
.collect::<PyResult<Vec<_>>>()?,
)?;
let row_error_kind = typing.getattr("Union")?.get_item(row_error_kinds)?;
module.add("RowErrorKind", row_error_kind)
}
}
@ -81,6 +98,35 @@ pub(crate) fn native_module(py: Python<'_>) -> Bound<'_, PyModule> {
mod tests {
use super::*;
fn singleton_literal_values(py: Python<'_>, alias: &Bound<'_, PyAny>) -> (bool, Vec<String>) {
let typing = py.import("typing").unwrap();
let literal = typing.getattr("Literal").unwrap();
let components = typing.call_method1("get_args", (alias,)).unwrap();
let inspected = components
.try_iter()
.unwrap()
.map(|component| {
let component = component.unwrap();
let origin = typing.call_method1("get_origin", (&component,)).unwrap();
let arguments = typing.call_method1("get_args", (&component,)).unwrap();
let values = arguments.extract::<Vec<String>>().unwrap();
(
origin.is(&literal) && values.len() == 1,
values.first().cloned(),
)
})
.collect::<Vec<_>>();
(
inspected
.iter()
.all(|(is_singleton_literal, _)| *is_singleton_literal),
inspected
.into_iter()
.filter_map(|(_, value)| value)
.collect(),
)
}
#[rstest::rstest]
fn module_registration_preserves_the_public_surface() {
Python::initialize();
@ -107,6 +153,12 @@ mod tests {
"aresponses",
"ResponsesWebSocketConnection",
"NativeDiagnosticProcessor",
"Catalog",
"Fetched",
"Installed",
"Unavailable",
"RowError",
"RowErrorKind",
"NativeClickHouseSpendConfig",
"NativeClickHouseSpendStorage",
"trace_encode_error",
@ -132,4 +184,48 @@ mod tests {
assert_eq!(public_names, expected);
});
}
#[rstest::rstest]
fn row_error_kind_is_a_union_of_singleton_literals() {
Python::initialize();
Python::attach(|py| {
let module = native_module(py);
let typing = py.import("typing").unwrap();
let union = typing.getattr("Union").unwrap();
let alias = module.getattr("RowErrorKind").unwrap();
let origin = typing.call_method1("get_origin", (alias.clone(),)).unwrap();
assert!(origin.is(&union));
let (all_singleton_literals, actual) = singleton_literal_values(py, &alias);
assert!(all_singleton_literals);
assert_eq!(
actual,
Vec::from([
"json".to_owned(),
"empty".to_owned(),
"below_minimum".to_owned(),
"shrunk".to_owned(),
"not_an_object".to_owned(),
"invalid".to_owned(),
"unknown_block_key".to_owned(),
"missing_provider".to_owned(),
"unknown_timezone".to_owned(),
"alias_shadows_model".to_owned(),
"alias_taken".to_owned(),
"duplicate_price".to_owned(),
"suffix_inside_block".to_owned(),
"tier_range".to_owned(),
"tier_reversed".to_owned(),
"tier_gap".to_owned(),
"mixed_tier_basis".to_owned(),
"not_priced_by_results".to_owned(),
"bands_from_two_sources".to_owned(),
"not_an_off_peak_rate".to_owned(),
"no_off_peak_prices".to_owned(),
"no_off_peak_hours".to_owned(),
"empty_list".to_owned(),
"invalid_date".to_owned(),
])
);
});
}
}

View file

@ -0,0 +1,298 @@
mod install;
mod row_error;
use std::collections::HashMap;
use std::sync::{Arc, LazyLock, Mutex, PoisonError};
use litellm_host_python::{release_gil, to_py};
use litellm_model_catalog::wire::Fields;
use litellm_model_catalog::{BUNDLED_REGISTRY, Catalog, IntegrityLimits, Layers};
use pyo3::exceptions::PyTypeError;
use pyo3::prelude::*;
use pyo3::pybacked::PyBackedBytes;
use pyo3::types::{PyDict, PyModule};
use serde::Serialize;
use sha1::{Digest, Sha1};
use install::{Record, write_into};
pub(crate) use row_error::{RowError, RowErrorKind, row_error};
static BUNDLED: LazyLock<Arc<Parsed>> = LazyLock::new(|| {
let loaded = Catalog::parse(BUNDLED_REGISTRY).expect("the bundled model registry parses");
let catalog = Arc::new(loaded.value);
Arc::new(Parsed {
raw: Arc::clone(&catalog),
catalog,
errors: loaded.errors.iter().map(row_error).collect(),
rejected: Vec::new(),
})
});
static BUNDLED_REVISION: LazyLock<String> = LazyLock::new(|| git_blob_id(BUNDLED_REGISTRY));
struct Parsed {
/// What install layers over bundled, rows decision 15 rejects replaced or left out
catalog: Arc<Catalog>,
/// The document as written
raw: Arc<Catalog>,
errors: Vec<RowError>,
rejected: Vec<RowError>,
}
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
pub(crate) struct Fetched {
parsed: Arc<Parsed>,
#[pyo3(get)]
revision: String,
#[pyo3(get)]
etag: Option<String>,
}
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
pub(crate) struct Installed {
/// Rows the composed map serves, what main's adopt returns as the fetched map's length
#[pyo3(get)]
count: usize,
/// Keys set because the row differed or was new
#[pyo3(get)]
written: usize,
/// Keys deleted because the map no longer serves them
#[pyo3(get)]
removed: usize,
/// Decision 15, empty for the bundled map
#[pyo3(get)]
rejected: Vec<RowError>,
}
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
pub(crate) struct Unavailable {
#[pyo3(get)]
reason: String,
}
struct State {
layers: Layers,
record: Option<Record>,
}
#[pyclass(frozen, name = "Catalog", module = "litellm.rust_bridge._native")]
pub(crate) struct CatalogHandle {
module: Py<PyModule>,
attr: String,
state: Mutex<State>,
}
#[pymethods]
impl Fetched {
/// The rows of this one document as install would serve them, without layers
fn served(&self, py: Python<'_>) -> PyResult<Py<PyDict>> {
let served = release_gil(py, || served_rows(&self.parsed.catalog));
let rows = PyDict::new(py);
let mut shared: HashMap<String, Py<PyDict>> = HashMap::new();
for (key, owner, fields) in served {
let row = match shared.get(&owner) {
Some(row) => row.clone_ref(py),
None => {
let row = fields_to_dict(py, &fields)?;
shared.insert(owner, row.clone_ref(py));
row
}
};
rows.set_item(key, row)?;
}
Ok(rows.unbind())
}
/// The raw document, `sample_spec` and `fallback_generalizations` included
fn document(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
to_py(py, &WrittenDocument(self.parsed.raw.as_ref()))
}
fn fallback_rules(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
to_py(py, &self.parsed.raw.fallback_rules())
}
fn errors(&self) -> Vec<RowError> {
self.parsed.errors.clone()
}
}
#[pymethods]
impl CatalogHandle {
#[staticmethod]
fn bound_to(module: Bound<'_, PyModule>, attr: String) -> Self {
Self {
module: module.unbind(),
attr,
state: Mutex::new(State {
layers: Layers::new(Arc::clone(&BUNDLED.catalog)),
record: None,
}),
}
}
fn bundled(&self) -> Fetched {
Fetched {
parsed: Arc::clone(&BUNDLED),
revision: BUNDLED_REVISION.clone(),
etag: None,
}
}
/// Parses a fetched body and applies main's whole-map checks against the bundled model count
#[staticmethod]
#[pyo3(signature = (body, *, etag, min_model_count, max_shrink_ratio))]
fn parse(
py: Python<'_>,
body: PyBackedBytes,
etag: Option<String>,
min_model_count: usize,
max_shrink_ratio: f64,
) -> PyResult<Py<PyAny>> {
let parsed = release_gil(py, || {
parse_remote(&body, min_model_count, max_shrink_ratio)
});
match parsed {
Ok((parsed, revision)) => Ok(Fetched {
parsed: Arc::new(parsed),
revision,
etag,
}
.into_pyobject(py)?
.into_any()
.unbind()),
Err(reason) => Ok(Unavailable { reason }
.into_pyobject(py)?
.into_any()
.unbind()),
}
}
/// Writes the composed rows into the dict bound to the attribute, never over a field the user owns (decision 25)
fn install(&self, py: Python<'_>, fetched: &Fetched) -> PyResult<Installed> {
let bound = self.module.bind(py).getattr(self.attr.as_str())?;
let dict = bound
.cast::<PyDict>()
.map_err(|_| PyTypeError::new_err(format!("{} must be a dict", self.attr)))?;
let dict_id = dict.as_ptr() as usize;
let (layers, record) = {
let mut state = self.lock();
state
.layers
.replace_fetched(Arc::clone(&fetched.parsed.catalog));
let record = state
.record
.clone()
.filter(|record| record.dict_id == dict_id);
(state.layers.clone(), record)
};
let composed = release_gil(py, || layers.compose());
let outcome = write_into(py, dict, &composed, &layers, record)?;
self.lock().record = Some(outcome.record);
Ok(Installed {
count: composed.len(),
written: outcome.written,
removed: outcome.removed,
rejected: fetched.parsed.rejected.clone(),
})
}
}
impl CatalogHandle {
fn lock(&self) -> std::sync::MutexGuard<'_, State> {
self.state.lock().unwrap_or_else(PoisonError::into_inner)
}
}
fn parse_remote(
body: &[u8],
min_model_count: usize,
max_shrink_ratio: f64,
) -> Result<(Parsed, String), String> {
let limits = IntegrityLimits::new(min_model_count, max_shrink_ratio)
.ok_or_else(|| format!("invalid shrink ratio {max_shrink_ratio}"))?;
let loaded = Catalog::parse(body).map_err(|error| error.to_string())?;
loaded
.value
.check_size(BUNDLED.raw.model_count(), limits)
.map_err(|error| error.to_string())?;
let raw = Arc::new(loaded.value.clone());
let errors = loaded.errors.iter().map(row_error).collect();
let (catalog, rejections) = loaded
.value
.with_rejected_replaced(&loaded.errors, &BUNDLED.raw);
let rejected = rejections
.iter()
.flat_map(|rejection| rejection.errors.iter().copied().map(row_error))
.collect();
Ok((
Parsed {
catalog: Arc::new(catalog),
raw,
errors,
rejected,
},
git_blob_id(body),
))
}
fn git_blob_id(body: &[u8]) -> String {
let mut hasher = Sha1::new();
hasher.update(format!("blob {}\0", body.len()).as_bytes());
hasher.update(body);
hasher
.finalize()
.iter()
.map(|byte| format!("{byte:02x}"))
.collect()
}
fn served_rows(catalog: &Catalog) -> Vec<(String, String, Fields)> {
catalog
.served()
.map(|(key, fields)| {
let owner = catalog
.aliases()
.get(key)
.cloned()
.unwrap_or_else(|| key.to_owned());
let fields = fields
.iter()
.filter(|(field, _)| *field != "aliases")
.map(|(field, value)| (field.clone(), value.clone()))
.collect();
(key.to_owned(), owner, fields)
})
.collect()
}
fn fields_to_dict(py: Python<'_>, fields: &Fields) -> PyResult<Py<PyDict>> {
let row = PyDict::new(py);
for (key, value) in fields {
row.set_item(key, to_py(py, value)?)?;
}
Ok(row.unbind())
}
struct WrittenDocument<'a>(&'a Catalog);
impl Serialize for WrittenDocument<'_> {
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
serializer.collect_map(self.0.written())
}
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use super::*;
#[rstest]
fn the_revision_is_the_git_blob_id_of_the_body() {
assert_eq!(
git_blob_id(b"hello\n"),
"ce013625030ba8dba906f756967f9e9ca394464a"
);
}
}

View file

@ -0,0 +1,234 @@
use std::collections::HashMap;
use indexmap::IndexMap;
use litellm_host_python::{from_py, to_py};
use litellm_model_catalog::Layers;
use litellm_model_catalog::wire::Fields;
use pyo3::prelude::*;
use pyo3::types::PyDict;
use serde_json::Value;
/// The field values litellm last wrote into one dict object (decision 25)
#[derive(Clone, Debug)]
pub(crate) struct Record {
pub(crate) dict_id: usize,
rows: HashMap<String, Fields>,
}
pub(crate) struct Outcome {
pub(crate) record: Record,
pub(crate) written: usize,
pub(crate) removed: usize,
}
/// What the dict holds for one field, as litellm compares it
#[derive(PartialEq)]
enum Held {
Absent,
Json(Value),
/// A value that does not convert to JSON, which only the user can have written
Foreign,
}
/// One pass over the bound dict under one GIL hold. A field is written only while litellm still owns it, meaning the
/// dict holds the value last written there, or nothing where nothing was written. A dict that is not the object of the
/// last install is written in full, which is main's result after rebind then adopt
pub(crate) fn write_into(
py: Python<'_>,
dict: &Bound<'_, PyDict>,
composed: &IndexMap<String, Fields>,
layers: &Layers,
previous: Option<Record>,
) -> PyResult<Outcome> {
let dict_id = dict.as_ptr() as usize;
let Some(previous) = previous else {
return write_in_full(py, dict, composed, layers, dict_id);
};
let mut rows = previous.rows;
let mut written = 0;
let mut removed = 0;
let mut fresh: HashMap<String, Py<PyDict>> = HashMap::new();
let keys: Vec<String> = composed
.keys()
.chain(rows.keys().filter(|key| !composed.contains_key(*key)))
.cloned()
.collect();
for key in keys {
let target = composed.get(&key);
let last = rows.remove(&key);
let Some(current) = dict.get_item(&key)? else {
match (target, last) {
(Some(target), None) => {
dict.set_item(&key, shared_row(py, &mut fresh, layers, &key, target)?)?;
rows.insert(key, target.clone());
written += 1;
}
(_, Some(last)) if target.is_some() => {
rows.insert(key, last);
}
_ => {}
}
continue;
};
let Ok(row) = current.cast::<PyDict>() else {
if let Some(last) = last {
rows.insert(key, last);
}
continue;
};
let last = last.unwrap_or_default();
let (owned, changed) = write_owned_fields(py, row, target, &last)?;
if target.is_none() && changed && row.is_empty() {
dict.del_item(&key)?;
removed += 1;
continue;
}
written += usize::from(changed);
if !owned.is_empty() {
rows.insert(key, owned);
}
}
Ok(Outcome {
record: Record { dict_id, rows },
written,
removed,
})
}
fn write_in_full(
py: Python<'_>,
dict: &Bound<'_, PyDict>,
composed: &IndexMap<String, Fields>,
layers: &Layers,
dict_id: usize,
) -> PyResult<Outcome> {
let stale: Vec<Bound<'_, PyAny>> = dict
.keys()
.iter()
.filter(|key| {
key.extract::<String>()
.map_or(true, |key| !composed.contains_key(&key))
})
.collect();
let removed = stale.len();
for key in stale {
dict.del_item(key)?;
}
let mut fresh: HashMap<String, Py<PyDict>> = HashMap::new();
let mut written = 0;
for (key, target) in composed {
let current = dict.get_item(key)?;
let row = current.as_ref().and_then(|row| row.cast::<PyDict>().ok());
match row {
Some(row) => {
let owned_by_litellm: Fields = row
.iter()
.filter_map(|(field, value)| {
let field = field.extract::<String>().ok()?;
Some((field, from_py::<Value>(&value).ok()?))
})
.collect();
let extra: Vec<String> = row
.keys()
.iter()
.filter_map(|field| field.extract::<String>().ok())
.filter(|field| !owned_by_litellm.contains_key(field))
.collect();
for field in extra {
row.del_item(field)?;
}
let (_, changed) = write_owned_fields(py, row, Some(target), &owned_by_litellm)?;
written += usize::from(changed);
}
None => {
dict.set_item(key, shared_row(py, &mut fresh, layers, key, target)?)?;
written += 1;
}
}
}
Ok(Outcome {
record: Record {
dict_id,
rows: composed
.iter()
.map(|(key, fields)| (key.clone(), fields.clone()))
.collect(),
},
written,
removed,
})
}
/// Sets or removes each field litellm owns, and returns the owned fields as now written and whether any changed
fn write_owned_fields(
py: Python<'_>,
row: &Bound<'_, PyDict>,
target: Option<&Fields>,
last: &Fields,
) -> PyResult<(Fields, bool)> {
let empty = Fields::new();
let target = target.unwrap_or(&empty);
let fields: Vec<&String> = target
.keys()
.chain(last.keys().filter(|field| !target.contains_key(*field)))
.collect();
let mut owned = Fields::new();
let mut changed = false;
for field in fields {
let held = held(row, field)?;
let litellm_owns = match (&held, last.get(field)) {
(Held::Absent, None) => true,
(Held::Json(value), Some(last)) => value == last,
_ => false,
};
if !litellm_owns {
if let Some(last) = last.get(field) {
owned.insert(field.clone(), last.clone());
}
continue;
}
match target.get(field) {
Some(value) => {
if held != Held::Json(value.clone()) {
row.set_item(field, to_py(py, value)?)?;
changed = true;
}
owned.insert(field.clone(), value.clone());
}
None if held != Held::Absent => {
row.del_item(field)?;
changed = true;
}
None => {}
}
}
Ok((owned, changed))
}
fn held(row: &Bound<'_, PyDict>, field: &str) -> PyResult<Held> {
Ok(match row.get_item(field)? {
None => Held::Absent,
Some(value) => from_py::<Value>(&value).map_or(Held::Foreign, Held::Json),
})
}
/// A new row, the same object for every key of one alias group, as main's alias expansion gives them
fn shared_row(
py: Python<'_>,
fresh: &mut HashMap<String, Py<PyDict>>,
layers: &Layers,
key: &str,
fields: &Fields,
) -> PyResult<Py<PyDict>> {
let owner = layers.alias_owner(key).unwrap_or(key).to_owned();
if let Some(row) = fresh.get(&owner) {
return Ok(row.clone_ref(py));
}
let row = PyDict::new(py);
for (field, value) in fields {
row.set_item(field, to_py(py, value)?)?;
}
let row = row.unbind();
fresh.insert(owner, row.clone_ref(py));
Ok(row)
}

View file

@ -0,0 +1,117 @@
use litellm_model_catalog::{Error, Located};
use pyo3::prelude::*;
use strum::{IntoStaticStr, VariantArray};
#[derive(Clone, Copy, Debug, IntoStaticStr, PartialEq, Eq, VariantArray)]
pub(crate) enum RowErrorKind {
#[strum(serialize = "json")]
Json,
#[strum(serialize = "empty")]
Empty,
#[strum(serialize = "below_minimum")]
BelowMinimum,
#[strum(serialize = "shrunk")]
Shrunk,
#[strum(serialize = "not_an_object")]
NotAnObject,
#[strum(serialize = "invalid")]
Invalid,
#[strum(serialize = "unknown_block_key")]
UnknownBlockKey,
#[strum(serialize = "missing_provider")]
MissingProvider,
#[strum(serialize = "unknown_timezone")]
UnknownTimezone,
#[strum(serialize = "alias_shadows_model")]
AliasShadowsModel,
#[strum(serialize = "alias_taken")]
AliasTaken,
#[strum(serialize = "duplicate_price")]
DuplicatePrice,
#[strum(serialize = "suffix_inside_block")]
SuffixInsideBlock,
#[strum(serialize = "tier_range")]
TierRange,
#[strum(serialize = "tier_reversed")]
TierReversed,
#[strum(serialize = "tier_gap")]
TierGap,
#[strum(serialize = "mixed_tier_basis")]
MixedTierBasis,
#[strum(serialize = "not_priced_by_results")]
NotPricedByResults,
#[strum(serialize = "bands_from_two_sources")]
BandsFromTwoSources,
#[strum(serialize = "not_an_off_peak_rate")]
NotAnOffPeakRate,
#[strum(serialize = "no_off_peak_prices")]
NoOffPeakPrices,
#[strum(serialize = "no_off_peak_hours")]
NoOffPeakHours,
#[strum(serialize = "empty_list")]
EmptyList,
#[strum(serialize = "invalid_date")]
InvalidDate,
}
#[pyclass(frozen, skip_from_py_object, module = "litellm.rust_bridge._native")]
#[derive(Clone, Debug)]
pub(crate) struct RowError {
#[pyo3(get)]
pub(crate) key: String,
#[pyo3(get)]
pub(crate) field: Option<String>,
#[pyo3(get)]
pub(crate) kind: String,
#[pyo3(get)]
pub(crate) rejects: bool,
#[pyo3(get)]
pub(crate) message: String,
}
pub(crate) fn row_error(error: &Located) -> RowError {
let mut path = error.path.split('/').skip(1);
let key = unescape_pointer_segment(path.next().unwrap_or_default());
let field = path.map(unescape_pointer_segment).collect::<Vec<_>>();
let field = (!field.is_empty()).then(|| field.join("/"));
RowError {
key,
field,
kind: <&'static str>::from(error_kind(&error.error)).to_owned(),
rejects: litellm_model_catalog::rejects_row(&error.error),
message: error.error.to_string(),
}
}
fn unescape_pointer_segment(segment: &str) -> String {
segment.replace("~1", "/").replace("~0", "~")
}
fn error_kind(error: &Error) -> RowErrorKind {
match error {
Error::Json(_) => RowErrorKind::Json,
Error::Empty => RowErrorKind::Empty,
Error::BelowMinimum { .. } => RowErrorKind::BelowMinimum,
Error::Shrunk { .. } => RowErrorKind::Shrunk,
Error::NotAnObject => RowErrorKind::NotAnObject,
Error::Invalid(_) => RowErrorKind::Invalid,
Error::UnknownBlockKey => RowErrorKind::UnknownBlockKey,
Error::MissingProvider => RowErrorKind::MissingProvider,
Error::UnknownTimezone { .. } => RowErrorKind::UnknownTimezone,
Error::AliasShadowsModel { .. } => RowErrorKind::AliasShadowsModel,
Error::AliasTaken { .. } => RowErrorKind::AliasTaken,
Error::DuplicatePrice { .. } => RowErrorKind::DuplicatePrice,
Error::SuffixInsideBlock { .. } => RowErrorKind::SuffixInsideBlock,
Error::TierRange => RowErrorKind::TierRange,
Error::TierReversed => RowErrorKind::TierReversed,
Error::TierGap { .. } => RowErrorKind::TierGap,
Error::MixedTierBasis => RowErrorKind::MixedTierBasis,
Error::NotPricedByResults => RowErrorKind::NotPricedByResults,
Error::BandsFromTwoSources => RowErrorKind::BandsFromTwoSources,
Error::NotAnOffPeakRate => RowErrorKind::NotAnOffPeakRate,
Error::NoOffPeakPrices => RowErrorKind::NoOffPeakPrices,
Error::NoOffPeakHours => RowErrorKind::NoOffPeakHours,
Error::EmptyList => RowErrorKind::EmptyList,
Error::InvalidDate => RowErrorKind::InvalidDate,
}
}

View file

@ -548,9 +548,14 @@ _key_management_system: Optional["KeyManagementSystem"] = None
#### PII MASKING ####
output_parse_pii: bool = False
#############################################
from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map, mark_litellm_import_complete
from litellm.litellm_core_utils.get_model_cost_map import (
get_model_cost_map as get_model_cost_map,
install_initial_model_cost_map,
mark_litellm_import_complete,
)
model_cost = get_model_cost_map(url=model_cost_map_url)
model_cost: dict[str, dict[str, object]] = {}
install_initial_model_cost_map(url=model_cost_map_url)
cost_discount_config: Dict[str, float] = {} # Provider-specific cost discounts {"vertex_ai": 0.05} = 5% discount
apply_search_free_tier: bool = False
cost_margin_config: Dict[

View file

@ -0,0 +1,10 @@
"""Internal catalog API backed by the Rust ``model-catalog`` crate. Not public, its shape may change in any release"""
import sys
from typing import Final
from litellm.rust_bridge._native import Catalog, Fetched, Installed, RowError, Unavailable
CATALOG: Final = Catalog.bound_to(sys.modules["litellm"], "model_cost")
__all__ = ["CATALOG", "Catalog", "Fetched", "Installed", "RowError", "Unavailable"]

View file

@ -2794,7 +2794,7 @@ def default_image_cost_calculator(
price * units
for price_table in price_tables
for cost_key, units in unit_counts
if (price := price_table.get(cost_key)) is not None
if isinstance(price := price_table.get(cost_key), (int, float))
),
None,
)

View file

@ -10,8 +10,6 @@ export LITELLM_LOCAL_MODEL_COST_MAP=True
"""
import asyncio
import hashlib
import json
import os
import random
import sys
@ -19,19 +17,19 @@ import threading
import time
import urllib.parse
from collections.abc import Awaitable, Callable, Mapping
from dataclasses import dataclass, replace
from dataclasses import dataclass
from datetime import datetime, timezone
from importlib.resources import files
from itertools import chain
from pathlib import Path
from types import MappingProxyType
from typing import Final, Protocol, cast # noqa: TID251 # adapts boto3 and botocore's dynamic client responses
import httpx
from pydantic import ConfigDict, TypeAdapter
from pydantic import JsonValue, TypeAdapter
from typing_extensions import NotRequired, ReadOnly, TypedDict
from litellm import verbose_logger
from litellm._catalog import CATALOG, Catalog, Fetched, Installed, Unavailable
from litellm.constants import (
MODEL_COST_MAP_MAX_SHRINK_RATIO,
MODEL_COST_MAP_MIN_MODEL_COUNT,
@ -43,8 +41,6 @@ from litellm.litellm_core_utils.offline_mode import is_offline_mode
FALLBACK_GENERALIZATIONS_KEY: Final = "fallback_generalizations"
_CATALOG_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]])
_DECODED_JSON: Final = TypeAdapter(object)
_BUNDLED_CATALOG_ADAPTER: Final = TypeAdapter(dict[str, object], config=ConfigDict(hide_input_in_errors=True))
_CLI_ENTRYPOINT_NAMES: Final = frozenset({"lite", "litellm-proxy"})
@ -52,49 +48,8 @@ def _is_cli_process() -> bool:
return Path(sys.argv[0]).stem in _CLI_ENTRYPOINT_NAMES
# Reserved top-level keys that are not model entries. They must be excluded
# from the model-count integrity check so a real upstream shrink can't be masked.
RESERVED_TOP_LEVEL_KEYS: Final = frozenset({"sample_spec", FALLBACK_GENERALIZATIONS_KEY})
def _count_model_entries(model_cost: dict) -> int:
"""Count actual model entries, excluding reserved meta keys."""
return sum(1 for key in model_cost if key not in RESERVED_TOP_LEVEL_KEYS)
def git_blob_id(body: bytes) -> str:
return hashlib.sha1(b"blob %d\0" % len(body) + body, usedforsecurity=False).hexdigest()
class GetModelCostMap:
"""
Handles fetching, validating, and loading the model cost map.
Only the backup model *count* is cached (a single int). The full
backup dict is never held in memory — it is only parsed when it
needs to be *returned* as a fallback.
"""
_backup_model_count: int = -1 # -1 = not yet loaded
@staticmethod
def read_local_model_cost_map_bytes() -> bytes:
return files("litellm").joinpath("model_prices_and_context_window_backup.json").read_bytes()
@staticmethod
def read_local_model_cost_map_text() -> str:
return GetModelCostMap.read_local_model_cost_map_bytes().decode("utf-8")
@staticmethod
def load_local_model_cost_map_with_revision() -> "ModelCostMapReloaded":
body: Final = GetModelCostMap.read_local_model_cost_map_bytes()
content: Final = _BUNDLED_CATALOG_ADAPTER.validate_python(json.loads(body))
return ModelCostMapReloaded(model_cost_map=content, revision=git_blob_id(body))
@staticmethod
def load_local_model_cost_map() -> dict:
"""Load the local backup model cost map bundled with the package."""
return GetModelCostMap.load_local_model_cost_map_with_revision().model_cost_map
"""The catalog as last loaded, for readers that want it without later registrations"""
_loaded_catalog: Mapping[str, Mapping[str, object]] = MappingProxyType({})
_loaded_catalog_fields: frozenset[str] = frozenset()
@ -115,102 +70,6 @@ class GetModelCostMap:
cls._loaded_catalog = MappingProxyType({key: MappingProxyType(entry) for key, entry in raw.items()})
cls._loaded_catalog_fields = frozenset(chain.from_iterable(entry.keys() for entry in raw.values()))
@classmethod
def get_backup_model_count(cls) -> int:
"""Return the number of models in the local backup (cached int)."""
if cls._backup_model_count < 0:
backup: Final = cls.load_local_model_cost_map()
cls._backup_model_count = _count_model_entries(backup)
return cls._backup_model_count
_get_backup_model_count = get_backup_model_count
@staticmethod
def _check_is_valid_dict(fetched_map: dict) -> bool:
"""Check 1: fetched map is a non-empty dict."""
if not isinstance(fetched_map, dict):
verbose_logger.warning(
"LiteLLM: Fetched model cost map is not a dict (type=%s). Falling back to local backup.",
type(fetched_map).__name__,
)
return False
if len(fetched_map) == 0:
verbose_logger.warning(
"LiteLLM: Fetched model cost map is empty. Falling back to local backup.",
)
return False
return True
@classmethod
def _check_model_count_not_reduced(
cls,
fetched_map: dict,
backup_model_count: int,
min_model_count: int = MODEL_COST_MAP_MIN_MODEL_COUNT,
max_shrink_ratio: float = MODEL_COST_MAP_MAX_SHRINK_RATIO,
) -> bool:
"""Check 2: model count has not reduced significantly vs backup."""
fetched_count: Final = _count_model_entries(fetched_map)
if fetched_count < min_model_count:
verbose_logger.warning(
"LiteLLM: Fetched model cost map has only %d models (minimum=%d). "
"This may indicate a corrupted upstream file. "
"Falling back to local backup.",
fetched_count,
min_model_count,
)
return False
if backup_model_count > 0 and fetched_count < backup_model_count * max_shrink_ratio:
verbose_logger.warning(
"LiteLLM: Fetched model cost map shrank significantly "
"(fetched=%d, backup=%d, threshold=%.0f%%). "
"This may indicate a corrupted upstream file. "
"Falling back to local backup.",
fetched_count,
backup_model_count,
max_shrink_ratio * 100,
)
return False
return True
@classmethod
def validate_model_cost_map(
cls,
fetched_map: dict,
backup_model_count: int,
min_model_count: int = MODEL_COST_MAP_MIN_MODEL_COUNT,
max_shrink_ratio: float = MODEL_COST_MAP_MAX_SHRINK_RATIO,
) -> bool:
"""
Validate the integrity of a fetched model cost map.
Runs each check in order and returns False on the first failure.
Checks:
1. ``_check_is_valid_dict`` -- fetched map is a non-empty dict.
2. ``_check_model_count_not_reduced`` -- model count meets minimum
and has not shrunk >``max_shrink_ratio`` vs backup.
Returns True if all checks pass, False otherwise.
"""
if not cls._check_is_valid_dict(fetched_map):
return False
if not cls._check_model_count_not_reduced(
fetched_map=fetched_map,
backup_model_count=backup_model_count,
min_model_count=min_model_count,
max_shrink_ratio=max_shrink_ratio,
):
return False
return True
RETRYABLE_FETCH_STATUS_CODES: Final = frozenset({429, 500, 502, 503, 504})
MODEL_COST_MAP_FETCH_MAX_ATTEMPTS: Final = 3
@ -224,9 +83,15 @@ def mark_litellm_import_complete() -> None:
@dataclass(frozen=True, slots=True)
class ModelCostMapReloaded:
model_cost_map: dict # mutable-ok: adopted as litellm.model_cost, whose consumer contract is a plain mutable dict
revision: str | None = None
etag: str | None = None
fetched: Fetched
@property
def revision(self) -> str:
return self.fetched.revision
@property
def etag(self) -> str | None:
return self.fetched.etag
@dataclass(frozen=True, slots=True)
@ -314,16 +179,15 @@ def _classify_fetch_error(error: httpx.HTTPError | httpx.InvalidURL, url: str) -
def _classify_fetch_content(content: bytes, url: str, etag: str | None) -> _FetchAttemptOutcome:
try:
parsed: Final = _DECODED_JSON.validate_python(json.loads(content))
except ValueError as e:
return ModelCostMapReloadUnavailable(reason=f"invalid JSON from {url}: {e}")
if not isinstance(parsed, dict):
return ModelCostMapReloadUnavailable(reason=f"expected a JSON object from {url}, got {type(parsed).__name__}")
model_cost_map: Final = cast( # cast-ok: JSON object keys are strings by definition
dict[str, object], parsed
parsed: Final = CATALOG.parse(
content,
etag=etag,
min_model_count=MODEL_COST_MAP_MIN_MODEL_COUNT,
max_shrink_ratio=MODEL_COST_MAP_MAX_SHRINK_RATIO,
)
return ModelCostMapReloaded(model_cost_map=model_cost_map, revision=git_blob_id(content), etag=etag)
if isinstance(parsed, Unavailable):
return ModelCostMapReloadUnavailable(reason=f"model cost map from {url} is unusable: {parsed.reason}")
return ModelCostMapReloaded(fetched=parsed)
def _split_url(url: str) -> urllib.parse.SplitResult | ModelCostMapReloadUnavailable:
@ -465,7 +329,7 @@ async def refetch_model_cost_map(
_cost_map_source_info.url = None
_cost_map_source_info.is_env_forced = True
_cost_map_source_info.fallback_reason = None
return _finalize_loaded_model_cost_map(GetModelCostMap.load_local_model_cost_map_with_revision())
return _note_loaded(ModelCostMapReloaded(fetched=CATALOG.bundled()))
result: Final = await _fetch_remote_model_cost_map_with_retry(
url=url,
@ -481,17 +345,12 @@ async def refetch_model_cost_map(
result.reason,
)
return result
if not GetModelCostMap.validate_model_cost_map(
fetched_map=result.model_cost_map,
backup_model_count=GetModelCostMap.get_backup_model_count(),
):
return ModelCostMapReloadUnavailable(reason=f"model cost map from {url} failed integrity validation")
_cost_map_source_info.loaded_at = datetime.now(timezone.utc)
_cost_map_source_info.source = "remote"
_cost_map_source_info.url = url
_cost_map_source_info.is_env_forced = False
_cost_map_source_info.fallback_reason = None
return _finalize_loaded_model_cost_map(result)
return _note_loaded(result)
class ModelCostMapSourceInfo:
@ -571,103 +430,46 @@ class AliasedCostEntry(dict[str, object]):
self.registered_fields: frozenset[str] = registered_fields
def _alias_groups(alias_owners: Mapping[str, str]) -> Mapping[str, tuple[str, ...]]:
return MappingProxyType(
{
owner: (owner, *(alias for alias, alias_owner in alias_owners.items() if alias_owner == owner))
for owner in dict.fromkeys(alias_owners.values())
}
def _note_loaded(loaded: ModelCostMapReloaded) -> ModelCostMapReloaded:
_cost_map_source_info.source_revision = loaded.fetched.revision
_cost_map_source_info.etag = loaded.fetched.etag
set_fallback_generalizations(loaded.fetched.fallback_rules())
GetModelCostMap._snapshot_loaded_catalog( # pyright: ignore[reportPrivateUsage] # same module
loaded.fetched.served()
)
return loaded
def _expand_model_aliases(model_cost: dict[str, dict[str, object]]) -> dict[str, dict[str, object]]:
"""
Expand ``aliases`` lists in model cost entries into top-level entries.
class _InstallGeneration:
"""Bumped by every install, so a registration racing a reload into the same dict can rebuild its entry"""
Each alias gets a reference to the **same** ``AliasedCostEntry`` as the
canonical entry, which carries every name it is served under. The
``aliases`` key is removed from the entry so downstream code never sees it.
If an alias collides with an existing canonical entry the alias is
skipped and a warning is logged.
"""
alias_owners: Final[dict[str, str]] = {}
keys_with_aliases: Final[list[str]] = []
for model_name, model_info in model_cost.items():
aliases: object = model_info.get("aliases")
if aliases is None:
continue
keys_with_aliases.append(model_name)
if not isinstance(aliases, list):
verbose_logger.warning(
"LiteLLM model alias field for '%s' is not a list (got %s) — skipping.",
model_name,
type(aliases).__name__,
)
continue
if not aliases:
continue
for alias in aliases:
if alias in model_cost:
verbose_logger.warning(
"LiteLLM model alias conflict: alias '%s' (from '%s') "
"already exists as a canonical entry — skipping.",
alias,
model_name,
)
continue
if alias in alias_owners:
verbose_logger.warning(
"LiteLLM model alias conflict: alias '%s' (from '%s') "
"was already claimed by another entry — skipping.",
alias,
model_name,
)
continue
alias_owners[alias] = model_name
# Remove the ``aliases`` key from entries so it doesn't pollute model info
for key in keys_with_aliases:
model_cost[key].pop("aliases", None)
for owner, names in _alias_groups(alias_owners).items():
model_cost.update(dict.fromkeys(names, AliasedCostEntry(model_cost[owner], names)))
return model_cost
value: int = 0
def _finalize_model_cost_map(model_cost: dict) -> dict:
"""Extract fallback generalizations out of the raw map, then expand aliases.
The ``fallback_generalizations`` block is installed into the generalizations
module and removed from the map so it is never treated as a model entry.
"""
raw: Final = model_cost.pop(FALLBACK_GENERALIZATIONS_KEY, None)
rules: Final = raw.get("rules") if isinstance(raw, dict) else None
set_fallback_generalizations(rules)
return _expand_model_aliases(model_cost)
def model_cost_install_generation() -> int:
return _InstallGeneration.value
def _finalize_loaded_model_cost_map(loaded: ModelCostMapReloaded) -> ModelCostMapReloaded:
_cost_map_source_info.source_revision = loaded.revision
_cost_map_source_info.etag = loaded.etag
finalized: Final = _finalize_model_cost_map(loaded.model_cost_map)
GetModelCostMap._snapshot_loaded_catalog(finalized) # pyright: ignore[reportPrivateUsage] # same module
return replace(loaded, model_cost_map=finalized)
def _install(fetched: Fetched, catalog: Catalog = CATALOG) -> Installed:
installed: Final = catalog.install(fetched)
_InstallGeneration.value += 1
if installed.rejected:
verbose_logger.warning(
"LiteLLM: model cost map rows failed validation and are served from the bundled map or left out: %s",
"; ".join(f"{error.key}: {error.message}" for error in installed.rejected),
)
return installed
def adopt_model_cost_map(
new_model_cost_map: dict, # mutable-ok: public API preserves the mutable cost-map contract
) -> int:
def adopt_model_cost_map(fetched: Fetched) -> int:
import litellm
from litellm import utils
litellm.model_cost = new_model_cost_map
installed: Final = _install(fetched)
utils._invalidate_model_cost_lowercase_map() # pyright: ignore[reportPrivateUsage] # required cache invalidation
litellm.add_known_models(model_cost_map=new_model_cost_map)
fetched_model_count: Final = len(new_model_cost_map) if new_model_cost_map else 0
litellm.add_known_models(model_cost_map=litellm.model_cost)
utils.reapply_runtime_model_cost_registrations()
return fetched_model_count
return installed.count
def _retry_remote_fetch_in_background(
@ -700,45 +502,27 @@ def _retry_remote_fetch_in_background(
)
return
_litellm_import_complete.wait()
if not GetModelCostMap.validate_model_cost_map(
fetched_map=result.model_cost_map,
backup_model_count=GetModelCostMap.get_backup_model_count(),
):
verbose_logger.warning(
"LiteLLM: Fetched model cost map failed integrity check. Using local backup instead. url=%s",
url,
)
return
finalized: Final = _finalize_loaded_model_cost_map(result).model_cost_map
_note_loaded(result)
_cost_map_source_info.source = "remote"
_cost_map_source_info.fallback_reason = None
_cost_map_source_info.loaded_at = datetime.now(timezone.utc)
adopt_model_cost_map(finalized)
adopt_model_cost_map(result.fetched)
except Exception as e: # noqa: BLE001 # a failed background retry must not kill the task; the backup stays
verbose_logger.warning("LiteLLM: Background model cost map retry failed: %s", e)
def get_model_cost_map(
def _pick_model_cost_map(
url: str,
timeout: int = 5,
max_attempts: int = MODEL_COST_MAP_FETCH_MAX_ATTEMPTS,
sleep: Callable[[float], None] = time.sleep,
rng: random.Random | None = None,
client: "_SyncGetClient | None" = None,
) -> dict:
timeout: int,
max_attempts: int,
sleep: Callable[[float], None],
rng: random.Random | None,
client: "_SyncGetClient | None",
) -> Fetched:
"""
Public entry point — returns the model cost map dict.
1. If ``LITELLM_LOCAL_MODEL_COST_MAP`` is set or this is a ``lite`` /
``litellm-proxy`` CLI process, uses the local backup only.
2. Otherwise fetches from ``url``, retrying transient HTTP errors
(429/5xx/transport) with Retry-After-aware backoff in a background
thread, validates integrity, and falls back to the local backup on any
failure.
Only the backup model count is cached (a single int) for validation.
The full backup dict is only parsed when it must be *returned* as a
fallback — it is never held in memory long-term.
The map main loads at import. The bundled map when ``LITELLM_LOCAL_MODEL_COST_MAP`` is set, in offline mode or in a
``lite`` / ``litellm-proxy`` CLI process. Otherwise one remote attempt, with a background retry on a transient
failure, and the bundled map on any failure
"""
_cost_map_source_info.loaded_at = datetime.now(timezone.utc)
# Note: can't use get_secret_bool here — this runs during litellm.__init__
@ -748,7 +532,7 @@ def get_model_cost_map(
_cost_map_source_info.url = None
_cost_map_source_info.is_env_forced = True
_cost_map_source_info.fallback_reason = None
return _finalize_loaded_model_cost_map(GetModelCostMap.load_local_model_cost_map_with_revision()).model_cost_map
return _note_loaded(ModelCostMapReloaded(fetched=CATALOG.bundled())).fetched
_cost_map_source_info.url = url
_cost_map_source_info.is_env_forced = False
@ -779,22 +563,39 @@ def get_model_cost_map(
)
_cost_map_source_info.source = "local"
_cost_map_source_info.fallback_reason = f"Remote fetch failed: {outcome.reason}"
return _finalize_loaded_model_cost_map(GetModelCostMap.load_local_model_cost_map_with_revision()).model_cost_map
content: Final = outcome.model_cost_map
# Validate using cached count (cheap int comparison, no file I/O)
if not GetModelCostMap.validate_model_cost_map(
fetched_map=content,
backup_model_count=GetModelCostMap.get_backup_model_count(),
):
verbose_logger.warning(
"LiteLLM: Fetched model cost map failed integrity check. Using local backup instead. url=%s",
url,
)
_cost_map_source_info.source = "local"
_cost_map_source_info.fallback_reason = "Remote data failed integrity validation"
return _finalize_loaded_model_cost_map(GetModelCostMap.load_local_model_cost_map_with_revision()).model_cost_map
return _note_loaded(ModelCostMapReloaded(fetched=CATALOG.bundled())).fetched
_cost_map_source_info.source = "remote"
_cost_map_source_info.fallback_reason = None
return _finalize_loaded_model_cost_map(outcome).model_cost_map
return _note_loaded(outcome).fetched
def install_initial_model_cost_map(url: str) -> None:
"""Fills ``litellm.model_cost`` at import with the map main's rules pick"""
_install(
_pick_model_cost_map(
url=url,
timeout=5,
max_attempts=MODEL_COST_MAP_FETCH_MAX_ATTEMPTS,
sleep=time.sleep,
rng=None,
client=None,
)
)
def get_model_cost_map(
url: str,
timeout: int = 5,
max_attempts: int = MODEL_COST_MAP_FETCH_MAX_ATTEMPTS,
sleep: Callable[[float], None] = time.sleep,
rng: random.Random | None = None,
client: "_SyncGetClient | None" = None,
) -> dict[str, dict[str, JsonValue]]:
"""
Public entry point, returns a fresh model cost map dict picked by main's rules. A remote body is parsed and
validated by the Rust catalog, and the bundled map is returned on any failure
"""
return _pick_model_cost_map(
url=url, timeout=timeout, max_attempts=max_attempts, sleep=sleep, rng=rng, client=client
).served()

View file

@ -805,7 +805,8 @@ class StandardBuiltInToolCostTracking:
model_key = model_info.get("key") if isinstance(model_info, dict) else getattr(model_info, "key", None)
if model_key and model_key in litellm.model_cost:
return litellm.model_cost[model_key].get("code_interpreter_cost_per_session")
session_cost: Final = litellm.model_cost[model_key].get("code_interpreter_cost_per_session")
return float(session_cost) if isinstance(session_cost, (int, float)) else None
except Exception:
pass

View file

@ -18,7 +18,8 @@ def _input_cost_per_pixel(resolved: ModelInfo) -> float:
shared_entry: Final = litellm.model_cost.get(model_cost_key) if model_cost_key is not None else None
if shared_entry is None:
return 0.0
return shared_entry.get("input_cost_per_pixel") or 0.0
shared_price: Final = shared_entry.get("input_cost_per_pixel")
return float(shared_price) if isinstance(shared_price, (int, float)) else 0.0
def cost_calculator(

View file

@ -200,13 +200,6 @@ def get_cached_model_info():
return _get_model_info
@functools.lru_cache(maxsize=1)
def _get_local_model_cost_map() -> dict:
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
return GetModelCostMap.load_local_model_cost_map()
def pop_bedrock_invoke_output_config_format(request_body: dict[str, object]) -> dict[str, object] | None:
"""
Remove and return Anthropic's nested ``output_config.format`` field.
@ -1532,12 +1525,6 @@ def _bedrock_converse_model_flag(model: str, key: _BedrockConverseModelFlag) ->
flag = model_info.get(key)
if isinstance(flag, bool):
return flag
model_cost_key = model_info.get("key")
if isinstance(model_cost_key, str):
local_flag = _get_local_model_cost_map().get(model_cost_key, {}).get(key)
if isinstance(local_flag, bool):
return local_flag
return None
@ -1579,16 +1566,7 @@ def _get_bedrock_output_config_effort_ceiling(
except Exception:
return None
ceiling = model_info.get("bedrock_output_config_effort_ceiling")
if isinstance(ceiling, str) and ceiling in _BEDROCK_OUTPUT_CONFIG_EFFORT_ORDER:
return ceiling
model_cost_key: Final = model_info.get("key")
if not isinstance(model_cost_key, str):
return None
local_model_info: Final = _get_local_model_cost_map().get(model_cost_key, {})
ceiling = local_model_info.get("bedrock_output_config_effort_ceiling")
ceiling: Final = model_info.get("bedrock_output_config_effort_ceiling")
if isinstance(ceiling, str) and ceiling in _BEDROCK_OUTPUT_CONFIG_EFFORT_ORDER:
return ceiling
return None

View file

@ -12,7 +12,6 @@ import litellm
from litellm._logging import verbose_logger
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook, targets_openai_api
from litellm.litellm_core_utils.core_helpers import normalize_drop_params, process_response_headers
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
safe_convert_created_field,
)
@ -209,7 +208,9 @@ def _bundled_openai_reasoning_models() -> frozenset[str]:
map it fetches), and a lagging entry must never strip `reasoning` from a model this
release knows accepts it.
"""
bundled: Final = _BUNDLED_COST_MAP.validate_json(GetModelCostMap.read_local_model_cost_map_text())
from litellm._catalog import CATALOG
bundled: Final = _BUNDLED_COST_MAP.validate_python(CATALOG.bundled().document())
return frozenset(
name
for name, entry in bundled.items()

View file

@ -3,7 +3,6 @@ from collections.abc import Callable, Iterable, Mapping, Sequence
from copy import deepcopy
from datetime import datetime, timezone
from enum import Enum
from functools import lru_cache
from itertools import accumulate, chain, repeat, takewhile
from types import MappingProxyType
from typing import Any, Final, Literal, TypeVar, cast, get_type_hints
@ -77,19 +76,6 @@ def _validate_vertex_ai_lyria_model_info(raw_model_info: object) -> VertexAILyri
return None
@lru_cache(maxsize=1)
def _bundled_vertex_ai_lyria_model_infos() -> Mapping[str, VertexAILyriaModelInfo]:
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
return MappingProxyType(
{
model_key: lyria_model_info
for model_key, raw_model_info in GetModelCostMap.load_local_model_cost_map().items()
if (lyria_model_info := _validate_vertex_ai_lyria_model_info(raw_model_info)) is not None
}
)
def _vertex_ai_lyria_model_key(model: str) -> str:
return model if model.startswith("vertex_ai/") else f"vertex_ai/{model}"
@ -100,18 +86,12 @@ def _vertex_ai_lyria_generation_cost(model_info: VertexAILyriaModelInfo | None)
def get_vertex_ai_lyria_model_info(model: str) -> VertexAILyriaModelInfo | None:
model_key: Final = _vertex_ai_lyria_model_key(model)
runtime_model_info: Final = _validate_vertex_ai_lyria_model_info(litellm.model_cost.get(model_key))
return runtime_model_info or _bundled_vertex_ai_lyria_model_infos().get(model_key)
return _validate_vertex_ai_lyria_model_info(litellm.model_cost.get(model_key))
def get_vertex_ai_lyria_generation_cost(model: str) -> float | None:
model_key: Final = _vertex_ai_lyria_model_key(model)
runtime_cost: Final = _vertex_ai_lyria_generation_cost(
_validate_vertex_ai_lyria_model_info(litellm.model_cost.get(model_key))
)
if runtime_cost is not None:
return runtime_cost
return _vertex_ai_lyria_generation_cost(_bundled_vertex_ai_lyria_model_infos().get(model_key))
return _vertex_ai_lyria_generation_cost(_validate_vertex_ai_lyria_model_info(litellm.model_cost.get(model_key)))
class VertexAIError(BaseLLMException):

View file

@ -12,6 +12,8 @@ from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol
from urllib.parse import urlparse
from pydantic import TypeAdapter, ValidationError
import litellm
from litellm._logging import verbose_logger
from litellm.constants import DEFAULT_MAX_LRU_CACHE_SIZE
@ -57,6 +59,8 @@ else:
GoogleCredentialsObject = Any
_SUPPORTED_REGIONS: Final = TypeAdapter(tuple[str, ...])
class _VertexCredentialsObject(Protocol):
"""Structural view of the google-auth credentials handle that this class caches and refreshes."""
@ -114,8 +118,12 @@ class VertexBase:
# Try to get supported_regions directly from model_cost
# Check both with and without vertex_ai/ prefix
model_key: Final = f"vertex_ai/{model}" if not model.startswith("vertex_ai/") else model
model_info: Final = litellm.model_cost.get(model_key, {})
supported_regions: Final = model_info.get("supported_regions")
try:
supported_regions: Final = _SUPPORTED_REGIONS.validate_python(
litellm.model_cost.get(model_key, {}).get("supported_regions") or ()
)
except ValidationError:
return vertex_region or "us-central1"
if supported_regions and len(supported_regions) > 0:
# If user didn't specify region, use the first supported region

View file

@ -711,7 +711,7 @@ def _is_cost_explicitly_configured(deployments: Sequence[DeploymentTypedDict]) -
"""
for deployment in deployments:
model_id = (deployment.get("model_info") or _EMPTY_COST_ENTRY).get("id")
if model_id is None:
if not isinstance(model_id, str):
continue
raw_entry = litellm.model_cost.get(model_id, _EMPTY_COST_ENTRY)
if "input_cost_per_token" in raw_entry or "output_cost_per_token" in raw_entry:
@ -729,7 +729,7 @@ def _has_explicit_positive_cost(deployments: Sequence[DeploymentTypedDict]) -> b
"""
for deployment in deployments:
model_id = (deployment.get("model_info") or _EMPTY_COST_ENTRY).get("id")
if model_id is None:
if not isinstance(model_id, str):
continue
raw_entry = litellm.model_cost.get(model_id, _EMPTY_COST_ENTRY)
if _is_positive_cost(raw_entry.get("input_cost_per_token")) or _is_positive_cost(

View file

@ -211,6 +211,7 @@ if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
from prisma import models as prisma_models
from litellm._catalog import Fetched
from litellm.integrations.opentelemetry import OpenTelemetry
from litellm.proxy.health_check_utils.shared_health_check_manager import SharedHealthCheckManager
@ -5385,10 +5386,10 @@ def resolve_classifier_plugin(
return resolved
def _swap_in_model_cost_map(new_model_cost_map: dict) -> int:
def _swap_in_model_cost_map(fetched: "Fetched") -> int:
from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map
return adopt_model_cost_map(new_model_cost_map)
return adopt_model_cost_map(fetched)
def _websearch_handler_params(stored: Mapping[str, object]) -> dict[str, object]:
@ -9104,7 +9105,7 @@ class ProxyConfig:
)
return
models_count = _swap_in_model_cost_map(reload_result.model_cost_map)
models_count = _swap_in_model_cost_map(reload_result.fetched)
self.model_cost_map_loaded_at = current_time
await record_reload_run(prisma_client, MODEL_COST_MAP_RELOAD_PARAM_NAME, current_time)
# Adopted last, so neither a failed fetch nor a failed status write is recorded
@ -21011,7 +21012,7 @@ async def reload_model_cost_map(
detail=f"Failed to reload model cost map: {reload_result.reason}. Current pricing data was kept.",
)
models_count = _swap_in_model_cost_map(reload_result.model_cost_map)
models_count = _swap_in_model_cost_map(reload_result.fetched)
current_time = utc_now()
proxy_config.model_cost_map_loaded_at = current_time
provenance: Final = get_model_cost_map_provenance()

View file

@ -12284,9 +12284,9 @@ class Router:
# tell a resolvable name apart; require usable limits/costs.
_azure_fallback_key = _model if _model.startswith("azure/") else f"azure/{_model}"
_fallback_entry = litellm.model_cost.get(_azure_fallback_key)
_fallback_resolves = _fallback_entry is not None and (
(_fallback_entry.get("max_input_tokens") or 0) > 0
or (_fallback_entry.get("input_cost_per_token") or 0) > 0
_fallback_resolves = _fallback_entry is not None and any(
isinstance(limit, (int, float)) and limit > 0
for limit in (_fallback_entry.get("max_input_tokens"), _fallback_entry.get("input_cost_per_token"))
)
if _fallback_resolves:
verbose_router_logger.debug(

View file

@ -1,6 +1,7 @@
from asyncio import Future
from collections.abc import Mapping, Sequence
from typing import Never, final
from types import ModuleType
from typing import Literal, Never, final
import httpx
from pydantic import JsonValue
@ -15,6 +16,72 @@ class ProcessReservedForForking(RuntimeError): ...
def trace_encode_error(message: str) -> bytes: ...
RowErrorKind = Literal[
"json",
"empty",
"below_minimum",
"shrunk",
"not_an_object",
"invalid",
"unknown_block_key",
"missing_provider",
"unknown_timezone",
"alias_shadows_model",
"alias_taken",
"duplicate_price",
"suffix_inside_block",
"tier_range",
"tier_reversed",
"tier_gap",
"mixed_tier_basis",
"not_priced_by_results",
"bands_from_two_sources",
"not_an_off_peak_rate",
"no_off_peak_prices",
"no_off_peak_hours",
"empty_list",
"invalid_date",
]
@final
class RowError:
key: str
field: str | None
kind: RowErrorKind
rejects: bool
message: str
@final
class Fetched:
revision: str
etag: str | None
def served(self) -> dict[str, dict[str, JsonValue]]: ...
def document(self) -> dict[str, JsonValue]: ...
def fallback_rules(self) -> list[JsonValue] | None: ...
def errors(self) -> list[RowError]: ...
@final
class Installed:
count: int
written: int
removed: int
rejected: list[RowError]
@final
class Unavailable:
reason: str
@final
class Catalog:
@staticmethod
def bound_to(module: ModuleType, attr: str) -> Catalog: ...
def bundled(self) -> Fetched: ...
@staticmethod
def parse(
body: bytes, *, etag: str | None, min_model_count: int, max_shrink_ratio: float
) -> Fetched | Unavailable: ...
def install(self, fetched: Fetched) -> Installed: ...
@final
class NativeClickHouseSpendConfig:
def __new__(cls, database: str, url: str, retention_days: int) -> NativeClickHouseSpendConfig: ...

View file

@ -3830,10 +3830,19 @@ def _registration_attempt(
warning_display_name: str | None,
final_attempt: bool,
) -> _RegistrationWrite | None:
from litellm.litellm_core_utils.get_model_cost_map import model_cost_install_generation
source: Final = _builtin_cost_map()
generation: Final = model_cost_install_generation()
write: Final = _registration_write(key, value, custom_llm_provider, warning_display_name)
if not final_attempt and model_cost_install_generation() != generation:
return None
_replace_cost_map_entry(_builtin_cost_map() if final_attempt else source, write.key, write.entry, value)
return write if final_attempt or litellm.model_cost is source else None
return (
write
if final_attempt or (litellm.model_cost is source and model_cost_install_generation() == generation)
else None
)
def _register_on_current_map(
@ -6041,6 +6050,10 @@ def get_utc_datetime() -> datetime.datetime:
return datetime.datetime.now(datetime.timezone.utc)
def _as_max_tokens(value: object) -> int | None:
return value if isinstance(value, int) else None
def get_max_tokens(model: str) -> int | None:
"""
Get the maximum number of output tokens allowed for a given model.
@ -6080,7 +6093,7 @@ def get_max_tokens(model: str) -> int | None:
try:
if model in litellm.model_cost and "max_output_tokens" in litellm.model_cost[model]:
return litellm.model_cost[model]["max_output_tokens"]
return _as_max_tokens(litellm.model_cost[model]["max_output_tokens"])
get_llm_provider: Final = litellm_utils.get_llm_provider
model, custom_llm_provider, _, _ = get_llm_provider(model=model)
if custom_llm_provider == "huggingface":
@ -6088,7 +6101,7 @@ def get_max_tokens(model: str) -> int | None:
return max_tokens
if model in litellm.model_cost: # check if extracted model is in model_list
if "max_output_tokens" in litellm.model_cost[model]:
return litellm.model_cost[model]["max_output_tokens"]
return _as_max_tokens(litellm.model_cost[model]["max_output_tokens"])
else:
raise Exception()
return None
@ -8236,8 +8249,9 @@ def trim_messages(
try:
if max_tokens is None:
# Check if model is valid
if model in litellm.model_cost:
max_tokens_for_model: Final = litellm.model_cost[model]["max_input_tokens"]
trimmed_row: Final = None if model is None else litellm.model_cost.get(model)
max_tokens_for_model: Final = None if trimmed_row is None else trimmed_row.get("max_input_tokens")
if isinstance(max_tokens_for_model, (int, float)):
max_tokens = int(max_tokens_for_model * trim_ratio)
else:
# if user did not specify max (input) tokens
@ -9054,11 +9068,9 @@ def validate_openai_optional_params(stop: str | list[str] | None = None, **kwarg
@lru_cache(maxsize=1)
def get_bundled_model_cost_map() -> dict[str, Any]:
try:
model_cost_path: Final = resources.files("litellm").joinpath("model_prices_and_context_window_backup.json")
return json.loads(model_cost_path.read_text())
except Exception:
return {}
from litellm._catalog import CATALOG
return CATALOG.bundled().document()
_get_bundled_model_cost_map = get_bundled_model_cost_map

View file

@ -335,6 +335,7 @@ include = [
"litellm/proxy/public_endpoints/whats_new.json",
"litellm/proxy/common_utils/codex_base_instructions.md",
"litellm/proxy/common_utils/codex_bundled_models_0.159.3.json",
{ path = "model_prices_and_context_window.json", format = "sdist" },
]
exclude = [
"litellm/proxy/enterprise",
@ -357,6 +358,7 @@ cache-keys = [
{ file = "litellm-rust/Cargo.lock" },
{ file = "litellm-rust/Cargo.toml" },
{ file = "litellm-rust/crates/**/*" },
{ file = "model_prices_and_context_window.json" },
]
constraint-dependencies = [
"tornado>=6.5.8",

View file

@ -1,28 +1,30 @@
import asyncio
import importlib
import os
from collections.abc import Iterator
from pathlib import Path
from typing import Final
from unittest.mock import patch
import respx
import asyncio, httpx, importlib, os
import httpx
import pytest
import respx
import litellm
from litellm import CustomLLM
from litellm.litellm_core_utils import get_llm_provider_logic
from litellm.litellm_core_utils.get_llm_provider_logic import (
get_llm_provider,
resolve_model_and_provider_for_metadata,
get_provider_affinity_header,
inferred_provider,
is_registered_custom_provider,
resolve_model_and_provider_for_metadata,
)
from litellm.llms.custom_httpx.http_handler import HTTPHandler
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.llms.custom_httpx.http_handler import HTTPHandler
from litellm.types.router import LiteLLM_Params
from litellm.utils import _invalidate_model_cost_lowercase_map
from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome
from unittest.mock import patch
CUSTOM_PROVIDER: Final = "test-onprem-llm"
@ -634,15 +636,14 @@ def shipped_generalizations():
added on this branch, so these tests install the rule the branch actually
ships rather than depending on whatever the live URL returns.
"""
from litellm._catalog import CATALOG
from litellm.litellm_core_utils.fallback_generalizations import (
get_fallback_generalization_rules,
set_fallback_generalizations,
)
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
previous = list(get_fallback_generalization_rules())
backup = GetModelCostMap.load_local_model_cost_map()
rules = backup.get("fallback_generalizations", {}).get("rules", [])
rules = CATALOG.bundled().fallback_rules() or []
set_fallback_generalizations(rules)
try:
yield rules

View file

@ -4,16 +4,14 @@ count actual model entries, not reserved meta keys) and the extraction of the
``fallback_generalizations`` block out of the raw map.
"""
import asyncio, importlib, importlib.resources as importlib_get_model, json, litellm
import asyncio, hashlib, importlib, json, litellm
import os
import sys
import threading
from pathlib import Path
from typing import Final
import boto3
import pytest
from pydantic import ValidationError
from typing_extensions import NotRequired, ReadOnly, TypedDict
from litellm.litellm_core_utils import get_model_cost_map as get_model_cost_map_module
@ -23,13 +21,11 @@ from litellm.litellm_core_utils.fallback_generalizations import (
match_routing_generalization,
set_fallback_generalizations,
)
from litellm._catalog import CATALOG, Catalog, Fetched, Unavailable
from litellm.litellm_core_utils.get_model_cost_map import (
FALLBACK_GENERALIZATIONS_KEY,
GetModelCostMap,
_count_model_entries,
_finalize_model_cost_map,
get_model_cost_map_provenance,
git_blob_id,
)
@ -39,75 +35,65 @@ def _load_root_cost_map() -> dict:
return json.load(f)
def git_blob_id(body: bytes) -> str:
return hashlib.sha1(b"blob %d\0" % len(body) + body, usedforsecurity=False).hexdigest()
def _bundled_blob_id() -> str:
path = os.path.join(os.path.dirname(__file__), "../../../litellm/model_prices_and_context_window_backup.json")
path = os.path.join(os.path.dirname(__file__), "../../../model_prices_and_context_window.json")
with open(path, "rb") as f:
return git_blob_id(f.read())
def test_git_blob_id_is_what_git_hash_object_prints():
assert git_blob_id(b'{"gpt-5.4-mini": {"mode": "chat"}}\n') == "18b9a8381e13a3b38a2128f184f631f95829e987"
def _parse(document: dict, min_model_count: int = 50, max_shrink_ratio: float = 0.5) -> Fetched | Unavailable:
return Catalog.parse(
json.dumps(document).encode(),
etag=None,
min_model_count=min_model_count,
max_shrink_ratio=max_shrink_ratio,
)
def test_the_revision_is_what_git_hash_object_prints():
parsed = Catalog.parse(
b'{"gpt-5.4-mini": {"mode": "chat"}}\n', etag=None, min_model_count=0, max_shrink_ratio=0.0
)
assert isinstance(parsed, Fetched)
assert parsed.revision == "18b9a8381e13a3b38a2128f184f631f95829e987"
def test_the_bundled_revision_is_the_root_json_blob_id():
assert CATALOG.bundled().revision == _bundled_blob_id()
def _make_models(n: int) -> dict:
return {f"model-{i}": {"litellm_provider": "openai", "mode": "chat"} for i in range(n)}
def test_count_model_entries_excludes_reserved_keys():
m = _make_models(3)
m["sample_spec"] = {"foo": "bar"}
m[FALLBACK_GENERALIZATIONS_KEY] = {"rules": []}
assert _count_model_entries(m) == 3
def test_validation_rejects_truly_shrunk_file_even_with_meta_keys():
"""A file with only a handful of real models must be rejected as corrupt,
and the extra meta keys must not inflate the count past the minimum."""
"""Meta keys must not inflate the model count past the minimum"""
shrunk = _make_models(5)
shrunk["sample_spec"] = {"foo": "bar"}
shrunk[FALLBACK_GENERALIZATIONS_KEY] = {"rules": [{"name": "x"}]}
assert (
GetModelCostMap.validate_model_cost_map(
fetched_map=shrunk,
backup_model_count=2000,
min_model_count=50,
)
is False
)
assert isinstance(_parse(shrunk, max_shrink_ratio=0.0), Unavailable)
def test_validation_accepts_healthy_file_with_meta_keys():
healthy = _make_models(2000)
healthy = _make_models(len(CATALOG.bundled().served()))
healthy["sample_spec"] = {"foo": "bar"}
healthy[FALLBACK_GENERALIZATIONS_KEY] = {"rules": []}
assert (
GetModelCostMap.validate_model_cost_map(
fetched_map=healthy,
backup_model_count=2000,
min_model_count=50,
)
is True
)
assert isinstance(_parse(healthy), Fetched)
def test_validation_rejects_significant_shrink_vs_backup():
# 600 real models vs a 2000-model backup is below the 50% shrink threshold.
shrunk = _make_models(600)
shrunk[FALLBACK_GENERALIZATIONS_KEY] = {"rules": []}
assert (
GetModelCostMap.validate_model_cost_map(
fetched_map=shrunk,
backup_model_count=2000,
min_model_count=50,
max_shrink_ratio=0.5,
)
is False
)
def test_validation_rejects_significant_shrink_vs_the_bundled_map():
shrunk = _make_models(len(CATALOG.bundled().served()) // 3)
assert isinstance(_parse(shrunk), Unavailable)
def test_finalize_pops_key_and_installs_rules():
def test_a_loaded_map_installs_its_rules_and_never_serves_the_block():
previous = list(get_fallback_generalization_rules())
try:
raw = _make_models(2)
@ -120,21 +106,24 @@ def test_finalize_pops_key_and_installs_rules():
}
]
}
finalized = _finalize_model_cost_map(raw)
fetched = _parse(raw, min_model_count=0, max_shrink_ratio=0.0)
assert isinstance(fetched, Fetched)
# The reserved key is removed from the returned model map ...
assert FALLBACK_GENERALIZATIONS_KEY not in finalized
# ... and its rules are installed into the generalizations module.
get_model_cost_map_module._note_loaded(get_model_cost_map_module.ModelCostMapReloaded(fetched=fetched))
assert FALLBACK_GENERALIZATIONS_KEY not in fetched.served()
assert match_routing_generalization("widget-9") == "openai"
finally:
set_fallback_generalizations(previous)
def test_finalize_with_no_block_clears_rules():
def test_a_loaded_map_with_no_block_clears_rules():
previous = list(get_fallback_generalization_rules())
try:
set_fallback_generalizations([{"name": "stale", "pattern": r"^x", "model_info": {"a": 1}}])
_finalize_model_cost_map(_make_models(2))
fetched = _parse(_make_models(2), min_model_count=0, max_shrink_ratio=0.0)
assert isinstance(fetched, Fetched)
get_model_cost_map_module._note_loaded(get_model_cost_map_module.ModelCostMapReloaded(fetched=fetched))
assert match_capability_generalizations("x-1") is None
finally:
set_fallback_generalizations(previous)
@ -144,7 +133,7 @@ def test_shipped_backup_carries_the_claude_routing_rules():
"""The bundled backup must ship the Claude routing rules so a fresh install
(or an offline fallback) routes unknown Claude models without code changes.
Bedrock-syntax ids must hit the bedrock rule before the bare-id Anthropic rule."""
backup = GetModelCostMap.load_local_model_cost_map()
backup = CATALOG.bundled().document()
rules = backup.get(FALLBACK_GENERALIZATIONS_KEY, {}).get("rules", [])
names = [r.get("name") for r in rules]
assert names.index("bedrock-claude-ids") < names.index("anthropic-claude-ids")
@ -166,7 +155,7 @@ def test_shipped_routing_rules_never_match_through_an_unrecognized_namespace():
``bedrockz/anthropic.claude-...`` resolve to bedrock and slip through a
``bedrock/*`` key, so every shipped routing rule must anchor to the start of
the name and never match an id carrying an unrecognized namespace prefix."""
backup = GetModelCostMap.load_local_model_cost_map()
backup = CATALOG.bundled().document()
rules = backup[FALLBACK_GENERALIZATIONS_KEY]["rules"]
routing_rules = [r for r in rules if "litellm_provider" in r["model_info"]]
@ -204,7 +193,7 @@ def test_shipped_backup_marks_claude_4_6_plus_adaptive_not_4_0():
it, so an unmapped sub-4.6 name resolves but stays non-adaptive. The adaptive rule
carries only its delta; capability unioning stacks it onto the baseline, so the
baseline block is never duplicated across rules and no rule needs ``extends``."""
backup = GetModelCostMap.load_local_model_cost_map()
backup = CATALOG.bundled().document()
rules = backup[FALLBACK_GENERALIZATIONS_KEY]["rules"]
baseline_rule = next(r for r in rules if r.get("name") == "claude-family-baseline")
@ -314,7 +303,7 @@ async def test_refetch_retries_429_honoring_retry_after():
sleeper = _SleepRecorder()
result = await refetch_model_cost_map(url=_URL, sleep=sleeper, rng=random.Random(0), client=client)
assert isinstance(result, ModelCostMapReloaded)
assert len(result.model_cost_map) > 100
assert len(result.fetched.served()) > 100
assert calls["count"] == 3
assert sleeper.waits == [7.0, 7.0]
@ -384,6 +373,7 @@ async def test_refetch_invalid_json_fails_immediately():
result = await refetch_model_cost_map(url=_URL, sleep=sleeper, rng=random.Random(0), client=client)
assert isinstance(result, ModelCostMapReloadUnavailable)
assert "invalid JSON" in result.reason
assert "unusable" in result.reason
assert calls["count"] == 1
assert sleeper.waits == []
@ -395,7 +385,7 @@ async def test_refetch_shrunk_map_fails_integrity_not_swapped_in():
client, _calls = _mock_client([httpx.Response(200, content=tiny)])
result = await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0), client=client)
assert isinstance(result, ModelCostMapReloadUnavailable)
assert "integrity validation" in result.reason
assert "below 0.5 of reference count" in result.reason
@pytest.mark.asyncio
@ -414,7 +404,7 @@ async def test_refetch_respects_local_env_override(monkeypatch, flag):
client=httpx.AsyncClient(transport=httpx.MockTransport(_fail)),
)
assert isinstance(result, ModelCostMapReloaded)
assert len(result.model_cost_map) > 100
assert len(result.fetched.served()) > 100
@pytest.mark.asyncio
@ -441,7 +431,7 @@ async def test_loaded_catalog_snapshot_follows_the_fetched_map_and_ignores_later
result = await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0), client=client)
assert isinstance(result, ModelCostMapReloaded)
monkeypatch.setattr(litellm, "model_cost", result.model_cost_map)
monkeypatch.setattr(litellm, "model_cost", result.fetched.served())
litellm.register_model({"gpt-5.4-mini": {"max_input_tokens": 2048}}, persist_across_reloads=False)
assert litellm.model_cost["gpt-5.4-mini"]["max_input_tokens"] == 2048
assert GetModelCostMap.loaded_model_cost_map()["gpt-5.4-mini"]["max_input_tokens"] == 777
@ -555,7 +545,7 @@ def test_get_model_cost_map_fetches_s3_catalog(monkeypatch: pytest.MonkeyPatch)
for name in ("source", "url", "is_env_forced", "fallback_reason", "loaded_at", "source_revision", "etag"):
monkeypatch.setattr(source_info, name, getattr(source_info, name))
s3_content: Final = _make_models(GetModelCostMap.get_backup_model_count())
s3_content: Final = _make_models(len(CATALOG.bundled().served()))
s3_client: Final = _FakeS3Client(json.dumps(s3_content).encode())
monkeypatch.setattr(boto3, "client", lambda *args, **kwargs: s3_client)
@ -623,8 +613,10 @@ def test_boot_load_transient_failure_returns_local_then_background_retry_adopts_
for name in ("source", "url", "is_env_forced", "fallback_reason", "loaded_at", "source_revision", "etag"):
monkeypatch.setattr(source_info, name, getattr(source_info, name))
held = litellm.model_cost
remote_map = _load_root_cost_map()
remote_map["claude-remote-only-test"] = {"litellm_provider": "anthropic", "mode": "chat"}
remote_map["gpt-4o"]["input_cost_per_token"] = 0.125
client, calls = _mock_client(
[
httpx.ConnectError("connection refused"),
@ -650,7 +642,7 @@ def test_boot_load_transient_failure_returns_local_then_background_retry_adopts_
try:
assert len(threads) == 1
assert "claude-remote-only-test" not in cost_map
assert cost_map.keys() == _finalize_model_cost_map(GetModelCostMap.load_local_model_cost_map()).keys()
assert cost_map.keys() == CATALOG.bundled().served().keys()
source = get_model_cost_map_source_info()
assert source["source"] == "local"
assert source["fallback_reason"].startswith("Remote fetch failed:")
@ -661,6 +653,8 @@ def test_boot_load_transient_failure_returns_local_then_background_retry_adopts_
assert sleeper.waits and 2.0 <= sleeper.waits[0] < 3.0
assert calls["count"] == 2
assert "claude-remote-only-test" in litellm.model_cost
assert litellm.model_cost is held
assert held["gpt-4o"]["input_cost_per_token"] == 0.125
assert "claude-remote-only-test" in litellm.anthropic_models
assert "my-runtime-model" in litellm.model_cost
source = get_model_cost_map_source_info()
@ -768,7 +762,8 @@ def test_boot_load_that_fails_the_integrity_check_reports_the_backup_not_the_rej
source = get_model_cost_map_source_info()
assert source["source"] == "local"
assert source["fallback_reason"] == "Remote data failed integrity validation"
assert source["fallback_reason"].startswith("Remote fetch failed:")
assert "below minimum" in source["fallback_reason"]
assert source["etag"] is None
assert source["source_revision"] == _bundled_blob_id()
assert source["source_revision"] != git_blob_id(shrunk_body)
@ -862,119 +857,24 @@ _SCALAR_DEFAULTS = {
"cohere_key": getattr(litellm, "cohere_key", None),
}
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
class TestCheckIsValidDict:
"""Unit tests for _check_is_valid_dict."""
def test_should_reject_non_dict(self):
"""Non-dict should fail."""
assert GetModelCostMap._check_is_valid_dict("not a dict") is False
@pytest.mark.parametrize(
("body", "min_model_count", "max_shrink_ratio", "usable"),
[
pytest.param(b'"not a dict"', 10, 0.5, False, id="not_an_object"),
pytest.param(b"{}", 10, 0.5, False, id="empty"),
pytest.param(b"[1, 2, 3]", 10, 0.5, False, id="list"),
pytest.param(b"null", 10, 0.5, False, id="null"),
pytest.param(json.dumps(_make_models(5)).encode(), 10, 0.0, False, id="below_minimum"),
pytest.param(json.dumps(_make_models(60)).encode(), 10, 0.5, False, id="shrunk"),
pytest.param(json.dumps(_make_models(60)).encode(), 10, 0.0, True, id="no_ratio_floor"),
pytest.param(json.dumps(_make_models(60)).encode(), 61, 0.0, False, id="one_below_minimum"),
],
)
def test_parse_applies_mains_whole_map_checks(body, min_model_count, max_shrink_ratio, usable):
parsed = Catalog.parse(body, etag=None, min_model_count=min_model_count, max_shrink_ratio=max_shrink_ratio)
assert isinstance(parsed, Fetched) is usable
def test_should_reject_empty_dict(self):
"""Empty dict should fail."""
assert GetModelCostMap._check_is_valid_dict({}) is False
def test_should_reject_list(self):
"""List should fail."""
assert GetModelCostMap._check_is_valid_dict([1, 2, 3]) is False
def test_should_reject_none(self):
"""None should fail."""
assert GetModelCostMap._check_is_valid_dict(None) is False
def test_should_accept_non_empty_dict(self):
"""Non-empty dict should pass."""
assert GetModelCostMap._check_is_valid_dict({"model": {}}) is True
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
class TestCheckModelCountNotReduced:
"""Unit tests for _check_model_count_not_reduced."""
def test_should_reject_too_few_models(self):
"""Fetched map with fewer models than min_model_count should fail."""
small_map = {f"model-{i}": {} for i in range(5)}
assert (
GetModelCostMap._check_model_count_not_reduced(
fetched_map=small_map, backup_model_count=0, min_model_count=10
)
is False
)
def test_should_reject_significant_shrinkage(self):
"""Fetched map that shrunk >50% vs backup should fail."""
fetched = {f"model-{i}": {} for i in range(40)} # 40% of 100
assert (
GetModelCostMap._check_model_count_not_reduced(
fetched_map=fetched, backup_model_count=100, min_model_count=10
)
is False
)
def test_should_accept_when_above_threshold(self):
"""Fetched map at 60% of backup (above 50% threshold) should pass."""
fetched = {f"model-{i}": {} for i in range(60)}
assert (
GetModelCostMap._check_model_count_not_reduced(
fetched_map=fetched, backup_model_count=100, min_model_count=10
)
is True
)
def test_should_accept_growth(self):
"""Fetched map larger than backup should pass."""
fetched = {f"model-{i}": {} for i in range(120)}
assert (
GetModelCostMap._check_model_count_not_reduced(
fetched_map=fetched, backup_model_count=100, min_model_count=10
)
is True
)
def test_should_accept_with_empty_backup(self):
"""When backup is empty, only min_model_count matters."""
fetched = {f"model-{i}": {} for i in range(15)}
assert (
GetModelCostMap._check_model_count_not_reduced(
fetched_map=fetched, backup_model_count=0, min_model_count=10
)
is True
)
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
class TestValidateModelCostMap:
"""Unit tests for validate_model_cost_map (combines both checks)."""
def test_should_reject_non_dict(self):
"""Non-dict should fail at check 1."""
assert GetModelCostMap.validate_model_cost_map(fetched_map="not a dict", backup_model_count=0) is False
def test_should_reject_empty_map(self):
"""Empty dict should fail at check 1."""
assert GetModelCostMap.validate_model_cost_map(fetched_map={}, backup_model_count=0) is False
def test_should_reject_significant_shrinkage(self):
"""Should fail at check 2 (shrinkage)."""
fetched = {f"model-{i}": {} for i in range(40)}
assert (
GetModelCostMap.validate_model_cost_map(fetched_map=fetched, backup_model_count=100, min_model_count=10)
is False
)
def test_should_accept_valid_map(self):
"""Should pass both checks."""
fetched = {f"model-{i}": {} for i in range(120)}
assert (
GetModelCostMap.validate_model_cost_map(fetched_map=fetched, backup_model_count=100, min_model_count=10)
is True
)
def test_should_accept_equal_size_map(self):
"""Equal size should pass both checks."""
fetched = {f"model-{i}": {} for i in range(100)}
assert (
GetModelCostMap.validate_model_cost_map(fetched_map=fetched, backup_model_count=100, min_model_count=10)
is True
)
def _fetch_with_single_outcome(monkeypatch, outcome):
from litellm.litellm_core_utils import get_model_cost_map as module
@ -989,7 +889,7 @@ def _fetch_with_single_outcome(monkeypatch, outcome):
def _backup_keys():
return _finalize_model_cost_map(GetModelCostMap.load_local_model_cost_map()).keys()
return CATALOG.bundled().served().keys()
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
@ -1028,7 +928,7 @@ class TestGetModelCostMapFallback:
assert fetches == 1
assert result.keys() == _backup_keys()
assert source["source"] == "local"
assert source["fallback_reason"] == f"Remote fetch failed: expected a JSON object from {_URL}, got list"
assert source["fallback_reason"].startswith(f"Remote fetch failed: model cost map from {_URL} is unusable")
def test_should_fallback_when_fetched_map_shrinks_dramatically(self, monkeypatch):
"""When upstream returns far fewer models than backup, should fall back."""
@ -1039,7 +939,8 @@ class TestGetModelCostMapFallback:
assert fetches == 1
assert result.keys() == _backup_keys()
assert source == {**source, "source": "local", "fallback_reason": "Remote data failed integrity validation"}
assert source["source"] == "local"
assert "below minimum" in source["fallback_reason"]
def test_should_use_local_map_when_env_var_set(self):
"""LITELLM_LOCAL_MODEL_COST_MAP=True should skip remote fetch entirely."""
@ -1056,14 +957,14 @@ class TestBackupModelCostMapExists:
"""Validates the local backup file is always present and valid."""
def test_should_have_backup_file(self):
"""The backup model cost map must exist and be loadable."""
backup = GetModelCostMap.load_local_model_cost_map()
"""The bundled model cost map must be embedded and loadable."""
backup = CATALOG.bundled().served()
assert isinstance(backup, dict)
assert len(backup) > 0, "Backup model cost map is empty"
def test_should_have_minimum_models_in_backup(self):
"""The backup must contain a reasonable number of models."""
backup = GetModelCostMap.load_local_model_cost_map()
backup = CATALOG.bundled().served()
assert len(backup) > 100, f"Backup has only {len(backup)} models, expected > 100"
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
@ -1241,26 +1142,156 @@ def test_get_model_cost_map():
except Exception as e:
pytest.fail(f"An exception occurred: {e}")
@pytest.mark.usefixtures(
"_vcr_outcome_gate_local_testing",
"isolate_litellm_state",
"setup_and_teardown_local_testing",
)
def test_get_backup_model_cost_map():
with importlib_get_model.open_text("litellm", "model_prices_and_context_window_backup.json") as f:
print("inside backup")
content = json.load(f)
print("content", content)
@pytest.mark.parametrize("backup", [b'["only-in-the-backup"]', b'"only-in-the-backup"', b"null"])
def test_a_bundled_backup_that_is_not_a_json_object_is_rejected_without_quoting_it(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, backup: bytes
):
(tmp_path / "model_prices_and_context_window_backup.json").write_bytes(backup)
monkeypatch.setattr(get_model_cost_map_module, "files", lambda _package: tmp_path)
# ---------------------------------------------------------------------------
# Install through the Rust catalog (decisions 1, 15, 23 and 25)
# ---------------------------------------------------------------------------
with pytest.raises(ValidationError) as rejected:
GetModelCostMap.load_local_model_cost_map_with_revision()
import logging
import types
assert "only-in-the-backup" not in str(rejected.value)
@functools.lru_cache(maxsize=1)
def _bundled_document() -> dict:
return CATALOG.bundled().document()
def _remote(edit) -> Fetched:
document = json.loads(json.dumps(_bundled_document()))
edit(document)
parsed = Catalog.parse(json.dumps(document).encode(), etag=None, min_model_count=50, max_shrink_ratio=0.5)
assert isinstance(parsed, Fetched)
return parsed
def _bound_catalog() -> tuple[types.ModuleType, Catalog]:
module = types.ModuleType("fake_litellm")
module.model_cost = {}
catalog = Catalog.bound_to(module, "model_cost")
catalog.install(catalog.bundled())
return module, catalog
def test_sample_spec_is_not_served_but_stays_in_the_bundled_document():
assert "sample_spec" not in litellm.model_cost
assert "sample_spec" in CATALOG.bundled().document()
def test_a_fetched_row_that_fails_validation_is_replaced_by_bundled_or_left_out(caplog):
module, catalog = _bound_catalog()
bundled_gpt_4o = dict(module.model_cost["gpt-4o"])
def edit(document):
document["gpt-4o"]["input_cost_per_token"] = -1
document["acme/new"] = {"litellm_provider": "acme", "input_cost_per_token": "free"}
with caplog.at_level(logging.WARNING, logger="LiteLLM"):
installed = get_model_cost_map_module._install( # pyright: ignore[reportPrivateUsage] # only the install step logs rejected rows
_remote(edit), catalog
)
assert module.model_cost["gpt-4o"] == bundled_gpt_4o
assert "acme/new" not in module.model_cost
assert {error.key for error in installed.rejected} == {"gpt-4o", "acme/new"}
warnings = [record for record in caplog.records if "failed validation" in record.getMessage()]
assert len(warnings) == 1
assert "gpt-4o" in warnings[0].getMessage() and "acme/new" in warnings[0].getMessage()
def test_a_field_the_fetched_row_lacks_is_served_from_bundled():
module, catalog = _bound_catalog()
bundled_max_output_tokens = module.model_cost["gpt-4o"]["max_output_tokens"]
catalog.install(_remote(lambda document: document["gpt-4o"].pop("max_output_tokens")))
assert module.model_cost["gpt-4o"]["max_output_tokens"] == bundled_max_output_tokens
def test_a_key_the_fetched_map_lacks_stays_until_the_map_writes_it_as_null():
module, catalog = _bound_catalog()
catalog.install(_remote(lambda document: document.pop("gpt-4o")))
assert "gpt-4o" in module.model_cost
catalog.install(_remote(lambda document: document.update({"gpt-4o": None})))
assert "gpt-4o" not in module.model_cost
def test_a_null_field_in_the_fetched_map_removes_it():
module, catalog = _bound_catalog()
catalog.install(_remote(lambda document: document["gpt-4o"].update({"max_output_tokens": None})))
assert "max_output_tokens" not in module.model_cost["gpt-4o"]
def _direct_writes(model_cost: dict) -> None:
model_cost["gpt-4o"]["input_cost_per_token"] = 1.0
model_cost["my-own-model"] = {"litellm_provider": "openai", "input_cost_per_token": 3.0}
model_cost["gpt-4o-mini"] = {"litellm_provider": "openai", "input_cost_per_token": 4.0}
del model_cost["gpt-4.1"]
def _changed_upstream(document: dict) -> None:
document["gpt-4o"]["input_cost_per_token"] = 9.0
document["gpt-4o-mini"]["input_cost_per_token"] = 9.0
document["gpt-4o-mini"]["max_input_tokens"] = 1
def test_direct_writes_survive_a_reload():
module, catalog = _bound_catalog()
held = module.model_cost
_direct_writes(module.model_cost)
catalog.install(_remote(_changed_upstream))
assert module.model_cost is held
assert module.model_cost["gpt-4o"]["input_cost_per_token"] == 1.0
assert module.model_cost["my-own-model"]["input_cost_per_token"] == 3.0
assert module.model_cost["gpt-4o-mini"]["input_cost_per_token"] == 4.0
assert "max_input_tokens" not in module.model_cost["gpt-4o-mini"]
assert "gpt-4.1" not in module.model_cost
def test_rebinding_the_dict_then_reloading_drops_direct_writes():
module, catalog = _bound_catalog()
_direct_writes(module.model_cost)
module.model_cost = dict(module.model_cost)
catalog.install(_remote(_changed_upstream))
assert module.model_cost["gpt-4o"]["input_cost_per_token"] == 9.0
assert "my-own-model" not in module.model_cost
assert module.model_cost["gpt-4o-mini"]["max_input_tokens"] == 1
assert "gpt-4.1" in module.model_cost
def test_install_keeps_a_row_object_whose_fields_did_not_change():
module, catalog = _bound_catalog()
row = module.model_cost["gpt-4o"]
catalog.install(_remote(lambda document: document["gpt-4o-mini"].update({"input_cost_per_token": 9.0})))
assert module.model_cost["gpt-4o"] is row
def test_install_refuses_a_bound_attribute_that_is_not_a_dict():
module, catalog = _bound_catalog()
module.model_cost = types.MappingProxyType({})
with pytest.raises(TypeError):
catalog.install(catalog.bundled())
def test_the_local_map_is_the_embedded_one_not_the_backup_file(monkeypatch, tmp_path):
(tmp_path / "model_prices_and_context_window_backup.json").write_text(
json.dumps({"only-in-the-backup": {"litellm_provider": "openai"}})
)
monkeypatch.setattr(get_model_cost_map_module, "files", lambda _package: tmp_path, raising=False)
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
cost_map = get_model_cost_map(url=_URL)
assert "only-in-the-backup" not in cost_map
assert cost_map.keys() == CATALOG.bundled().served().keys()

View file

@ -241,21 +241,15 @@ print("compatible")
@pytest.mark.parametrize("rust", (None, "0", "1"))
def test_tokenization_without_native_extension_stays_offline(tmp_path: Path, rust: str | None) -> None:
def test_tokenization_stays_offline(tmp_path: Path, rust: str | None) -> None:
script: Final = """
import importlib.abc
import sys
sys.path.insert(0, sys.argv[1])
import litellm
def reject_network(event, args):
if event == "socket.connect":
raise AssertionError("tokenizer attempted a network connection")
sys.addaudithook(reject_network)
class Block(importlib.abc.MetaPathFinder):
def find_spec(self, fullname, path=None, target=None):
if fullname == "litellm.rust_bridge._native":
raise ImportError("native extension is unavailable")
sys.meta_path.insert(0, Block())
import litellm
from litellm.rust_bridge.tokenizer import get_encoding
import tiktoken
from tokenizers import Tokenizer

View file

@ -6,7 +6,7 @@ import httpx
import pytest
import litellm
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
from litellm._catalog import CATALOG
from litellm.litellm_core_utils.llm_cost_calc.utils import CostCalculatorUtils
from litellm.llms.azure.azure import AzureChatCompletion
from litellm.llms.azure.image_generation import get_azure_image_generation_config
@ -20,7 +20,7 @@ from litellm.utils import _invalidate_model_cost_lowercase_map, get_optional_par
@pytest.fixture(autouse=True)
def use_local_model_cost_map(monkeypatch):
monkeypatch.setattr(litellm, "model_cost", GetModelCostMap.load_local_model_cost_map())
monkeypatch.setattr(litellm, "model_cost", CATALOG.bundled().served())
litellm.get_model_info.cache_clear()
_invalidate_model_cost_lowercase_map()
yield

View file

@ -8,10 +8,10 @@ the sharded CI (coverage collection runs against this tree).
import json
import os
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
from unittest.mock import AsyncMock, patch, MagicMock
import litellm
from litellm.llms.base_llm.search.transformation import SearchResponse
@ -632,10 +632,10 @@ class TestAgentCoreSearchEdgeCases:
def test_search_cost_lookup_is_mapped(self, monkeypatch):
"""Assert against the map in this checkout: the remote cost map litellm loads by
default only carries providers already released."""
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
from litellm._catalog import CATALOG
from litellm.search.cost_calculator import search_provider_cost_per_query
monkeypatch.setattr(litellm, "model_cost", GetModelCostMap.load_local_model_cost_map())
monkeypatch.setattr(litellm, "model_cost", CATALOG.bundled().served())
price_per_query = litellm.model_cost["agentcore/search"]["input_cost_per_query"]
assert price_per_query > 0
assert search_provider_cost_per_query(

View file

@ -1,9 +1,16 @@
import asyncio, importlib, litellm, litellm.litellm_core_utils.get_model_cost_map as bedrock_govcloud_model_cost_map, pytest
import asyncio
import importlib
from collections.abc import Iterator
from typing import Final
from unittest.mock import Mock, patch
import pytest
from litellm.llms.bedrock.common_utils import(
import litellm
import litellm.litellm_core_utils.get_model_cost_map as bedrock_govcloud_model_cost_map
from litellm import completion
from litellm.llms.bedrock.common_utils import (
AmazonBedrockGlobalConfig,
BedrockModelInfo,
bedrock_native_output_is_opt_in,
@ -16,11 +23,8 @@ from litellm.llms.bedrock.common_utils import(
strip_bedrock_throughput_suffix,
without_bedrock_native_output_opt_in,
)
from collections.abc import Iterator
from litellm import completion
from litellm.llms.bedrock.count_tokens.bedrock_token_counter import BedrockTokenCounter
from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome
from unittest.mock import Mock, patch
# --------------------------------------------------------------------------- #
# get_bedrock_response_stream_shape lazy-load tests #
@ -33,10 +37,8 @@ def _reset_bedrock_response_stream_shape_cache():
import litellm.llms.bedrock.common_utils as mod
mod.get_bedrock_response_stream_shape.cache_clear()
mod._get_local_model_cost_map.cache_clear()
yield
mod.get_bedrock_response_stream_shape.cache_clear()
mod._get_local_model_cost_map.cache_clear()
def test_bedrock_response_stream_shape_lazy_loads_once():
@ -342,9 +344,9 @@ def test_output_config_effort_normalization_uses_model_info_ceiling(monkeypatch)
def test_bundled_bedrock_opus_model_info_declares_output_config_effort_ceiling(
model, expected_ceiling
):
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
from litellm._catalog import CATALOG
model_info = GetModelCostMap.load_local_model_cost_map()[model]
model_info = CATALOG.bundled().served()[model]
assert model_info["bedrock_output_config_effort_ceiling"] == expected_ceiling
@ -2034,6 +2036,7 @@ class TestBedrockGovCloudSupport:
@pytest.mark.parametrize("missing", ["botocore", "unrelated_dependency"])
def test_event_decoder_reports_only_missing_aws_dependency(missing):
from unittest.mock import patch
from litellm.llms.bedrock.common_utils import BedrockEventStreamDecoderBase
failure = ModuleNotFoundError("missing dependency", name=missing)

View file

@ -7,12 +7,11 @@ Moonshot AI is an OpenAI-compatible provider with minor customizations.
from unittest.mock import patch
import pytest
import litellm
import litellm.utils
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
from litellm._catalog import CATALOG
from litellm.llms.moonshot.chat.transformation import MoonshotChatConfig
@ -681,7 +680,7 @@ class TestKimiK26ModelRegistry:
@pytest.fixture(autouse=True)
def model_cost_map(self):
"""Load directly from the bundled backup so tests don't depend on remote fetch."""
return GetModelCostMap.load_local_model_cost_map()
return CATALOG.bundled().served()
class TestMoonshotResponseSchemaSupport:
@ -703,7 +702,7 @@ class TestMoonshotResponseSchemaSupport:
@pytest.fixture(autouse=True)
def model_cost_map(self):
return GetModelCostMap.load_local_model_cost_map()
return CATALOG.bundled().served()
class TestMoonshotReasoningEffort:
@ -713,7 +712,7 @@ class TestMoonshotReasoningEffort:
@pytest.fixture(autouse=True)
def force_local_model_cost(self, monkeypatch):
monkeypatch.setattr(litellm, "model_cost", GetModelCostMap.load_local_model_cost_map())
monkeypatch.setattr(litellm, "model_cost", CATALOG.bundled().served())
@pytest.mark.parametrize("model", ["kimi-k3", "kimi-k2.5", "kimi-k2.6"])
def test_reasoning_model_supports_reasoning_effort(self, model):

View file

@ -64,6 +64,15 @@ _EXPECTED_RESYNC_APPLIED_CONFIG_PARAM_NAMES = frozenset(
_STARTUP_ONLY_CONFIG_PARAM_NAMES = ("environment_variables",)
def _reloaded(cost_map: dict) -> "ModelCostMapReloaded":
from litellm._catalog import Catalog, Fetched
from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded
parsed = Catalog.parse(json.dumps(cost_map).encode(), etag=None, min_model_count=0, max_shrink_ratio=0.0)
assert isinstance(parsed, Fetched)
return ModelCostMapReloaded(fetched=parsed)
class _RecordingRedisClient(Redis):
def __init__(self) -> None:
self.published: List[Tuple[str, str]] = []
@ -860,7 +869,7 @@ async def test_model_cost_map_reload_does_not_publish_config_change() -> None:
with patch(
"litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map",
new=AsyncMock(
return_value=ModelCostMapReloaded(model_cost_map={"gpt-5.2": {"input_cost_per_token": 0.001}})
return_value=_reloaded({"gpt-5.2": {"input_cost_per_token": 0.001}})
),
):
await ProxyConfig()._check_and_reload_model_cost_map(prisma_client=prisma_client)

View file

@ -32,6 +32,15 @@ _ROOT_COST_MAP = Path(__file__).resolve().parents[4] / "model_prices_and_context
# ---------------------------------------------------------------------------
def _reloaded(cost_map: dict) -> "ModelCostMapReloaded":
from litellm._catalog import Catalog, Fetched
from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded
parsed = Catalog.parse(json.dumps(cost_map).encode(), etag=None, min_model_count=0, max_shrink_ratio=0.0)
assert isinstance(parsed, Fetched)
return ModelCostMapReloaded(fetched=parsed)
def _attach_litellm_config(mock_prisma):
"""Attach a litellm_config table mock (not in conftest's _PRISMA_TABLES)."""
table = MagicMock()
@ -55,7 +64,7 @@ def _attach_litellm_config(mock_prisma):
def test_reload_model_cost_map_happy(client, auth_as, monkeypatch, mock_prisma):
"""Admin can trigger a manual reload; handler returns model count + status."""
from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded
from litellm._catalog import CATALOG
from litellm.proxy import proxy_server as ps
from litellm.proxy._types import LitellmUserRoles
@ -65,7 +74,7 @@ def test_reload_model_cost_map_happy(client, auth_as, monkeypatch, mock_prisma):
fake_cost_map = {"gpt-4": {"input_cost": 0.03}, "gpt-3.5": {"input_cost": 0.002}}
monkeypatch.setattr(
"litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map",
AsyncMock(return_value=ModelCostMapReloaded(model_cost_map=fake_cost_map)),
AsyncMock(return_value=_reloaded(fake_cost_map)),
)
monkeypatch.setattr("litellm.add_known_models", lambda model_cost_map=None: None)
monkeypatch.setattr("litellm.model_cost", {}, raising=False)
@ -83,11 +92,12 @@ def test_reload_model_cost_map_happy(client, auth_as, monkeypatch, mock_prisma):
with auth_as(LitellmUserRoles.PROXY_ADMIN):
response = client.post("/reload/model_cost_map")
assert response.status_code == 200
served = len(CATALOG.bundled().served().keys() | fake_cost_map.keys())
body = normalize(response.json(), volatile=_VOLATILE)
assert body == {
"message": "Price data reloaded successfully! 2 models updated.",
"message": f"Price data reloaded successfully! {served} models updated.",
"status": "success",
"models_count": 2,
"models_count": served,
"timestamp": "<VOLATILE>",
**get_model_cost_map_provenance(),
}
@ -103,7 +113,7 @@ def test_reload_model_cost_map_surfaces_the_blob_id_of_the_bytes_served_on_every
import httpx
import litellm
from litellm.litellm_core_utils.get_model_cost_map import _finalize_model_cost_map, git_blob_id
import hashlib
from litellm.proxy import proxy_server as ps
from litellm.proxy._types import LitellmUserRoles
@ -111,7 +121,10 @@ def test_reload_model_cost_map_surfaces_the_blob_id_of_the_bytes_served_on_every
monkeypatch.setattr(ps, "prisma_client", mock_prisma)
monkeypatch.delenv("LITELLM_LOCAL_MODEL_COST_MAP", raising=False)
body = _ROOT_COST_MAP.read_bytes()
expected = {"source_revision": git_blob_id(body), "etag": _SERVED_ETAG}
expected = {
"source_revision": hashlib.sha1(b"blob %d\0" % len(body) + body, usedforsecurity=False).hexdigest(),
"etag": _SERVED_ETAG,
}
served = httpx.Response(200, headers={"ETag": _SERVED_ETAG}, content=body)
monkeypatch.setattr(
"litellm.litellm_core_utils.get_model_cost_map._default_reload_client",
@ -142,7 +155,7 @@ def test_reload_model_cost_map_surfaces_the_blob_id_of_the_bytes_served_on_every
assert {key: status_response.json()[key] for key in expected} == expected
assert public_response.status_code == 200
assert "gpt-4o" in public_response.json()
assert reload_body["models_count"] == len(_finalize_model_cost_map(json.loads(body)))
assert reload_body["models_count"] == len(litellm.model_cost)
def test_reload_model_cost_map_fetch_failure_502_keeps_map(

View file

@ -80,6 +80,15 @@ example_embedding_result = {
}
def _reloaded(cost_map: dict) -> "ModelCostMapReloaded":
from litellm._catalog import Catalog, Fetched
from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded
parsed = Catalog.parse(json.dumps(cost_map).encode(), etag=None, min_model_count=0, max_shrink_ratio=0.0)
assert isinstance(parsed, Fetched)
return ModelCostMapReloaded(fetched=parsed)
def mock_patch_aembedding():
return mock.patch(
"litellm.proxy.proxy_server.llm_router.aembedding",
@ -4551,7 +4560,7 @@ class TestPriceDataReloadAPI:
with patch(
"litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map",
new=AsyncMock(
return_value=ModelCostMapReloaded(model_cost_map={"gpt-3.5-turbo": {"input_cost_per_token": 0.001}})
return_value=_reloaded({"gpt-3.5-turbo": {"input_cost_per_token": 0.001}})
),
):
# Mock the database connection
@ -4569,8 +4578,11 @@ class TestPriceDataReloadAPI:
assert "timestamp" in data
assert "models_count" in data
# The new implementation immediately reloads and returns the count
assert "Price data reloaded successfully! 1 models updated." in data["message"]
assert data["models_count"] == 1
from litellm._catalog import CATALOG
served = len(CATALOG.bundled().served())
assert f"Price data reloaded successfully! {served} models updated." in data["message"]
assert data["models_count"] == served
finally:
# Restore the full model cost map so subsequent tests are not affected
litellm.model_cost = original_model_cost
@ -4822,7 +4834,7 @@ class TestPriceDataReloadIntegration:
try:
with patch(
"litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map",
new=AsyncMock(return_value=ModelCostMapReloaded(model_cost_map=mock_cost_map)),
new=AsyncMock(return_value=_reloaded(mock_cost_map)),
):
# Mock the database connection
with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma:
@ -4895,13 +4907,12 @@ class TestPriceDataReloadIntegration:
) as mock_get_map,
patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now),
):
mock_get_map.return_value = ModelCostMapReloaded(
model_cost_map={"gpt-3.5-turbo": {"input_cost_per_token": 0.001}}
mock_get_map.return_value = _reloaded({"gpt-3.5-turbo": {"input_cost_per_token": 0.001}}
)
asyncio.run(proxy_config._check_and_reload_model_cost_map(mock_prisma))
assert litellm.model_cost["gpt-3.5-turbo"] == {"input_cost_per_token": 0.001}
assert litellm.model_cost["gpt-3.5-turbo"]["input_cost_per_token"] == 0.001
assert proxy_config.model_cost_map_loaded_at == frozen_now
assert mock_prisma.db.litellm_config.update_many.call_args[1] == {
"data": {"last_run_at": frozen_now},
@ -4980,13 +4991,12 @@ class TestPriceDataReloadIntegration:
) as mock_get_map,
patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now),
):
mock_get_map.return_value = ModelCostMapReloaded(
model_cost_map={"gpt-4-test": {"input_cost_per_token": 0.5}}
mock_get_map.return_value = _reloaded({"gpt-4-test": {"input_cost_per_token": 0.5}}
)
asyncio.run(proxy_config._check_and_reload_model_cost_map(mock_prisma))
assert litellm.model_cost["gpt-4-test"] == {"input_cost_per_token": 0.5}
assert litellm.model_cost["gpt-4-test"]["input_cost_per_token"] == 0.5
assert proxy_config.model_cost_map_loaded_at == frozen_now
assert mock_prisma.db.litellm_config.update_many.call_args[1]["data"] == {"last_run_at": frozen_now}
@ -5025,8 +5035,7 @@ class TestPriceDataReloadIntegration:
) as mock_get_map,
patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now),
):
mock_get_map.return_value = ModelCostMapReloaded(
model_cost_map={"gpt-4": {"input_cost_per_token": 0.001}}
mock_get_map.return_value = _reloaded({"gpt-4": {"input_cost_per_token": 0.001}}
)
for _ in range(3):
@ -5069,8 +5078,7 @@ class TestPriceDataReloadIntegration:
) as mock_get_map,
patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now),
):
mock_get_map.return_value = ModelCostMapReloaded(
model_cost_map={"gpt-4": {"input_cost_per_token": 0.001}}
mock_get_map.return_value = _reloaded({"gpt-4": {"input_cost_per_token": 0.001}}
)
asyncio.run(proxy_config._check_and_reload_model_cost_map(mock_prisma))
@ -5105,8 +5113,7 @@ class TestPriceDataReloadIntegration:
) as mock_get_map,
patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now),
):
mock_get_map.return_value = ModelCostMapReloaded(
model_cost_map={"gpt-4": {"input_cost_per_token": 0.001}}
mock_get_map.return_value = _reloaded({"gpt-4": {"input_cost_per_token": 0.001}}
)
asyncio.run(proxy_config._check_and_reload_model_cost_map(mock_prisma))
@ -5147,8 +5154,7 @@ class TestPriceDataReloadIntegration:
) as mock_get_map,
patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now),
):
mock_get_map.return_value = ModelCostMapReloaded(
model_cost_map={"gpt-4": {"input_cost_per_token": 0.1}}
mock_get_map.return_value = _reloaded({"gpt-4": {"input_cost_per_token": 0.1}}
)
asyncio.run(proxy_config._check_and_reload_model_cost_map(mock_prisma))
@ -5236,8 +5242,7 @@ class TestPriceDataReloadIntegration:
patch(
"litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map",
new=AsyncMock(
return_value=ModelCostMapReloaded(
model_cost_map={"gpt-4o": {"litellm_provider": "openai", "mode": "chat"}}
return_value=_reloaded({"gpt-4o": {"litellm_provider": "openai", "mode": "chat"}}
)
),
),
@ -5271,9 +5276,9 @@ class TestPriceDataReloadIntegration:
model_cost={"custom/deployment-model": {"litellm_provider": "custom", "max_input_tokens": 4321}}
)
models_count = _swap_in_model_cost_map({"gpt-4o": {"litellm_provider": "openai", "mode": "chat"}})
models_count = _swap_in_model_cost_map(_reloaded({"gpt-4o": {"litellm_provider": "openai", "mode": "chat"}}).fetched)
assert models_count == 1
assert models_count == len(litellm.model_cost) - 1
assert litellm.model_cost["custom/deployment-model"]["max_input_tokens"] == 4321
finally:
litellm.model_cost = original_model_cost
@ -5311,8 +5316,7 @@ class TestPriceDataReloadIntegration:
patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma,
patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now),
):
mock_get_map.return_value = ModelCostMapReloaded(
model_cost_map={"gpt-4": {"input_cost_per_token": 0.001}}
mock_get_map.return_value = _reloaded({"gpt-4": {"input_cost_per_token": 0.001}}
)
mock_prisma.db.litellm_config.upsert = AsyncMock(
return_value=_reload_schedule_row({}, reload_revision=9)

View file

@ -8,6 +8,7 @@ import pytest
import respx
import litellm
from litellm._catalog import CATALOG
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.search.cost_calculator import search_provider_cost_per_query
@ -29,7 +30,7 @@ SEARCH_MODE_PRICES: Final = {
@pytest.fixture
def _local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(litellm, "model_cost", GetModelCostMap.load_local_model_cost_map())
monkeypatch.setattr(litellm, "model_cost", CATALOG.bundled().served())
@pytest.fixture

View file

@ -14,8 +14,8 @@ import os
import pytest
from litellm._catalog import CATALOG
from litellm.constants import BEDROCK_CONVERSE_MODELS
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
REPO_ROOT = os.path.join(os.path.dirname(__file__), "../..")
@ -64,7 +64,7 @@ FABLE_5_1_VARIANTS = (
def test_fable_5_1_present_in_bundled_backup():
backup = GetModelCostMap.load_local_model_cost_map()
backup = CATALOG.bundled().served()
root = _load_root_cost_map()
for model_name in FABLE_5_1_VARIANTS:
assert model_name in backup, f"Missing from backup cost map: {model_name}"

View file

@ -16,7 +16,7 @@ from typing import Final, cast
import pytest
import litellm
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
from litellm._catalog import CATALOG
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
REPO_ROOT: Final = os.path.join(os.path.dirname(__file__), "../..")
@ -74,7 +74,7 @@ def test_haiku_5_5_rows_allow_disabling_thinking_and_forced_tools(
model_name: str,
) -> None:
root: Final = _load_root_cost_map()
backup: Final = GetModelCostMap.load_local_model_cost_map()
backup: Final = CATALOG.bundled().served()
assert model_name in root
row: Final = root[model_name]
# https://platform.claude.com/docs/en/models/haiku-5-5/whats-new-haiku-5-5 (2026-10-07):

View file

@ -19,8 +19,8 @@ import os
import pytest
from litellm._catalog import CATALOG
from litellm.constants import BEDROCK_CONVERSE_MODELS
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
REPO_ROOT = os.path.join(os.path.dirname(__file__), "../..")
@ -82,7 +82,7 @@ OPUS_5_5_VARIANTS = (
@pytest.mark.parametrize("model_name", OPUS_5_5_VARIANTS)
def test_opus_5_5_present_in_bundled_backup(model_name):
backup = GetModelCostMap.load_local_model_cost_map()
backup = CATALOG.bundled().served()
root = _load_root_cost_map()
assert model_name in backup
assert model_name in root

View file

@ -19,8 +19,8 @@ import os
import pytest
from litellm._catalog import CATALOG
from litellm.constants import BEDROCK_CONVERSE_MODELS
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
REPO_ROOT = os.path.join(os.path.dirname(__file__), "../..")
@ -60,7 +60,7 @@ SONNET_5_5_VARIANTS = (
@pytest.mark.parametrize("model_name", SONNET_5_5_VARIANTS)
def test_sonnet_5_5_present_in_bundled_backup(model_name):
backup = GetModelCostMap.load_local_model_cost_map()
backup = CATALOG.bundled().served()
root = _load_root_cost_map()
assert model_name in backup
assert model_name in root

View file

@ -186,16 +186,32 @@ def test_bot_may_only_touch_the_cost_map_files() -> None:
)
_REMOVED_B: Final = "openrouter/b is removed without a null; write it as null so older wheels stop serving it"
_REMOVED_VISION: Final = (
"openrouter/a.supports_vision is removed without a null; write it as null so older wheels stop serving it"
)
def test_bot_may_not_remove_models() -> None:
head = _snapshot({key: value for key, value in BASE_MAP.items() if key != "openrouter/b"})
assert _failures(head, bot=False) == ()
assert _failures(head) == ("bot PRs may not remove models: openrouter/b",)
assert _failures(head, bot=False) == (_REMOVED_B,)
assert _failures(head) == (_REMOVED_B, "bot PRs may not remove models: openrouter/b")
def test_bot_may_not_remove_fields() -> None:
head = _snapshot({**BASE_MAP, "openrouter/a": _entry()})
assert _failures(head, bot=False) == (_REMOVED_VISION,)
assert _failures(head) == (_REMOVED_VISION, "bot PRs may not remove fields: openrouter/a.supports_vision")
def test_a_human_pr_retires_a_model_or_a_price_field_by_writing_null() -> None:
head = _snapshot({**BASE_MAP, "openrouter/a": {**_entry(), "supports_vision": None}, "openrouter/b": None})
assert _failures(head, bot=False) == ()
def test_a_human_pr_may_drop_a_field_that_is_not_a_price_or_capability() -> None:
head = _snapshot({**BASE_MAP, "openrouter/b": {k: v for k, v in _entry(2e-06).items() if k != "max_tokens"}})
assert _failures(head, bot=False) == ()
assert _failures(head) == ("bot PRs may not remove fields: openrouter/a.supports_vision",)
def test_bot_may_not_change_special_root_keys() -> None:
@ -244,7 +260,7 @@ def _run_guard(repo: Path, base: str, head: str, head_ref: str) -> subprocess.Co
("head_ref", "expected_code", "expected_line"),
[
(BOT_REF, 1, "- bot PRs may not remove models: openrouter/b"),
("litellm_fix_pricing", 0, "cost map guard passed (human PR, file checks only)"),
("litellm_fix_pricing", 1, f"- {_REMOVED_B}"),
],
)
def test_main_reads_both_revisions_from_git(

View file

@ -1,14 +1,28 @@
"""
Tests for the ``aliases`` feature in the model cost map.
The ``_expand_model_aliases`` function processes ``aliases`` lists from model
entries, creating shared dict references for alias entries at load time.
The Rust catalog serves every name in an ``aliases`` list as its own key, sharing one dict with the canonical entry
"""
from unittest.mock import patch
import json
from litellm import verbose_logger
from litellm.litellm_core_utils.get_model_cost_map import _expand_model_aliases
from litellm._catalog import Catalog, Fetched, Unavailable
def _parsed(model_cost: dict) -> Fetched | Unavailable:
return Catalog.parse(json.dumps(model_cost).encode(), etag=None, min_model_count=0, max_shrink_ratio=0.0)
def _expand(model_cost: dict) -> dict:
parsed = _parsed(model_cost)
assert isinstance(parsed, Fetched)
return parsed.served()
def _error_kinds(model_cost: dict) -> list[str]:
parsed = _parsed(model_cost)
assert isinstance(parsed, Fetched)
return [error.kind for error in parsed.errors()]
# ---------------------------------------------------------------------------
@ -29,7 +43,7 @@ class TestExpandModelAliases:
"mode": "chat",
},
}
result = _expand_model_aliases(model_cost)
result = _expand(model_cost)
assert "my-model-20250101" in result
assert result["my-model-20250101"]["input_cost_per_token"] == 1e-06
@ -45,7 +59,7 @@ class TestExpandModelAliases:
"mode": "chat",
},
}
result = _expand_model_aliases(model_cost)
result = _expand(model_cost)
assert "provider/model-v1" in result
assert "provider/model-v2" in result
@ -60,7 +74,7 @@ class TestExpandModelAliases:
"mode": "chat",
},
}
result = _expand_model_aliases(model_cost)
result = _expand(model_cost)
assert result["alias-model"] is result["canonical-model"]
@ -74,7 +88,7 @@ class TestExpandModelAliases:
"mode": "chat",
},
}
result = _expand_model_aliases(model_cost)
result = _expand(model_cost)
assert "aliases" not in result["my-model"]
assert "aliases" not in result["my-model-alias"]
@ -88,7 +102,7 @@ class TestExpandModelAliases:
"mode": "chat",
},
}
result = _expand_model_aliases(model_cost)
result = _expand(model_cost)
assert "plain-model" in result
assert result["plain-model"]["input_cost_per_token"] == 3e-06
@ -104,7 +118,7 @@ class TestExpandModelAliases:
"mode": "chat",
},
}
result = _expand_model_aliases(model_cost)
result = _expand(model_cost)
assert len(result) == 1
assert "model-a" in result
@ -134,15 +148,11 @@ class TestAliasConflicts:
"mode": "chat",
},
}
with patch.object(verbose_logger, "warning") as mock_warn:
result = _expand_model_aliases(model_cost)
result = _expand(model_cost)
# The canonical "model-dated" entry is preserved, not overwritten
assert "model-dated" in result
# Verify a warning about the alias conflict was logged
mock_warn.assert_called()
warning_messages = " ".join(str(c) for c in mock_warn.call_args_list)
assert "alias conflict" in warning_messages.lower()
assert "alias_shadows_model" in _error_kinds(model_cost)
def test_duplicate_alias_across_entries(self):
"""Same alias claimed by two different entries: second one is skipped."""
@ -160,16 +170,12 @@ class TestAliasConflicts:
"mode": "chat",
},
}
with patch.object(verbose_logger, "warning") as mock_warn:
result = _expand_model_aliases(model_cost)
result = _expand(model_cost)
# "shared-alias" should point to model-a (first one wins)
assert "shared-alias" in result
assert result["shared-alias"]["input_cost_per_token"] == 1e-06
# Verify a warning about the alias conflict was logged
mock_warn.assert_called()
warning_messages = " ".join(str(c) for c in mock_warn.call_args_list)
assert "alias conflict" in warning_messages.lower()
assert "alias_taken" in _error_kinds(model_cost)
def test_canonical_entry_not_overwritten_by_alias(self):
"""An alias must never overwrite an existing canonical entry's data."""
@ -187,7 +193,7 @@ class TestAliasConflicts:
"mode": "chat",
},
}
result = _expand_model_aliases(model_cost)
result = _expand(model_cost)
# Original entry must be preserved
assert result["existing-model"]["input_cost_per_token"] == original_cost
@ -211,7 +217,7 @@ class TestAliasIntegration:
"mode": "chat",
},
}
result = _expand_model_aliases(model_cost)
result = _expand(model_cost)
# Mutate via alias
result["alias"]["input_cost_per_token"] = 999
@ -232,7 +238,7 @@ class TestAliasIntegration:
"mode": "chat",
},
}
result = _expand_model_aliases(model_cost)
result = _expand(model_cost)
assert len(result) == 4 # 2 canonical + 2 aliases
assert "alias-1" in result
@ -240,6 +246,5 @@ class TestAliasIntegration:
assert "plain-model" in result
assert "model-with-alias" in result
def test_expand_on_empty_dict(self):
"""Expanding an empty dict returns an empty dict."""
assert _expand_model_aliases({}) == {}
def test_an_empty_map_is_unavailable(self):
assert isinstance(_parsed({}), Unavailable)

View file

@ -1384,13 +1384,6 @@ ALIAS_CASES: Final = (
"is_non_openai_azure_model",
False,
),
(
"litellm.litellm_core_utils.get_model_cost_map",
"GetModelCostMap",
"_get_backup_model_count",
"get_backup_model_count",
False,
),
(
"litellm.litellm_core_utils.health_check_helpers",
"HealthCheckHelpers",

View file

@ -11,6 +11,7 @@ calculations for DB-sourced models with prompt caching pricing.
import copy
import itertools
import json
import os
from collections.abc import Callable, Mapping
from typing import Final, NoReturn
@ -19,11 +20,18 @@ import pytest
import litellm
from litellm.litellm_core_utils.get_model_cost_map import _expand_model_aliases, adopt_model_cost_map
from litellm._catalog import Catalog, Fetched
from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry, adopt_model_cost_map
from litellm.main import _build_custom_pricing_entry
from litellm.utils import _invalidate_model_cost_lowercase_map
def _fetched(cost_map: Mapping[str, object]) -> Fetched:
parsed: Final = Catalog.parse(json.dumps(cost_map).encode(), etag=None, min_model_count=0, max_shrink_ratio=0.0)
assert isinstance(parsed, Fetched)
return parsed
def _snapshot_model_cost_entries(keys):
return {key: copy.deepcopy(litellm.model_cost.get(key)) for key in keys}
@ -1038,11 +1046,12 @@ def test_register_model_keeps_the_prices_a_reload_adopts_while_it_registers(monk
catalog_key: {**catalog_row, "input_cost_per_token": reloaded_input_cost},
}
real_lookup: Final = litellm.utils.get_model_info
reloads: Final[list[int]] = [] # mutable-ok: counts reloads the racing lookup triggered
def lookup_racing_a_price_data_reload(model: str, custom_llm_provider: str | None = None) -> object:
info: Final = real_lookup(model=model, custom_llm_provider=custom_llm_provider)
if litellm.model_cost is not reloaded_cost_map:
adopt_model_cost_map(reloaded_cost_map)
if not reloads:
reloads.append(adopt_model_cost_map(_fetched(reloaded_cost_map)))
return info
with pytest.MonkeyPatch.context() as lookup_patch:
@ -1056,52 +1065,6 @@ def test_register_model_keeps_the_prices_a_reload_adopts_while_it_registers(monk
_invalidate_model_cost_lowercase_map()
def test_register_model_keeps_its_price_when_a_reload_adopts_a_map_as_it_writes(
monkeypatch: pytest.MonkeyPatch,
) -> None:
catalog_key: Final = "au.anthropic.lit-5853-reloaded-as-written-model"
deployment_key: Final = f"bedrock/{catalog_key}"
catalog_row: Final = {
"litellm_provider": "bedrock_converse",
"mode": "chat",
"input_cost_per_token": 3.3e-06,
"output_cost_per_token": 1.65e-05,
}
reloaded_input_cost: Final = 6.6e-06
registered_output_cost: Final = 2e-05
builtin_cost_map: Final = litellm.get_model_cost_map(url="")
reloaded_cost_map: Final = {
**builtin_cost_map,
catalog_key: {**catalog_row, "input_cost_per_token": reloaded_input_cost},
}
class CostMapAReloadReplacesAsItIsWritten(dict[str, dict[str, object]]):
def update(self, other: Mapping[str, dict[str, object]], /) -> None: # pyright: ignore[reportIncompatibleMethodOverride] # registration writes pass one mapping
if litellm.model_cost is self:
adopt_model_cost_map(reloaded_cost_map)
super().update(other)
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
monkeypatch.setattr(
litellm,
"model_cost",
CostMapAReloadReplacesAsItIsWritten({**builtin_cost_map, catalog_key: dict(catalog_row)}),
)
_invalidate_model_cost_lowercase_map()
litellm.register_model(
{deployment_key: {"litellm_provider": "bedrock", "output_cost_per_token": registered_output_cost}},
persist_across_reloads=False,
)
try:
assert litellm.model_cost is reloaded_cost_map
assert litellm.get_model_info(deployment_key)["input_cost_per_token"] == reloaded_input_cost
assert litellm.get_model_info(deployment_key)["output_cost_per_token"] == registered_output_cost
finally:
_invalidate_model_cost_lowercase_map()
def test_register_model_leaves_the_entry_a_lookup_is_walking_intact(monkeypatch: pytest.MonkeyPatch) -> None:
catalog_key: Final = "lit-5853-live-entry-model"
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
@ -1139,21 +1102,14 @@ _ALIAS_KEY: Final = "lit-5853-alias-model"
def _install_aliased_cost_map(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
aliased: Final = AliasedCostEntry(
{"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 2e-06, "output_cost_per_token": 8e-06},
(_CANONICAL_KEY, _ALIAS_KEY),
)
monkeypatch.setattr(
litellm,
"model_cost",
_expand_model_aliases(
{
**litellm.get_model_cost_map(url=""),
_CANONICAL_KEY: {
"aliases": [_ALIAS_KEY],
"litellm_provider": "openai",
"mode": "chat",
"input_cost_per_token": 2e-06,
"output_cost_per_token": 8e-06,
},
}
),
{**litellm.get_model_cost_map(url=""), _CANONICAL_KEY: aliased, _ALIAS_KEY: aliased},
)
_invalidate_model_cost_lowercase_map()

View file

@ -20337,9 +20337,9 @@ class TestTierParamsTheTargetAccepts:
@pytest.fixture(autouse=True)
def force_local_model_cost(self, monkeypatch):
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
from litellm._catalog import CATALOG
monkeypatch.setattr(litellm, "model_cost", GetModelCostMap.load_local_model_cost_map())
monkeypatch.setattr(litellm, "model_cost", CATALOG.bundled().served())
@staticmethod
def _router(model: str) -> litellm.Router: